def _make_block(self, offset):
url, size = self._mlk_url(offset)
url = https_check(url)
headers = self.__generate_headers()
try:
mkblk_retries = int(self.cfg.mkblk_retries)
except ValueError as e:
warning(
'parameter mkblk_retries is invalid, so use default value 3')
mkblk_retries = 3
with open(self.path, 'rb') as f:
bput = readfile(f, offset, self.bput_size)
blkcode, blktext, _ = _post(url=url, headers=headers, data=bput)
while mkblk_retries and self.__need_retry(blkcode):
blkcode, blktext, _ = _post(url=url,
headers=headers,
data=bput)
mkblk_retries -= 1
if blkcode != 200:
result = [offset, blkcode, blktext['message']]
debug('make block fail,code :{0},message :{1}'.format(
blkcode, blktext))
else:
result = self._make_bput(f, blktext['ctx'], offset)
self._record_upload_progress(result, size)
return blkcode
def _make_bput(self, f, ctx, offset):
bputnum = 1
offset_next = offset + self.bput_size
bput_next = readfile(f, offset_next, self.bput_size)
bputcode = 200
bputtext = {'ctx': ctx}
try:
bput_retries = int(self.cfg.bput_retries)
except ValueError as e:
warning(
'parameter bput_retries is invalid, so use default value 3')
bput_retries = 3
while bput_next and bputnum < self.block_size / self.bput_size:
bputcode, bputtext, _ = self._make_bput_post(
ctx, bputnum, bput_next)
while bput_retries and self.__need_retry(bputcode):
bputcode, bputtext, _ = self._make_bput_post(
ctx, bputnum, bput_next)
bput_retries -= 1
if bputcode != 200:
return offset, bputcode, bputtext['message']
ctx = bputtext['ctx']
offset_next = offset + bputtext['offset']
bput_next = readfile(f, offset_next, self.bput_size)
bputnum += 1
return offset, bputcode, bputtext['ctx']
def _make_file(self, ctx_string):
try:
mkfile_retries = int(self.cfg.mkfile_retries)
except ValueError as e:
warning(
u"parameter mkfile_retries is invalid, so use default value 3")
mkfile_retries = 3
url = https_check(self.__file_url())
body = ctx_string #','.join(blkstatus)
headers = self.__generate_headers()
code, text, logid = _post(url=url, headers=headers, data=body)
while mkfile_retries and self.__need_retry(code):
debug('make file fail.retry upload')
code, text, logid = _post(url=url, headers=headers, data=body)
mkfile_retries -= 1
return code, text, logid
def _make_file(self):
try:
mkfile_retries = int(self.cfg.mkfile_retries)
except ValueError as e:
warning(
u"parameter mkfile_retries is invalid, so use default value 3")
mkfile_retries = 3
blkstatus = self._get_blkstatus()
url = https_check(self.__file_url())
body = ','.join(blkstatus)
headers = self.__generate_headers()
code, text, logid = _post(url=url, headers=headers, data=body)
while mkfile_retries and self.__need_retry(code):
code, text, logid = _post(url=url, headers=headers, data=body)
mkfile_retries -= 1
self.recorder.delete_upload_record()
return code, text, logid
def parse_file(self, file, sections=[]):
#debug("ConfigParse: Reading file '%s'" % file)
if type(sections) != type([]):
sections = [sections]
in_our_section = True
r_section = re.compile("^\[([^\]]+)\]")
r_comment = re.compile("^\s*#.*")
r_empty = re.compile("\s*$")
r_data = re.compile("^\s*(?P<key>\w+)\s*=\s*(?P<value>.*)")
r_quotes = re.compile("^\"(.*)\"\s*$") #匹配双引号
with io.open(file, "r", encoding=self.get('encoding', 'UTF-8')) as fp:
for line in fp:
if r_comment.match(line) or r_empty.match(line):
continue
is_section = r_section.match(line)
if is_section:
section = is_section.group()[0]
in_our_section = (section in sections) or (len(sections)
== 0)
continue
is_data = r_data.match(line)
if is_data and in_our_section:
data = is_data.groupdict()
if r_quotes.match(data["value"]):
data["value"] = data["value"][1:-1]
self.__setitem__(data["key"], data["value"])
if data["key"] in ("access_key", "secret_key"):
try: #caiyz 20180315 添加异常处理
print_value = ("%s...%d_chars...%s") % (
data["value"][-2], len(data["value"]) - 3,
data["value"][-1:1])
except IndexError, e:
debug(u"{0} is empty".format(data["key"]))
else:
print_value = data["value"]
#debug("ConfigParser: %s->%s" % (data["key"], print_value))
continue
warning("Ingnoring invalid line in '%s': %s" % (file, line))