def get_pcs_filelist(pcspath, startindex, endindex):
logger.debug('Start get pcs file list(%d-%d) of "%s".' % (startindex, endindex, pcspath))
sycurl = SynCurl()
url = 'https://pcs.baidu.com/rest/2.0/pcs/file'
querydata = {
'method' : 'list',
'access_token': SynConfig.token['access_token'],
'path' : pcspath,
'by' : 'name',
'order' : 'asc',
'limit' : '%d-%d' % (startindex, endindex),
}
retcode, responses = sycurl.request(url, querydata, '', 'GET', SynCurl.Normal)
try:
responses = json.loads(responses)
if retcode != 200 or 'error_code' in responses:
if responses['error_code'] == 31066:
return 31066, []
else:
logger.error('Errno:%d: Get PCS file list of "%s" failed: %s.' % (retcode, pcspath, responses['error_msg']))
return 1, []
return 0, responses['list']
except Exception as e:
logger.error('Get PCS file list of "%s" failed. return code: %d, response body: %s.\n%s\n%s' % (pcspath, retcode, str(responses), e, traceback.format_exc()))
return 1, []
finally:
del responses
logger.debug('Complete get pcs file list(%d-%d) of "%s".' % (startindex, endindex, pcspath))
def rapid_uploadfile(filepath, pcspath):
if os.stat(filepath).st_size <= 262144:
logger.error(
'Rapid upload file "%s" failed: flie size must be greater than or equal to 256KB.'
% (filepath))
return 1
logger.debug('start rapid upload file "%s".' % (filepath))
crc, contentmd5, slicemd5 = BaiduPcsApi.__rapid_checkcode(filepath)
sycurl = SynCurl()
url = 'https://pcs.baidu.com/rest/2.0/pcs/file'
querydata = {
'method': 'rapidupload',
'access_token': SynConfig.token['access_token'],
'path': pcspath,
'content-length': os.stat(filepath).st_size,
'content-md5': contentmd5,
'slice-md5': slicemd5,
'content-crc32': crc,
'ondup': 'newcopy'
}
retcode, responses = sycurl.request(url, querydata, '', 'POST',
SynCurl.Normal)
responses = json.loads(responses)
if retcode != 200 or 'error_code' in responses:
if responses['error_code'] == 31079:
logger.info(
' File md5 not found, will upload the whole file "%s".' %
(filepath))
return 1
else:
logger.error('Errno:%d: Rapid upload file "%s" failed: %s.' %
(retcode, filepath, responses['error_msg']))
return 1
else:
time.sleep(1)
url = 'https://pcs.baidu.com/rest/2.0/pcs/file'
querydata = {
'method': 'meta',
'access_token': SynConfig.token['access_token'],
'path': pcspath
}
retcode, responses = sycurl.request(url, querydata, '', 'GET',
SynCurl.Normal)
responses = json.loads(responses)
if retcode != 200 or 'error_code' in responses:
logger.error(
'Errno:%d: File "%s" is rapid uploaded, but get remote file\'s mate failed: %s.'
% (retcode, filepath, responses['error_msg']))
return 1
responses = responses['list'][0]
if responses['size'] == os.stat(filepath).st_size:
logger.info(' Rapid upload file "%s" completed.' % (filepath))
return 0
else:
logger.error(
'File "%s" is rapid uploaded, but remote file size not equal to local.'
% (filepath))
return 1
def rapid_uploadfile(filepath, pcspath):
if os.stat(filepath).st_size <= 262144:
logger.error('Rapid upload file "%s" failed: flie size must be greater than or equal to 256KB.' % ( filepath))
return 1
logger.debug('start rapid upload file "%s".' % (filepath))
crc, contentmd5, slicemd5 = BaiduPcsApi.__rapid_checkcode(filepath)
sycurl = SynCurl()
url = 'https://pcs.baidu.com/rest/2.0/pcs/file'
querydata = {
'method': 'rapidupload',
'access_token': SynConfig.token['access_token'],
'path': pcspath,
'content-length': os.stat(filepath).st_size,
'content-md5': contentmd5,
'slice-md5': slicemd5,
'content-crc32': crc,
'ondup': 'newcopy'
}
retcode, responses = sycurl.request(url, querydata, '', 'POST', SynCurl.Normal)
responses = json.loads(responses)
if retcode != 200 or 'error_code' in responses:
if responses['error_code'] == 31079:
logger.info(' File md5 not found, will upload the whole file "%s".' % (filepath))
return 1
else:
logger.error('Errno:%d: Rapid upload file "%s" failed: %s.' % (retcode, filepath, responses['error_msg']))
return 1
else:
time.sleep(1)
url = 'https://pcs.baidu.com/rest/2.0/pcs/file'
querydata = {
'method': 'meta',
'access_token': SynConfig.token['access_token'],
'path': pcspath
}
retcode, responses = sycurl.request(url, querydata, '', 'GET', SynCurl.Normal)
responses = json.loads(responses)
if retcode != 200 or 'error_code' in responses:
logger.error('Errno:%d: File "%s" is rapid uploaded, but get remote file\'s mate failed: %s.' % (retcode, filepath, responses['error_msg']))
return 1
responses = responses['list'][0]
if responses['size'] == os.stat(filepath).st_size:
logger.info(' Rapid upload file "%s" completed.' % (filepath))
return 0
else:
logger.error('File "%s" is rapid uploaded, but remote file size not equal to local.' % (filepath))
return 1
def download_file(filepath, pcspath, filerange):
logger.debug('start download file "%s" : range %s.' % (filepath, filerange))
sycurl = SynCurl()
url = 'https://d.pcs.baidu.com/rest/2.0/pcs/file'
querydata = {
'method': 'download',
'access_token': SynConfig.token['access_token'],
'path': pcspath
}
retcode, responses = sycurl.request(url, querydata, filerange, 'GET', SynCurl.Download, filepath)
if (retcode != 200 and retcode != 206) or responses != '':
responses = json.loads(responses)
logger.error('Errno:%d: Download file "%s" failed: %s.' % (retcode, pcspath, responses['error_msg']))
return 1
logger.info(' download file "%s" completed.' % (filepath))
return 0
def slice_upload_tmpfile(filepath, filerange):
logger.debug('start slice upload file "%s".' % (filepath))
sycurl = SynCurl()
url = 'https://c.pcs.baidu.com/rest/2.0/pcs/file'
querydata = {
'method': 'upload',
'access_token': SynConfig.token['access_token'],
'type': 'tmpfile'
}
retcode, responses = sycurl.request(url, querydata, filerange, 'POST', SynCurl.Upload, filepath)
responses = json.loads(responses)
if retcode == 200:
return 0, responses['md5']
else:
logger.error('Errno:%d: Upload file "%s"\'s extra slice failed: %s.' % (retcode, filepath, responses['error_msg']))
return 1, None
def upload_file(filepath, pcspath):
logger.debug('start upload whole file "%s".' % (filepath))
sycurl = SynCurl()
url = 'https://c.pcs.baidu.com/rest/2.0/pcs/file'
querydata = {
'method': 'upload',
'access_token': SynConfig.token['access_token'],
'path': pcspath,
'ondup': 'newcopy'
}
retcode, responses = sycurl.request(url, querydata, '0-%d' % (os.stat(filepath).st_size - 1), 'POST', SynCurl.Upload, filepath)
responses = json.loads(responses)
if retcode != 200 or 'error_code' in responses:
logger.error('Errno:%d: Upload file to pcs failed: %s, %s.' % (retcode, filepath, responses['error_msg']))
return 1
logger.info(' Upload file "%s" completed.' % (filepath))
return 0
def download_file(filepath, pcspath, filerange):
logger.debug('start download file "%s" : range %s.' %
(filepath, filerange))
sycurl = SynCurl()
url = 'https://d.pcs.baidu.com/rest/2.0/pcs/file'
querydata = {
'method': 'download',
'access_token': SynConfig.token['access_token'],
'path': pcspath
}
retcode, responses = sycurl.request(url, querydata, filerange, 'GET',
SynCurl.Download, filepath)
if (retcode != 200 and retcode != 206) or responses != '':
responses = json.loads(responses)
logger.error('Errno:%d: Download file "%s" failed: %s.' %
(retcode, pcspath, responses['error_msg']))
return 1
logger.info(' download file "%s" completed.' % (filepath))
return 0
def slice_upload_tmpfile(filepath, filerange):
logger.debug('start slice upload file "%s".' % (filepath))
sycurl = SynCurl()
url = 'https://c.pcs.baidu.com/rest/2.0/pcs/file'
querydata = {
'method': 'upload',
'access_token': SynConfig.token['access_token'],
'type': 'tmpfile'
}
retcode, responses = sycurl.request(url, querydata, filerange, 'POST',
SynCurl.Upload, filepath)
responses = json.loads(responses)
if retcode == 200:
return 0, responses['md5']
else:
logger.error(
'Errno:%d: Upload file "%s"\'s extra slice failed: %s.' %
(retcode, filepath, responses['error_msg']))
return 1, None
def upload_file(filepath, pcspath):
logger.debug('start upload whole file "%s".' % (filepath))
sycurl = SynCurl()
url = 'https://c.pcs.baidu.com/rest/2.0/pcs/file'
querydata = {
'method': 'upload',
'access_token': SynConfig.token['access_token'],
'path': pcspath,
'ondup': 'newcopy'
}
retcode, responses = sycurl.request(
url, querydata, '0-%d' % (os.stat(filepath).st_size - 1), 'POST',
SynCurl.Upload, filepath)
responses = json.loads(responses)
if retcode != 200 or 'error_code' in responses:
logger.error('Errno:%d: Upload file to pcs failed: %s, %s.' %
(retcode, filepath, responses['error_msg']))
return 1
logger.info(' Upload file "%s" completed.' % (filepath))
return 0
def get_pcs_filelist(pcspath, startindex, endindex):
logger.debug('Start get pcs file list(%d-%d) of "%s".' %
(startindex, endindex, pcspath))
sycurl = SynCurl()
url = 'https://pcs.baidu.com/rest/2.0/pcs/file'
querydata = {
'method': 'list',
'access_token': SynConfig.token['access_token'],
'path': pcspath,
'by': 'name',
'order': 'asc',
'limit': '%d-%d' % (startindex, endindex),
}
retcode, responses = sycurl.request(url, querydata, '', 'GET',
SynCurl.Normal)
try:
responses = json.loads(responses)
if retcode != 200 or 'error_code' in responses:
if responses['error_code'] == 31066:
return 31066, []
else:
logger.error(
'Errno:%d: Get PCS file list of "%s" failed: %s.' %
(retcode, pcspath, responses['error_msg']))
return 1, []
return 0, responses['list']
except Exception as e:
logger.error(
'Get PCS file list of "%s" failed. return code: %d, response body: %s.\n%s\n%s'
%
(pcspath, retcode, str(responses), e, traceback.format_exc()))
return 1, []
finally:
del responses
logger.debug('Complete get pcs file list(%d-%d) of "%s".' %
(startindex, endindex, pcspath))
def request(self,
url,
querydata,
rdata,
method='POST',
rtype=0,
fnname=''):
retrycnt = -1
self.__op = rtype
if querydata:
if 'path' in querydata:
querydata['path'] = querydata['path'].encode('utf-8')
while retrycnt < SynConfig.config['retrytimes']:
retrycnt += 1
logger.debug('Start curl request(%s) %d times for %s.' %
(rdata, retrycnt, fnname))
if self.__op != SynCurl.Normal:
startpos, self.__endpos = rdata.split('-', 1)
startpos = self.__startpos = int(startpos)
self.__endpos = int(self.__endpos)
self.__response = ''
curl = pycurl.Curl()
try:
if querydata:
url += '?%s' % urlencode(querydata)
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.SSL_VERIFYPEER, 0)
curl.setopt(pycurl.SSL_VERIFYHOST, 0)
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.setopt(pycurl.CONNECTTIMEOUT, 15)
curl.setopt(pycurl.LOW_SPEED_LIMIT, 1)
curl.setopt(pycurl.LOW_SPEED_TIME, 30)
curl.setopt(pycurl.USERAGENT, '')
curl.setopt(pycurl.HEADER, 0)
curl.setopt(pycurl.NOSIGNAL, 1)
curl.setopt(pycurl.WRITEFUNCTION, self.__write_data)
starthour, endhour = SynConfig.config[
'speedlimitperiod'].split('-', 1)
starthour = int(starthour)
endhour = int(endhour)
curhour = time.localtime().tm_hour
if (endhour > starthour and starthour <= curhour < endhour
) or (endhour < starthour and
(curhour < starthour or curhour >= endhour)):
curl.setopt(
pycurl.MAX_SEND_SPEED_LARGE,
SynConfig.config['maxsendspeed'] /
SynConfig.config['tasknumber'] /
SynConfig.config['threadnumber'])
curl.setopt(
pycurl.MAX_RECV_SPEED_LARGE,
SynConfig.config['maxrecvspeed'] /
SynConfig.config['tasknumber'] /
SynConfig.config['threadnumber'])
if self.__op == SynCurl.Upload:
curl.setopt(pycurl.UPLOAD, 1)
curl.setopt(pycurl.READFUNCTION, self.__read_data)
curl.setopt(pycurl.INFILESIZE,
self.__endpos - startpos + 1)
with open(fnname, 'rb') as self.__fd:
self.__fd.seek(startpos)
flock(self.__fd, LOCK_SH)
curl.perform()
flock(self.__fd, LOCK_UN)
elif self.__op == SynCurl.Download:
curl.setopt(pycurl.RANGE, rdata)
with open(fnname, 'rb+') as self.__fd:
self.__fd.seek(startpos)
lockf(self.__fd, LOCK_EX, self.__endpos - startpos + 1,
startpos, 0)
curl.perform()
self.__fd.flush()
os.fsync(self.__fd.fileno())
lockf(self.__fd, LOCK_UN, self.__endpos - startpos + 1,
startpos, 0)
else:
curl.setopt(pycurl.CUSTOMREQUEST, method)
if method == 'POST':
curl.setopt(pycurl.POSTFIELDS, urlencode(rdata))
curl.perform()
retcode = curl.getinfo(pycurl.HTTP_CODE)
if retcode < 400 or retcode == 404 or retrycnt == SynConfig.config[
'retrytimes']:
if retcode != 200 and retcode != 206 and self.__response == '':
self.__response = '{"error_code":%d,"error_msg":"Returned by the server is not in the expected results."}' % retcode
return retcode, self.__response
else:
time.sleep(SynConfig.config['retrydelay'])
except pycurl.error as error:
if retrycnt == SynConfig.config['retrytimes']:
(errno, errstr) = error
return errno, '{"error_code":%d,"error_msg":"%s"}' % (
errno, errstr)
except Exception as e:
return -1, '{"error_code":%d,"error_msg":"%s"}' % (
-1, traceback.format_exc().replace('\n', '\\n').replace(
'"', '\''))
finally:
curl.close()
logger.debug('Complete curl request(%s) %d times for %s.' %
(rdata, retrycnt, fnname))
def request(self, url, querydata, rdata, method='POST', rtype=0, fnname=''):
retrycnt = -1
self.__op = rtype
if querydata:
if 'path' in querydata:
querydata['path'] = querydata['path'].encode('utf-8')
while retrycnt < SynConfig.config['retrytimes']:
retrycnt += 1
logger.debug('Start curl request(%s) %d times for %s.' % (rdata, retrycnt, fnname))
if self.__op != SynCurl.Normal:
startpos, self.__endpos = rdata.split('-', 1)
startpos = self.__startpos = int(startpos)
self.__endpos = int(self.__endpos)
self.__response = ''
curl = pycurl.Curl()
try:
if querydata:
url += '?%s' % urlencode(querydata)
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.SSL_VERIFYPEER, 0)
curl.setopt(pycurl.SSL_VERIFYHOST, 0)
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.setopt(pycurl.CONNECTTIMEOUT, 15)
curl.setopt(pycurl.LOW_SPEED_LIMIT, 1)
curl.setopt(pycurl.LOW_SPEED_TIME, 30)
curl.setopt(pycurl.USERAGENT, '')
curl.setopt(pycurl.HEADER, 0)
curl.setopt(pycurl.NOSIGNAL, 1)
curl.setopt(pycurl.WRITEFUNCTION, self.__write_data)
starthour, endhour = SynConfig.config['speedlimitperiod'].split('-', 1)
starthour = int(starthour)
endhour = int(endhour)
curhour = time.localtime().tm_hour
if (endhour > starthour and starthour <= curhour < endhour) or (endhour < starthour and (curhour < starthour or curhour >= endhour)):
curl.setopt(pycurl.MAX_SEND_SPEED_LARGE, SynConfig.config['maxsendspeed'] / SynConfig.config['tasknumber'] / SynConfig.config['threadnumber'])
curl.setopt(pycurl.MAX_RECV_SPEED_LARGE, SynConfig.config['maxrecvspeed'] / SynConfig.config['tasknumber'] / SynConfig.config['threadnumber'])
if self.__op == SynCurl.Upload:
curl.setopt(pycurl.UPLOAD, 1)
curl.setopt(pycurl.READFUNCTION, self.__read_data)
curl.setopt(pycurl.INFILESIZE, self.__endpos - startpos + 1)
with open(fnname, 'rb') as self.__fd:
self.__fd.seek(startpos)
flock(self.__fd, LOCK_SH)
curl.perform()
flock(self.__fd, LOCK_UN)
elif self.__op == SynCurl.Download:
curl.setopt(pycurl.RANGE, rdata)
with open(fnname, 'rb+') as self.__fd:
self.__fd.seek(startpos)
lockf(self.__fd, LOCK_EX, self.__endpos - startpos + 1, startpos, 0)
curl.perform()
self.__fd.flush()
os.fsync(self.__fd.fileno())
lockf(self.__fd, LOCK_UN, self.__endpos - startpos + 1, startpos, 0)
else:
curl.setopt(pycurl.CUSTOMREQUEST, method)
if method == 'POST':
curl.setopt(pycurl.POSTFIELDS, urlencode(rdata))
curl.perform()
retcode = curl.getinfo(pycurl.HTTP_CODE)
if retcode < 400 or retcode == 404 or retrycnt == SynConfig.config['retrytimes']:
if retcode != 200 and retcode != 206 and self.__response == '':
self.__response = '{"error_code":%d,"error_msg":"Returned by the server is not in the expected results."}' % retcode
return retcode, self.__response
else:
time.sleep(SynConfig.config['retrydelay'])
except pycurl.error as error:
if retrycnt == SynConfig.config['retrytimes']:
(errno, errstr) = error
return errno, '{"error_code":%d,"error_msg":"%s"}' % (errno, errstr)
except Exception as e:
return -1, '{"error_code":%d,"error_msg":"%s"}' % (-1, traceback.format_exc().replace('\n', '\\n').replace('"', '\''))
finally:
curl.close()
logger.debug('Complete curl request(%s) %d times for %s.' % (rdata, retrycnt, fnname))