def get_pcs_filelist(pcspath, startindex, endindex): logger.debug('Start get pcs file list(%d-%d) of "%s".' % (startindex, endindex, pcspath)) sycurl = SynCurl() url = 'https://pcs.baidu.com/rest/2.0/pcs/file' querydata = { 'method' : 'list', 'access_token': SynConfig.token['access_token'], 'path' : pcspath, 'by' : 'name', 'order' : 'asc', 'limit' : '%d-%d' % (startindex, endindex), } retcode, responses = sycurl.request(url, querydata, '', 'GET', SynCurl.Normal) try: responses = json.loads(responses) if retcode != 200 or 'error_code' in responses: if responses['error_code'] == 31066: return 31066, [] else: logger.error('Errno:%d: Get PCS file list of "%s" failed: %s.' % (retcode, pcspath, responses['error_msg'])) return 1, [] return 0, responses['list'] except Exception as e: logger.error('Get PCS file list of "%s" failed. return code: %d, response body: %s.\n%s\n%s' % (pcspath, retcode, str(responses), e, traceback.format_exc())) return 1, [] finally: del responses logger.debug('Complete get pcs file list(%d-%d) of "%s".' % (startindex, endindex, pcspath))
def rapid_uploadfile(filepath, pcspath): if os.stat(filepath).st_size <= 262144: logger.error( 'Rapid upload file "%s" failed: flie size must be greater than or equal to 256KB.' % (filepath)) return 1 logger.debug('start rapid upload file "%s".' % (filepath)) crc, contentmd5, slicemd5 = BaiduPcsApi.__rapid_checkcode(filepath) sycurl = SynCurl() url = 'https://pcs.baidu.com/rest/2.0/pcs/file' querydata = { 'method': 'rapidupload', 'access_token': SynConfig.token['access_token'], 'path': pcspath, 'content-length': os.stat(filepath).st_size, 'content-md5': contentmd5, 'slice-md5': slicemd5, 'content-crc32': crc, 'ondup': 'newcopy' } retcode, responses = sycurl.request(url, querydata, '', 'POST', SynCurl.Normal) responses = json.loads(responses) if retcode != 200 or 'error_code' in responses: if responses['error_code'] == 31079: logger.info( ' File md5 not found, will upload the whole file "%s".' % (filepath)) return 1 else: logger.error('Errno:%d: Rapid upload file "%s" failed: %s.' % (retcode, filepath, responses['error_msg'])) return 1 else: time.sleep(1) url = 'https://pcs.baidu.com/rest/2.0/pcs/file' querydata = { 'method': 'meta', 'access_token': SynConfig.token['access_token'], 'path': pcspath } retcode, responses = sycurl.request(url, querydata, '', 'GET', SynCurl.Normal) responses = json.loads(responses) if retcode != 200 or 'error_code' in responses: logger.error( 'Errno:%d: File "%s" is rapid uploaded, but get remote file\'s mate failed: %s.' % (retcode, filepath, responses['error_msg'])) return 1 responses = responses['list'][0] if responses['size'] == os.stat(filepath).st_size: logger.info(' Rapid upload file "%s" completed.' % (filepath)) return 0 else: logger.error( 'File "%s" is rapid uploaded, but remote file size not equal to local.' % (filepath)) return 1
def rapid_uploadfile(filepath, pcspath): if os.stat(filepath).st_size <= 262144: logger.error('Rapid upload file "%s" failed: flie size must be greater than or equal to 256KB.' % ( filepath)) return 1 logger.debug('start rapid upload file "%s".' % (filepath)) crc, contentmd5, slicemd5 = BaiduPcsApi.__rapid_checkcode(filepath) sycurl = SynCurl() url = 'https://pcs.baidu.com/rest/2.0/pcs/file' querydata = { 'method': 'rapidupload', 'access_token': SynConfig.token['access_token'], 'path': pcspath, 'content-length': os.stat(filepath).st_size, 'content-md5': contentmd5, 'slice-md5': slicemd5, 'content-crc32': crc, 'ondup': 'newcopy' } retcode, responses = sycurl.request(url, querydata, '', 'POST', SynCurl.Normal) responses = json.loads(responses) if retcode != 200 or 'error_code' in responses: if responses['error_code'] == 31079: logger.info(' File md5 not found, will upload the whole file "%s".' % (filepath)) return 1 else: logger.error('Errno:%d: Rapid upload file "%s" failed: %s.' % (retcode, filepath, responses['error_msg'])) return 1 else: time.sleep(1) url = 'https://pcs.baidu.com/rest/2.0/pcs/file' querydata = { 'method': 'meta', 'access_token': SynConfig.token['access_token'], 'path': pcspath } retcode, responses = sycurl.request(url, querydata, '', 'GET', SynCurl.Normal) responses = json.loads(responses) if retcode != 200 or 'error_code' in responses: logger.error('Errno:%d: File "%s" is rapid uploaded, but get remote file\'s mate failed: %s.' % (retcode, filepath, responses['error_msg'])) return 1 responses = responses['list'][0] if responses['size'] == os.stat(filepath).st_size: logger.info(' Rapid upload file "%s" completed.' % (filepath)) return 0 else: logger.error('File "%s" is rapid uploaded, but remote file size not equal to local.' % (filepath)) return 1
def download_file(filepath, pcspath, filerange): logger.debug('start download file "%s" : range %s.' % (filepath, filerange)) sycurl = SynCurl() url = 'https://d.pcs.baidu.com/rest/2.0/pcs/file' querydata = { 'method': 'download', 'access_token': SynConfig.token['access_token'], 'path': pcspath } retcode, responses = sycurl.request(url, querydata, filerange, 'GET', SynCurl.Download, filepath) if (retcode != 200 and retcode != 206) or responses != '': responses = json.loads(responses) logger.error('Errno:%d: Download file "%s" failed: %s.' % (retcode, pcspath, responses['error_msg'])) return 1 logger.info(' download file "%s" completed.' % (filepath)) return 0
def slice_upload_tmpfile(filepath, filerange): logger.debug('start slice upload file "%s".' % (filepath)) sycurl = SynCurl() url = 'https://c.pcs.baidu.com/rest/2.0/pcs/file' querydata = { 'method': 'upload', 'access_token': SynConfig.token['access_token'], 'type': 'tmpfile' } retcode, responses = sycurl.request(url, querydata, filerange, 'POST', SynCurl.Upload, filepath) responses = json.loads(responses) if retcode == 200: return 0, responses['md5'] else: logger.error('Errno:%d: Upload file "%s"\'s extra slice failed: %s.' % (retcode, filepath, responses['error_msg'])) return 1, None
def upload_file(filepath, pcspath): logger.debug('start upload whole file "%s".' % (filepath)) sycurl = SynCurl() url = 'https://c.pcs.baidu.com/rest/2.0/pcs/file' querydata = { 'method': 'upload', 'access_token': SynConfig.token['access_token'], 'path': pcspath, 'ondup': 'newcopy' } retcode, responses = sycurl.request(url, querydata, '0-%d' % (os.stat(filepath).st_size - 1), 'POST', SynCurl.Upload, filepath) responses = json.loads(responses) if retcode != 200 or 'error_code' in responses: logger.error('Errno:%d: Upload file to pcs failed: %s, %s.' % (retcode, filepath, responses['error_msg'])) return 1 logger.info(' Upload file "%s" completed.' % (filepath)) return 0
def slice_upload_tmpfile(filepath, filerange): logger.debug('start slice upload file "%s".' % (filepath)) sycurl = SynCurl() url = 'https://c.pcs.baidu.com/rest/2.0/pcs/file' querydata = { 'method': 'upload', 'access_token': SynConfig.token['access_token'], 'type': 'tmpfile' } retcode, responses = sycurl.request(url, querydata, filerange, 'POST', SynCurl.Upload, filepath) responses = json.loads(responses) if retcode == 200: return 0, responses['md5'] else: logger.error( 'Errno:%d: Upload file "%s"\'s extra slice failed: %s.' % (retcode, filepath, responses['error_msg'])) return 1, None
def upload_file(filepath, pcspath): logger.debug('start upload whole file "%s".' % (filepath)) sycurl = SynCurl() url = 'https://c.pcs.baidu.com/rest/2.0/pcs/file' querydata = { 'method': 'upload', 'access_token': SynConfig.token['access_token'], 'path': pcspath, 'ondup': 'newcopy' } retcode, responses = sycurl.request( url, querydata, '0-%d' % (os.stat(filepath).st_size - 1), 'POST', SynCurl.Upload, filepath) responses = json.loads(responses) if retcode != 200 or 'error_code' in responses: logger.error('Errno:%d: Upload file to pcs failed: %s, %s.' % (retcode, filepath, responses['error_msg'])) return 1 logger.info(' Upload file "%s" completed.' % (filepath)) return 0
def get_pcs_filelist(pcspath, startindex, endindex): logger.debug('Start get pcs file list(%d-%d) of "%s".' % (startindex, endindex, pcspath)) sycurl = SynCurl() url = 'https://pcs.baidu.com/rest/2.0/pcs/file' querydata = { 'method': 'list', 'access_token': SynConfig.token['access_token'], 'path': pcspath, 'by': 'name', 'order': 'asc', 'limit': '%d-%d' % (startindex, endindex), } retcode, responses = sycurl.request(url, querydata, '', 'GET', SynCurl.Normal) try: responses = json.loads(responses) if retcode != 200 or 'error_code' in responses: if responses['error_code'] == 31066: return 31066, [] else: logger.error( 'Errno:%d: Get PCS file list of "%s" failed: %s.' % (retcode, pcspath, responses['error_msg'])) return 1, [] return 0, responses['list'] except Exception as e: logger.error( 'Get PCS file list of "%s" failed. return code: %d, response body: %s.\n%s\n%s' % (pcspath, retcode, str(responses), e, traceback.format_exc())) return 1, [] finally: del responses logger.debug('Complete get pcs file list(%d-%d) of "%s".' % (startindex, endindex, pcspath))
def request(self, url, querydata, rdata, method='POST', rtype=0, fnname=''): retrycnt = -1 self.__op = rtype if querydata: if 'path' in querydata: querydata['path'] = querydata['path'].encode('utf-8') while retrycnt < SynConfig.config['retrytimes']: retrycnt += 1 logger.debug('Start curl request(%s) %d times for %s.' % (rdata, retrycnt, fnname)) if self.__op != SynCurl.Normal: startpos, self.__endpos = rdata.split('-', 1) startpos = self.__startpos = int(startpos) self.__endpos = int(self.__endpos) self.__response = '' curl = pycurl.Curl() try: if querydata: url += '?%s' % urlencode(querydata) curl.setopt(pycurl.URL, url) curl.setopt(pycurl.SSL_VERIFYPEER, 0) curl.setopt(pycurl.SSL_VERIFYHOST, 0) curl.setopt(pycurl.FOLLOWLOCATION, 1) curl.setopt(pycurl.CONNECTTIMEOUT, 15) curl.setopt(pycurl.LOW_SPEED_LIMIT, 1) curl.setopt(pycurl.LOW_SPEED_TIME, 30) curl.setopt(pycurl.USERAGENT, '') curl.setopt(pycurl.HEADER, 0) curl.setopt(pycurl.NOSIGNAL, 1) curl.setopt(pycurl.WRITEFUNCTION, self.__write_data) starthour, endhour = SynConfig.config[ 'speedlimitperiod'].split('-', 1) starthour = int(starthour) endhour = int(endhour) curhour = time.localtime().tm_hour if (endhour > starthour and starthour <= curhour < endhour ) or (endhour < starthour and (curhour < starthour or curhour >= endhour)): curl.setopt( pycurl.MAX_SEND_SPEED_LARGE, SynConfig.config['maxsendspeed'] / SynConfig.config['tasknumber'] / SynConfig.config['threadnumber']) curl.setopt( pycurl.MAX_RECV_SPEED_LARGE, SynConfig.config['maxrecvspeed'] / SynConfig.config['tasknumber'] / SynConfig.config['threadnumber']) if self.__op == SynCurl.Upload: curl.setopt(pycurl.UPLOAD, 1) curl.setopt(pycurl.READFUNCTION, self.__read_data) curl.setopt(pycurl.INFILESIZE, self.__endpos - startpos + 1) with open(fnname, 'rb') as self.__fd: self.__fd.seek(startpos) flock(self.__fd, LOCK_SH) curl.perform() flock(self.__fd, LOCK_UN) elif self.__op == SynCurl.Download: curl.setopt(pycurl.RANGE, rdata) with open(fnname, 'rb+') as self.__fd: self.__fd.seek(startpos) lockf(self.__fd, LOCK_EX, self.__endpos - startpos + 1, startpos, 0) curl.perform() self.__fd.flush() os.fsync(self.__fd.fileno()) lockf(self.__fd, LOCK_UN, self.__endpos - startpos + 1, startpos, 0) else: curl.setopt(pycurl.CUSTOMREQUEST, method) if method == 'POST': curl.setopt(pycurl.POSTFIELDS, urlencode(rdata)) curl.perform() retcode = curl.getinfo(pycurl.HTTP_CODE) if retcode < 400 or retcode == 404 or retrycnt == SynConfig.config[ 'retrytimes']: if retcode != 200 and retcode != 206 and self.__response == '': self.__response = '{"error_code":%d,"error_msg":"Returned by the server is not in the expected results."}' % retcode return retcode, self.__response else: time.sleep(SynConfig.config['retrydelay']) except pycurl.error as error: if retrycnt == SynConfig.config['retrytimes']: (errno, errstr) = error return errno, '{"error_code":%d,"error_msg":"%s"}' % ( errno, errstr) except Exception as e: return -1, '{"error_code":%d,"error_msg":"%s"}' % ( -1, traceback.format_exc().replace('\n', '\\n').replace( '"', '\'')) finally: curl.close() logger.debug('Complete curl request(%s) %d times for %s.' % (rdata, retrycnt, fnname))
def request(self, url, querydata, rdata, method='POST', rtype=0, fnname=''): retrycnt = -1 self.__op = rtype if querydata: if 'path' in querydata: querydata['path'] = querydata['path'].encode('utf-8') while retrycnt < SynConfig.config['retrytimes']: retrycnt += 1 logger.debug('Start curl request(%s) %d times for %s.' % (rdata, retrycnt, fnname)) if self.__op != SynCurl.Normal: startpos, self.__endpos = rdata.split('-', 1) startpos = self.__startpos = int(startpos) self.__endpos = int(self.__endpos) self.__response = '' curl = pycurl.Curl() try: if querydata: url += '?%s' % urlencode(querydata) curl.setopt(pycurl.URL, url) curl.setopt(pycurl.SSL_VERIFYPEER, 0) curl.setopt(pycurl.SSL_VERIFYHOST, 0) curl.setopt(pycurl.FOLLOWLOCATION, 1) curl.setopt(pycurl.CONNECTTIMEOUT, 15) curl.setopt(pycurl.LOW_SPEED_LIMIT, 1) curl.setopt(pycurl.LOW_SPEED_TIME, 30) curl.setopt(pycurl.USERAGENT, '') curl.setopt(pycurl.HEADER, 0) curl.setopt(pycurl.NOSIGNAL, 1) curl.setopt(pycurl.WRITEFUNCTION, self.__write_data) starthour, endhour = SynConfig.config['speedlimitperiod'].split('-', 1) starthour = int(starthour) endhour = int(endhour) curhour = time.localtime().tm_hour if (endhour > starthour and starthour <= curhour < endhour) or (endhour < starthour and (curhour < starthour or curhour >= endhour)): curl.setopt(pycurl.MAX_SEND_SPEED_LARGE, SynConfig.config['maxsendspeed'] / SynConfig.config['tasknumber'] / SynConfig.config['threadnumber']) curl.setopt(pycurl.MAX_RECV_SPEED_LARGE, SynConfig.config['maxrecvspeed'] / SynConfig.config['tasknumber'] / SynConfig.config['threadnumber']) if self.__op == SynCurl.Upload: curl.setopt(pycurl.UPLOAD, 1) curl.setopt(pycurl.READFUNCTION, self.__read_data) curl.setopt(pycurl.INFILESIZE, self.__endpos - startpos + 1) with open(fnname, 'rb') as self.__fd: self.__fd.seek(startpos) flock(self.__fd, LOCK_SH) curl.perform() flock(self.__fd, LOCK_UN) elif self.__op == SynCurl.Download: curl.setopt(pycurl.RANGE, rdata) with open(fnname, 'rb+') as self.__fd: self.__fd.seek(startpos) lockf(self.__fd, LOCK_EX, self.__endpos - startpos + 1, startpos, 0) curl.perform() self.__fd.flush() os.fsync(self.__fd.fileno()) lockf(self.__fd, LOCK_UN, self.__endpos - startpos + 1, startpos, 0) else: curl.setopt(pycurl.CUSTOMREQUEST, method) if method == 'POST': curl.setopt(pycurl.POSTFIELDS, urlencode(rdata)) curl.perform() retcode = curl.getinfo(pycurl.HTTP_CODE) if retcode < 400 or retcode == 404 or retrycnt == SynConfig.config['retrytimes']: if retcode != 200 and retcode != 206 and self.__response == '': self.__response = '{"error_code":%d,"error_msg":"Returned by the server is not in the expected results."}' % retcode return retcode, self.__response else: time.sleep(SynConfig.config['retrydelay']) except pycurl.error as error: if retrycnt == SynConfig.config['retrytimes']: (errno, errstr) = error return errno, '{"error_code":%d,"error_msg":"%s"}' % (errno, errstr) except Exception as e: return -1, '{"error_code":%d,"error_msg":"%s"}' % (-1, traceback.format_exc().replace('\n', '\\n').replace('"', '\'')) finally: curl.close() logger.debug('Complete curl request(%s) %d times for %s.' % (rdata, retrycnt, fnname))