def upload_stream(key, data): start = time.time() status = -1 retry_times = 3 while status != 200 and retry_times >= 0: retry_times -= 1 try: upload_handler = putufile.PutUFile(public_key=public_key, private_key=private_key) ret, resp = upload_handler.putstream(bucket_name, key, data) status = resp.status_code except Exception as exc: logger.exception(msg="[upload file error]", exc_info=exc) status = -1 if status == 200: logger.debug( "[Succeed][upload file][bucket: {0}][key: {1}][takes: {2}]".format( bucket_name, key, time.time() - start)) return True else: logger.debug( "[Failed][upload file][bucket: {0}][key: {1}][status: {3}][takes: {2}]" .format(bucket_name, key, time.time() - start, status)) return False
def ufile_set_cache(b64url, url_hash, batch_id, groups, content, refresh=False): if not hasattr(ufile_set_cache, '_auth'): put_auth = putufile.PutUFile(public_key, private_key) setattr(ufile_set_cache, '_auth', put_auth) try: sio = StringIO(content) filename = '{}_{}'.format(batch_id, url_hash) batch_key = batch_id.rsplit('-', 1)[0] ret, resp = ufile_set_cache._auth.putstream(batch_key, filename, sio) if resp.status_code != 200: if resp.status_code == 400: if json.loads(resp.content)[u'ErrMsg'] == u'bucket not exist': ensure_bucket(batch_key) raise Exception( '{} bucket not exist, create, upload again'.format( batch_key)) raise Exception('{} upload ufile error: {}'.format( batch_id, b64url)) _groups = str(map(string.strip, groups.split(',')))[1:-1].replace('\'', '"') sql1 = ( "insert into accessed (batch_id, groups, status, b64url, url_hash) " "values ('{}', '{{{}}}', '{}', '{}', '{}');".format( batch_id, _groups, 0, b64url, url_hash)) sql2 = ("insert into cached (b64url, url_hash) values ('{}', '{}')" ";".format(b64url, url_hash)) dbwrapper.execute(sql1) dbwrapper.execute(sql2) now = datetime.now() log_line = json.dumps({ 'date': str(now), 'batch_id': batch_id, 'groups': groups, 'url': base64.urlsafe_b64decode(b64url), }) cachelog.get_logger(batch_id, now.strftime('%Y%m%d'), FSCACHEDIR).info(log_line) except Exception as e: return {'success': False, 'error': e} return {'success': True}
def upload_task(appname, stream, filepath): print 'uploadtask ', appname, stream, filepath file_key = os.path.basename(filepath) handler = putufile.PutUFile(public_key, private_key) ret, resp = handler.putfile(bucket_name, file_key, filepath) print 'upload result ', ret, resp data = { 'action': 'on_upload', 'app': appname, 'stream': stream, 'file_key': file_key } http_callback.delay(data)
def putfile(dir, file): # 构造上传对象,并设置公私钥 handler = putufile.PutUFile(public_key, private_key) # upload small file to public bucket logger.info('start upload file to public bucket') # 要上传的目标空间 bucket = bucketname # 上传到目标空间后保存的文件名 key = file # 要上传文件的本地路径 local_file = dir + '/' + file print(local_file) # 请求上传 ret, resp = handler.putfile(bucket, key, local_file) assert resp.status_code == 200
class UFile(object): config.ufile['public_key'] __public_key = config.ufile['public_key'] __private_key = config.ufile['private_key'] __bucket = config.ufile['bucket'] __pf = putufile.PutUFile(__public_key, __private_key) __dl = downloadufile.DownloadUFile(__public_key, __private_key) @classmethod def put_stream(cls, path, stream): return cls.__pf.putstream(cls.__bucket, path, stream) @classmethod def get_dl_url(cls, path): return cls.__dl.private_download_url(cls.__bucket, path, expires=None)
def UploadUfileSingle(public_key, private_key, bucket, key, local_file): try: isExists = os.path.exists(local_file) if not isExists: print("file is not exist. path: {0}. ".format(local_file)) return False except Exception as e: print(e) return False handler = putufile.PutUFile(public_key, private_key) for i in range(retry_max_fail): ret, resp = handler.putfile(bucket, key, local_file) if resp.status_code == 200: break if resp.status_code != 200: print("fail to upload file to ufile. file name: {0}. [Err Info]: {1}". format(key, resp)) print("upload to ufile succ. path: {0}, ufile: [bucket: {1}, key: {2}]". format(local_file, bucket, key)) return True
def _upload_file(self, file_path): """ Upload the file to the ufile """ public_key = self.conf_params['docker']['ufile']['public_key'] private_key = self.conf_params['docker']['ufile']['private_key'] bucket = self.conf_params['docker']['ufile']['bucket'] uai_logger.debug('Start upload file to the bucket {0}'.format( self.conf_params['docker']['ufile']['bucket'])) handler = putufile.PutUFile(public_key, private_key) local_file = file_path key = local_file uai_logger.info('Upload >> key: {0}, local file: {1}'.format( key, local_file)) ret, resp = handler.putfile(bucket, key, local_file) assert resp.status_code == 200, 'upload seems something error' uai_logger.debug( 'upload local file :{0} to ufile key= {1} successful'.format( local_file, key)) self.files.append(key)
def _upload_to_ufile(self): public_key = self.public_key private_key = self.private_key bucket = self.bucket uai_logger.debug('Start upload file to the bucket {0}'.format(bucket)) handler = putufile.PutUFile(public_key, private_key) local_file = os.path.join(CURRENT_PATH, self.pack_file_path, self.upload_name) local_file = local_file.replace('\\', '/') key = self.upload_name uai_logger.info('Upload >> key: {0}, local file: {1}'.format( key, local_file)) ret, resp = handler.putfile(bucket, key, local_file) uai_logger.debug('Ufile response: {0}'.format(resp)) assert resp.status_code == 200, 'upload seems something error' uai_logger.debug( 'upload local file :{0} to ufile key= {1} successful'.format( local_file, key)) current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S') self.ufile_info.append('Upload Time: {0}\n'.format(current_time)) self.ufile_info.append('Path of Upload File: {0}\n'.format(key))
def _upload_file(self): """ Upload tar file to certain bucket """ public_key = self.conf_params['docker']['ufile']['public_key'] private_key = self.conf_params['docker']['ufile']['private_key'] bucket = self.conf_params['docker']['ufile']['bucket'] uai_logger.debug('Start upload file to the bucket {0}'.format(bucket)) handler = putufile.PutUFile(public_key, private_key) local_file = os.path.join(self.params['pack_file_path'], self.params['upload_name']) key = self.params['upload_name'] uai_logger.info('Upload >> key: {0}, local file: {1}'.format( key, local_file)) ret, resp = handler.putfile(bucket, key, local_file) uai_logger.debug('Ufile response: {0}'.format(resp)) assert resp.status_code == 200, 'upload seems something error' uai_logger.debug( 'upload local file :{0} to ufile key= {1} successful'.format( local_file, key)) current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S') self.ufile_info.write('Upload Time: {0}\n'.format(current_time)) self.ufile_info.write('Path of Upload File: {0}\n'.format(key))
def uploadUfileSingle(public_key, private_key, bucket, key, local_file, res_queue): try: isExists = os.path.exists(local_file) if not isExists: err_info = "file is not exist. path: {0}. ".format(local_file) res_queue.put(err_info) return except Exception as e: err_info = e res_queue.put(err_info) return handler = putufile.PutUFile(public_key, private_key) for i in range(retry_max_fail): ret, resp = handler.putfile(bucket, key, local_file) if resp.status_code == 200: break if resp.status_code != 200: err_info = "fail to download ufile. file name: {0}. [Err Info]: {1}".format( key, resp) res_queue.put(err_info) else: res_queue.put(True)
if local_ip.startswith('10.10'): config.set_default(uploadsuffix='.ufile.cn-north-03.ucloud.cn') config.set_default(downloadsuffix='.ufile.cn-north-03.ucloud.cn') elif local_ip.startswith('10.19'): config.set_default(uploadsuffix='.ufile.cn-north-04.ucloud.cn') config.set_default(downloadsuffix='.ufile.cn-north-04.ucloud.cn') else: logger.debug("[no ucloud machine][use public ip]") config.set_default(connection_timeout=60) # public_key = 'vCuKhG6UcHvB1UswK/q5zjMMHxt7PjKIu3/6q1JtxiZHtAuv' # private_key = 'fdfbdf9cb0ebfeed522f664efc44f752694b15f6' public_key = 'M7jIsudUE4Nvn6zQGjNMWxReCrSpc8HcWdBztizB38qvbXkS' private_key = '3fe4dbb2f16a86a8a58c6c9aac061d83c43ff468' put_handler = putufile.PutUFile(public_key, private_key) bucket_name = "verify-case" def do_not_print_http_debug_log(f): @functools.wraps(f) def wrapper(*args, **kwargs): temp = httplib.HTTPConnection.debuglevel httplib.HTTPConnection.debuglevel = 0 res = f(*args, **kwargs) httplib.HTTPConnection.debuglevel = temp return res return wrapper
class PutUFileTestCase(unittest.TestCase): putufile_handler = putufile.PutUFile(public_key, private_key) def test_putufile(self): self.putufile_handler.set_keys(public_key, private_key) # put small file to public bucket logger.info('\nput small file to public bucket') ret, resp = self.putufile_handler.putfile(public_bucket, put_small_key, small_local_file) assert resp.status_code == 200 # put big file to public bucket logger.info('\nput big file to public bucket') ret, resp = self.putufile_handler.putfile(public_bucket, put_big_key, big_local_file) assert resp.status_code == 200 # put small file to private bucket logger.info('\nput small file to private bucket') ret, resp = self.putufile_handler.putfile(private_bucket, put_small_key, small_local_file) assert resp.status_code == 200 # put big file to private bucket logger.info('\nput big file to private bucket') ret, resp = self.putufile_handler.putfile(private_bucket, put_big_key, big_local_file) assert resp.status_code == 200 def test_putstream(self): self.putufile_handler.set_keys(public_key, private_key) logger.info('\nput stream to public bucket') ret, resp = self.putufile_handler.putstream(public_bucket, put_stream_key, bio) assert resp.status_code == 200 bio.seek(0, os.SEEK_SET) logger.info('\nput stream to private bucket') ret, resp = self.putufile_handler.putstream(private_bucket, put_stream_key, bio) assert resp.status_code == 200 def test_putfiletowrongbucket(self): self.putufile_handler.set_keys(public_key, private_key) # put file to wrong bucket logger.info('\nput file to wrong bucket') ret, resp = self.putufile_handler.putfile(wrong_bucket, put_small_key, small_local_file) assert resp.status_code == 400 logger.info(resp.error) def test_putufilewithwrongkey(self): self.putufile_handler.set_keys(wrong_public_key, wrong_private_key) logger.info( '\nput small file to public bucket with wrong api keys pair') # put small file to public bucket with wrong api keys pair ret, resp = self.putufile_handler.putfile(public_bucket, put_small_key, small_local_file) assert resp.status_code == 403 logger.info(resp.error) # put small file to private bucket with wrong api keys pair logger.info( '\nput small file to private bucket with wrong api keys pair') ret, resp = self.putufile_handler.putfile(private_bucket, put_small_key, small_local_file) logger.error('status_code:{0}'.format(resp.status_code)) assert resp.status_code == 403 logger.info(resp.error)