def upload_stream(key, data):
start = time.time()
status = -1
retry_times = 3
while status != 200 and retry_times >= 0:
retry_times -= 1
try:
upload_handler = putufile.PutUFile(public_key=public_key,
private_key=private_key)
ret, resp = upload_handler.putstream(bucket_name, key, data)
status = resp.status_code
except Exception as exc:
logger.exception(msg="[upload file error]", exc_info=exc)
status = -1
if status == 200:
logger.debug(
"[Succeed][upload file][bucket: {0}][key: {1}][takes: {2}]".format(
bucket_name, key,
time.time() - start))
return True
else:
logger.debug(
"[Failed][upload file][bucket: {0}][key: {1}][status: {3}][takes: {2}]"
.format(bucket_name, key,
time.time() - start, status))
return False
def ufile_set_cache(b64url,
url_hash,
batch_id,
groups,
content,
refresh=False):
if not hasattr(ufile_set_cache, '_auth'):
put_auth = putufile.PutUFile(public_key, private_key)
setattr(ufile_set_cache, '_auth', put_auth)
try:
sio = StringIO(content)
filename = '{}_{}'.format(batch_id, url_hash)
batch_key = batch_id.rsplit('-', 1)[0]
ret, resp = ufile_set_cache._auth.putstream(batch_key, filename, sio)
if resp.status_code != 200:
if resp.status_code == 400:
if json.loads(resp.content)[u'ErrMsg'] == u'bucket not exist':
ensure_bucket(batch_key)
raise Exception(
'{} bucket not exist, create, upload again'.format(
batch_key))
raise Exception('{} upload ufile error: {}'.format(
batch_id, b64url))
_groups = str(map(string.strip,
groups.split(',')))[1:-1].replace('\'', '"')
sql1 = (
"insert into accessed (batch_id, groups, status, b64url, url_hash) "
"values ('{}', '{{{}}}', '{}', '{}', '{}');".format(
batch_id, _groups, 0, b64url, url_hash))
sql2 = ("insert into cached (b64url, url_hash) values ('{}', '{}')"
";".format(b64url, url_hash))
dbwrapper.execute(sql1)
dbwrapper.execute(sql2)
now = datetime.now()
log_line = json.dumps({
'date': str(now),
'batch_id': batch_id,
'groups': groups,
'url': base64.urlsafe_b64decode(b64url),
})
cachelog.get_logger(batch_id, now.strftime('%Y%m%d'),
FSCACHEDIR).info(log_line)
except Exception as e:
return {'success': False, 'error': e}
return {'success': True}
def upload_task(appname, stream, filepath):
print 'uploadtask ', appname, stream, filepath
file_key = os.path.basename(filepath)
handler = putufile.PutUFile(public_key, private_key)
ret, resp = handler.putfile(bucket_name, file_key, filepath)
print 'upload result ', ret, resp
data = {
'action': 'on_upload',
'app': appname,
'stream': stream,
'file_key': file_key
}
http_callback.delay(data)
def putfile(dir, file):
# 构造上传对象，并设置公私钥
handler = putufile.PutUFile(public_key, private_key)
# upload small file to public bucket
logger.info('start upload file to public bucket')
# 要上传的目标空间
bucket = bucketname
# 上传到目标空间后保存的文件名
key = file
# 要上传文件的本地路径
local_file = dir + '/' + file
print(local_file)
# 请求上传
ret, resp = handler.putfile(bucket, key, local_file)
assert resp.status_code == 200
class UFile(object):
config.ufile['public_key']
__public_key = config.ufile['public_key']
__private_key = config.ufile['private_key']
__bucket = config.ufile['bucket']
__pf = putufile.PutUFile(__public_key, __private_key)
__dl = downloadufile.DownloadUFile(__public_key, __private_key)
@classmethod
def put_stream(cls, path, stream):
return cls.__pf.putstream(cls.__bucket, path, stream)
@classmethod
def get_dl_url(cls, path):
return cls.__dl.private_download_url(cls.__bucket, path, expires=None)
def UploadUfileSingle(public_key, private_key, bucket, key, local_file):
try:
isExists = os.path.exists(local_file)
if not isExists:
print("file is not exist. path: {0}. ".format(local_file))
return False
except Exception as e:
print(e)
return False
handler = putufile.PutUFile(public_key, private_key)
for i in range(retry_max_fail):
ret, resp = handler.putfile(bucket, key, local_file)
if resp.status_code == 200:
break
if resp.status_code != 200:
print("fail to upload file to ufile. file name: {0}. [Err Info]: {1}".
format(key, resp))
print("upload to ufile succ. path: {0}, ufile: [bucket: {1}, key: {2}]".
format(local_file, bucket, key))
return True
def _upload_file(self, file_path):
""" Upload the file to the ufile
"""
public_key = self.conf_params['docker']['ufile']['public_key']
private_key = self.conf_params['docker']['ufile']['private_key']
bucket = self.conf_params['docker']['ufile']['bucket']
uai_logger.debug('Start upload file to the bucket {0}'.format(
self.conf_params['docker']['ufile']['bucket']))
handler = putufile.PutUFile(public_key, private_key)
local_file = file_path
key = local_file
uai_logger.info('Upload >> key: {0}, local file: {1}'.format(
key, local_file))
ret, resp = handler.putfile(bucket, key, local_file)
assert resp.status_code == 200, 'upload seems something error'
uai_logger.debug(
'upload local file :{0} to ufile key= {1} successful'.format(
local_file, key))
self.files.append(key)
def _upload_to_ufile(self):
public_key = self.public_key
private_key = self.private_key
bucket = self.bucket
uai_logger.debug('Start upload file to the bucket {0}'.format(bucket))
handler = putufile.PutUFile(public_key, private_key)
local_file = os.path.join(CURRENT_PATH, self.pack_file_path,
self.upload_name)
local_file = local_file.replace('\\', '/')
key = self.upload_name
uai_logger.info('Upload >> key: {0}, local file: {1}'.format(
key, local_file))
ret, resp = handler.putfile(bucket, key, local_file)
uai_logger.debug('Ufile response: {0}'.format(resp))
assert resp.status_code == 200, 'upload seems something error'
uai_logger.debug(
'upload local file :{0} to ufile key= {1} successful'.format(
local_file, key))
current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
self.ufile_info.append('Upload Time: {0}\n'.format(current_time))
self.ufile_info.append('Path of Upload File: {0}\n'.format(key))
def _upload_file(self):
""" Upload tar file to certain bucket
"""
public_key = self.conf_params['docker']['ufile']['public_key']
private_key = self.conf_params['docker']['ufile']['private_key']
bucket = self.conf_params['docker']['ufile']['bucket']
uai_logger.debug('Start upload file to the bucket {0}'.format(bucket))
handler = putufile.PutUFile(public_key, private_key)
local_file = os.path.join(self.params['pack_file_path'],
self.params['upload_name'])
key = self.params['upload_name']
uai_logger.info('Upload >> key: {0}, local file: {1}'.format(
key, local_file))
ret, resp = handler.putfile(bucket, key, local_file)
uai_logger.debug('Ufile response: {0}'.format(resp))
assert resp.status_code == 200, 'upload seems something error'
uai_logger.debug(
'upload local file :{0} to ufile key= {1} successful'.format(
local_file, key))
current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
self.ufile_info.write('Upload Time: {0}\n'.format(current_time))
self.ufile_info.write('Path of Upload File: {0}\n'.format(key))
def uploadUfileSingle(public_key, private_key, bucket, key, local_file,
res_queue):
try:
isExists = os.path.exists(local_file)
if not isExists:
err_info = "file is not exist. path: {0}. ".format(local_file)
res_queue.put(err_info)
return
except Exception as e:
err_info = e
res_queue.put(err_info)
return
handler = putufile.PutUFile(public_key, private_key)
for i in range(retry_max_fail):
ret, resp = handler.putfile(bucket, key, local_file)
if resp.status_code == 200:
break
if resp.status_code != 200:
err_info = "fail to download ufile. file name: {0}. [Err Info]: {1}".format(
key, resp)
res_queue.put(err_info)
else:
res_queue.put(True)
if local_ip.startswith('10.10'):
config.set_default(uploadsuffix='.ufile.cn-north-03.ucloud.cn')
config.set_default(downloadsuffix='.ufile.cn-north-03.ucloud.cn')
elif local_ip.startswith('10.19'):
config.set_default(uploadsuffix='.ufile.cn-north-04.ucloud.cn')
config.set_default(downloadsuffix='.ufile.cn-north-04.ucloud.cn')
else:
logger.debug("[no ucloud machine][use public ip]")
config.set_default(connection_timeout=60)
# public_key = 'vCuKhG6UcHvB1UswK/q5zjMMHxt7PjKIu3/6q1JtxiZHtAuv'
# private_key = 'fdfbdf9cb0ebfeed522f664efc44f752694b15f6'
public_key = 'M7jIsudUE4Nvn6zQGjNMWxReCrSpc8HcWdBztizB38qvbXkS'
private_key = '3fe4dbb2f16a86a8a58c6c9aac061d83c43ff468'
put_handler = putufile.PutUFile(public_key, private_key)
bucket_name = "verify-case"
def do_not_print_http_debug_log(f):
@functools.wraps(f)
def wrapper(*args, **kwargs):
temp = httplib.HTTPConnection.debuglevel
httplib.HTTPConnection.debuglevel = 0
res = f(*args, **kwargs)
httplib.HTTPConnection.debuglevel = temp
return res
return wrapper
class PutUFileTestCase(unittest.TestCase):
putufile_handler = putufile.PutUFile(public_key, private_key)
def test_putufile(self):
self.putufile_handler.set_keys(public_key, private_key)
# put small file to public bucket
logger.info('\nput small file to public bucket')
ret, resp = self.putufile_handler.putfile(public_bucket, put_small_key,
small_local_file)
assert resp.status_code == 200
# put big file to public bucket
logger.info('\nput big file to public bucket')
ret, resp = self.putufile_handler.putfile(public_bucket, put_big_key,
big_local_file)
assert resp.status_code == 200
# put small file to private bucket
logger.info('\nput small file to private bucket')
ret, resp = self.putufile_handler.putfile(private_bucket,
put_small_key,
small_local_file)
assert resp.status_code == 200
# put big file to private bucket
logger.info('\nput big file to private bucket')
ret, resp = self.putufile_handler.putfile(private_bucket, put_big_key,
big_local_file)
assert resp.status_code == 200
def test_putstream(self):
self.putufile_handler.set_keys(public_key, private_key)
logger.info('\nput stream to public bucket')
ret, resp = self.putufile_handler.putstream(public_bucket,
put_stream_key, bio)
assert resp.status_code == 200
bio.seek(0, os.SEEK_SET)
logger.info('\nput stream to private bucket')
ret, resp = self.putufile_handler.putstream(private_bucket,
put_stream_key, bio)
assert resp.status_code == 200
def test_putfiletowrongbucket(self):
self.putufile_handler.set_keys(public_key, private_key)
# put file to wrong bucket
logger.info('\nput file to wrong bucket')
ret, resp = self.putufile_handler.putfile(wrong_bucket, put_small_key,
small_local_file)
assert resp.status_code == 400
logger.info(resp.error)
def test_putufilewithwrongkey(self):
self.putufile_handler.set_keys(wrong_public_key, wrong_private_key)
logger.info(
'\nput small file to public bucket with wrong api keys pair')
# put small file to public bucket with wrong api keys pair
ret, resp = self.putufile_handler.putfile(public_bucket, put_small_key,
small_local_file)
assert resp.status_code == 403
logger.info(resp.error)
# put small file to private bucket with wrong api keys pair
logger.info(
'\nput small file to private bucket with wrong api keys pair')
ret, resp = self.putufile_handler.putfile(private_bucket,
put_small_key,
small_local_file)
logger.error('status_code:{0}'.format(resp.status_code))
assert resp.status_code == 403
logger.info(resp.error)
