class CosV4StorageService(storage_service.StorageService):
def __init__(self, *args, **kwargs):
appid = int(kwargs['appid'])
region = kwargs['region']
accesskeyid = unicode(kwargs['accesskeyid'])
accesskeysecret = unicode(kwargs['accesskeysecret'])
bucket = unicode(kwargs['bucket'])
if 'prefix_dir' in kwargs:
self._prefix_dir = kwargs['prefix_dir']
else:
self._prefix_dir = None
self._cos_api = CosClient(appid,
accesskeyid,
accesskeysecret,
region=region)
self._bucket = bucket
self._overwrite = kwargs[
'overwrite'] == 'true' if 'overwrite' in kwargs else False
self._max_retry = 20
def download(self, task, local_path):
# self._oss_api.get_object_to_file(urllib.unquote(cos_path).encode('utf-8'), local_path)
for i in range(20):
logger.info("download file with rety {0}".format(i))
import os
try:
os.remove(task.key)
except:
pass
req = DownloadFileRequest(self._bucket, task.key, local_path)
ret = self._cos_api.download_file(req)
# self._oss_api.get_object_to_file(task.key, local_path)
logger.debug(str(ret))
if task.size is None:
logger.info(
"task's size is None, skip check file size on local")
break
from os import path
if path.getsize(local_path) != int(task.size):
logger.error(
"Download Failed, size1: {size1}, size2: {size2}".format(
size1=path.getsize(local_path), size2=task.size))
else:
logger.info("Download Successfully, break")
break
else:
raise IOError("Download Failed with 20 retry")
def upload(self, task, local_path):
cos_path = task.key
if not cos_path.startswith('/'):
cos_path = '/' + cos_path
if self._prefix_dir:
cos_path = self._prefix_dir + cos_path
if isinstance(local_path, unicode):
local_path.encode('utf-8')
insert_only = 0 if self._overwrite else 1
for i in range(10):
try:
upload_request = UploadFileRequest(self._bucket,
unicode(cos_path),
local_path,
insert_only=insert_only)
upload_file_ret = self._cos_api.upload_file(upload_request)
if upload_file_ret[u'code'] != 0:
logger.warn("upload failed:" + str(upload_file_ret))
raise OSError("UploadError: " + str(upload_file_ret))
else:
break
except:
pass
else:
raise IOError("upload failed")
def list(self):
if self._prefix_dir is None:
for i in self.dfs('/'):
yield i
else:
for i in self.dfs(self._prefix_dir):
yield i
def dfs(self, path):
print "DFS: {path}".format(path=to_utf8(path))
_finish = False
_context = u''
max_retry = self._max_retry
path = to_unicode(path)
while not _finish:
request = ListFolderRequest(bucket_name=self._bucket,
cos_path=path,
context=_context)
ret = self._cos_api.list_folder(request)
if ret['code'] != 0:
max_retry -= 1
else:
_finish = ret['data']['listover']
_context = ret['data']['context']
for item in ret['data']['infos']:
if 'filelen' in item:
try:
key = "{prefix}{filename}".format(
prefix=path, filename=item['name'])
yield Task(key, item['filelen'], None)
except:
pass
else:
_sub_dir = "{prefix}{filename}".format(
prefix=path.encode('utf-8'),
filename=item['name'].encode('utf-8'))
for i in self.dfs(_sub_dir):
yield i
if max_retry == 0:
_finish = True
def exists(self, task):
_path = task.key
_size = task.size
if not _path.startswith('/'):
_path = '/' + _path
logger = getLogger(__name__)
# logger.info("func: exists: " + str(_path))
if self._prefix_dir:
_path = self._prefix_dir + _path
if isinstance(_path, str):
_path = _path.decode('utf-8')
request = StatFileRequest(self._bucket, _path)
ret = self._cos_api.stat_file(request)
logger.debug("ret: " + str(ret))
# import json
# v = json.loads(ret)
if ret['code'] != 0:
# logger.warn("error code: " + str(ret['code']))
return False
if ret['data']['filelen'] != ret['data']['filesize']:
logger.warn(
"file is broken, filelen: {len}, filesize: {size}".format(
len=ret['data']['filelen'], size=ret['data']['filesize']))
return False
elif _size is not None and ret['data']['filelen'] != _size:
return False
elif ret['data']['filelen'] != _size:
return False
return True
class CosV4StorageService(storage_service.StorageService):
def __init__(self, *args, **kwargs):
appid = int(kwargs['appid'])
region = kwargs['region']
accesskeyid = unicode(kwargs['accesskeyid'])
accesskeysecret = unicode(kwargs['accesskeysecret'])
bucket = unicode(kwargs['bucket'])
if 'prefix_dir' in kwargs:
self._prefix_dir = kwargs['prefix_dir']
else:
self._prefix_dir = None
# filelist存在时, prefix_dir不生效
if 'filelist' in kwargs:
self._filelist = kwargs['filelist']
self._prefix_dir = None
else:
self._filelist = None
if 'sync_files' in kwargs:
self._sync_files_dir = kwargs['sync_files']
else:
self._sync_files_dir = '/tmp/workspace/sync_files'
self._cos_api = CosClient(appid, accesskeyid, accesskeysecret, region=region)
self._bucket = bucket
self._overwrite = kwargs['overwrite'] == 'true' if 'overwrite' in kwargs else False
self._max_retry = 20
def download(self, task, local_path):
return self._download(task, local_path, False)
def _download(self, task, local_path, is_ignore_size):
for i in range(20):
logger.info("download file with rety {0}".format(i))
import os
try:
os.remove(task.key)
except:
pass
req = DownloadFileRequest(self._bucket, task.key, local_path)
ret = self._cos_api.download_file(req)
logger.debug(str(ret))
if ret['code'] != 0:
logger.error("error code: " + str(ret['code']))
raise IOError("Download Failed, ret=" + str(ret))
if is_ignore_size:
logger.info("Download Successfully, break")
break
if task.size is None:
logger.info("task's size is None, skip check file size on local")
break
from os import path
task_size = 0
if isinstance(task.size, int):
task_size = task.size
else:
task_size = int(task.size['filelen'])
if path.getsize(local_path) != task_size:
logger.error("Download Failed, size1: {size1}, size2: {size2}".format(size1=path.getsize(local_path),
size2=task_size))
else:
logger.info("Download Successfully, break")
break
else:
raise IOError("Download Failed with 20 retry")
def upload(self, task, local_path):
cos_path = task.key
if not cos_path.startswith('/'):
cos_path = '/' + cos_path
if self._prefix_dir:
cos_path = self._prefix_dir + cos_path
if isinstance(local_path, unicode):
local_path.encode('utf-8')
insert_only = 0 if self._overwrite else 1
for i in range(10):
try:
upload_request = UploadFileRequest(self._bucket, unicode(cos_path), local_path, insert_only=insert_only)
upload_file_ret = self._cos_api.upload_file(upload_request)
if upload_file_ret[u'code'] != 0:
logger.warn("upload failed:" + str(upload_file_ret))
raise OSError("UploadError: " + str(upload_file_ret))
else:
break
except:
pass
else:
raise IOError("upload failed")
def list(self, marker):
if self._filelist is not None and len(self._filelist) > 0:
filelist_path = self._sync_files_dir + '/' + os.path.basename(self._filelist)
filelist_task = Task(self._filelist, None, None, None)
self._download(filelist_task, filelist_path, True)
with open(filelist_path) as f:
line = f.readline()
while line:
if not line.startswith('/'):
line = '/' + line
task = self.get_task_by_key(line.strip())
yield task
line = f.readline()
else:
if self._prefix_dir is None:
for i in self.dfs('/'):
yield i
else:
for i in self.dfs(self._prefix_dir):
yield i
def dfs(self, path):
print "DFS: {path}".format(path=to_utf8(path))
_finish = False
_context = u''
max_retry = self._max_retry
path = to_unicode(path)
while not _finish:
request = ListFolderRequest(bucket_name=self._bucket, cos_path=path, context=_context)
request.set_delimiter = '/'
ret = self._cos_api.list_folder(request)
logger.debug(str(ret))
if ret['code'] != 0:
max_retry -= 1
else:
_finish = ret['data']['listover']
_context = ret['data']['context']
for item in ret['data']['infos']:
if 'filelen' in item:
try:
key = "{prefix}{filename}".format(prefix=path, filename=item['name'])
logger.info("key=%s", key)
yield Task(key, item, None, None)
except:
pass
else:
_sub_dir = "{prefix}{filename}".format(prefix=path.encode('utf-8'),
filename=item['name'].encode('utf-8'))
for i in self.dfs(_sub_dir):
yield i
if max_retry == 0:
_finish = True
def exists(self, task):
_path = task.key
_size = task.size
if not _path.startswith('/'):
_path = '/' + _path
logger = getLogger(__name__)
# logger.info("func: exists: " + str(_path))
if self._prefix_dir:
_path = self._prefix_dir + _path
if isinstance(_path, str):
_path = _path.decode('utf-8')
request = StatFileRequest(self._bucket, _path)
ret = self._cos_api.stat_file(request)
logger.debug("ret: " + str(ret))
# import json
# v = json.loads(ret)
if ret['code'] != 0:
# logger.warn("error code: " + str(ret['code']))
return False
if int(ret['data']['filelen']) != int(ret['data']['filesize']):
logger.warn("file is broken, filelen: {len}, filesize: {size}".format(
len=ret['data']['filelen'],
size=ret['data']['filesize']
))
return False
elif _size is not None and int(ret['data']['filelen']) != int(_size):
return False
return True
def get_task_by_key(self, key):
_path = key
if not _path.startswith('/'):
_path = '/' + _path
logger = getLogger(__name__)
if isinstance(_path, str):
_path = _path.decode('utf-8')
request = StatFileRequest(self._bucket, _path)
ret = self._cos_api.stat_file(request)
logger.info("ret: " + str(ret))
# import json
# v = json.loads(ret)
if ret['code'] != 0:
logger.warn("get task by key error, key = {},error code: {}".format(key, str(ret['code'])))
return Task(key, None, None, None)
return Task(key, int(ret['data']['filesize']), None, None)
