def _download(src_url, dst):
src_bucket_name = src_url.bucket_name()
src_object_name = src_url.object_name()
if os.path.isdir(dst):
if dst == "." or dst == "..":
dst_name = src_url.file_name()
elif dst.endswith("/"):
dst_name = dst + src_url.file_name()
else:
dst_name = dst + "/" + src_object_name.split("/")[-1]
else:
dst_name = dst
mtime = None
if os.path.isfile(dst_name):
local_md5 = file_md5(dst_name)
remote_md5 = fds_client.get_object_metadata(
src_bucket_name, src_object_name).metadata.get(Common.CONTENT_MD5)
if remote_md5 is not None and local_md5 == remote_md5:
CLIPrinter.done(
"download %s/%s to local(skip because of same md5)" %
(src_bucket_name, src_object_name))
return
mtime = os.path.getmtime(dst_name)
try:
fds_object = fds_client.get_object(bucket_name=src_bucket_name,
object_name=src_object_name,
stream=True)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
lm = fds_object.metadata.metadata["last-modified"]
remote_modified = rfc822_timestamp(lm)
# if last-modified of local file is not less last-modified of remote file, skip
if mtime is not None and datetime.fromtimestamp(mtime) >= remote_modified:
CLIPrinter.done("download %s/%s to local(skip because of updated)" %
(src_bucket_name, src_object_name))
return
length_left = IS_PY3 and sys.maxsize or sys.maxint
try:
with open(dst_name, "wb") as f:
for chunk in fds_object.stream:
length = min(length_left, len(chunk))
f.write(chunk[0:length])
length_left -= length
if length_left <= 0:
break
except Exception as exception:
print(exception)
finally:
fds_object.stream.close()
CLIPrinter.done("download %s/%s to local" %
(src_bucket_name, src_object_name))
def _upload(fpath, object_name, dst_url, autodetect_mimetype):
if not fpath.exists():
CLIPrinter.warn("{} is a bad file".format(str(fpath)))
return
try:
dst_bucket_name = dst_url.bucket_name()
dst_object_name = object_name
if fds_client.does_object_exists(dst_bucket_name, dst_object_name):
# check md5 firstly
metadata = fds_client.get_object_metadata(dst_bucket_name,
dst_object_name)
if metadata.metadata.get(Common.CONTENT_MD5) is not None:
local_md5 = file_md5(str(fpath.resolve()))
if local_md5 == metadata.metadata.get(Common.CONTENT_MD5):
CLIPrinter.done(
"upload object %s/%s(skip because of same md5)" %
(dst_bucket_name, dst_object_name))
return
# check last-modified
mtime = None
if fpath.is_file():
mtime = os.path.getmtime(str(fpath.resolve()))
lm = metadata.metadata[Common.LAST_MODIFIED]
remote_modified = rfc822_timestamp(lm)
# if last-modified of local file is not less last-modified of remote file, skip
if mtime is not None and datetime.fromtimestamp(
mtime) <= remote_modified:
CLIPrinter.done(
"upload object %s/%s(skip because of updated)" %
(dst_bucket_name, dst_object_name))
return
except Exception as e:
CLIPrinter.fail(e.message)
return
mimetype = None
if autodetect_mimetype:
mimetype = mimetypes.guess_type(str(fpath.resolve()))[0]
metadata = FDSObjectMetadata()
if mimetype is not None:
metadata.add_header(Common.CONTENT_TYPE, mimetype)
with open(str(fpath.resolve()), "rb") as f:
try:
fds_client.put_object(dst_bucket_name,
dst_object_name,
f,
metadata=metadata)
CLIPrinter.done("upload object %s/%s" %
(dst_bucket_name, dst_object_name))
except GalaxyFDSClientException as e:
CLIPrinter.fail("upload object %s/%s, %s" %
(dst_bucket_name, dst_object_name, e.message))
def _download(self, src_url, dst):
src_bucket_name = src_url.bucket_name()
src_object_name = src_url.object_name()
if os.path.isdir(dst):
if dst == '.' or dst == '..':
dst_name = src_url.file_name()
elif dst.endswith('/'):
dst_name = dst + src_url.file_name()
else:
dst_name = dst + '/' + src_object_name.split('/')[-1]
else:
dst_name = dst
mtime = None
if os.path.isfile(dst_name):
local_md5 = file_md5(dst_name)
remote_md5 = self._fds.get_object_metadata(src_bucket_name, src_object_name).metadata.get(Common.CONTENT_MD5)
if remote_md5 is not None and local_md5 == remote_md5:
CLIPrinter.done("download %s/%s to local(skip because of same md5)" % (src_bucket_name, src_object_name))
return
mtime = os.path.getmtime(dst_name)
try:
fds_object = self._fds.get_object(bucket_name=src_bucket_name, object_name=src_object_name,
stream=True)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
lm = fds_object.metadata.metadata['last-modified']
remote_modified = rfc822_timestamp(lm)
# if last-modified of local file is not less last-modified of remote file, skip
if mtime is not None and datetime.fromtimestamp(mtime) >= remote_modified:
CLIPrinter.done("download %s/%s to local(skip because of updated)" % (src_bucket_name, src_object_name))
return
length_left = IS_PY3 and sys.maxsize or sys.maxint
try:
with open(dst_name, 'wb') as f:
for chunk in fds_object.stream:
length = min(length_left, len(chunk))
f.write(chunk[0:length])
length_left -= length
if length_left <= 0:
break
except Exception as exception:
print(exception)
finally:
fds_object.stream.close()
CLIPrinter.done("download %s/%s to local" % (src_bucket_name, src_object_name))
def _upload(self, filename, dst_url, autodetect_mimetype, sync=False):
if not os.path.exists(filename):
CLIPrinter.warn("{} is a bad file".format(filename))
return
dst_bucket_name = dst_url.bucket_name()
if dst_url.is_object_url():
dst_object_name = dst_url.object_name()
elif sync:
dst_object_name = filename[2:]
elif dst_url.is_object_dir():
dst_object_name = dst_url.object_dir() + os.path.basename(filename)
else:
dst_object_name = os.path.basename(filename)
try:
if self._fds.does_object_exists(dst_bucket_name, dst_object_name):
# check md5 firstly
metadata = self._fds.get_object_metadata(
dst_bucket_name, dst_object_name)
if metadata.metadata.get(Common.CONTENT_MD5) is not None:
local_md5 = file_md5(filename)
if local_md5 == metadata.metadata.get(Common.CONTENT_MD5):
CLIPrinter.done(
'upload object %s/%s(skip because of same md5)' %
(dst_bucket_name, dst_object_name))
return
# check last-modified
mtime = None
if os.path.isfile(filename):
mtime = os.path.getmtime(filename)
lm = metadata.metadata[Common.LAST_MODIFIED]
remote_modified = rfc822_timestamp(lm)
# if last-modified of local file is not less last-modified of remote file, skip
if mtime is not None and datetime.fromtimestamp(
mtime) <= remote_modified:
CLIPrinter.done(
'upload object %s/%s(skip because of updated)' %
(dst_bucket_name, dst_object_name))
return
except Exception as e:
CLIPrinter.fail(e.message)
return
mimetype = None
if autodetect_mimetype:
mimetype = mimetypes.guess_type(filename)[0]
metadata = FDSObjectMetadata()
if mimetype is not None:
metadata.add_header(Common.CONTENT_TYPE, mimetype)
result = None
with open(filename, "rb") as f:
file_length = os.path.getsize(filename)
if file_length < multipart_upload_buffer_size:
try:
result = self._fds.put_object(dst_bucket_name,
dst_object_name,
f,
metadata=metadata)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
else:
try:
upload_token = self._fds.init_multipart_upload(
dst_bucket_name, dst_object_name)
part_number = 1
result_list = []
while True:
data = f.read(multipart_upload_buffer_size)
if len(data) <= 0:
break
for i in range(max_upload_retry_time):
upload_result = None
try:
upload_result = self._fds.upload_part(
dst_bucket_name, dst_object_name,
upload_token.upload_id, part_number, data)
result_list.append(upload_result)
break
except GalaxyFDSClientException as e:
sleep_seconds = (i + 1) * 10
CLIPrinter.warn(
"upload part %d failed, retry after %d seconds"
% (part_number, sleep_seconds))
time.sleep(sleep_seconds)
part_number = part_number + 1
upload_part_result = UploadPartResultList(
{"uploadPartResultList": result_list})
result = self._fds.complete_multipart_upload(
upload_token.bucket_name, upload_token.object_name,
upload_token.upload_id, metadata,
json.dumps(upload_part_result))
except Exception as e:
self._fds.abort_multipart_upload(dst_bucket_name,
dst_object_name,
upload_token.upload_id)
CLIPrinter.fail(e.message)
if result is not None:
CLIPrinter.done('upload object %s/%s' %
(dst_bucket_name, dst_object_name))
else:
CLIPrinter.fail('upload object %s/%s' %
(dst_bucket_name, dst_object_name))
def _upload(self, filename, dst_url, autodetect_mimetype, sync=False):
if not os.path.exists(filename):
CLIPrinter.warn("{} is a bad file".format(filename))
return
dst_bucket_name = dst_url.bucket_name()
if dst_url.is_object_url():
dst_object_name = dst_url.object_name()
elif sync:
dst_object_name = filename[2:]
elif dst_url.is_object_dir():
dst_object_name = dst_url.object_dir() + os.path.basename(filename)
else:
dst_object_name = os.path.basename(filename)
try:
if self._fds.does_object_exists(dst_bucket_name, dst_object_name):
# check md5 firstly
metadata = self._fds.get_object_metadata(dst_bucket_name, dst_object_name)
if metadata.metadata.get(Common.CONTENT_MD5) is not None:
local_md5 = file_md5(filename)
if local_md5 == metadata.metadata.get(Common.CONTENT_MD5):
CLIPrinter.done('upload object %s/%s(skip because of same md5)' % (dst_bucket_name, dst_object_name))
return
# check last-modified
mtime = None
if os.path.isfile(filename):
mtime = os.path.getmtime(filename)
lm = metadata.metadata[Common.LAST_MODIFIED]
remote_modified = rfc822_timestamp(lm)
# if last-modified of local file is not less last-modified of remote file, skip
if mtime is not None and datetime.fromtimestamp(mtime) <= remote_modified:
CLIPrinter.done('upload object %s/%s(skip because of updated)' % (dst_bucket_name, dst_object_name))
return
except Exception as e:
CLIPrinter.fail(e.message)
return
mimetype = None
if autodetect_mimetype:
mimetype = mimetypes.guess_type(filename)[0]
metadata = FDSObjectMetadata()
if mimetype is not None:
metadata.add_header(Common.CONTENT_TYPE, mimetype)
result = None
with open(filename, "rb") as f:
file_length = os.path.getsize(filename)
if file_length < multipart_upload_buffer_size:
try:
result = self._fds.put_object(dst_bucket_name, dst_object_name, f, metadata=metadata)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
else:
try:
upload_token = self._fds.init_multipart_upload(dst_bucket_name, dst_object_name)
part_number = 1
result_list = []
while True:
data = f.read(multipart_upload_buffer_size)
if len(data) <= 0:
break
for i in range(max_upload_retry_time):
upload_result = None
try:
upload_result = self._fds.upload_part(dst_bucket_name, dst_object_name, upload_token.upload_id, part_number, data)
result_list.append(upload_result)
break
except GalaxyFDSClientException as e:
sleep_seconds = (i + 1) * 10
CLIPrinter.warn("upload part %d failed, retry after %d seconds" % (
part_number, sleep_seconds))
time.sleep(sleep_seconds)
part_number = part_number + 1
upload_part_result = UploadPartResultList({"uploadPartResultList": result_list})
result = self._fds.complete_multipart_upload(upload_token.bucket_name, upload_token.object_name,
upload_token.upload_id, metadata,
json.dumps(upload_part_result))
except Exception as e:
self._fds.abort_multipart_upload(dst_bucket_name, dst_object_name, upload_token.upload_id)
CLIPrinter.fail(e.message)
if result is not None:
CLIPrinter.done('upload object %s/%s' % (dst_bucket_name, dst_object_name))
else:
CLIPrinter.fail('upload object %s/%s' % (dst_bucket_name, dst_object_name))