def _download(src_url, dst): src_bucket_name = src_url.bucket_name() src_object_name = src_url.object_name() if os.path.isdir(dst): if dst == "." or dst == "..": dst_name = src_url.file_name() elif dst.endswith("/"): dst_name = dst + src_url.file_name() else: dst_name = dst + "/" + src_object_name.split("/")[-1] else: dst_name = dst mtime = None if os.path.isfile(dst_name): local_md5 = file_md5(dst_name) remote_md5 = fds_client.get_object_metadata( src_bucket_name, src_object_name).metadata.get(Common.CONTENT_MD5) if remote_md5 is not None and local_md5 == remote_md5: CLIPrinter.done( "download %s/%s to local(skip because of same md5)" % (src_bucket_name, src_object_name)) return mtime = os.path.getmtime(dst_name) try: fds_object = fds_client.get_object(bucket_name=src_bucket_name, object_name=src_object_name, stream=True) except GalaxyFDSClientException as e: CLIPrinter.fail(e.message) return lm = fds_object.metadata.metadata["last-modified"] remote_modified = rfc822_timestamp(lm) # if last-modified of local file is not less last-modified of remote file, skip if mtime is not None and datetime.fromtimestamp(mtime) >= remote_modified: CLIPrinter.done("download %s/%s to local(skip because of updated)" % (src_bucket_name, src_object_name)) return length_left = IS_PY3 and sys.maxsize or sys.maxint try: with open(dst_name, "wb") as f: for chunk in fds_object.stream: length = min(length_left, len(chunk)) f.write(chunk[0:length]) length_left -= length if length_left <= 0: break except Exception as exception: print(exception) finally: fds_object.stream.close() CLIPrinter.done("download %s/%s to local" % (src_bucket_name, src_object_name))
def _upload(fpath, object_name, dst_url, autodetect_mimetype): if not fpath.exists(): CLIPrinter.warn("{} is a bad file".format(str(fpath))) return try: dst_bucket_name = dst_url.bucket_name() dst_object_name = object_name if fds_client.does_object_exists(dst_bucket_name, dst_object_name): # check md5 firstly metadata = fds_client.get_object_metadata(dst_bucket_name, dst_object_name) if metadata.metadata.get(Common.CONTENT_MD5) is not None: local_md5 = file_md5(str(fpath.resolve())) if local_md5 == metadata.metadata.get(Common.CONTENT_MD5): CLIPrinter.done( "upload object %s/%s(skip because of same md5)" % (dst_bucket_name, dst_object_name)) return # check last-modified mtime = None if fpath.is_file(): mtime = os.path.getmtime(str(fpath.resolve())) lm = metadata.metadata[Common.LAST_MODIFIED] remote_modified = rfc822_timestamp(lm) # if last-modified of local file is not less last-modified of remote file, skip if mtime is not None and datetime.fromtimestamp( mtime) <= remote_modified: CLIPrinter.done( "upload object %s/%s(skip because of updated)" % (dst_bucket_name, dst_object_name)) return except Exception as e: CLIPrinter.fail(e.message) return mimetype = None if autodetect_mimetype: mimetype = mimetypes.guess_type(str(fpath.resolve()))[0] metadata = FDSObjectMetadata() if mimetype is not None: metadata.add_header(Common.CONTENT_TYPE, mimetype) with open(str(fpath.resolve()), "rb") as f: try: fds_client.put_object(dst_bucket_name, dst_object_name, f, metadata=metadata) CLIPrinter.done("upload object %s/%s" % (dst_bucket_name, dst_object_name)) except GalaxyFDSClientException as e: CLIPrinter.fail("upload object %s/%s, %s" % (dst_bucket_name, dst_object_name, e.message))
def _download(self, src_url, dst): src_bucket_name = src_url.bucket_name() src_object_name = src_url.object_name() if os.path.isdir(dst): if dst == '.' or dst == '..': dst_name = src_url.file_name() elif dst.endswith('/'): dst_name = dst + src_url.file_name() else: dst_name = dst + '/' + src_object_name.split('/')[-1] else: dst_name = dst mtime = None if os.path.isfile(dst_name): local_md5 = file_md5(dst_name) remote_md5 = self._fds.get_object_metadata(src_bucket_name, src_object_name).metadata.get(Common.CONTENT_MD5) if remote_md5 is not None and local_md5 == remote_md5: CLIPrinter.done("download %s/%s to local(skip because of same md5)" % (src_bucket_name, src_object_name)) return mtime = os.path.getmtime(dst_name) try: fds_object = self._fds.get_object(bucket_name=src_bucket_name, object_name=src_object_name, stream=True) except GalaxyFDSClientException as e: CLIPrinter.fail(e.message) return lm = fds_object.metadata.metadata['last-modified'] remote_modified = rfc822_timestamp(lm) # if last-modified of local file is not less last-modified of remote file, skip if mtime is not None and datetime.fromtimestamp(mtime) >= remote_modified: CLIPrinter.done("download %s/%s to local(skip because of updated)" % (src_bucket_name, src_object_name)) return length_left = IS_PY3 and sys.maxsize or sys.maxint try: with open(dst_name, 'wb') as f: for chunk in fds_object.stream: length = min(length_left, len(chunk)) f.write(chunk[0:length]) length_left -= length if length_left <= 0: break except Exception as exception: print(exception) finally: fds_object.stream.close() CLIPrinter.done("download %s/%s to local" % (src_bucket_name, src_object_name))
def _upload(self, filename, dst_url, autodetect_mimetype, sync=False): if not os.path.exists(filename): CLIPrinter.warn("{} is a bad file".format(filename)) return dst_bucket_name = dst_url.bucket_name() if dst_url.is_object_url(): dst_object_name = dst_url.object_name() elif sync: dst_object_name = filename[2:] elif dst_url.is_object_dir(): dst_object_name = dst_url.object_dir() + os.path.basename(filename) else: dst_object_name = os.path.basename(filename) try: if self._fds.does_object_exists(dst_bucket_name, dst_object_name): # check md5 firstly metadata = self._fds.get_object_metadata( dst_bucket_name, dst_object_name) if metadata.metadata.get(Common.CONTENT_MD5) is not None: local_md5 = file_md5(filename) if local_md5 == metadata.metadata.get(Common.CONTENT_MD5): CLIPrinter.done( 'upload object %s/%s(skip because of same md5)' % (dst_bucket_name, dst_object_name)) return # check last-modified mtime = None if os.path.isfile(filename): mtime = os.path.getmtime(filename) lm = metadata.metadata[Common.LAST_MODIFIED] remote_modified = rfc822_timestamp(lm) # if last-modified of local file is not less last-modified of remote file, skip if mtime is not None and datetime.fromtimestamp( mtime) <= remote_modified: CLIPrinter.done( 'upload object %s/%s(skip because of updated)' % (dst_bucket_name, dst_object_name)) return except Exception as e: CLIPrinter.fail(e.message) return mimetype = None if autodetect_mimetype: mimetype = mimetypes.guess_type(filename)[0] metadata = FDSObjectMetadata() if mimetype is not None: metadata.add_header(Common.CONTENT_TYPE, mimetype) result = None with open(filename, "rb") as f: file_length = os.path.getsize(filename) if file_length < multipart_upload_buffer_size: try: result = self._fds.put_object(dst_bucket_name, dst_object_name, f, metadata=metadata) except GalaxyFDSClientException as e: CLIPrinter.fail(e.message) else: try: upload_token = self._fds.init_multipart_upload( dst_bucket_name, dst_object_name) part_number = 1 result_list = [] while True: data = f.read(multipart_upload_buffer_size) if len(data) <= 0: break for i in range(max_upload_retry_time): upload_result = None try: upload_result = self._fds.upload_part( dst_bucket_name, dst_object_name, upload_token.upload_id, part_number, data) result_list.append(upload_result) break except GalaxyFDSClientException as e: sleep_seconds = (i + 1) * 10 CLIPrinter.warn( "upload part %d failed, retry after %d seconds" % (part_number, sleep_seconds)) time.sleep(sleep_seconds) part_number = part_number + 1 upload_part_result = UploadPartResultList( {"uploadPartResultList": result_list}) result = self._fds.complete_multipart_upload( upload_token.bucket_name, upload_token.object_name, upload_token.upload_id, metadata, json.dumps(upload_part_result)) except Exception as e: self._fds.abort_multipart_upload(dst_bucket_name, dst_object_name, upload_token.upload_id) CLIPrinter.fail(e.message) if result is not None: CLIPrinter.done('upload object %s/%s' % (dst_bucket_name, dst_object_name)) else: CLIPrinter.fail('upload object %s/%s' % (dst_bucket_name, dst_object_name))
def _upload(self, filename, dst_url, autodetect_mimetype, sync=False): if not os.path.exists(filename): CLIPrinter.warn("{} is a bad file".format(filename)) return dst_bucket_name = dst_url.bucket_name() if dst_url.is_object_url(): dst_object_name = dst_url.object_name() elif sync: dst_object_name = filename[2:] elif dst_url.is_object_dir(): dst_object_name = dst_url.object_dir() + os.path.basename(filename) else: dst_object_name = os.path.basename(filename) try: if self._fds.does_object_exists(dst_bucket_name, dst_object_name): # check md5 firstly metadata = self._fds.get_object_metadata(dst_bucket_name, dst_object_name) if metadata.metadata.get(Common.CONTENT_MD5) is not None: local_md5 = file_md5(filename) if local_md5 == metadata.metadata.get(Common.CONTENT_MD5): CLIPrinter.done('upload object %s/%s(skip because of same md5)' % (dst_bucket_name, dst_object_name)) return # check last-modified mtime = None if os.path.isfile(filename): mtime = os.path.getmtime(filename) lm = metadata.metadata[Common.LAST_MODIFIED] remote_modified = rfc822_timestamp(lm) # if last-modified of local file is not less last-modified of remote file, skip if mtime is not None and datetime.fromtimestamp(mtime) <= remote_modified: CLIPrinter.done('upload object %s/%s(skip because of updated)' % (dst_bucket_name, dst_object_name)) return except Exception as e: CLIPrinter.fail(e.message) return mimetype = None if autodetect_mimetype: mimetype = mimetypes.guess_type(filename)[0] metadata = FDSObjectMetadata() if mimetype is not None: metadata.add_header(Common.CONTENT_TYPE, mimetype) result = None with open(filename, "rb") as f: file_length = os.path.getsize(filename) if file_length < multipart_upload_buffer_size: try: result = self._fds.put_object(dst_bucket_name, dst_object_name, f, metadata=metadata) except GalaxyFDSClientException as e: CLIPrinter.fail(e.message) else: try: upload_token = self._fds.init_multipart_upload(dst_bucket_name, dst_object_name) part_number = 1 result_list = [] while True: data = f.read(multipart_upload_buffer_size) if len(data) <= 0: break for i in range(max_upload_retry_time): upload_result = None try: upload_result = self._fds.upload_part(dst_bucket_name, dst_object_name, upload_token.upload_id, part_number, data) result_list.append(upload_result) break except GalaxyFDSClientException as e: sleep_seconds = (i + 1) * 10 CLIPrinter.warn("upload part %d failed, retry after %d seconds" % ( part_number, sleep_seconds)) time.sleep(sleep_seconds) part_number = part_number + 1 upload_part_result = UploadPartResultList({"uploadPartResultList": result_list}) result = self._fds.complete_multipart_upload(upload_token.bucket_name, upload_token.object_name, upload_token.upload_id, metadata, json.dumps(upload_part_result)) except Exception as e: self._fds.abort_multipart_upload(dst_bucket_name, dst_object_name, upload_token.upload_id) CLIPrinter.fail(e.message) if result is not None: CLIPrinter.done('upload object %s/%s' % (dst_bucket_name, dst_object_name)) else: CLIPrinter.fail('upload object %s/%s' % (dst_bucket_name, dst_object_name))