def post_object(self, bucket_name, data, metadata=None): ''' Post the object to a specified bucket. The object name will be generated by the server uniquely. :param bucket_name: The name of the bucket to whom the object is put :param data: The data to put, bytes or a file like object :param metadata: The metadata of the object :return: The result of posting action server returns ''' uri = '%s%s/' % (self._config.get_upload_base_uri(), bucket_name) if metadata is None: metadata = FDSObjectMetadata() if self._config.enable_md5_calculate: digest = hashlib.md5() digest.update(data) metadata.add_header(Common.CONTENT_MD5,digest.hexdigest()) response = self._request.post(uri, data=data, auth=self._auth, headers=metadata.metadata) if response.status_code == requests.codes.ok: return PutObjectResult(to_json_object(response.content)) headers = "" if self._config.debug: headers = ' header=%s' % response.headers message = 'Post object failed, status=%s, reason=%s%s' % ( response.status_code, response.content, headers) raise GalaxyFDSClientException(message)
def _upload(fpath, object_name, dst_url, autodetect_mimetype): if not fpath.exists(): CLIPrinter.warn("{} is a bad file".format(str(fpath))) return try: dst_bucket_name = dst_url.bucket_name() dst_object_name = object_name if fds_client.does_object_exists(dst_bucket_name, dst_object_name): # check md5 firstly metadata = fds_client.get_object_metadata(dst_bucket_name, dst_object_name) if metadata.metadata.get(Common.CONTENT_MD5) is not None: local_md5 = file_md5(str(fpath.resolve())) if local_md5 == metadata.metadata.get(Common.CONTENT_MD5): CLIPrinter.done( "upload object %s/%s(skip because of same md5)" % (dst_bucket_name, dst_object_name)) return # check last-modified mtime = None if fpath.is_file(): mtime = os.path.getmtime(str(fpath.resolve())) lm = metadata.metadata[Common.LAST_MODIFIED] remote_modified = rfc822_timestamp(lm) # if last-modified of local file is not less last-modified of remote file, skip if mtime is not None and datetime.fromtimestamp( mtime) <= remote_modified: CLIPrinter.done( "upload object %s/%s(skip because of updated)" % (dst_bucket_name, dst_object_name)) return except Exception as e: CLIPrinter.fail(e.message) return mimetype = None if autodetect_mimetype: mimetype = mimetypes.guess_type(str(fpath.resolve()))[0] metadata = FDSObjectMetadata() if mimetype is not None: metadata.add_header(Common.CONTENT_TYPE, mimetype) with open(str(fpath.resolve()), "rb") as f: try: fds_client.put_object(dst_bucket_name, dst_object_name, f, metadata=metadata) CLIPrinter.done("upload object %s/%s" % (dst_bucket_name, dst_object_name)) except GalaxyFDSClientException as e: CLIPrinter.fail("upload object %s/%s, %s" % (dst_bucket_name, dst_object_name, e.message))
def test_invalid_object_metadata(self): metadata = FDSObjectMetadata() metadata.add_user_metadata(FDSObjectMetadata.USER_DEFINED_METADATA_PREFIX + "test", "test-value") metadata.add_header(Common.CACHE_CONTROL, "no-cache") try: metadata.add_user_metadata("test-key", "test-vale") self.fail("Exception should not be thrown here") except: pass
def parse_metadata_from_str(metadata): fds_metadata = None if metadata: fds_metadata = FDSObjectMetadata() for i in metadata.split(';'): key, value = i.split(':', 1) if key and value: if key.startswith(FDSObjectMetadata.USER_DEFINED_METADATA_PREFIX): fds_metadata.add_user_metadata(key, value) else: fds_metadata.add_header(key, value) return fds_metadata
def _parse_object_metadata_from_headers(self, response_headers): ''' Parse object metadata from the response headers. ''' metadata = FDSObjectMetadata() header_keys = [c.lower() for c in response_headers.keys()]; for key in FDSObjectMetadata.PRE_DEFINED_METADATA: if key.lower() in header_keys: metadata.add_header(key, response_headers[key]) for key in response_headers: if key.lower().startswith(FDSObjectMetadata.USER_DEFINED_METADATA_PREFIX): metadata.add_user_metadata(key, response_headers[key]) return metadata
def put_object(self, bucket_name, object_name, data, metadata=None): ''' Put the object to a specified bucket. If a object with the same name already existed, it will be overwritten. :param bucket_name: The name of the bucket to whom the object is put :param object_name: The name of the object to put :param data: The data to put, bytes or a file like object :param metadata: The metadata of the object :return: The result of putting action server returns ''' uri = '%s%s/%s' % (self._config.get_upload_base_uri(), bucket_name, object_name) if metadata is None: metadata = FDSObjectMetadata() if self._config.enable_md5_calculate: digest = hashlib.md5() if IS_PY3: if isinstance(data, str): data = data.encode(encoding="UTF-8") digest.update(data) elif data.seekable and data.seekable(): pos = data.tell() digest.update(data.read()) data.seek(0, pos) else: raise GalaxyFDSClientException("Cannot digest data") else: digest.update(data) metadata.add_header(Common.CONTENT_MD5, digest.hexdigest()) response = self._request.put(uri, data=data, auth=self._auth, headers=metadata.metadata) if response.status_code == requests.codes.ok: return PutObjectResult(to_json_object(response.content)) headers = "" if self._config.debug: headers = ' header=%s' % response.headers message = 'Put object failed, status=%s, reason=%s%s' % ( response.status_code, response.content, headers) raise GalaxyFDSClientException(message)
# endpoint is auto-read from ~/.config/xiaomi/config client = GalaxyFDSClient() bucket = os.environ.get('XIAOMI_FDS_DEFAULT_BUCKET') or 'johndoe' #LOG_DIR = '/home/mi/Documents/github/changbinglin/aicontest/logs' DIRS = [ 'camera_data/raw_train', 'camera_data/raw_val', 'keras_model', 'tensorflow_model' ] if len(sys.argv) > 1: LOG_DIR = sys.argv[1] metadata = FDSObjectMetadata() metadata.add_header('x-xiaomi-meta-mode', '33188') # give rights: rw-r--r-- try: for directory in DIRS: for log in glob.glob(directory + '/*'): if os.path.isfile(log): print log.split('/')[-1] if not client.does_object_exists(bucket, log): with open(log, 'r') as f: data = f.read() #path_to = '/'.join(LOG_DIR.split('/')[-3:]) res = client.put_object(bucket, log, data, metadata) print 'Put Object: ', res.signature, res.expires client.set_public(bucket, log) print 'Set public', log except GalaxyFDSClientException as e:
def _upload(self, filename, dst_url, autodetect_mimetype, sync=False): if not os.path.exists(filename): CLIPrinter.warn("{} is a bad file".format(filename)) return dst_bucket_name = dst_url.bucket_name() if dst_url.is_object_url(): dst_object_name = dst_url.object_name() elif sync: dst_object_name = filename[2:] elif dst_url.is_object_dir(): dst_object_name = dst_url.object_dir() + os.path.basename(filename) else: dst_object_name = os.path.basename(filename) try: if self._fds.does_object_exists(dst_bucket_name, dst_object_name): # check md5 firstly metadata = self._fds.get_object_metadata( dst_bucket_name, dst_object_name) if metadata.metadata.get(Common.CONTENT_MD5) is not None: local_md5 = file_md5(filename) if local_md5 == metadata.metadata.get(Common.CONTENT_MD5): CLIPrinter.done( 'upload object %s/%s(skip because of same md5)' % (dst_bucket_name, dst_object_name)) return # check last-modified mtime = None if os.path.isfile(filename): mtime = os.path.getmtime(filename) lm = metadata.metadata[Common.LAST_MODIFIED] remote_modified = rfc822_timestamp(lm) # if last-modified of local file is not less last-modified of remote file, skip if mtime is not None and datetime.fromtimestamp( mtime) <= remote_modified: CLIPrinter.done( 'upload object %s/%s(skip because of updated)' % (dst_bucket_name, dst_object_name)) return except Exception as e: CLIPrinter.fail(e.message) return mimetype = None if autodetect_mimetype: mimetype = mimetypes.guess_type(filename)[0] metadata = FDSObjectMetadata() if mimetype is not None: metadata.add_header(Common.CONTENT_TYPE, mimetype) result = None with open(filename, "rb") as f: file_length = os.path.getsize(filename) if file_length < multipart_upload_buffer_size: try: result = self._fds.put_object(dst_bucket_name, dst_object_name, f, metadata=metadata) except GalaxyFDSClientException as e: CLIPrinter.fail(e.message) else: try: upload_token = self._fds.init_multipart_upload( dst_bucket_name, dst_object_name) part_number = 1 result_list = [] while True: data = f.read(multipart_upload_buffer_size) if len(data) <= 0: break for i in range(max_upload_retry_time): upload_result = None try: upload_result = self._fds.upload_part( dst_bucket_name, dst_object_name, upload_token.upload_id, part_number, data) result_list.append(upload_result) break except GalaxyFDSClientException as e: sleep_seconds = (i + 1) * 10 CLIPrinter.warn( "upload part %d failed, retry after %d seconds" % (part_number, sleep_seconds)) time.sleep(sleep_seconds) part_number = part_number + 1 upload_part_result = UploadPartResultList( {"uploadPartResultList": result_list}) result = self._fds.complete_multipart_upload( upload_token.bucket_name, upload_token.object_name, upload_token.upload_id, metadata, json.dumps(upload_part_result)) except Exception as e: self._fds.abort_multipart_upload(dst_bucket_name, dst_object_name, upload_token.upload_id) CLIPrinter.fail(e.message) if result is not None: CLIPrinter.done('upload object %s/%s' % (dst_bucket_name, dst_object_name)) else: CLIPrinter.fail('upload object %s/%s' % (dst_bucket_name, dst_object_name))
def _upload(self, filename, dst_url, autodetect_mimetype, sync=False): if not os.path.exists(filename): CLIPrinter.warn("{} is a bad file".format(filename)) return dst_bucket_name = dst_url.bucket_name() if dst_url.is_object_url(): dst_object_name = dst_url.object_name() elif sync: dst_object_name = filename[2:] elif dst_url.is_object_dir(): dst_object_name = dst_url.object_dir() + os.path.basename(filename) else: dst_object_name = os.path.basename(filename) try: if self._fds.does_object_exists(dst_bucket_name, dst_object_name): # check md5 firstly metadata = self._fds.get_object_metadata(dst_bucket_name, dst_object_name) if metadata.metadata.get(Common.CONTENT_MD5) is not None: local_md5 = file_md5(filename) if local_md5 == metadata.metadata.get(Common.CONTENT_MD5): CLIPrinter.done('upload object %s/%s(skip because of same md5)' % (dst_bucket_name, dst_object_name)) return # check last-modified mtime = None if os.path.isfile(filename): mtime = os.path.getmtime(filename) lm = metadata.metadata[Common.LAST_MODIFIED] remote_modified = rfc822_timestamp(lm) # if last-modified of local file is not less last-modified of remote file, skip if mtime is not None and datetime.fromtimestamp(mtime) <= remote_modified: CLIPrinter.done('upload object %s/%s(skip because of updated)' % (dst_bucket_name, dst_object_name)) return except Exception as e: CLIPrinter.fail(e.message) return mimetype = None if autodetect_mimetype: mimetype = mimetypes.guess_type(filename)[0] metadata = FDSObjectMetadata() if mimetype is not None: metadata.add_header(Common.CONTENT_TYPE, mimetype) result = None with open(filename, "rb") as f: file_length = os.path.getsize(filename) if file_length < multipart_upload_buffer_size: try: result = self._fds.put_object(dst_bucket_name, dst_object_name, f, metadata=metadata) except GalaxyFDSClientException as e: CLIPrinter.fail(e.message) else: try: upload_token = self._fds.init_multipart_upload(dst_bucket_name, dst_object_name) part_number = 1 result_list = [] while True: data = f.read(multipart_upload_buffer_size) if len(data) <= 0: break for i in range(max_upload_retry_time): upload_result = None try: upload_result = self._fds.upload_part(dst_bucket_name, dst_object_name, upload_token.upload_id, part_number, data) result_list.append(upload_result) break except GalaxyFDSClientException as e: sleep_seconds = (i + 1) * 10 CLIPrinter.warn("upload part %d failed, retry after %d seconds" % ( part_number, sleep_seconds)) time.sleep(sleep_seconds) part_number = part_number + 1 upload_part_result = UploadPartResultList({"uploadPartResultList": result_list}) result = self._fds.complete_multipart_upload(upload_token.bucket_name, upload_token.object_name, upload_token.upload_id, metadata, json.dumps(upload_part_result)) except Exception as e: self._fds.abort_multipart_upload(dst_bucket_name, dst_object_name, upload_token.upload_id) CLIPrinter.fail(e.message) if result is not None: CLIPrinter.done('upload object %s/%s' % (dst_bucket_name, dst_object_name)) else: CLIPrinter.fail('upload object %s/%s' % (dst_bucket_name, dst_object_name))