def post_object(self, bucket_name, data, metadata=None):
'''
Post the object to a specified bucket. The object name will be generated
by the server uniquely.
:param bucket_name: The name of the bucket to whom the object is put
:param data: The data to put, bytes or a file like object
:param metadata: The metadata of the object
:return: The result of posting action server returns
'''
uri = '%s%s/' % (self._config.get_upload_base_uri(), bucket_name)
if metadata is None:
metadata = FDSObjectMetadata()
if self._config.enable_md5_calculate:
digest = hashlib.md5()
digest.update(data)
metadata.add_header(Common.CONTENT_MD5,digest.hexdigest())
response = self._request.post(uri, data=data, auth=self._auth,
headers=metadata.metadata)
if response.status_code == requests.codes.ok:
return PutObjectResult(to_json_object(response.content))
headers = ""
if self._config.debug:
headers = ' header=%s' % response.headers
message = 'Post object failed, status=%s, reason=%s%s' % (
response.status_code, response.content, headers)
raise GalaxyFDSClientException(message)
def _upload(fpath, object_name, dst_url, autodetect_mimetype):
if not fpath.exists():
CLIPrinter.warn("{} is a bad file".format(str(fpath)))
return
try:
dst_bucket_name = dst_url.bucket_name()
dst_object_name = object_name
if fds_client.does_object_exists(dst_bucket_name, dst_object_name):
# check md5 firstly
metadata = fds_client.get_object_metadata(dst_bucket_name,
dst_object_name)
if metadata.metadata.get(Common.CONTENT_MD5) is not None:
local_md5 = file_md5(str(fpath.resolve()))
if local_md5 == metadata.metadata.get(Common.CONTENT_MD5):
CLIPrinter.done(
"upload object %s/%s(skip because of same md5)" %
(dst_bucket_name, dst_object_name))
return
# check last-modified
mtime = None
if fpath.is_file():
mtime = os.path.getmtime(str(fpath.resolve()))
lm = metadata.metadata[Common.LAST_MODIFIED]
remote_modified = rfc822_timestamp(lm)
# if last-modified of local file is not less last-modified of remote file, skip
if mtime is not None and datetime.fromtimestamp(
mtime) <= remote_modified:
CLIPrinter.done(
"upload object %s/%s(skip because of updated)" %
(dst_bucket_name, dst_object_name))
return
except Exception as e:
CLIPrinter.fail(e.message)
return
mimetype = None
if autodetect_mimetype:
mimetype = mimetypes.guess_type(str(fpath.resolve()))[0]
metadata = FDSObjectMetadata()
if mimetype is not None:
metadata.add_header(Common.CONTENT_TYPE, mimetype)
with open(str(fpath.resolve()), "rb") as f:
try:
fds_client.put_object(dst_bucket_name,
dst_object_name,
f,
metadata=metadata)
CLIPrinter.done("upload object %s/%s" %
(dst_bucket_name, dst_object_name))
except GalaxyFDSClientException as e:
CLIPrinter.fail("upload object %s/%s, %s" %
(dst_bucket_name, dst_object_name, e.message))
def parse_metadata_from_str(metadata):
fds_metadata = None
if metadata:
fds_metadata = FDSObjectMetadata()
for i in metadata.split(';'):
key, value = i.split(':', 1)
if key and value:
if key.startswith(FDSObjectMetadata.USER_DEFINED_METADATA_PREFIX):
fds_metadata.add_user_metadata(key, value)
else:
fds_metadata.add_header(key, value)
return fds_metadata
def complete_multipart_upload(self, bucket_name, object_name, upload_id,
metadata, upload_part_result_list):
'''
Complete a multipart upload
:param bucket_name:
:param object_name:
:param upload_id:
:param metadata:
:param upload_part_result_list:
:return:
'''
uri = '%s%s/%s?%s' % (
self._config.get_base_uri(), bucket_name, object_name, "uploadId=" +
upload_id)
if metadata is None:
metadata = FDSObjectMetadata()
response = self._request.put(uri, auth=self._auth,
data=upload_part_result_list, headers=metadata.metadata)
if response.status_code == requests.codes.ok:
result = PutObjectResult(to_json_object(response.content))
return result
else:
headers = ""
if self._config.debug:
headers = ' headers=%s' % response.headers
message = 'Complete multipart upload failed, status=%s, reason=%s%s' % (
response.status_code, response.content, headers)
raise GalaxyFDSClientException(message)
def test_invalid_object_metadata(self):
metadata = FDSObjectMetadata()
metadata.add_user_metadata(FDSObjectMetadata.USER_DEFINED_METADATA_PREFIX
+ "test", "test-value")
metadata.add_header(Common.CACHE_CONTROL, "no-cache")
try:
metadata.add_user_metadata("test-key", "test-vale")
self.fail("Exception should not be thrown here")
except:
pass
def put_object(self, bucket_name, object_name, data, metadata=None):
'''
Put the object to a specified bucket. If a object with the same name already
existed, it will be overwritten.
:param bucket_name: The name of the bucket to whom the object is put
:param object_name: The name of the object to put
:param data: The data to put, bytes or a file like object
:param metadata: The metadata of the object
:return: The result of putting action server returns
'''
uri = '%s%s/%s' % (self._config.get_upload_base_uri(), bucket_name,
object_name)
if metadata is None:
metadata = FDSObjectMetadata()
if self._config.enable_md5_calculate:
digest = hashlib.md5()
if IS_PY3:
if isinstance(data, str):
data = data.encode(encoding="UTF-8")
digest.update(data)
elif data.seekable and data.seekable():
pos = data.tell()
digest.update(data.read())
data.seek(0, pos)
else:
raise GalaxyFDSClientException("Cannot digest data")
else:
digest.update(data)
metadata.add_header(Common.CONTENT_MD5, digest.hexdigest())
response = self._request.put(uri, data=data, auth=self._auth,
headers=metadata.metadata)
if response.status_code == requests.codes.ok:
return PutObjectResult(to_json_object(response.content))
headers = ""
if self._config.debug:
headers = ' header=%s' % response.headers
message = 'Put object failed, status=%s, reason=%s%s' % (
response.status_code, response.content, headers)
raise GalaxyFDSClientException(message)
def test_invalid_object_metadata(self):
metadata = FDSObjectMetadata()
metadata.add_user_metadata(FDSObjectMetadata.USER_DEFINED_METADATA_PREFIX
+ "test", "test-value")
metadata.add_header(Common.CACHE_CONTROL, "no-cache")
try:
metadata.add_user_metadata("test-key", "test-vale")
self.fail("Exception should not be thrown here")
except:
pass
def parse_metadata_from_str(metadata):
fds_metadata = None
if metadata:
fds_metadata = FDSObjectMetadata()
for i in metadata.split(';'):
key, value = i.split(':', 1)
if key and value:
if key.startswith(FDSObjectMetadata.USER_DEFINED_METADATA_PREFIX):
fds_metadata.add_user_metadata(key, value)
else:
fds_metadata.add_header(key, value)
return fds_metadata
def _parse_object_metadata_from_headers(self, response_headers):
'''
Parse object metadata from the response headers.
'''
metadata = FDSObjectMetadata()
header_keys = [c.lower() for c in response_headers.keys()];
for key in FDSObjectMetadata.PRE_DEFINED_METADATA:
if key.lower() in header_keys:
metadata.add_header(key, response_headers[key])
for key in response_headers:
if key.lower().startswith(FDSObjectMetadata.USER_DEFINED_METADATA_PREFIX):
metadata.add_user_metadata(key, response_headers[key])
return metadata
from fds.model.fds_object_metadata import FDSObjectMetadata
# endpoint is auto-read from ~/.config/xiaomi/config
client = GalaxyFDSClient()
bucket = os.environ.get('XIAOMI_FDS_DEFAULT_BUCKET') or 'johndoe'
#LOG_DIR = '/home/mi/Documents/github/changbinglin/aicontest/logs'
DIRS = [
'camera_data/raw_train', 'camera_data/raw_val', 'keras_model',
'tensorflow_model'
]
if len(sys.argv) > 1:
LOG_DIR = sys.argv[1]
metadata = FDSObjectMetadata()
metadata.add_header('x-xiaomi-meta-mode', '33188') # give rights: rw-r--r--
try:
for directory in DIRS:
for log in glob.glob(directory + '/*'):
if os.path.isfile(log):
print log.split('/')[-1]
if not client.does_object_exists(bucket, log):
with open(log, 'r') as f:
data = f.read()
#path_to = '/'.join(LOG_DIR.split('/')[-3:])
res = client.put_object(bucket, log, data, metadata)
print 'Put Object: ', res.signature, res.expires
client.set_public(bucket, log)
print 'Set public', log
def _upload(self, filename, dst_url, autodetect_mimetype, sync=False):
if not os.path.exists(filename):
CLIPrinter.warn("{} is a bad file".format(filename))
return
dst_bucket_name = dst_url.bucket_name()
if dst_url.is_object_url():
dst_object_name = dst_url.object_name()
elif sync:
dst_object_name = filename[2:]
elif dst_url.is_object_dir():
dst_object_name = dst_url.object_dir() + os.path.basename(filename)
else:
dst_object_name = os.path.basename(filename)
try:
if self._fds.does_object_exists(dst_bucket_name, dst_object_name):
# check md5 firstly
metadata = self._fds.get_object_metadata(
dst_bucket_name, dst_object_name)
if metadata.metadata.get(Common.CONTENT_MD5) is not None:
local_md5 = file_md5(filename)
if local_md5 == metadata.metadata.get(Common.CONTENT_MD5):
CLIPrinter.done(
'upload object %s/%s(skip because of same md5)' %
(dst_bucket_name, dst_object_name))
return
# check last-modified
mtime = None
if os.path.isfile(filename):
mtime = os.path.getmtime(filename)
lm = metadata.metadata[Common.LAST_MODIFIED]
remote_modified = rfc822_timestamp(lm)
# if last-modified of local file is not less last-modified of remote file, skip
if mtime is not None and datetime.fromtimestamp(
mtime) <= remote_modified:
CLIPrinter.done(
'upload object %s/%s(skip because of updated)' %
(dst_bucket_name, dst_object_name))
return
except Exception as e:
CLIPrinter.fail(e.message)
return
mimetype = None
if autodetect_mimetype:
mimetype = mimetypes.guess_type(filename)[0]
metadata = FDSObjectMetadata()
if mimetype is not None:
metadata.add_header(Common.CONTENT_TYPE, mimetype)
result = None
with open(filename, "rb") as f:
file_length = os.path.getsize(filename)
if file_length < multipart_upload_buffer_size:
try:
result = self._fds.put_object(dst_bucket_name,
dst_object_name,
f,
metadata=metadata)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
else:
try:
upload_token = self._fds.init_multipart_upload(
dst_bucket_name, dst_object_name)
part_number = 1
result_list = []
while True:
data = f.read(multipart_upload_buffer_size)
if len(data) <= 0:
break
for i in range(max_upload_retry_time):
upload_result = None
try:
upload_result = self._fds.upload_part(
dst_bucket_name, dst_object_name,
upload_token.upload_id, part_number, data)
result_list.append(upload_result)
break
except GalaxyFDSClientException as e:
sleep_seconds = (i + 1) * 10
CLIPrinter.warn(
"upload part %d failed, retry after %d seconds"
% (part_number, sleep_seconds))
time.sleep(sleep_seconds)
part_number = part_number + 1
upload_part_result = UploadPartResultList(
{"uploadPartResultList": result_list})
result = self._fds.complete_multipart_upload(
upload_token.bucket_name, upload_token.object_name,
upload_token.upload_id, metadata,
json.dumps(upload_part_result))
except Exception as e:
self._fds.abort_multipart_upload(dst_bucket_name,
dst_object_name,
upload_token.upload_id)
CLIPrinter.fail(e.message)
if result is not None:
CLIPrinter.done('upload object %s/%s' %
(dst_bucket_name, dst_object_name))
else:
CLIPrinter.fail('upload object %s/%s' %
(dst_bucket_name, dst_object_name))
def _upload(self, filename, dst_url, autodetect_mimetype, sync=False):
if not os.path.exists(filename):
CLIPrinter.warn("{} is a bad file".format(filename))
return
dst_bucket_name = dst_url.bucket_name()
if dst_url.is_object_url():
dst_object_name = dst_url.object_name()
elif sync:
dst_object_name = filename[2:]
elif dst_url.is_object_dir():
dst_object_name = dst_url.object_dir() + os.path.basename(filename)
else:
dst_object_name = os.path.basename(filename)
try:
if self._fds.does_object_exists(dst_bucket_name, dst_object_name):
# check md5 firstly
metadata = self._fds.get_object_metadata(dst_bucket_name, dst_object_name)
if metadata.metadata.get(Common.CONTENT_MD5) is not None:
local_md5 = file_md5(filename)
if local_md5 == metadata.metadata.get(Common.CONTENT_MD5):
CLIPrinter.done('upload object %s/%s(skip because of same md5)' % (dst_bucket_name, dst_object_name))
return
# check last-modified
mtime = None
if os.path.isfile(filename):
mtime = os.path.getmtime(filename)
lm = metadata.metadata[Common.LAST_MODIFIED]
remote_modified = rfc822_timestamp(lm)
# if last-modified of local file is not less last-modified of remote file, skip
if mtime is not None and datetime.fromtimestamp(mtime) <= remote_modified:
CLIPrinter.done('upload object %s/%s(skip because of updated)' % (dst_bucket_name, dst_object_name))
return
except Exception as e:
CLIPrinter.fail(e.message)
return
mimetype = None
if autodetect_mimetype:
mimetype = mimetypes.guess_type(filename)[0]
metadata = FDSObjectMetadata()
if mimetype is not None:
metadata.add_header(Common.CONTENT_TYPE, mimetype)
result = None
with open(filename, "rb") as f:
file_length = os.path.getsize(filename)
if file_length < multipart_upload_buffer_size:
try:
result = self._fds.put_object(dst_bucket_name, dst_object_name, f, metadata=metadata)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
else:
try:
upload_token = self._fds.init_multipart_upload(dst_bucket_name, dst_object_name)
part_number = 1
result_list = []
while True:
data = f.read(multipart_upload_buffer_size)
if len(data) <= 0:
break
for i in range(max_upload_retry_time):
upload_result = None
try:
upload_result = self._fds.upload_part(dst_bucket_name, dst_object_name, upload_token.upload_id, part_number, data)
result_list.append(upload_result)
break
except GalaxyFDSClientException as e:
sleep_seconds = (i + 1) * 10
CLIPrinter.warn("upload part %d failed, retry after %d seconds" % (
part_number, sleep_seconds))
time.sleep(sleep_seconds)
part_number = part_number + 1
upload_part_result = UploadPartResultList({"uploadPartResultList": result_list})
result = self._fds.complete_multipart_upload(upload_token.bucket_name, upload_token.object_name,
upload_token.upload_id, metadata,
json.dumps(upload_part_result))
except Exception as e:
self._fds.abort_multipart_upload(dst_bucket_name, dst_object_name, upload_token.upload_id)
CLIPrinter.fail(e.message)
if result is not None:
CLIPrinter.done('upload object %s/%s' % (dst_bucket_name, dst_object_name))
else:
CLIPrinter.fail('upload object %s/%s' % (dst_bucket_name, dst_object_name))