class multithreadingClientTest(unittest.TestCase):
def setUp(self):
config = FDSClientConfiguration(region_name, False, False, False)
self.client = GalaxyFDSClient(access_key, access_secret, config)
self.bucket_name = "1024"
def checkAndDeleteBucket(self, index):
for i in range(50):
self.client.list_objects(self.bucket_name, "", "/")
print index
def testMultithreadingClient(self):
pool = ThreadPool(50)
if not self.client.does_bucket_exist(self.bucket_name):
self.client.create_bucket(self.bucket_name)
pool.map(self.checkAndDeleteBucket, range(50))
pool.close()
pool.join()
class multithreadingClientTest(unittest.TestCase):
def setUp(self):
config = FDSClientConfiguration(region_name, False, False, False)
self.client = GalaxyFDSClient(access_key, access_secret, config)
self.bucket_name = "1024"
def checkAndDeleteBucket(self, index):
for i in range(50):
self.client.list_objects(self.bucket_name, "", "/")
print
index
def testMultithreadingClient(self):
pool = ThreadPool(50)
if not self.client.does_bucket_exist(self.bucket_name):
self.client.create_bucket(self.bucket_name)
pool.map(self.checkAndDeleteBucket, range(50))
pool.close()
pool.join()
# the settings for its parents
#os.environ['XIAOMI_FDS_DEFAULT_BUCKET'] = bucket
#os.system("export XIAOMI_FDS_DEFAULT_BUCKET='%s'"%bucket)
# permanently for next restart
with open(os.path.expanduser("~/.bashrc"), "a") as f:
f.write("export XIAOMI_FDS_DEFAULT_BUCKET='%s'\n" % bucket)
print 'Warning: You must restart a terminal window to take into effect.'
# endpoint is read from ~/.config/xiaomi/config
#client = GalaxyFDSClient(XIAOMI_ACCESS_KEY_ID, XIAOMI_SECRET_ACCESS_KEY)
client = GalaxyFDSClient()
if not client.does_bucket_exist(bucket):
try:
print 'Create bucket ', bucket
client.create_bucket(bucket)
except GalaxyFDSClientException as e:
print e.message
# no need to delete since duplicate objects will be replaced when put
# try:
# client.delete_object(bucket, "tf_train-1.0.tar.gz")
# except GalaxyFDSClientException as e:
# print e.message
tensorflow_model = 'TFTrainerPredictor-1.4.tar.gz'
keras_model = 'KerasTrainer-1.0.tar.gz'
try:
with open('../dist/' + tensorflow_model, 'rb') as f:
data = f.read()
class FDSCli(object):
"""
Advanced fds cli you deserved!
"""
def __init__(self, ak=None, sk=None, endpoint=None):
self._fds_prefix = r'fds://'
self._fds_prefix_len = len(self._fds_prefix)
self._local_config = LocalConfig()
self._ak = self._local_config.ak if ak is None else ak
if self._ak is None:
self.config()
self._sk = self._local_config.sk if sk is None else sk
if self._sk is None:
self.config()
self._endpoint = self._local_config.endpoint if endpoint is None else endpoint
if self._sk is None:
self.config()
logger.debug("endpoint: " + self._endpoint)
self._fds_config = FDSClientConfiguration(
region_name="awsde0",
enable_https=False,
enable_cdn_for_download=False,
enable_cdn_for_upload=False,
endpoint=self._endpoint)
self._fds = GalaxyFDSClient(access_key=self._ak,
access_secret=self._sk,
config=self._fds_config)
def config(self):
"""
config command configures ak sk and endpoint
:return:
"""
default_ak = self._local_config.ak
default_sk = self._local_config.sk
default_endpoint = self._local_config.endpoint
ak = input("enter access key id[default: %s]: " % default_ak)
if ak == '':
ak = default_ak
sk = input("enter secret access key[default: %s]: " % default_sk)
if sk == '':
sk = default_sk
endpoint = input("enter endpoint[default: %s]: " % default_endpoint)
if endpoint == '':
endpoint = default_endpoint
self._local_config.ak = ak
self._local_config.sk = sk
self._local_config.endpoint = endpoint
def mb(self, fds_url):
"""
create(make) a bucket
:param fds_url: fds url format like fds://bucket_name_to_make
"""
url = FDSURL(fds_url)
if not url.is_bucket_url():
CLIPrinter.wrong_format()
bucket_name = url.bucket_name()
try:
self._fds.create_bucket(bucket_name)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
CLIPrinter.done("create bucket [%s]" % bucket_name)
def rb(self, fds_url, force=False):
"""
delete(remove) a bucket
:param fds_url: fds url format like fds://bucket_name_to_delete
:param force: remove a bucket even if this bucket is not empty
"""
url = FDSURL(fds_url)
if not url.is_bucket_url():
CLIPrinter.wrong_format()
return
bucket_name = url.bucket_name()
if force:
all_objects = self._fds.list_all_objects(bucket_name, '', '')
names = []
try:
for o in all_objects:
names.append(o.object_name)
except GalaxyFDSClientException as e:
CLIPrinter.warn(e.message)
try:
self._fds.delete_objects(bucket_name, names)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
try:
self._fds.delete_bucket(bucket_name)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
CLIPrinter.done("remove bucket [%s]" % bucket_name)
def rm(self, fds_url):
"""
delete(remove) a object
:param fds_url: fds url format like fds://bucket_name/object_name_to_delete
"""
url = FDSURL(fds_url)
bucket_name = url.bucket_name()
if url.is_bucket_url():
CLIPrinter.fail("please enter a object resource address to remove")
return
object_name = url.object_name()
try:
self._fds.delete_object(bucket_name, object_name)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
CLIPrinter.done("remove object: [%s] in bucket [%s]" %
(object_name, bucket_name))
def ls(self, fds_url=None, recursive=False, human='k'):
"""
list all buckets or objects in a bucket
:param fds_url: fds url format like fds://bucket
:param recursive: recursive listing
"""
if human != 'k' and human != 'm' and human != 'g':
CLIPrinter.fail("human should be in 'k|m|g'")
return
# bucket_url is None means listing all bucket name
if fds_url is None:
buckets = None
try:
buckets = self._fds.list_authorized_buckets()
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
for bucket in buckets:
CLIPrinter.print_bucket(bucket)
else:
delimiter = "/"
if recursive:
delimiter = ""
url = FDSURL(fds_url)
bucket_name = url.bucket_name()
prefix = ''
if url.is_bucket_url() or url.is_object_dir():
if url.is_object_dir():
prefix = url.object_dir()
results = self._fds.list_all_objects(bucket_name, prefix,
delimiter)
try:
for result in results:
metadata = self._fds.get_object_metadata(
bucket_name, result.object_name)
CLIPrinter.print_object(result.object_name, metadata,
human)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
else:
object_name = url.object_name()
metadata = None
try:
metadata = self._fds.get_object_metadata(
bucket_name, object_name)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
if metadata is not None:
CLIPrinter.print_object(object_name, metadata, human)
def presigned(self, fds_url, expires=1, cdn=False):
"""
presigned command generates presigned url for download project
:param fds_url: format url like fds://bucket/a.txt
:param expires: expiration time in minutes
:return: presigned url for downloading
"""
url = FDSURL(fds_url)
bucket_name = url.bucket_name()
if url.is_bucket_url():
CLIPrinter.fail("%uri is illegal" % fds_url)
object_name = url.object_name()
expiration = int(1000 * (float(expires) * 3600 + float(
(datetime.utcnow() -
datetime(1970, 1, 1, 0, 0, 0, 0)).total_seconds())))
try:
if cdn:
base_uri = self._fds_config.get_cdn_base_uri()
else:
base_uri = self._fds_config.get_base_uri()
u = self._fds.generate_presigned_uri(base_uri, bucket_name,
object_name, expiration)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
CLIPrinter.done('generated presigned url: ' + u)
def ttl(self, fds_url):
"""
ttl command shows the lifecycle information of a bucket or a object
:param fds_url: format url like fds://bucket/a.txt or fds://bucket/
"""
url = FDSURL(fds_url)
bucket_name = url.bucket_name()
try:
ttl = self._fds.get_lifecycle_config(bucket_name)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
if url.is_bucket_url():
CLIPrinter.print_lifecycle(ttl)
elif url.is_object_url():
if not self._fds.does_object_exists(bucket_name,
url.object_name()):
CLIPrinter.fail("object does not exists")
if url.is_object_dir():
prefix = url.object_dir()
else:
prefix = url.object_name()
rules = [rule for rule in ttl['rules'] if rule['prefix'] in prefix]
CLIPrinter.print_lifecycle({"rules": rules})
else:
CLIPrinter.wrong_format()
def cp(self, src, dst, recursive=False, autodetect_mimetype=False):
"""
cp command do lots of things.
1. file upload
2. batch files upload
3. file download
4. batch files download
5. rename object
6. object copy
7. batch objects copy
:param src: source fds url format like fds://bucket
:param dst: target fds url format like fds://bucket
:param recursive: recursive listing
"""
if FDSURL.is_fds_url(src) and FDSURL.is_fds_url(dst):
src_url = FDSURL(src)
dst_url = FDSURL(dst)
if src_url.is_object_url():
self._cp(src_url, dst_url)
elif not src_url.is_object_url() and not dst_url.is_object_url():
self._cp_batch(src_url, dst_url, recursive)
else:
CLIPrinter.wrong_format()
elif FDSURL.is_fds_url(src) and not FDSURL.is_fds_url(dst):
src_url = FDSURL(src)
if src_url.is_object_url():
self._download(src_url, dst)
elif src_url.is_object_dir() and os.path.isdir(dst):
self._download_batch(src_url, dst, recursive)
else:
CLIPrinter.wrong_format()
elif not FDSURL.is_fds_url(src) and FDSURL.is_fds_url(dst):
dst_url = FDSURL(dst)
if os.path.isfile(src):
self._upload(src,
dst_url,
autodetect_mimetype=autodetect_mimetype)
elif os.path.isdir(src) and not dst_url.is_object_url():
self._upload_batch(src,
dst_url,
recursive,
autodetect_mimetype=autodetect_mimetype)
else:
CLIPrinter.wrong_format()
else:
CLIPrinter.fail("don't support copy file from local to local")
def _cp(self, src_url, dst_url):
src_bucket_name = src_url.bucket_name()
src_object_name = src_url.object_name()
dst_bucket_name = dst_url.bucket_name()
if dst_url.is_object_url():
dst_object_name = dst_url.object_name()
else:
dst_object_name = src_object_name
try:
self._fds.copy_object(src_bucket_name, src_object_name,
dst_bucket_name, dst_object_name)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
CLIPrinter.done("copy %s/%s to %s/%s" %
(src_bucket_name, src_object_name, dst_bucket_name,
dst_object_name))
def _cp_batch(self, src_url, dst_url, recursive):
src_bucket_name = src_url.bucket_name()
dst_bucket_name = dst_url.bucket_name()
prefix = ""
if src_url.is_object_dir():
prefix = src_url.object_dir()
delimiter = "/"
if recursive:
delimiter = ""
all_objects = self._fds.list_all_objects(bucket_name=src_bucket_name,
prefix=prefix,
delimiter=delimiter)
try:
for o in all_objects:
o_name = o.object_name
self._fds.copy_object(src_bucket_name, o_name, dst_bucket_name,
o_name)
CLIPrinter.done(
"copy %s/%s to %s/%s" %
(src_bucket_name, o_name, dst_bucket_name, o_name))
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
def _download(self, src_url, dst):
src_bucket_name = src_url.bucket_name()
src_object_name = src_url.object_name()
if os.path.isdir(dst):
if dst == '.' or dst == '..':
dst_name = src_url.file_name()
elif dst.endswith('/'):
dst_name = dst + src_url.file_name()
else:
dst_name = dst + '/' + src_object_name.split('/')[-1]
else:
dst_name = dst
mtime = None
if os.path.isfile(dst_name):
local_md5 = file_md5(dst_name)
remote_md5 = self._fds.get_object_metadata(
src_bucket_name,
src_object_name).metadata.get(Common.CONTENT_MD5)
if remote_md5 is not None and local_md5 == remote_md5:
CLIPrinter.done(
"download %s/%s to local(skip because of same md5)" %
(src_bucket_name, src_object_name))
return
mtime = os.path.getmtime(dst_name)
try:
fds_object = self._fds.get_object(bucket_name=src_bucket_name,
object_name=src_object_name,
stream=True)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
lm = fds_object.metadata.metadata['last-modified']
remote_modified = rfc822_timestamp(lm)
# if last-modified of local file is not less last-modified of remote file, skip
if mtime is not None and datetime.fromtimestamp(
mtime) >= remote_modified:
CLIPrinter.done(
"download %s/%s to local(skip because of updated)" %
(src_bucket_name, src_object_name))
return
length_left = IS_PY3 and sys.maxsize or sys.maxint
try:
with open(dst_name, 'wb') as f:
for chunk in fds_object.stream:
length = min(length_left, len(chunk))
f.write(chunk[0:length])
length_left -= length
if length_left <= 0:
break
except Exception as exception:
print(exception)
finally:
fds_object.stream.close()
CLIPrinter.done("download %s/%s to local" %
(src_bucket_name, src_object_name))
def _download_batch(self, src_url, dst, recursive):
src_bucket_name = src_url.bucket_name()
prefix = ""
if src_url.is_object_dir():
prefix = src_url.object_dir()
delimiter = "/"
if recursive:
delimiter = ""
all_objects = self._fds.list_all_objects(bucket_name=src_bucket_name,
prefix=prefix,
delimiter=delimiter)
try:
for o in all_objects:
o_name = o.object_name
url = FDSURL(fds_prefix + src_bucket_name + "/" + o_name)
if url.is_object_url():
self._download(url, dst)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
def _upload(self, filename, dst_url, autodetect_mimetype, sync=False):
if not os.path.exists(filename):
CLIPrinter.warn("{} is a bad file".format(filename))
return
dst_bucket_name = dst_url.bucket_name()
if dst_url.is_object_url():
dst_object_name = dst_url.object_name()
elif sync:
dst_object_name = filename[2:]
elif dst_url.is_object_dir():
dst_object_name = dst_url.object_dir() + os.path.basename(filename)
else:
dst_object_name = os.path.basename(filename)
try:
if self._fds.does_object_exists(dst_bucket_name, dst_object_name):
# check md5 firstly
metadata = self._fds.get_object_metadata(
dst_bucket_name, dst_object_name)
if metadata.metadata.get(Common.CONTENT_MD5) is not None:
local_md5 = file_md5(filename)
if local_md5 == metadata.metadata.get(Common.CONTENT_MD5):
CLIPrinter.done(
'upload object %s/%s(skip because of same md5)' %
(dst_bucket_name, dst_object_name))
return
# check last-modified
mtime = None
if os.path.isfile(filename):
mtime = os.path.getmtime(filename)
lm = metadata.metadata[Common.LAST_MODIFIED]
remote_modified = rfc822_timestamp(lm)
# if last-modified of local file is not less last-modified of remote file, skip
if mtime is not None and datetime.fromtimestamp(
mtime) <= remote_modified:
CLIPrinter.done(
'upload object %s/%s(skip because of updated)' %
(dst_bucket_name, dst_object_name))
return
except Exception as e:
CLIPrinter.fail(e.message)
return
mimetype = None
if autodetect_mimetype:
mimetype = mimetypes.guess_type(filename)[0]
metadata = FDSObjectMetadata()
if mimetype is not None:
metadata.add_header(Common.CONTENT_TYPE, mimetype)
result = None
with open(filename, "rb") as f:
file_length = os.path.getsize(filename)
if file_length < multipart_upload_buffer_size:
try:
result = self._fds.put_object(dst_bucket_name,
dst_object_name,
f,
metadata=metadata)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
else:
try:
upload_token = self._fds.init_multipart_upload(
dst_bucket_name, dst_object_name)
part_number = 1
result_list = []
while True:
data = f.read(multipart_upload_buffer_size)
if len(data) <= 0:
break
for i in range(max_upload_retry_time):
upload_result = None
try:
upload_result = self._fds.upload_part(
dst_bucket_name, dst_object_name,
upload_token.upload_id, part_number, data)
result_list.append(upload_result)
break
except GalaxyFDSClientException as e:
sleep_seconds = (i + 1) * 10
CLIPrinter.warn(
"upload part %d failed, retry after %d seconds"
% (part_number, sleep_seconds))
time.sleep(sleep_seconds)
part_number = part_number + 1
upload_part_result = UploadPartResultList(
{"uploadPartResultList": result_list})
result = self._fds.complete_multipart_upload(
upload_token.bucket_name, upload_token.object_name,
upload_token.upload_id, metadata,
json.dumps(upload_part_result))
except Exception as e:
self._fds.abort_multipart_upload(dst_bucket_name,
dst_object_name,
upload_token.upload_id)
CLIPrinter.fail(e.message)
if result is not None:
CLIPrinter.done('upload object %s/%s' %
(dst_bucket_name, dst_object_name))
else:
CLIPrinter.fail('upload object %s/%s' %
(dst_bucket_name, dst_object_name))
def _upload_batch(self,
d,
dst_url,
recursive,
autodetect_mimetype,
sync=False):
for root, dirs, files in os.walk(d):
relative_dir = os.path.relpath(root, d)
if relative_dir != '.' and relative_dir != '..' and relative_dir.startswith(
'.'):
CLIPrinter.warn('skipping hidden dir ' + relative_dir)
continue
for filename in files:
object_name = os.path.join(root, filename)
object_name = '/'.join(object_name.split('\\'))
self._upload(object_name, dst_url, autodetect_mimetype, sync)
if not recursive:
break
def sync(self, src, dst, autodetect_mimetype=False):
"""
sync command syncs between (local directory and fds) (fds and local directory) (fds and fds)
:param src: src can be a fds bucket url like fds://bucketname or '.'
:param dst: src can be a fds bucket url like fds://bucketname or '.'
:param delete: todo delete target file if source file is deleted
:param exclude: todo
:param include: todo
"""
if FDSURL.is_fds_url(src) and not FDSURL.is_fds_url(dst):
src_url = FDSURL(src)
if not src_url.is_bucket_url() or not dst.strip() == '.':
CLIPrinter.wrong_format()
src_bucket_name = src_url.bucket_name()
all_objects = self._fds.list_all_objects(
bucket_name=src_bucket_name, prefix='', delimiter='')
try:
for o in all_objects:
o_name = o.object_name
url = FDSURL(fds_prefix + src_bucket_name + '/' + o_name)
if '/' not in o_name:
self._download(url, dst)
elif url.is_object_url():
o_file_name = o_name.split('/')[-1]
o_dir = o_name.split(o_file_name)[0]
mkdirs(o_dir)
self._download(url, o_dir)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
elif not FDSURL.is_fds_url(src) and FDSURL.is_fds_url(dst):
dst_url = FDSURL(dst)
if not src.strip() == '.' or not dst_url.is_bucket_url():
CLIPrinter.wrong_format()
self._upload_batch(src, dst_url, True, autodetect_mimetype, True)
elif FDSURL.is_fds_url(src) and FDSURL.is_fds_url(dst):
self.cp(src, dst)
else:
CLIPrinter.wrong_format()
def make_public(self, url):
if not FDSURL.is_fds_url(url):
CLIPrinter.wrong_format()
return
url = FDSURL(url)
if url.is_object_url():
try:
self._fds.set_public(url.bucket_name(), url.object_name())
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
elif url.is_bucket_url():
try:
acl = AccessControlList()
grant = Grant(Grantee(UserGroups.ALL_USERS), Permission.READ)
grant.type = GrantType.GROUP
acl.add_grant(grant)
self._fds.set_bucket_acl(url.bucket_name(), acl)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
else:
CLIPrinter.wrong_format()
return
def make_outside(self, url, close=False):
if not FDSURL.is_fds_url(url):
CLIPrinter.wrong_format()
return
url = FDSURL(url)
if url.is_bucket_url():
try:
if close:
self._fds.set_bucket_outside_access(
url.bucket_name(), False)
else:
self._fds.set_bucket_outside_access(
url.bucket_name(), True)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
elif url.is_object_url():
try:
if close:
self._fds.set_object_outside_access(
url.bucket_name(), url.object_name(), False)
else:
self._fds.set_object_outside_access(
url.bucket_name(), url.object_name(), True)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
else:
CLIPrinter.wrong_format()
return
def info(self):
print("Access Key ID: {}".format(self._local_config.ak))
print("Access Secret Key: {}".format(self._local_config.sk))
print("Endpoint: {}".format(self._local_config.endpoint))
class FDSCli(object):
"""
Advanced fds cli you deserved!
"""
def __init__(self, ak=None, sk=None, endpoint=None):
self._fds_prefix = r'fds://'
self._fds_prefix_len = len(self._fds_prefix)
self._local_config = LocalConfig()
self._ak = self._local_config.ak if ak is None else ak
if self._ak is None:
self.config()
self._sk = self._local_config.sk if sk is None else sk
if self._sk is None:
self.config()
self._endpoint = self._local_config.endpoint if endpoint is None else endpoint
if self._endpoint is None:
self.config()
self._fds_config = FDSClientConfiguration(region_name="awsde0",
enable_https=False,
enable_cdn_for_download=False,
enable_cdn_for_upload=False,
endpoint=self._endpoint)
self._fds = GalaxyFDSClient(access_key=self._ak,
access_secret=self._sk,
config=self._fds_config)
def config(self):
"""
config command configures ak sk and endpoint
:return:
"""
default_ak = self._local_config.ak
default_sk = self._local_config.sk
default_endpoint = self._local_config.endpoint
ak = input("enter access key id[default: %s]: " % default_ak)
if ak == '':
ak = default_ak
sk = input("enter secret access key[default: %s]: " % default_sk)
if sk == '':
sk = default_sk
endpoint = input("enter endpoint[default: %s]: " % default_endpoint)
if endpoint == '':
endpoint = default_endpoint
self._local_config.ak = ak
self._local_config.sk = sk
self._local_config.endpoint = endpoint
def mb(self, fds_url):
"""
create(make) a bucket
:param fds_url: fds url format like fds://bucket_name_to_make
"""
url = FDSURL(fds_url)
if not url.is_bucket_url():
CLIPrinter.wrong_format()
bucket_name = url.bucket_name()
try:
self._fds.create_bucket(bucket_name)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
CLIPrinter.done("create bucket [%s]" % bucket_name)
def rb(self, fds_url, force=False):
"""
delete(remove) a bucket
:param fds_url: fds url format like fds://bucket_name_to_delete
:param force: remove a bucket even if this bucket is not empty
"""
url = FDSURL(fds_url)
if not url.is_bucket_url():
CLIPrinter.wrong_format()
return
bucket_name = url.bucket_name()
if force:
result = self._fds.list_objects(bucket_name, '', '')
while True:
names = []
try:
for object_summary in result.objects:
names.append(object_summary.object_name)
except GalaxyFDSClientException as e:
CLIPrinter.warn(e.message)
try:
self._fds.delete_objects(bucket_name, names)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
if result.is_truncated:
result = self._fds.list_next_batch_of_objects(result)
else:
break
try:
self._fds.delete_bucket(bucket_name)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
CLIPrinter.done("remove bucket [%s]" % bucket_name)
def rm(self, fds_url):
"""
delete(remove) a object
:param fds_url: fds url format like fds://bucket_name/object_name_to_delete
"""
url = FDSURL(fds_url)
bucket_name = url.bucket_name()
if url.is_bucket_url():
CLIPrinter.fail("please enter a object resource address to remove")
return
object_name = url.object_name()
try:
self._fds.delete_object(bucket_name, object_name)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
CLIPrinter.done("remove object: [%s] in bucket [%s]" % (
object_name, bucket_name))
def ls(self, fds_url=None, recursive=False, human='k'):
"""
list all buckets or objects in a bucket
:param fds_url: fds url format like fds://bucket
:param recursive: recursive listing
"""
if human != 'k' and human != 'm' and human != 'g':
CLIPrinter.fail("human should be in 'k|m|g'")
return
# bucket_url is None means listing all bucket name
if fds_url is None:
buckets = None
try:
buckets = self._fds.list_authorized_buckets()
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
for bucket in buckets:
CLIPrinter.print_bucket(bucket)
else:
delimiter = "/"
if recursive:
delimiter = ""
url = FDSURL(fds_url)
bucket_name = url.bucket_name()
prefix = ''
if url.is_bucket_url() or url.is_object_dir():
if url.is_object_dir():
prefix = url.object_dir()
results = self._fds.list_all_objects(
bucket_name, prefix, delimiter)
try:
for result in results:
metadata = self._fds.get_object_metadata(bucket_name, result.object_name)
CLIPrinter.print_object(result.object_name, metadata, human)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
else:
object_name = url.object_name()
metadata = None
try:
metadata = self._fds.get_object_metadata(bucket_name, object_name)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
if metadata is not None:
CLIPrinter.print_object(object_name, metadata, human)
def presigned(self, fds_url, expires=1, cdn=False):
"""
presigned command generates presigned url for download project
:param fds_url: format url like fds://bucket/a.txt
:param expires: expiration time in minutes
:return: presigned url for downloading
"""
url = FDSURL(fds_url)
bucket_name = url.bucket_name()
if url.is_bucket_url():
CLIPrinter.fail("%uri is illegal" % fds_url)
object_name = url.object_name()
expiration = int(1000 * (float(expires) * 3600 +
float((datetime.utcnow() - datetime(1970, 1, 1, 0, 0, 0,
0)).total_seconds())))
try:
if cdn:
base_uri = self._fds_config.get_cdn_base_uri()
else:
base_uri = self._fds_config.get_base_uri()
u = self._fds.generate_presigned_uri(base_uri, bucket_name, object_name, expiration)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
CLIPrinter.done('generated presigned url: ' + u)
def ttl(self, fds_url):
"""
ttl command shows the lifecycle information of a bucket or a object
:param fds_url: format url like fds://bucket/a.txt or fds://bucket/
"""
url = FDSURL(fds_url)
bucket_name = url.bucket_name()
try:
ttl = self._fds.get_lifecycle_config(bucket_name)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
if url.is_bucket_url():
CLIPrinter.print_lifecycle(ttl)
elif url.is_object_url():
if not self._fds.does_object_exists(bucket_name, url.object_name()):
CLIPrinter.fail("object does not exists")
if url.is_object_dir():
prefix = url.object_dir()
else:
prefix = url.object_name()
rules = [rule for rule in ttl['rules'] if rule['prefix'] in prefix]
CLIPrinter.print_lifecycle({"rules": rules})
else:
CLIPrinter.wrong_format()
def cp(self, src, dst, recursive=False, autodetect_mimetype=False):
"""
cp command do lots of things.
1. file upload
2. batch files upload
3. file download
4. batch files download
5. rename object
6. object copy
7. batch objects copy
:param src: source fds url format like fds://bucket
:param dst: target fds url format like fds://bucket
:param recursive: recursive listing
"""
if FDSURL.is_fds_url(src) and FDSURL.is_fds_url(dst):
src_url = FDSURL(src)
dst_url = FDSURL(dst)
if src_url.is_object_url():
self._cp(src_url, dst_url)
elif not src_url.is_object_url() and not dst_url.is_object_url():
self._cp_batch(src_url, dst_url, recursive)
else:
CLIPrinter.wrong_format()
elif FDSURL.is_fds_url(src) and not FDSURL.is_fds_url(dst):
src_url = FDSURL(src)
if src_url.is_object_url():
self._download(src_url, dst)
elif src_url.is_object_dir() and os.path.isdir(dst):
self._download_batch(src_url, dst, recursive)
else:
CLIPrinter.wrong_format()
elif not FDSURL.is_fds_url(src) and FDSURL.is_fds_url(dst):
dst_url = FDSURL(dst)
if os.path.isfile(src):
self._upload(src, dst_url, autodetect_mimetype=autodetect_mimetype)
elif os.path.isdir(src) and not dst_url.is_object_url():
self._upload_batch(src, dst_url, recursive, autodetect_mimetype=autodetect_mimetype)
else:
CLIPrinter.wrong_format()
else:
CLIPrinter.fail("don't support copy file from local to local")
def _cp(self, src_url, dst_url):
src_bucket_name = src_url.bucket_name()
src_object_name = src_url.object_name()
dst_bucket_name = dst_url.bucket_name()
if dst_url.is_object_url():
dst_object_name = dst_url.object_name()
else:
dst_object_name = src_object_name
try:
self._fds.copy_object(src_bucket_name, src_object_name, dst_bucket_name, dst_object_name)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
CLIPrinter.done(
"copy %s/%s to %s/%s" % (src_bucket_name, src_object_name, dst_bucket_name, dst_object_name))
def _cp_batch(self, src_url, dst_url, recursive):
src_bucket_name = src_url.bucket_name()
dst_bucket_name = dst_url.bucket_name()
prefix = ""
if src_url.is_object_dir():
prefix = src_url.object_dir()
delimiter = "/"
if recursive:
delimiter = ""
all_objects = self._fds.list_all_objects(bucket_name=src_bucket_name, prefix=prefix,
delimiter=delimiter)
try:
for o in all_objects:
o_name = o.object_name
self._fds.copy_object(src_bucket_name, o_name, dst_bucket_name, o_name)
CLIPrinter.done("copy %s/%s to %s/%s" % (src_bucket_name, o_name, dst_bucket_name, o_name))
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
def _download(self, src_url, dst):
src_bucket_name = src_url.bucket_name()
src_object_name = src_url.object_name()
if os.path.isdir(dst):
if dst == '.' or dst == '..':
dst_name = src_url.file_name()
elif dst.endswith('/'):
dst_name = dst + src_url.file_name()
else:
dst_name = dst + '/' + src_object_name.split('/')[-1]
else:
dst_name = dst
mtime = None
if os.path.isfile(dst_name):
local_md5 = file_md5(dst_name)
remote_md5 = self._fds.get_object_metadata(src_bucket_name, src_object_name).metadata.get(Common.CONTENT_MD5)
if remote_md5 is not None and local_md5 == remote_md5:
CLIPrinter.done("download %s/%s to local(skip because of same md5)" % (src_bucket_name, src_object_name))
return
mtime = os.path.getmtime(dst_name)
try:
fds_object = self._fds.get_object(bucket_name=src_bucket_name, object_name=src_object_name,
stream=True)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
lm = fds_object.metadata.metadata['last-modified']
remote_modified = rfc822_timestamp(lm)
# if last-modified of local file is not less last-modified of remote file, skip
if mtime is not None and datetime.fromtimestamp(mtime) >= remote_modified:
CLIPrinter.done("download %s/%s to local(skip because of updated)" % (src_bucket_name, src_object_name))
return
length_left = IS_PY3 and sys.maxsize or sys.maxint
try:
with open(dst_name, 'wb') as f:
for chunk in fds_object.stream:
length = min(length_left, len(chunk))
f.write(chunk[0:length])
length_left -= length
if length_left <= 0:
break
except Exception as exception:
print(exception)
finally:
fds_object.stream.close()
CLIPrinter.done("download %s/%s to local" % (src_bucket_name, src_object_name))
def _download_batch(self, src_url, dst, recursive):
src_bucket_name = src_url.bucket_name()
prefix = ""
if src_url.is_object_dir():
prefix = src_url.object_dir()
delimiter = "/"
if recursive:
delimiter = ""
all_objects = self._fds.list_all_objects(bucket_name=src_bucket_name, prefix=prefix, delimiter=delimiter)
try:
for o in all_objects:
o_name = o.object_name
url = FDSURL(fds_prefix + src_bucket_name + "/" + o_name)
if url.is_object_url():
self._download(url, dst)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
def _upload(self, filename, dst_url, autodetect_mimetype, sync=False):
if not os.path.exists(filename):
CLIPrinter.warn("{} is a bad file".format(filename))
return
dst_bucket_name = dst_url.bucket_name()
if dst_url.is_object_url():
dst_object_name = dst_url.object_name()
elif sync:
dst_object_name = filename[2:]
elif dst_url.is_object_dir():
dst_object_name = dst_url.object_dir() + os.path.basename(filename)
else:
dst_object_name = os.path.basename(filename)
try:
if self._fds.does_object_exists(dst_bucket_name, dst_object_name):
# check md5 firstly
metadata = self._fds.get_object_metadata(dst_bucket_name, dst_object_name)
if metadata.metadata.get(Common.CONTENT_MD5) is not None:
local_md5 = file_md5(filename)
if local_md5 == metadata.metadata.get(Common.CONTENT_MD5):
CLIPrinter.done('upload object %s/%s(skip because of same md5)' % (dst_bucket_name, dst_object_name))
return
# check last-modified
mtime = None
if os.path.isfile(filename):
mtime = os.path.getmtime(filename)
lm = metadata.metadata[Common.LAST_MODIFIED]
remote_modified = rfc822_timestamp(lm)
# if last-modified of local file is not less last-modified of remote file, skip
if mtime is not None and datetime.fromtimestamp(mtime) <= remote_modified:
CLIPrinter.done('upload object %s/%s(skip because of updated)' % (dst_bucket_name, dst_object_name))
return
except Exception as e:
CLIPrinter.fail(e.message)
return
mimetype = None
if autodetect_mimetype:
mimetype = mimetypes.guess_type(filename)[0]
metadata = FDSObjectMetadata()
if mimetype is not None:
metadata.add_header(Common.CONTENT_TYPE, mimetype)
result = None
with open(filename, "rb") as f:
file_length = os.path.getsize(filename)
if file_length < multipart_upload_buffer_size:
try:
result = self._fds.put_object(dst_bucket_name, dst_object_name, f, metadata=metadata)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
else:
try:
upload_token = self._fds.init_multipart_upload(dst_bucket_name, dst_object_name)
part_number = 1
result_list = []
while True:
data = f.read(multipart_upload_buffer_size)
if len(data) <= 0:
break
for i in range(max_upload_retry_time):
upload_result = None
try:
upload_result = self._fds.upload_part(dst_bucket_name, dst_object_name, upload_token.upload_id, part_number, data)
result_list.append(upload_result)
break
except GalaxyFDSClientException as e:
sleep_seconds = (i + 1) * 10
CLIPrinter.warn("upload part %d failed, retry after %d seconds" % (
part_number, sleep_seconds))
time.sleep(sleep_seconds)
part_number = part_number + 1
upload_part_result = UploadPartResultList({"uploadPartResultList": result_list})
result = self._fds.complete_multipart_upload(upload_token.bucket_name, upload_token.object_name,
upload_token.upload_id, metadata,
json.dumps(upload_part_result))
except Exception as e:
self._fds.abort_multipart_upload(dst_bucket_name, dst_object_name, upload_token.upload_id)
CLIPrinter.fail(e.message)
if result is not None:
CLIPrinter.done('upload object %s/%s' % (dst_bucket_name, dst_object_name))
else:
CLIPrinter.fail('upload object %s/%s' % (dst_bucket_name, dst_object_name))
def _upload_batch(self, d, dst_url, recursive, autodetect_mimetype, sync=False):
for root, dirs, files in os.walk(d):
relative_dir = os.path.relpath(root, d)
if relative_dir != '.' and relative_dir != '..' and relative_dir.startswith('.'):
CLIPrinter.warn('skipping hidden dir ' + relative_dir)
continue
for filename in files:
object_name = os.path.join(root, filename)
object_name = '/'.join(object_name.split('\\'))
self._upload(object_name, dst_url, autodetect_mimetype, sync)
if not recursive:
break
def sync(self, src, dst, autodetect_mimetype=False):
"""
sync command syncs between (local directory and fds) (fds and local directory) (fds and fds)
:param src: src can be a fds bucket url like fds://bucketname or '.'
:param dst: src can be a fds bucket url like fds://bucketname or '.'
:param delete: todo delete target file if source file is deleted
:param exclude: todo
:param include: todo
"""
if FDSURL.is_fds_url(src) and not FDSURL.is_fds_url(dst):
src_url = FDSURL(src)
if not src_url.is_bucket_url() or not dst.strip() == '.':
CLIPrinter.wrong_format()
src_bucket_name = src_url.bucket_name()
all_objects = self._fds.list_all_objects(bucket_name=src_bucket_name, prefix='', delimiter='')
try:
for o in all_objects:
o_name = o.object_name
url = FDSURL(fds_prefix + src_bucket_name + '/' + o_name)
if '/' not in o_name:
self._download(url, dst)
elif url.is_object_url():
o_file_name = o_name.split('/')[-1]
o_dir = o_name.split(o_file_name)[0]
mkdirs(o_dir)
self._download(url, o_dir)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
elif not FDSURL.is_fds_url(src) and FDSURL.is_fds_url(dst):
dst_url = FDSURL(dst)
if not src.strip() == '.' or not dst_url.is_bucket_url():
CLIPrinter.wrong_format()
self._upload_batch(src, dst_url, True, autodetect_mimetype, True)
elif FDSURL.is_fds_url(src) and FDSURL.is_fds_url(dst):
self.cp(src, dst)
else:
CLIPrinter.wrong_format()
def make_public(self, url):
if not FDSURL.is_fds_url(url):
CLIPrinter.wrong_format()
return
url = FDSURL(url)
if url.is_object_url():
try:
self._fds.set_public(url.bucket_name(), url.object_name())
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
elif url.is_bucket_url():
try:
acl = AccessControlList()
grant = Grant(Grantee(UserGroups.ALL_USERS), Permission.READ)
grant.type = GrantType.GROUP
acl.add_grant(grant)
self._fds.set_bucket_acl(url.bucket_name(), acl)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
else:
CLIPrinter.wrong_format()
return
def make_outside(self, url, close=False):
if not FDSURL.is_fds_url(url):
CLIPrinter.wrong_format()
return
url = FDSURL(url)
if url.is_bucket_url():
try:
if close:
self._fds.set_bucket_outside_access(url.bucket_name(), False)
else:
self._fds.set_bucket_outside_access(url.bucket_name(), True)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
elif url.is_object_url():
try:
if close:
self._fds.set_object_outside_access(url.bucket_name(), url.object_name(), False)
else:
self._fds.set_object_outside_access(url.bucket_name(), url.object_name(), True)
except GalaxyFDSClientException as e:
CLIPrinter.fail(e.message)
return
else:
CLIPrinter.wrong_format()
return
def info(self):
print("Access Key ID: {}".format(self._local_config.ak))
print("Access Secret Key: {}".format(self._local_config.sk))
print("Endpoint: {}".format(self._local_config.endpoint))