def __init__(self, access_key=None, access_secret=None, config=None): ''' :param access_key: The app access key :param access_secret: The app access secret :param config: The FDS service's config ''' self._delimiter = "/" if access_key == None or access_secret == None: # Get keys from environment variables if "XIAOMI_ACCESS_KEY" in os.environ and "XIAOMI_SECRET_KEY" in os.environ: self._access_key = os.environ["XIAOMI_ACCESS_KEY"] self._secret_key = os.environ["XIAOMI_SECRET_KEY"] else: # Read keys from configuration file config_filename = os.path.join(os.path.expanduser('~'), ".config/xiaomi/config") if os.path.exists(config_filename): with open(config_filename) as f: data = json.load(f) self._access_key = data["access_key"] self._secret_key = data["secret_key"] else: self._access_key = access_key self._secret_key = access_secret self._auth = Signer(self._access_key, self._secret_key) if config == None: config = FDSClientConfiguration() if "FDS_ENDPOINT" in os.environ: config.set_endpoint(os.environ["FDS_ENDPOINT"]) self._config = config self._request = FDSRequest(config.timeout, config.max_retries)
def setUpClass(cls): config = FDSClientConfiguration(region_name, False, False, False) config.enable_md5_calculate = True cls.client = GalaxyFDSClient(access_key, access_secret, config) cls.bucket_name = 'test-python-' + datetime.strftime( datetime.now(), "%Y%m%d%H%M%S%z") cls.delete_objects_and_bucket(cls.client, cls.bucket_name) cls.client.create_bucket(cls.bucket_name)
def test_set_endpoint(self): httpConfig = FDSClientConfiguration(region_name, False, False, False) endpoint = region_name + ".api.xiaomi.net" httpConfig.set_endpoint(endpoint) httpUri = httpConfig.get_base_uri() self.assertEqual("http://" + endpoint + "/", httpUri) httpsConfig = FDSClientConfiguration(region_name, False, False, True) httpsConfig.set_endpoint(endpoint) httpsUri = httpsConfig.get_base_uri() self.assertEqual("https://" + endpoint + "/", httpsUri)
def write_fds_file(object_content, name): bucket_name = "mlmodel" AK = "AKYX4NXAR6QJOK2RBW" SK = "MuS+R5nNehuCogsqtztD+HT0JTsmjHS88NW8J8Z+" ENDPOINT = "cnbj1-fds.api.xiaomi.net" config = FDSClientConfiguration() config.set_endpoint(ENDPOINT) client = GalaxyFDSClient(AK, SK, config) client.put_object(bucket_name, name, object_content)
def setUpClass(cls): ClientTest.init_from_local_config() config = FDSClientConfiguration(region_name=region_name, enable_https=False, enable_cdn_for_upload=False, enable_cdn_for_download=False, endpoint=endpoint) config.enable_md5_calculate = True ClientTest.client = GalaxyFDSClient(access_key, access_secret, config) ClientTest.bucket_name = 'test-python-' + datetime.strftime( datetime.now(), "%Y%m%d%H%M%S%z") ClientTest.delete_objects_and_bucket(cls.client, cls.bucket_name) ClientTest.client.create_bucket(cls.bucket_name)
def setUpClass(cls): ClientTest.init_from_local_config() config = FDSClientConfiguration( region_name=region_name, enable_https=False, enable_cdn_for_upload=False, enable_cdn_for_download=False, endpoint=endpoint) config.enable_md5_calculate = True ClientTest.client = GalaxyFDSClient(access_key, access_secret, config) ClientTest.bucket_name = 'test-python-' + datetime.strftime(datetime.now(), "%Y%m%d%H%M%S%z") ClientTest.delete_objects_and_bucket(cls.client, cls.bucket_name) ClientTest.client.create_bucket(cls.bucket_name)
def test_object_acl(self): object_name = "test1" content = "test1" self.client.put_object(self.bucket_name, object_name, content) for bucket in self.client.list_objects(self.bucket_name): print(bucket) print(self.client.get_object_acl(self.bucket_name, object_name)) objectAcl = AccessControlList() objectAcl.add_grant(Grant(Grantee("111"), Permission.READ)) objectAcl.add_grant(Grant(Grantee("109901"), Permission.FULL_CONTROL)) objectAcl.add_grant(Grant(Grantee(acl_ak), Permission.FULL_CONTROL)) self.client.set_object_acl(self.bucket_name, object_name, objectAcl) acl = self.client.get_object_acl(self.bucket_name, object_name) self.assertTrue(objectAcl.is_subset(acl)) acl_client = GalaxyFDSClient( acl_ak, acl_access_secret, FDSClientConfiguration(region_name, False, False, False, endpoint=endpoint)) self.assertTrue( self.client.does_object_exists(self.bucket_name, object_name)) print(acl_client.get_object(self.bucket_name, object_name)) self.client.delete_object(self.bucket_name, object_name) self.assertFalse( self.client.does_object_exists(self.bucket_name, object_name))
def test_uri(self): client = GalaxyFDSClient(access_key, access_secret, FDSClientConfiguration(region_name, False, False, False)) bucket_name = self.bucket_name + "1" if (client.does_bucket_exist(bucket_name)): client.delete_bucket(bucket_name) client.create_bucket(bucket_name) client.delete_bucket(bucket_name)
def test_set_endpoint(self): httpConfig = FDSClientConfiguration(region_name, False, False, False) endpoint = region_name + ".api.xiaomi.net" httpConfig.set_endpoint(endpoint) httpUri = httpConfig.get_base_uri() self.assertEqual("http://" + endpoint + "/", httpUri) httpsConfig = FDSClientConfiguration(region_name, False, False, True) httpsConfig.set_endpoint(endpoint) httpsUri = httpsConfig.get_base_uri() self.assertEqual("https://" + endpoint + "/", httpsUri)
def test_bucket_acl(self): print(self.bucket_name) self.client.get_bucket_acl(self.bucket_name) bucketAcl = AccessControlList() bucketAcl.add_grant(Grant(Grantee("111"), Permission.READ)) bucketAcl.add_grant(Grant(Grantee('109901'), Permission.FULL_CONTROL)) bucketAcl.add_grant(Grant(Grantee('123456'), Permission.SSO_WRITE)) bucketAcl.add_grant(Grant(Grantee(appid), Permission.FULL_CONTROL)) self.client.set_bucket_acl(self.bucket_name, bucketAcl) aclListGot = self.client.get_bucket_acl(self.bucket_name) readAclCnt = 0 fullControlCnt = 0 writeWithSSOCnt = 0 for i in aclListGot.get_grant_list(): if i['grantee']['id'] == '111': self.assertTrue(i['permission'].to_string() == Permission( Permission.READ).to_string()) readAclCnt += 1 elif i['grantee']['id'] == '109901': self.assertTrue(i['permission'].to_string() == Permission( Permission.FULL_CONTROL).to_string()) fullControlCnt += 1 elif i['grantee']['id'] == '123456': self.assertTrue(i['permission'].to_string() == Permission( Permission.SSO_WRITE).to_string()) writeWithSSOCnt += 1 self.assertTrue(readAclCnt == 1) self.assertTrue(fullControlCnt == 1) self.assertTrue(writeWithSSOCnt == 1) # self.client.set_bucket_acl(self.bucket_name, bucketAcl) acl = self.client.get_bucket_acl(self.bucket_name) self.assertTrue(bucketAcl.is_subset(acl)) acl_client = GalaxyFDSClient( acl_ak, acl_access_secret, FDSClientConfiguration(region_name, False, False, False, endpoint=endpoint)) object_name = "testBucketAcl7" acl_client.put_object(self.bucket_name, object_name, "hahhah") self.assertTrue( self.client.does_object_exists(self.bucket_name, object_name)) acl_client.list_objects(self.bucket_name) acl_client.delete_object(self.bucket_name, object_name) self.assertFalse( self.client.does_object_exists(self.bucket_name, object_name)) self.assertTrue(acl_client.does_bucket_exist(self.bucket_name)) try: acl_client.delete_bucket(self.bucket_name) except GalaxyFDSClientException as e: print(e.message) self.assertTrue(self.client.does_bucket_exist(self.bucket_name))
def __init__(self): self.config = FDSClientConfiguration( region_name=current_app.config['FDSREGIONNAME'], endpoint=current_app.config['FDSENDPOINT'], enable_https=current_app.config['ENABLEHTTPS'], enable_cdn_for_upload=current_app.config['ENABLECDNFORUPLOAD'], enable_cdn_for_download=current_app.config['ENABLECDNFORDOWNLOAD']) self.client = GalaxyFDSClient(current_app.config['FDSACCESSKEYID'], current_app.config['FDSACCESSKEYSECRET'], config=self.config)
def __init__(self, access_key=None, access_secret=None, config=None): ''' :param access_key: The app access key :param access_secret: The app access secret :param config: The FDS service's config ''' self._delimiter = "/" if access_key == None or access_secret == None: self._access_key = self.load_access_key() self._secret_key = self.load_secret_key() else: self._access_key = access_key self._secret_key = access_secret self._auth = Signer(self._access_key, self._secret_key) if config == None: config = FDSClientConfiguration() config.set_endpoint(self.load_endpoint()) self._config = config self._request = FDSRequest(config.timeout, config.max_retries)
def read_fds_file(paths): bucket_name = "mlmodel" AK = "" SK = "" ENDPOINT = "cnbj1-fds.api.xiaomi.net" config = FDSClientConfiguration() config.set_endpoint(ENDPOINT) client = GalaxyFDSClient(AK, SK, config) X = [] y = [] parts = [] for path in paths: print("read path:", path) try: obj = client.get_object(bucket_name, path) for chunk in obj.stream: parts.append(chunk) except GalaxyFDSClientException as e: print("reading fds something is wrong") print(e.message) whole_file = "".join(parts) del parts whole_lines = whole_file.split("\n") del whole_file gc.collect() whole_lines.pop() for i in range(len(whole_lines)): eles = whole_lines[i].strip().split(",") if len(eles) == 2: X.append(int(eles[0])) y.append([int(eles[1])]) del whole_lines gc.collect() return X, y
from __future__ import print_function import time import sys from fds.fds_client_configuration import FDSClientConfiguration from fds.galaxy_fds_client import GalaxyFDSClient from fds.galaxy_fds_client_exception import GalaxyFDSClientException from fds.model.permission import AccessControlList from fds.model.permission import Grant from fds.model.permission import Grantee from fds.model.permission import Permission # Create default client access_key = 'your_access_key' access_secret = 'your_access_secret' config = FDSClientConfiguration() bucket_name = 'fds-python-example-%d' % int(time.time()) fds_client = GalaxyFDSClient(access_key, access_secret, config) ##################### # List buckets buckets = fds_client.list_buckets() print('buckets list:') for bucket in buckets: print(bucket) print("---end---") # Check and create the bucket if not fds_client.does_bucket_exist(bucket_name): fds_client.create_bucket(bucket_name)
def setUp(self): config = FDSClientConfiguration(region_name, False, False, False) self.client = GalaxyFDSClient(access_key, access_secret, config) self.bucket_name = "1024"
#!/home/stansun/tools/p2tf/bin/python2 import yaml import os from fds.galaxy_fds_client import GalaxyFDSClient from fds.galaxy_fds_client_exception import GalaxyFDSClientException from fds.fds_client_configuration import FDSClientConfiguration config_filename = os.path.join(os.path.expanduser("~"), ".config/xiaomi/config") with open(config_filename) as f: config_data = yaml.safe_load(f) access_key = config_data["xiaomi_cloudml"]["xiaomi_access_key_id"] secret_key = config_data["xiaomi_cloudml"]["xiaomi_secret_access_key"] fds_endpoint = config_data["xiaomi_cloudml"]["xiaomi_fds_endpoint"] config = FDSClientConfiguration( region_name="cnbj1-fds", enable_https=False, enable_cdn_for_upload=False, enable_cdn_for_download=False, endpoint=fds_endpoint) fds_client = GalaxyFDSClient(access_key, access_secret, config) client.put_object( #cloudml jobs submit -n article-quality-local -m lstm_training -g 1 -M 8G -u fds://training/packages/article-quality-1.1.tar.gz