def __init__(self, access_key=None, access_secret=None, config=None):
'''
:param access_key: The app access key
:param access_secret: The app access secret
:param config: The FDS service's config
'''
self._delimiter = "/"
if access_key == None or access_secret == None:
# Get keys from environment variables
if "XIAOMI_ACCESS_KEY" in os.environ and "XIAOMI_SECRET_KEY" in os.environ:
self._access_key = os.environ["XIAOMI_ACCESS_KEY"]
self._secret_key = os.environ["XIAOMI_SECRET_KEY"]
else:
# Read keys from configuration file
config_filename = os.path.join(os.path.expanduser('~'),
".config/xiaomi/config")
if os.path.exists(config_filename):
with open(config_filename) as f:
data = json.load(f)
self._access_key = data["access_key"]
self._secret_key = data["secret_key"]
else:
self._access_key = access_key
self._secret_key = access_secret
self._auth = Signer(self._access_key, self._secret_key)
if config == None:
config = FDSClientConfiguration()
if "FDS_ENDPOINT" in os.environ:
config.set_endpoint(os.environ["FDS_ENDPOINT"])
self._config = config
self._request = FDSRequest(config.timeout, config.max_retries)
def setUpClass(cls):
config = FDSClientConfiguration(region_name, False, False, False)
config.enable_md5_calculate = True
cls.client = GalaxyFDSClient(access_key, access_secret, config)
cls.bucket_name = 'test-python-' + datetime.strftime(
datetime.now(), "%Y%m%d%H%M%S%z")
cls.delete_objects_and_bucket(cls.client, cls.bucket_name)
cls.client.create_bucket(cls.bucket_name)
def test_set_endpoint(self):
httpConfig = FDSClientConfiguration(region_name, False, False, False)
endpoint = region_name + ".api.xiaomi.net"
httpConfig.set_endpoint(endpoint)
httpUri = httpConfig.get_base_uri()
self.assertEqual("http://" + endpoint + "/", httpUri)
httpsConfig = FDSClientConfiguration(region_name, False, False, True)
httpsConfig.set_endpoint(endpoint)
httpsUri = httpsConfig.get_base_uri()
self.assertEqual("https://" + endpoint + "/", httpsUri)
def write_fds_file(object_content, name):
bucket_name = "mlmodel"
AK = "AKYX4NXAR6QJOK2RBW"
SK = "MuS+R5nNehuCogsqtztD+HT0JTsmjHS88NW8J8Z+"
ENDPOINT = "cnbj1-fds.api.xiaomi.net"
config = FDSClientConfiguration()
config.set_endpoint(ENDPOINT)
client = GalaxyFDSClient(AK, SK, config)
client.put_object(bucket_name, name, object_content)
def setUpClass(cls):
ClientTest.init_from_local_config()
config = FDSClientConfiguration(region_name=region_name,
enable_https=False,
enable_cdn_for_upload=False,
enable_cdn_for_download=False,
endpoint=endpoint)
config.enable_md5_calculate = True
ClientTest.client = GalaxyFDSClient(access_key, access_secret, config)
ClientTest.bucket_name = 'test-python-' + datetime.strftime(
datetime.now(), "%Y%m%d%H%M%S%z")
ClientTest.delete_objects_and_bucket(cls.client, cls.bucket_name)
ClientTest.client.create_bucket(cls.bucket_name)
def setUpClass(cls):
ClientTest.init_from_local_config()
config = FDSClientConfiguration(
region_name=region_name,
enable_https=False,
enable_cdn_for_upload=False,
enable_cdn_for_download=False,
endpoint=endpoint)
config.enable_md5_calculate = True
ClientTest.client = GalaxyFDSClient(access_key, access_secret, config)
ClientTest.bucket_name = 'test-python-' + datetime.strftime(datetime.now(), "%Y%m%d%H%M%S%z")
ClientTest.delete_objects_and_bucket(cls.client, cls.bucket_name)
ClientTest.client.create_bucket(cls.bucket_name)
def test_object_acl(self):
object_name = "test1"
content = "test1"
self.client.put_object(self.bucket_name, object_name, content)
for bucket in self.client.list_objects(self.bucket_name):
print(bucket)
print(self.client.get_object_acl(self.bucket_name, object_name))
objectAcl = AccessControlList()
objectAcl.add_grant(Grant(Grantee("111"), Permission.READ))
objectAcl.add_grant(Grant(Grantee("109901"), Permission.FULL_CONTROL))
objectAcl.add_grant(Grant(Grantee(acl_ak), Permission.FULL_CONTROL))
self.client.set_object_acl(self.bucket_name, object_name, objectAcl)
acl = self.client.get_object_acl(self.bucket_name, object_name)
self.assertTrue(objectAcl.is_subset(acl))
acl_client = GalaxyFDSClient(
acl_ak, acl_access_secret,
FDSClientConfiguration(region_name,
False,
False,
False,
endpoint=endpoint))
self.assertTrue(
self.client.does_object_exists(self.bucket_name, object_name))
print(acl_client.get_object(self.bucket_name, object_name))
self.client.delete_object(self.bucket_name, object_name)
self.assertFalse(
self.client.does_object_exists(self.bucket_name, object_name))
def test_uri(self):
client = GalaxyFDSClient(access_key, access_secret,
FDSClientConfiguration(region_name, False, False, False))
bucket_name = self.bucket_name + "1"
if (client.does_bucket_exist(bucket_name)):
client.delete_bucket(bucket_name)
client.create_bucket(bucket_name)
client.delete_bucket(bucket_name)
def test_set_endpoint(self):
httpConfig = FDSClientConfiguration(region_name, False, False, False)
endpoint = region_name + ".api.xiaomi.net"
httpConfig.set_endpoint(endpoint)
httpUri = httpConfig.get_base_uri()
self.assertEqual("http://" + endpoint + "/", httpUri)
httpsConfig = FDSClientConfiguration(region_name, False, False, True)
httpsConfig.set_endpoint(endpoint)
httpsUri = httpsConfig.get_base_uri()
self.assertEqual("https://" + endpoint + "/", httpsUri)
def test_bucket_acl(self):
print(self.bucket_name)
self.client.get_bucket_acl(self.bucket_name)
bucketAcl = AccessControlList()
bucketAcl.add_grant(Grant(Grantee("111"), Permission.READ))
bucketAcl.add_grant(Grant(Grantee('109901'), Permission.FULL_CONTROL))
bucketAcl.add_grant(Grant(Grantee('123456'), Permission.SSO_WRITE))
bucketAcl.add_grant(Grant(Grantee(appid), Permission.FULL_CONTROL))
self.client.set_bucket_acl(self.bucket_name, bucketAcl)
aclListGot = self.client.get_bucket_acl(self.bucket_name)
readAclCnt = 0
fullControlCnt = 0
writeWithSSOCnt = 0
for i in aclListGot.get_grant_list():
if i['grantee']['id'] == '111':
self.assertTrue(i['permission'].to_string() == Permission(
Permission.READ).to_string())
readAclCnt += 1
elif i['grantee']['id'] == '109901':
self.assertTrue(i['permission'].to_string() == Permission(
Permission.FULL_CONTROL).to_string())
fullControlCnt += 1
elif i['grantee']['id'] == '123456':
self.assertTrue(i['permission'].to_string() == Permission(
Permission.SSO_WRITE).to_string())
writeWithSSOCnt += 1
self.assertTrue(readAclCnt == 1)
self.assertTrue(fullControlCnt == 1)
self.assertTrue(writeWithSSOCnt == 1)
# self.client.set_bucket_acl(self.bucket_name, bucketAcl)
acl = self.client.get_bucket_acl(self.bucket_name)
self.assertTrue(bucketAcl.is_subset(acl))
acl_client = GalaxyFDSClient(
acl_ak, acl_access_secret,
FDSClientConfiguration(region_name,
False,
False,
False,
endpoint=endpoint))
object_name = "testBucketAcl7"
acl_client.put_object(self.bucket_name, object_name, "hahhah")
self.assertTrue(
self.client.does_object_exists(self.bucket_name, object_name))
acl_client.list_objects(self.bucket_name)
acl_client.delete_object(self.bucket_name, object_name)
self.assertFalse(
self.client.does_object_exists(self.bucket_name, object_name))
self.assertTrue(acl_client.does_bucket_exist(self.bucket_name))
try:
acl_client.delete_bucket(self.bucket_name)
except GalaxyFDSClientException as e:
print(e.message)
self.assertTrue(self.client.does_bucket_exist(self.bucket_name))
def __init__(self):
self.config = FDSClientConfiguration(
region_name=current_app.config['FDSREGIONNAME'],
endpoint=current_app.config['FDSENDPOINT'],
enable_https=current_app.config['ENABLEHTTPS'],
enable_cdn_for_upload=current_app.config['ENABLECDNFORUPLOAD'],
enable_cdn_for_download=current_app.config['ENABLECDNFORDOWNLOAD'])
self.client = GalaxyFDSClient(current_app.config['FDSACCESSKEYID'],
current_app.config['FDSACCESSKEYSECRET'],
config=self.config)
def __init__(self, access_key=None, access_secret=None, config=None):
'''
:param access_key: The app access key
:param access_secret: The app access secret
:param config: The FDS service's config
'''
self._delimiter = "/"
if access_key == None or access_secret == None:
self._access_key = self.load_access_key()
self._secret_key = self.load_secret_key()
else:
self._access_key = access_key
self._secret_key = access_secret
self._auth = Signer(self._access_key, self._secret_key)
if config == None:
config = FDSClientConfiguration()
config.set_endpoint(self.load_endpoint())
self._config = config
self._request = FDSRequest(config.timeout, config.max_retries)
def read_fds_file(paths):
bucket_name = "mlmodel"
AK = ""
SK = ""
ENDPOINT = "cnbj1-fds.api.xiaomi.net"
config = FDSClientConfiguration()
config.set_endpoint(ENDPOINT)
client = GalaxyFDSClient(AK, SK, config)
X = []
y = []
parts = []
for path in paths:
print("read path:", path)
try:
obj = client.get_object(bucket_name, path)
for chunk in obj.stream:
parts.append(chunk)
except GalaxyFDSClientException as e:
print("reading fds something is wrong")
print(e.message)
whole_file = "".join(parts)
del parts
whole_lines = whole_file.split("\n")
del whole_file
gc.collect()
whole_lines.pop()
for i in range(len(whole_lines)):
eles = whole_lines[i].strip().split(",")
if len(eles) == 2:
X.append(int(eles[0]))
y.append([int(eles[1])])
del whole_lines
gc.collect()
return X, y
from __future__ import print_function
import time
import sys
from fds.fds_client_configuration import FDSClientConfiguration
from fds.galaxy_fds_client import GalaxyFDSClient
from fds.galaxy_fds_client_exception import GalaxyFDSClientException
from fds.model.permission import AccessControlList
from fds.model.permission import Grant
from fds.model.permission import Grantee
from fds.model.permission import Permission
# Create default client
access_key = 'your_access_key'
access_secret = 'your_access_secret'
config = FDSClientConfiguration()
bucket_name = 'fds-python-example-%d' % int(time.time())
fds_client = GalaxyFDSClient(access_key, access_secret, config)
#####################
# List buckets
buckets = fds_client.list_buckets()
print('buckets list:')
for bucket in buckets:
print(bucket)
print("---end---")
# Check and create the bucket
if not fds_client.does_bucket_exist(bucket_name):
fds_client.create_bucket(bucket_name)
def setUp(self):
config = FDSClientConfiguration(region_name, False, False, False)
self.client = GalaxyFDSClient(access_key, access_secret, config)
self.bucket_name = "1024"
#!/home/stansun/tools/p2tf/bin/python2
import yaml
import os
from fds.galaxy_fds_client import GalaxyFDSClient
from fds.galaxy_fds_client_exception import GalaxyFDSClientException
from fds.fds_client_configuration import FDSClientConfiguration
config_filename = os.path.join(os.path.expanduser("~"), ".config/xiaomi/config")
with open(config_filename) as f:
config_data = yaml.safe_load(f)
access_key = config_data["xiaomi_cloudml"]["xiaomi_access_key_id"]
secret_key = config_data["xiaomi_cloudml"]["xiaomi_secret_access_key"]
fds_endpoint = config_data["xiaomi_cloudml"]["xiaomi_fds_endpoint"]
config = FDSClientConfiguration(
region_name="cnbj1-fds",
enable_https=False,
enable_cdn_for_upload=False,
enable_cdn_for_download=False,
endpoint=fds_endpoint)
fds_client = GalaxyFDSClient(access_key, access_secret, config)
client.put_object(
#cloudml jobs submit -n article-quality-local -m lstm_training -g 1 -M 8G -u fds://training/packages/article-quality-1.1.tar.gz