def main_2():
import struct
client = nos.Client(
access_key_id=NOS_ACCESS_KEY,
access_key_secret=NOS_ACCESS_SECRET,
end_point=NOS_ENDPOINT,
)
song_id = "songjun_test_song_1"
bucket = NOS_BUCKET
object_key = "cnn_feature_" + song_id
resp = client.get_object(
bucket=bucket,
key=object_key,
)
body = resp['body']
spos = 0
epos = 4
ndim = struct.unpack('i', body[spos:epos])
spos = epos
epos = spos + 4 * ndim
fea_size = struct.unpack('i', body[spos:epos])
fea_num = 1
for v in fea_size:
fea_num *= v
spos = epos
epos = spos + fea_num
feature = struct.unpack('f' * fea_num, body[spos:epos])
print(feature[0:10])
def test_client_http(self):
self.assertEquals(self.ACCESS_KEY_ID.startswith('xxxx'), False)
self.assertEquals(self.ACCESS_KEY_SECRET.startswith('xxxx'), False)
self.assertEquals(self.BUCKET.startswith('xxxx'), False)
self.clean_objects_http()
# create client
client = nos.Client(access_key_id=self.ACCESS_KEY_ID,
access_key_secret=self.ACCESS_KEY_SECRET,
enable_ssl=False)
# put object
r = client.put_object(self.BUCKET, self.KEYS[0], self.BODY_STR)
p_md5 = r['etag']
# get object
r = client.get_object(self.BUCKET, self.KEYS[0])
g_md5 = r['etag']
body = r['body'].read()
self.assertEquals(p_md5, g_md5)
self.assertEquals(self.BODY_STR, body)
md5sum = hashlib.md5(self.BODY_STR).hexdigest()
self.assertEquals(md5sum, p_md5)
def clean_objects_https(self):
client = nos.Client(access_key_id=self.ACCESS_KEY_ID,
access_key_secret=self.ACCESS_KEY_SECRET,
enable_ssl=True)
# delete objects
r = client.list_objects(bucket=self.BUCKET)
resp = r['response']
keys = [i.findtext('Key') for i in resp.findall('Contents')]
if keys:
client.delete_objects(self.BUCKET, keys)
# head object to check
for i in keys:
self.assertRaises(nos.exceptions.NotFoundError, client.head_object,
self.BUCKET, i)
# list objects to check
r = client.list_objects(bucket=self.BUCKET)
resp = r['response']
keys = [i.findtext('Key') for i in resp.findall('Contents')]
self.assertEquals([], keys)
# abort multipart upload
r = client.list_multipart_uploads(bucket=self.BUCKET)
resp = r['response']
for i in resp.findall('Upload'):
client.abort_multipart_upload(bucket=self.BUCKET,
key=i.findtext('Key'),
upload_id=i.findtext('UploadId'))
# list multipart uploads to check
r = client.list_multipart_uploads(bucket=self.BUCKET)
resp = r['response']
self.assertEquals([], [i for i in resp.findall('Upload')])
def main_2():
import tempfile
import struct
client = nos.Client(
access_key_id=NOS_ACCESS_KEY,
access_key_secret=NOS_ACCESS_SECRET,
end_point=NOS_ENDPOINT,
)
song_id = "songjun_test_song_1"
bucket = NOS_BUCKET
object_key = "cnn_feature_" + song_id
feature = np.random.random(256).flatten().tolist()
print(feature[0:10])
tempf = tempfile.TemporaryFile('wb')
tempf.write(struct.pack('i', 1))
tempf.write(struct.pack('i', 256))
tempf.write(struct.pack('f'*len(feature), *feature))
resp = client.put_object(
bucket=bucket,
key=object_key,
body=tempf
)
print('put NOS fihish.')
tempf.close()
def setup(self):
config = self.configs[self.domain]
config.update(self.options)
self.end_point = config.get('end_point', '')
self._protocol = config.pop('protocol', 'http')
self.access_key_id = config.get('access_key_id', '')
self.access_key_secret = config.get('access_key_secret', '')
self.client = nos.Client(**config)
def setup(self):
config = self.configs[self.domain]
config.update(self.options)
self.end_point = config.get('end_point', '')
self.enable_ssl = config.get('enable_ssl', False)
self.access_key_id = config.get('access_key_id', '')
self.access_key_secret = config.get('access_key_secret', '')
self.client = nos.Client(**config)
def __init__(self, basepath='test/', private=False):
self.basepath = basepath
self.private = private
self.nos_client = nos.Client(access_key_id=access_key_id,
access_key_secret=access_key_secret,
transport_class=nos.transport.Transport,
end_point=end_point,
num_pools=16,
timeout=20,
max_retries=3)
def upload(access_key, secret_key, end_point, bucket_name, prefix, image_path):
client = nos.Client(access_key, secret_key, end_point=end_point)
index = image_path.rfind('/')
if index == -1:
index = image_path.rfind('\\')
key = prefix + '/' + image_path[index + 1:]
client.put_object(bucket_name, key, open(image_path, "rb"))
return key
def __init__(self,
access_key,
secret_key,
bucket,
end_point,
access_point=None):
self.access_key = access_key
self.secret_key = secret_key
self.bucket = bucket
self.end_point = end_point
self.access_point = access_point if access_point else end_point
self.client = nos.Client(self.access_key,
self.secret_key,
end_point=self.end_point)
def main_1():
client = nos.Client(
access_key_id=NOS_ACCESS_KEY,
access_key_secret=NOS_ACCESS_SECRET,
end_point=NOS_ENDPOINT,
)
song_id = "songjun_test_song"
feature_info = {
"audio_id": song_id,
"version": "python",
"audio_url": "http://url.com",
"log_power_mel_feature": np.random.random((10, 3)).tolist(),
"cnn_feature": np.random.random(20).tolist()
}
try:
resp = client.put_object(
bucket=NOS_BUCKET,
key=NOS_OBJECT_KEY_PREFIX + song_id,
body=json.dumps(feature_info),
)
except nos.exceptions.ServiceException as e:
print (
'ServiceException: %s\n'
'status_code: %s\n'
'error_type: %s\n'
'error_code: %s\n'
'request_id: %s\n'
'message: %s\n'
) % (
e,
e.status_code, # error http code
e.error_type, # NOS server error type
e.error_code, # NOS server error code
e.request_id, # request id
e.message # error description message
)
except nos.exceptions.ClientException as e:
print (
'ClientException: %s\n'
'message: %s\n'
) % (
e,
e.message # client error message
)
print('put NOS fihish.')
def test_multipart_upload_http(self):
self.clean_objects_http()
info = {}
body_part1 = self.BODY_STR * 1024 * 1024 * 2
body_part2 = self.BODY_STR * 1024 * 1024 * 1
body_all = body_part1 + body_part2
md5sum_part1 = hashlib.md5(body_part1).hexdigest()
md5sum_part2 = hashlib.md5(body_part2).hexdigest()
md5sum = hashlib.md5("%s-%s-" %
(md5sum_part1, md5sum_part2)).hexdigest()
# create client
client = nos.Client(access_key_id=self.ACCESS_KEY_ID,
access_key_secret=self.ACCESS_KEY_SECRET,
enable_ssl=False)
# create multipart upload
r = client.create_multipart_upload(
self.BUCKET, self.KEYS[0], meta_data={'x-nos-meta-hello': 'world'})
resp = r['response']
self.assertEquals(self.BUCKET, resp.findtext('Bucket'))
self.assertEquals(self.KEYS[0], resp.findtext('Key'))
upload_id = resp.findtext('UploadId')
# upload part
r = client.upload_part(bucket=self.BUCKET,
key=self.KEYS[0],
part_num=1,
upload_id=upload_id,
body=body_part1)
info['1'] = r['etag']
self.assertEquals(md5sum_part1, r['etag'])
# upload part
r = client.upload_part(bucket=self.BUCKET,
key=self.KEYS[0],
part_num=2,
upload_id=upload_id,
body=body_part2)
info['2'] = r['etag']
self.assertEquals(md5sum_part2, r['etag'])
# list parts
r = client.list_parts(bucket=self.BUCKET,
key=self.KEYS[0],
upload_id=upload_id)
resp = r['response']
for i in resp.findall('Part'):
self.assertEquals(
i.findtext('ETag').strip('a'), info[i.findtext('PartNumber')])
# list multipart uploads
r = client.list_multipart_uploads(bucket=self.BUCKET)
resp = r['response']
for i in resp.findall('Upload'):
if upload_id == i.findtext('UploadId'):
self.assertEquals(self.KEYS[0], i.findtext('Key'))
# complete multipart upload
r = client.complete_multipart_upload(bucket=self.BUCKET,
key=self.KEYS[0],
upload_id=upload_id,
info=[{
'part_num': x,
'etag': y
} for x, y in info.iteritems()])
resp = r['response']
self.assertEquals(self.BUCKET, resp.findtext('Bucket'))
self.assertEquals(self.KEYS[0], resp.findtext('Key'))
self.assertEquals(md5sum, resp.findtext('ETag').strip())
# get object
r = client.get_object(self.BUCKET, self.KEYS[0])
g_md5 = r['etag'].split('-')[0]
body = r['body'].read()
self.assertEquals(body_all, body)
self.assertEquals(md5sum, g_md5)
# create multipart upload
r = client.create_multipart_upload(self.BUCKET, self.KEYS[1])
resp = r['response']
self.assertEquals(self.BUCKET, resp.findtext('Bucket'))
self.assertEquals(self.KEYS[1], resp.findtext('Key'))
upload_id = resp.findtext('UploadId')
# abort multipart upload
r = client.abort_multipart_upload(bucket=self.BUCKET,
key=self.KEYS[1],
upload_id=upload_id)
# list multipart uploads
r = client.list_multipart_uploads(bucket=self.BUCKET)
resp = r['response']
for i in resp.findall('Upload'):
if upload_id == i.findtext('UploadId'):
raise
def test_nos_client_https(self):
self.clean_objects_https()
info = {}
# create client
client = nos.Client(access_key_id=self.ACCESS_KEY_ID,
access_key_secret=self.ACCESS_KEY_SECRET,
enable_ssl=True)
# put invalid object name
self.assertRaises(nos.exceptions.InvalidObjectName, client.put_object,
'aa', '', '')
# put invalid bucket name
self.assertRaises(nos.exceptions.InvalidBucketName, client.put_object,
'', 'bb', '')
# put object
r = client.put_object(self.BUCKET,
self.KEYS[0],
self.BODY_STR,
meta_data={'x-nos-meta-hello': 'world'})
p_md5 = r['etag']
# get object
r = client.get_object(self.BUCKET, self.KEYS[0])
g_md5 = r['etag']
body = r['body'].read()
# head object
r = client.head_object(self.BUCKET, self.KEYS[0])
h_md5 = r['etag']
b_str = self.BODY_STR
md5_str = hashlib.md5(b_str).hexdigest()
info[self.KEYS[0]] = md5_str
self.assertEquals(h_md5, md5_str)
self.assertEquals(g_md5, md5_str)
self.assertEquals(p_md5, md5_str)
self.assertEquals(b_str, body)
# put object
r = client.put_object(self.BUCKET, self.KEYS[1], self.BODY_DICT)
p_md5 = r['etag']
# get object
r = client.get_object(self.BUCKET, self.KEYS[1])
g_md5 = r['etag']
body = r['body'].read()
# head object
r = client.head_object(self.BUCKET, self.KEYS[1])
h_md5 = r['etag']
b_dict = json.dumps(self.BODY_DICT)
md5_dict = hashlib.md5(b_dict).hexdigest()
info[self.KEYS[1]] = md5_dict
self.assertEquals(h_md5, md5_dict)
self.assertEquals(g_md5, md5_dict)
self.assertEquals(p_md5, md5_dict)
self.assertEquals(b_dict, body)
# put object
r = client.put_object(self.BUCKET, self.KEYS[2], self.BODY_LIST)
p_md5 = r['etag']
# get object
r = client.get_object(self.BUCKET, self.KEYS[2])
g_md5 = r['etag']
body = r['body'].read()
# head object
r = client.head_object(self.BUCKET, self.KEYS[2])
h_md5 = r['etag']
b_list = json.dumps(self.BODY_LIST)
md5_list = hashlib.md5(b_list).hexdigest()
info[self.KEYS[2]] = md5_list
self.assertEquals(h_md5, md5_list)
self.assertEquals(g_md5, md5_list)
self.assertEquals(p_md5, md5_list)
self.assertEquals(b_list, body)
# put object
self.BODY_FILE.seek(0)
r = client.put_object(self.BUCKET, self.KEYS[3], self.BODY_FILE)
p_md5 = r['etag']
# get object
r = client.get_object(self.BUCKET, self.KEYS[3])
g_md5 = r['etag']
body = r['body'].read()
# head object
r = client.head_object(self.BUCKET, self.KEYS[3])
h_md5 = r['etag']
self.BODY_FILE.seek(0)
b_file = self.BODY_FILE.read()
md5_file = hashlib.md5(b_file).hexdigest()
info[self.KEYS[3]] = md5_file
self.assertEquals(h_md5, md5_file)
self.assertEquals(g_md5, md5_file)
self.assertEquals(p_md5, md5_file)
self.assertEquals(b_file, body)
# list objects
r = client.list_objects(bucket=self.BUCKET)
resp = r['response']
for i in resp.findall('Contents'):
self.assertEquals(info[i.findtext('Key')],
i.findtext('ETag').strip('"'))
# move object
r = client.move_object(self.BUCKET, self.KEYS[0], self.BUCKET,
self.KEYS[3])
info[self.KEYS[3]] = info.pop(self.KEYS[0])
# list objects
r = client.list_objects(bucket=self.BUCKET)
resp = r['response']
for i in resp.findall('Contents'):
self.assertEquals(info[i.findtext('Key')],
i.findtext('ETag').strip('"'))
# copy object
r = client.copy_object(self.BUCKET, self.KEYS[1], self.BUCKET,
self.KEYS[0])
info[self.KEYS[0]] = info[self.KEYS[1]]
# list objects
r = client.list_objects(bucket=self.BUCKET)
resp = r['response']
for i in resp.findall('Contents'):
self.assertEquals(info[i.findtext('Key')],
i.findtext('ETag').strip('"'))
# delete object
r = client.delete_object(self.BUCKET, self.KEYS[3])
info.pop(self.KEYS[3], '')
self.assertRaises(nos.exceptions.NotFoundError, client.head_object,
self.BUCKET, self.KEYS[3])
# list objects
r = client.list_objects(bucket=self.BUCKET)
resp = r['response']
for i in resp.findall('Contents'):
self.assertEquals(info[i.findtext('Key')],
i.findtext('ETag').strip('"'))
# delete objects
r = client.delete_objects(self.BUCKET, [self.KEYS[1], self.KEYS[2]])
self.assertRaises(nos.exceptions.BadRequestError,
client.delete_objects, self.BUCKET, [])
self.assertRaises(nos.exceptions.MultiObjectDeleteException,
client.delete_objects, self.BUCKET, [self.KEYS[3]])
info.pop(self.KEYS[1], '')
info.pop(self.KEYS[2], '')
info.pop(self.KEYS[3], '')
self.assertRaises(nos.exceptions.NotFoundError, client.head_object,
self.BUCKET, self.KEYS[1])
self.assertRaises(nos.exceptions.NotFoundError, client.head_object,
self.BUCKET, self.KEYS[2])
self.assertRaises(nos.exceptions.NotFoundError, client.head_object,
self.BUCKET, self.KEYS[3])
# list objects
r = client.list_objects(bucket=self.BUCKET)
resp = r['response']
for i in resp.findall('Contents'):
self.assertEquals(info[i.findtext('Key')],
i.findtext('ETag').strip('"'))
# head object
client.head_object(bucket=self.BUCKET, key=self.KEYS[0])
#-*coding:utf-8-*-
import nos
access_key = ""
secret_key = ""
end_point = "nos-eastchina1.126.net" #保持默认即可，不需要修改。
bucket = ""
object = ""
file_name = ""
client = nos.Client(access_key, secret_key, end_point=end_point)
try:
client.put_object(bucket, object,open(file_name, "rb"))
except nos.exceptions.ServiceException as e:
print (
"ServiceException: %s\n"
"status_code: %s\n"
"error_type: %s\n"
"error_code: %s\n"
"request_id: %s\n"
"message: %s\n"
) % (
e,
e.status_code, # 错误 http 状态码
e.error_type, # NOS 服务器定义错误类型
e.error_code, # NOS 服务器定义错误码
e.request_id, # 请求 ID，有利于 nos 开发人员跟踪异常请求的错误原因
e.message # 错误描述信息
)
except nos.exceptions.ClientException as e:
def main_1():
client = nos.Client(
access_key_id=NOS_ACCESS_KEY,
access_key_secret=NOS_ACCESS_SECRET,
end_point=NOS_ENDPOINT,
)
# song_id = "songjun_test_song"
song_id = "1002166"
key_prefix = "meta_"
bucket = NOS_BUCKET
object_key = key_prefix + song_id
try:
resp = client.get_object(
bucket=bucket,
key=object_key,
)
print(type(resp))
print(resp)
body = resp['body']
feature_info = json.loads(body.read())
print("Bucket: %s, Key: %s" % (bucket, object_key))
for key in feature_info.keys():
val = feature_info[key]
if isinstance(val, list):
first_val = val[0]
last_val = val[-1]
size_str = str(len(val))
if isinstance(first_val, list):
size_str += " * " + str(len(first_val))
first_val = first_val[0]
last_val = last_val[-1]
val_str = "size: <%s>, first: %s, last: %s" % (
size_str, str(first_val), str(last_val))
else:
val_str = str(val)
print('key: %s, type: %s, value: %s' %
(key, str(type(feature_info[key])), val_str))
except nos.exceptions.ServiceException as e:
print('ServiceException: %s\n'
'status_code: %s\n'
'error_type: %s\n'
'error_code: %s\n'
'request_id: %s\n'
'message: %s\n') % (
e,
e.status_code, # error http code
e.error_type, # NOS server error type
e.error_code, # NOS server error code
e.request_id, # request id
e.message # error description message
)
except nos.exceptions.ClientException as e:
print('ClientException: %s\n'
'message: %s\n') % (
e,
e.message # client error message
)
print('get NOS fihish.')
