def main_2(): import struct client = nos.Client( access_key_id=NOS_ACCESS_KEY, access_key_secret=NOS_ACCESS_SECRET, end_point=NOS_ENDPOINT, ) song_id = "songjun_test_song_1" bucket = NOS_BUCKET object_key = "cnn_feature_" + song_id resp = client.get_object( bucket=bucket, key=object_key, ) body = resp['body'] spos = 0 epos = 4 ndim = struct.unpack('i', body[spos:epos]) spos = epos epos = spos + 4 * ndim fea_size = struct.unpack('i', body[spos:epos]) fea_num = 1 for v in fea_size: fea_num *= v spos = epos epos = spos + fea_num feature = struct.unpack('f' * fea_num, body[spos:epos]) print(feature[0:10])
def test_client_http(self): self.assertEquals(self.ACCESS_KEY_ID.startswith('xxxx'), False) self.assertEquals(self.ACCESS_KEY_SECRET.startswith('xxxx'), False) self.assertEquals(self.BUCKET.startswith('xxxx'), False) self.clean_objects_http() # create client client = nos.Client(access_key_id=self.ACCESS_KEY_ID, access_key_secret=self.ACCESS_KEY_SECRET, enable_ssl=False) # put object r = client.put_object(self.BUCKET, self.KEYS[0], self.BODY_STR) p_md5 = r['etag'] # get object r = client.get_object(self.BUCKET, self.KEYS[0]) g_md5 = r['etag'] body = r['body'].read() self.assertEquals(p_md5, g_md5) self.assertEquals(self.BODY_STR, body) md5sum = hashlib.md5(self.BODY_STR).hexdigest() self.assertEquals(md5sum, p_md5)
def clean_objects_https(self): client = nos.Client(access_key_id=self.ACCESS_KEY_ID, access_key_secret=self.ACCESS_KEY_SECRET, enable_ssl=True) # delete objects r = client.list_objects(bucket=self.BUCKET) resp = r['response'] keys = [i.findtext('Key') for i in resp.findall('Contents')] if keys: client.delete_objects(self.BUCKET, keys) # head object to check for i in keys: self.assertRaises(nos.exceptions.NotFoundError, client.head_object, self.BUCKET, i) # list objects to check r = client.list_objects(bucket=self.BUCKET) resp = r['response'] keys = [i.findtext('Key') for i in resp.findall('Contents')] self.assertEquals([], keys) # abort multipart upload r = client.list_multipart_uploads(bucket=self.BUCKET) resp = r['response'] for i in resp.findall('Upload'): client.abort_multipart_upload(bucket=self.BUCKET, key=i.findtext('Key'), upload_id=i.findtext('UploadId')) # list multipart uploads to check r = client.list_multipart_uploads(bucket=self.BUCKET) resp = r['response'] self.assertEquals([], [i for i in resp.findall('Upload')])
def main_2(): import tempfile import struct client = nos.Client( access_key_id=NOS_ACCESS_KEY, access_key_secret=NOS_ACCESS_SECRET, end_point=NOS_ENDPOINT, ) song_id = "songjun_test_song_1" bucket = NOS_BUCKET object_key = "cnn_feature_" + song_id feature = np.random.random(256).flatten().tolist() print(feature[0:10]) tempf = tempfile.TemporaryFile('wb') tempf.write(struct.pack('i', 1)) tempf.write(struct.pack('i', 256)) tempf.write(struct.pack('f'*len(feature), *feature)) resp = client.put_object( bucket=bucket, key=object_key, body=tempf ) print('put NOS fihish.') tempf.close()
def setup(self): config = self.configs[self.domain] config.update(self.options) self.end_point = config.get('end_point', '') self._protocol = config.pop('protocol', 'http') self.access_key_id = config.get('access_key_id', '') self.access_key_secret = config.get('access_key_secret', '') self.client = nos.Client(**config)
def setup(self): config = self.configs[self.domain] config.update(self.options) self.end_point = config.get('end_point', '') self.enable_ssl = config.get('enable_ssl', False) self.access_key_id = config.get('access_key_id', '') self.access_key_secret = config.get('access_key_secret', '') self.client = nos.Client(**config)
def __init__(self, basepath='test/', private=False): self.basepath = basepath self.private = private self.nos_client = nos.Client(access_key_id=access_key_id, access_key_secret=access_key_secret, transport_class=nos.transport.Transport, end_point=end_point, num_pools=16, timeout=20, max_retries=3)
def upload(access_key, secret_key, end_point, bucket_name, prefix, image_path): client = nos.Client(access_key, secret_key, end_point=end_point) index = image_path.rfind('/') if index == -1: index = image_path.rfind('\\') key = prefix + '/' + image_path[index + 1:] client.put_object(bucket_name, key, open(image_path, "rb")) return key
def __init__(self, access_key, secret_key, bucket, end_point, access_point=None): self.access_key = access_key self.secret_key = secret_key self.bucket = bucket self.end_point = end_point self.access_point = access_point if access_point else end_point self.client = nos.Client(self.access_key, self.secret_key, end_point=self.end_point)
def main_1(): client = nos.Client( access_key_id=NOS_ACCESS_KEY, access_key_secret=NOS_ACCESS_SECRET, end_point=NOS_ENDPOINT, ) song_id = "songjun_test_song" feature_info = { "audio_id": song_id, "version": "python", "audio_url": "http://url.com", "log_power_mel_feature": np.random.random((10, 3)).tolist(), "cnn_feature": np.random.random(20).tolist() } try: resp = client.put_object( bucket=NOS_BUCKET, key=NOS_OBJECT_KEY_PREFIX + song_id, body=json.dumps(feature_info), ) except nos.exceptions.ServiceException as e: print ( 'ServiceException: %s\n' 'status_code: %s\n' 'error_type: %s\n' 'error_code: %s\n' 'request_id: %s\n' 'message: %s\n' ) % ( e, e.status_code, # error http code e.error_type, # NOS server error type e.error_code, # NOS server error code e.request_id, # request id e.message # error description message ) except nos.exceptions.ClientException as e: print ( 'ClientException: %s\n' 'message: %s\n' ) % ( e, e.message # client error message ) print('put NOS fihish.')
def test_multipart_upload_http(self): self.clean_objects_http() info = {} body_part1 = self.BODY_STR * 1024 * 1024 * 2 body_part2 = self.BODY_STR * 1024 * 1024 * 1 body_all = body_part1 + body_part2 md5sum_part1 = hashlib.md5(body_part1).hexdigest() md5sum_part2 = hashlib.md5(body_part2).hexdigest() md5sum = hashlib.md5("%s-%s-" % (md5sum_part1, md5sum_part2)).hexdigest() # create client client = nos.Client(access_key_id=self.ACCESS_KEY_ID, access_key_secret=self.ACCESS_KEY_SECRET, enable_ssl=False) # create multipart upload r = client.create_multipart_upload( self.BUCKET, self.KEYS[0], meta_data={'x-nos-meta-hello': 'world'}) resp = r['response'] self.assertEquals(self.BUCKET, resp.findtext('Bucket')) self.assertEquals(self.KEYS[0], resp.findtext('Key')) upload_id = resp.findtext('UploadId') # upload part r = client.upload_part(bucket=self.BUCKET, key=self.KEYS[0], part_num=1, upload_id=upload_id, body=body_part1) info['1'] = r['etag'] self.assertEquals(md5sum_part1, r['etag']) # upload part r = client.upload_part(bucket=self.BUCKET, key=self.KEYS[0], part_num=2, upload_id=upload_id, body=body_part2) info['2'] = r['etag'] self.assertEquals(md5sum_part2, r['etag']) # list parts r = client.list_parts(bucket=self.BUCKET, key=self.KEYS[0], upload_id=upload_id) resp = r['response'] for i in resp.findall('Part'): self.assertEquals( i.findtext('ETag').strip('a'), info[i.findtext('PartNumber')]) # list multipart uploads r = client.list_multipart_uploads(bucket=self.BUCKET) resp = r['response'] for i in resp.findall('Upload'): if upload_id == i.findtext('UploadId'): self.assertEquals(self.KEYS[0], i.findtext('Key')) # complete multipart upload r = client.complete_multipart_upload(bucket=self.BUCKET, key=self.KEYS[0], upload_id=upload_id, info=[{ 'part_num': x, 'etag': y } for x, y in info.iteritems()]) resp = r['response'] self.assertEquals(self.BUCKET, resp.findtext('Bucket')) self.assertEquals(self.KEYS[0], resp.findtext('Key')) self.assertEquals(md5sum, resp.findtext('ETag').strip()) # get object r = client.get_object(self.BUCKET, self.KEYS[0]) g_md5 = r['etag'].split('-')[0] body = r['body'].read() self.assertEquals(body_all, body) self.assertEquals(md5sum, g_md5) # create multipart upload r = client.create_multipart_upload(self.BUCKET, self.KEYS[1]) resp = r['response'] self.assertEquals(self.BUCKET, resp.findtext('Bucket')) self.assertEquals(self.KEYS[1], resp.findtext('Key')) upload_id = resp.findtext('UploadId') # abort multipart upload r = client.abort_multipart_upload(bucket=self.BUCKET, key=self.KEYS[1], upload_id=upload_id) # list multipart uploads r = client.list_multipart_uploads(bucket=self.BUCKET) resp = r['response'] for i in resp.findall('Upload'): if upload_id == i.findtext('UploadId'): raise
def test_nos_client_https(self): self.clean_objects_https() info = {} # create client client = nos.Client(access_key_id=self.ACCESS_KEY_ID, access_key_secret=self.ACCESS_KEY_SECRET, enable_ssl=True) # put invalid object name self.assertRaises(nos.exceptions.InvalidObjectName, client.put_object, 'aa', '', '') # put invalid bucket name self.assertRaises(nos.exceptions.InvalidBucketName, client.put_object, '', 'bb', '') # put object r = client.put_object(self.BUCKET, self.KEYS[0], self.BODY_STR, meta_data={'x-nos-meta-hello': 'world'}) p_md5 = r['etag'] # get object r = client.get_object(self.BUCKET, self.KEYS[0]) g_md5 = r['etag'] body = r['body'].read() # head object r = client.head_object(self.BUCKET, self.KEYS[0]) h_md5 = r['etag'] b_str = self.BODY_STR md5_str = hashlib.md5(b_str).hexdigest() info[self.KEYS[0]] = md5_str self.assertEquals(h_md5, md5_str) self.assertEquals(g_md5, md5_str) self.assertEquals(p_md5, md5_str) self.assertEquals(b_str, body) # put object r = client.put_object(self.BUCKET, self.KEYS[1], self.BODY_DICT) p_md5 = r['etag'] # get object r = client.get_object(self.BUCKET, self.KEYS[1]) g_md5 = r['etag'] body = r['body'].read() # head object r = client.head_object(self.BUCKET, self.KEYS[1]) h_md5 = r['etag'] b_dict = json.dumps(self.BODY_DICT) md5_dict = hashlib.md5(b_dict).hexdigest() info[self.KEYS[1]] = md5_dict self.assertEquals(h_md5, md5_dict) self.assertEquals(g_md5, md5_dict) self.assertEquals(p_md5, md5_dict) self.assertEquals(b_dict, body) # put object r = client.put_object(self.BUCKET, self.KEYS[2], self.BODY_LIST) p_md5 = r['etag'] # get object r = client.get_object(self.BUCKET, self.KEYS[2]) g_md5 = r['etag'] body = r['body'].read() # head object r = client.head_object(self.BUCKET, self.KEYS[2]) h_md5 = r['etag'] b_list = json.dumps(self.BODY_LIST) md5_list = hashlib.md5(b_list).hexdigest() info[self.KEYS[2]] = md5_list self.assertEquals(h_md5, md5_list) self.assertEquals(g_md5, md5_list) self.assertEquals(p_md5, md5_list) self.assertEquals(b_list, body) # put object self.BODY_FILE.seek(0) r = client.put_object(self.BUCKET, self.KEYS[3], self.BODY_FILE) p_md5 = r['etag'] # get object r = client.get_object(self.BUCKET, self.KEYS[3]) g_md5 = r['etag'] body = r['body'].read() # head object r = client.head_object(self.BUCKET, self.KEYS[3]) h_md5 = r['etag'] self.BODY_FILE.seek(0) b_file = self.BODY_FILE.read() md5_file = hashlib.md5(b_file).hexdigest() info[self.KEYS[3]] = md5_file self.assertEquals(h_md5, md5_file) self.assertEquals(g_md5, md5_file) self.assertEquals(p_md5, md5_file) self.assertEquals(b_file, body) # list objects r = client.list_objects(bucket=self.BUCKET) resp = r['response'] for i in resp.findall('Contents'): self.assertEquals(info[i.findtext('Key')], i.findtext('ETag').strip('"')) # move object r = client.move_object(self.BUCKET, self.KEYS[0], self.BUCKET, self.KEYS[3]) info[self.KEYS[3]] = info.pop(self.KEYS[0]) # list objects r = client.list_objects(bucket=self.BUCKET) resp = r['response'] for i in resp.findall('Contents'): self.assertEquals(info[i.findtext('Key')], i.findtext('ETag').strip('"')) # copy object r = client.copy_object(self.BUCKET, self.KEYS[1], self.BUCKET, self.KEYS[0]) info[self.KEYS[0]] = info[self.KEYS[1]] # list objects r = client.list_objects(bucket=self.BUCKET) resp = r['response'] for i in resp.findall('Contents'): self.assertEquals(info[i.findtext('Key')], i.findtext('ETag').strip('"')) # delete object r = client.delete_object(self.BUCKET, self.KEYS[3]) info.pop(self.KEYS[3], '') self.assertRaises(nos.exceptions.NotFoundError, client.head_object, self.BUCKET, self.KEYS[3]) # list objects r = client.list_objects(bucket=self.BUCKET) resp = r['response'] for i in resp.findall('Contents'): self.assertEquals(info[i.findtext('Key')], i.findtext('ETag').strip('"')) # delete objects r = client.delete_objects(self.BUCKET, [self.KEYS[1], self.KEYS[2]]) self.assertRaises(nos.exceptions.BadRequestError, client.delete_objects, self.BUCKET, []) self.assertRaises(nos.exceptions.MultiObjectDeleteException, client.delete_objects, self.BUCKET, [self.KEYS[3]]) info.pop(self.KEYS[1], '') info.pop(self.KEYS[2], '') info.pop(self.KEYS[3], '') self.assertRaises(nos.exceptions.NotFoundError, client.head_object, self.BUCKET, self.KEYS[1]) self.assertRaises(nos.exceptions.NotFoundError, client.head_object, self.BUCKET, self.KEYS[2]) self.assertRaises(nos.exceptions.NotFoundError, client.head_object, self.BUCKET, self.KEYS[3]) # list objects r = client.list_objects(bucket=self.BUCKET) resp = r['response'] for i in resp.findall('Contents'): self.assertEquals(info[i.findtext('Key')], i.findtext('ETag').strip('"')) # head object client.head_object(bucket=self.BUCKET, key=self.KEYS[0])
#-*coding:utf-8-*- import nos access_key = "" secret_key = "" end_point = "nos-eastchina1.126.net" #保持默认即可,不需要修改。 bucket = "" object = "" file_name = "" client = nos.Client(access_key, secret_key, end_point=end_point) try: client.put_object(bucket, object,open(file_name, "rb")) except nos.exceptions.ServiceException as e: print ( "ServiceException: %s\n" "status_code: %s\n" "error_type: %s\n" "error_code: %s\n" "request_id: %s\n" "message: %s\n" ) % ( e, e.status_code, # 错误 http 状态码 e.error_type, # NOS 服务器定义错误类型 e.error_code, # NOS 服务器定义错误码 e.request_id, # 请求 ID,有利于 nos 开发人员跟踪异常请求的错误原因 e.message # 错误描述信息 ) except nos.exceptions.ClientException as e:
def main_1(): client = nos.Client( access_key_id=NOS_ACCESS_KEY, access_key_secret=NOS_ACCESS_SECRET, end_point=NOS_ENDPOINT, ) # song_id = "songjun_test_song" song_id = "1002166" key_prefix = "meta_" bucket = NOS_BUCKET object_key = key_prefix + song_id try: resp = client.get_object( bucket=bucket, key=object_key, ) print(type(resp)) print(resp) body = resp['body'] feature_info = json.loads(body.read()) print("Bucket: %s, Key: %s" % (bucket, object_key)) for key in feature_info.keys(): val = feature_info[key] if isinstance(val, list): first_val = val[0] last_val = val[-1] size_str = str(len(val)) if isinstance(first_val, list): size_str += " * " + str(len(first_val)) first_val = first_val[0] last_val = last_val[-1] val_str = "size: <%s>, first: %s, last: %s" % ( size_str, str(first_val), str(last_val)) else: val_str = str(val) print('key: %s, type: %s, value: %s' % (key, str(type(feature_info[key])), val_str)) except nos.exceptions.ServiceException as e: print('ServiceException: %s\n' 'status_code: %s\n' 'error_type: %s\n' 'error_code: %s\n' 'request_id: %s\n' 'message: %s\n') % ( e, e.status_code, # error http code e.error_type, # NOS server error type e.error_code, # NOS server error code e.request_id, # request id e.message # error description message ) except nos.exceptions.ClientException as e: print('ClientException: %s\n' 'message: %s\n') % ( e, e.message # client error message ) print('get NOS fihish.')