def download_files(id, destination):
"""Download files of data"""
api = rest.DataApi(configuration.get_api_client())
result = api.list_data_files(id, with_url=True)
pool_manager = api.api_client.rest_client.pool_manager
for x in result:
object_storage.download_file(pool_manager, x.url, destination, x.file_name)
def list_data(count, id, name, memo, created_at, created_by, tag):
"""List data filtered by conditions"""
api = rest.DataApi(configuration.get_api_client())
per_page = 1000
command_args = {
'id': id,
'name': name,
'memo': memo,
'created_at': created_at,
'created_by': created_by,
'tag': tag,
}
args = {key: value for key, value in command_args.items() if value is not None}
if count <= per_page:
result = api.list_data(per_page=count, **args)
else:
total_pages = (count - 1) // per_page + 1
result = []
for page in range(1, total_pages + 1):
page_result = api.list_data(page=page, **args)
result.extend(page_result)
if len(page_result) < per_page:
break
pprint.pp_table(['id', 'name', 'created_at', 'created_by', 'memo', 'tags'],
[[x.id, x.name, x.created_at, x.created_by, x.memo, x.tags] for x in result[:count]])
def upload_files(id, file):
"""Upload files to data"""
api = rest.DataApi(configuration.get_api_client())
for x in file:
upload_info = object_storage.upload_file(api.api_client, x, 'Data')
model = rest.ComponentsAddFileInputModel(file_name=upload_info.file_name, stored_path=upload_info.stored_path)
api.add_data_file(id, model=model)
def update(id, name, memo, tags):
"""Update data"""
api = rest.DataApi(configuration.get_api_client())
model = rest.DataApiModelsEditInputModel(name=name,
memo=memo,
tags=list(tags))
result = api.update_data(id, model=model)
print('updated', result.id)
def create(name, file, memo, tags):
"""Create new data"""
api = rest.DataApi(configuration.get_api_client())
check_upload_files(len(file))
model = rest.DataApiModelsCreateInputModel(
memo=memo,
name=name,
tags=list(tags)
)
result = api.create_data(model=model)
do_upload_files(api, result.id, file)
print('created', result.id)
def create(name, file, memo, tags):
"""Create new data"""
api = rest.DataApi(configuration.get_api_client())
model = rest.DataApiModelsCreateInputModel(memo=memo,
name=name,
tags=list(tags))
result = api.create_data(model=model)
for x in file:
upload_info = object_storage.upload_file(api.api_client, x, 'Data')
model = rest.ComponentsAddFileInputModel(
file_name=upload_info.file_name,
stored_path=upload_info.stored_path)
api.add_data_file(result.id, model=model)
print('created', result.id)
def get(id, destination):
"""Get details of data"""
api = rest.DataApi(configuration.get_api_client())
if destination is None:
result = api.get_data(id)
pprint.pp_dict(util.to_dict(result))
else:
with util.release_conn(api.get_data(id, _preload_content=False)) as result:
logging.info('open %s', destination)
with open(destination, 'wb') as f:
logging.info('begin io %s', destination)
f.write(result.data)
logging.info('end io %s', destination)
print('save', id, 'as', destination)
def list_files(id):
"""List files of data"""
api = rest.DataApi(configuration.get_api_client())
result = api.list_data_files(id)
pprint.pp_table(['file_id', 'file_name'],
[[x.file_id, x.file_name] for x in result])
def delete(id):
"""Delete data"""
api = rest.DataApi(configuration.get_api_client())
api.delete_data(id)
print('deleted', id)
def get(api_client, id):
api = rest.DataApi(api_client)
result = api.get_data(id)
return result
def download_files(api_client, id, dir_path):
api = rest.DataApi(api_client)
result = api.list_data_files(id, with_url=True)
pool_manager = api_client.rest_client.pool_manager
for x in result:
object_storage.download_file(pool_manager, x.url, dir_path, x.file_name)
def delete(api_client, id):
api = rest.DataApi(api_client)
api.delete_data(id)
def update(api_client, id, name=None, memo=None, tags=()):
api = rest.DataApi(api_client)
model = rest.DataApiModelsEditInputModel(name=name, memo=memo, tags=list(tags))
result = api.update_data(id, model=model)
return result
def create(api_client, name, memo=None, tags=()):
api = rest.DataApi(api_client)
model = rest.DataApiModelsCreateInputModel(memo=memo, name=name, tags=list(tags))
result = api.create_data(model=model)
return result
def upload_files(id, file):
"""Upload files to data"""
api = rest.DataApi(configuration.get_api_client())
result = api.list_data_files(id)
check_upload_files(len(result) + len(file))
do_upload_files(api, id, file)
def delete(id):
"""Delete data"""
api = rest.DataApi(configuration.get_api_client())
api.delete_data(id, _request_timeout=heavy_request_timeout(api.api_client))
print('deleted', id)
def delete_file(id, file_id):
"""Delete a file of data"""
api = rest.DataApi(configuration.get_api_client())
api.delete_data_file(id, file_id)
print('deleted', file_id)
def list_files(api_client, id):
api = rest.DataApi(api_client)
result = api.list_data_files(id, with_url=True)
return result
def upload_file(api_client, id, file_path):
api = rest.DataApi(api_client)
upload_info = object_storage.upload_file(api_client, file_path, 'Data')
model = rest.ComponentsAddFileInputModel(file_name=upload_info.file_name, stored_path=upload_info.stored_path)
result = api.add_data_file(id, model=model)
return result
def get(id):
"""Get details of data"""
api = rest.DataApi(configuration.get_api_client())
result = api.get_data(id)
pprint.pp_dict(util.to_dict(result))
