def list_datasets(count, id, name, memo, created_at):
"""List datasets filtered by conditions"""
api = rest.DataSetApi(configuration.get_api_client())
per_page = 1000
command_args = {
'id': id,
'name': name,
'memo': memo,
'created_at': created_at,
}
args = {
key: value
for key, value in command_args.items() if value is not None
}
if count <= per_page:
result = api.list_datasets(per_page=count, **args)
else:
total_pages = (count - 1) // per_page + 1
result = []
for page in range(1, total_pages + 1):
page_result = api.list_datasets(page=page, **args)
result.extend(page_result)
if len(page_result) < per_page:
break
pprint.pp_table(['id', 'name', 'created_at', 'memo'],
[[x.id, x.name, x.created_at, x.memo]
for x in result[:count]])
def list_path_pairs(id):
"""List path pairs of a dataset"""
api = rest.DataSetApi(configuration.get_api_client())
result = api.list_dataset_pathpairs(id)
for x in result:
print(repr(x.data_path)[1:-1])
print(repr(x.stored_path)[1:-1])
def download_files(api_client, id, dir_path):
api = rest.DataSetApi(api_client)
result = api.list_dataset_files(id, with_url=True)
pool_manager = api_client.rest_client.pool_manager
for entry in result.entries:
for file in entry.files:
destination_dir_path = os.path.join(dir_path, entry.type, str(file.id))
object_storage.download_file(pool_manager, file.url, destination_dir_path, file.file_name)
def update(id, file):
"""Update a dataset"""
api = rest.DataSetApi(configuration.get_api_client())
logging.info('open %s', file)
with io.open(file, 'r', encoding='utf-8') as f:
logging.info('begin io %s', file)
json_dict = json.load(f)
logging.info('end io %s', file)
result = api.update_dataset(id, model=json_dict)
print('updated', result.id)
def download_files(id, destination, data_type):
"""Download files of a dataset"""
api = rest.DataSetApi(configuration.get_api_client())
result = api.list_dataset_files(id, with_url=True)
pool_manager = api.api_client.rest_client.pool_manager
for entry in result.entries:
if not data_type or entry.type in data_type:
for file in entry.files:
destination_dir_path = os.path.join(destination, entry.type, str(file.id))
object_storage.download_file(pool_manager, file.url, destination_dir_path, file.file_name)
def get(id, destination):
"""Get details of a detaset"""
api = rest.DataSetApi(configuration.get_api_client())
if destination is None:
result = api.get_dataset(id)
pprint.pp_dict(util.to_dict(result))
else:
with util.release_conn(api.get_dataset(id, _preload_content=False)) as result:
logging.info('open %s', destination)
with open(destination, 'wb') as f:
logging.info('begin io %s', destination)
f.write(result.data)
logging.info('end io %s', destination)
print('save', id, 'as', destination)
def list_data_types():
"""List data types of a dataset"""
api = rest.DataSetApi(configuration.get_api_client())
result = api.list_dataset_datatypes()
for x in result:
print(x.name)
def delete(id):
"""Delete a dataset"""
api = rest.DataSetApi(configuration.get_api_client())
api.delete_dataset(id)
print('deleted', id)
def update_meta_info(id, name, memo):
"""Update meta information of a dataset"""
api = rest.DataSetApi(configuration.get_api_client())
model = rest.DataSetApiModelsEditInputModel(name=name, memo=memo)
result = api.patch_dataset(id, model=model)
print('meta-info updated', result.id)
def get(api_client, id):
api = rest.DataSetApi(api_client)
result = api.get_dataset(id)
return result
def delete(api_client, id):
api = rest.DataSetApi(api_client)
api.delete_dataset(id)
def update_meta_info(api_client, id, name=None, memo=None):
api = rest.DataSetApi(api_client)
model = rest.DataSetApiModelsEditInputModel(name=name, memo=memo)
result = api.patch_dataset(id, model=model)
return result
def update(api_client, id, model):
api = rest.DataSetApi(api_client)
result = api.update_dataset(id, model=model)
return result
def create(api_client, model):
api = rest.DataSetApi(api_client)
result = api.create_dataset(model=model)
return result
