def list_datasets(count, id, name, memo, created_at): """List datasets filtered by conditions""" api = rest.DataSetApi(configuration.get_api_client()) per_page = 1000 command_args = { 'id': id, 'name': name, 'memo': memo, 'created_at': created_at, } args = { key: value for key, value in command_args.items() if value is not None } if count <= per_page: result = api.list_datasets(per_page=count, **args) else: total_pages = (count - 1) // per_page + 1 result = [] for page in range(1, total_pages + 1): page_result = api.list_datasets(page=page, **args) result.extend(page_result) if len(page_result) < per_page: break pprint.pp_table(['id', 'name', 'created_at', 'memo'], [[x.id, x.name, x.created_at, x.memo] for x in result[:count]])
def list_path_pairs(id): """List path pairs of a dataset""" api = rest.DataSetApi(configuration.get_api_client()) result = api.list_dataset_pathpairs(id) for x in result: print(repr(x.data_path)[1:-1]) print(repr(x.stored_path)[1:-1])
def download_files(api_client, id, dir_path): api = rest.DataSetApi(api_client) result = api.list_dataset_files(id, with_url=True) pool_manager = api_client.rest_client.pool_manager for entry in result.entries: for file in entry.files: destination_dir_path = os.path.join(dir_path, entry.type, str(file.id)) object_storage.download_file(pool_manager, file.url, destination_dir_path, file.file_name)
def update(id, file): """Update a dataset""" api = rest.DataSetApi(configuration.get_api_client()) logging.info('open %s', file) with io.open(file, 'r', encoding='utf-8') as f: logging.info('begin io %s', file) json_dict = json.load(f) logging.info('end io %s', file) result = api.update_dataset(id, model=json_dict) print('updated', result.id)
def download_files(id, destination, data_type): """Download files of a dataset""" api = rest.DataSetApi(configuration.get_api_client()) result = api.list_dataset_files(id, with_url=True) pool_manager = api.api_client.rest_client.pool_manager for entry in result.entries: if not data_type or entry.type in data_type: for file in entry.files: destination_dir_path = os.path.join(destination, entry.type, str(file.id)) object_storage.download_file(pool_manager, file.url, destination_dir_path, file.file_name)
def get(id, destination): """Get details of a detaset""" api = rest.DataSetApi(configuration.get_api_client()) if destination is None: result = api.get_dataset(id) pprint.pp_dict(util.to_dict(result)) else: with util.release_conn(api.get_dataset(id, _preload_content=False)) as result: logging.info('open %s', destination) with open(destination, 'wb') as f: logging.info('begin io %s', destination) f.write(result.data) logging.info('end io %s', destination) print('save', id, 'as', destination)
def list_data_types(): """List data types of a dataset""" api = rest.DataSetApi(configuration.get_api_client()) result = api.list_dataset_datatypes() for x in result: print(x.name)
def delete(id): """Delete a dataset""" api = rest.DataSetApi(configuration.get_api_client()) api.delete_dataset(id) print('deleted', id)
def update_meta_info(id, name, memo): """Update meta information of a dataset""" api = rest.DataSetApi(configuration.get_api_client()) model = rest.DataSetApiModelsEditInputModel(name=name, memo=memo) result = api.patch_dataset(id, model=model) print('meta-info updated', result.id)
def get(api_client, id): api = rest.DataSetApi(api_client) result = api.get_dataset(id) return result
def delete(api_client, id): api = rest.DataSetApi(api_client) api.delete_dataset(id)
def update_meta_info(api_client, id, name=None, memo=None): api = rest.DataSetApi(api_client) model = rest.DataSetApiModelsEditInputModel(name=name, memo=memo) result = api.patch_dataset(id, model=model) return result
def update(api_client, id, model): api = rest.DataSetApi(api_client) result = api.update_dataset(id, model=model) return result
def create(api_client, model): api = rest.DataSetApi(api_client) result = api.create_dataset(model=model) return result