def list_preprocessings(count, id, name, created_at, memo):
"""List preprocessings filtered by conditions"""
api = rest.PreprocessingApi(configuration.get_api_client())
per_page = 1000
command_args = {
'id': id,
'name': name,
'memo': memo,
'created_at': created_at,
}
args = {
key: value
for key, value in command_args.items() if value is not None
}
if count <= per_page:
result = api.list_preprocessings(per_page=count, **args)
else:
total_pages = (count - 1) // per_page + 1
result = []
for page in range(1, total_pages + 1):
page_result = api.list_preprocessings(page=page, **args)
result.extend(page_result)
if len(page_result) < per_page:
break
pprint.pp_table(['id', 'name', 'created_at', 'memo'],
[[x.id, x.name, x.created_at, x.memo]
for x in result[:count]])
def list_histories(id):
"""List histories of a preprocessing"""
api = rest.PreprocessingApi(configuration.get_api_client())
result = api.list_preprocessing_histories(id)
pprint.pp_table(['data_id', 'data_name', 'created_at', 'status'],
[[x.data_id, x.data_name, x.created_at, x.status]
for x in result])
def patch(id, name, memo, cpu, memory, gpu):
"""Update meta information of a preprocessing"""
api = rest.PreprocessingApi(configuration.get_api_client())
model = rest.PreprocessingApiModelsEditInputModel(name=name,
memo=memo,
cpu=cpu,
memory=memory,
gpu=gpu)
result = api.patch_preprocessing(id, model=model)
print('meta-info updated', result.id)
def update(id, file):
"""Update a preprocessing"""
api = rest.PreprocessingApi(configuration.get_api_client())
logging.info('open %s', file)
with io.open(file, 'r', encoding='utf-8') as f:
logging.info('begin io %s', file)
json_dict = json.load(f)
logging.info('end io %s', file)
result = api.update_preprocessing(id, model=json_dict)
print('updated', result.id)
def run(id, data_id, cpu, memory, gpu, partition, options):
"""Run a preprocessing"""
api = rest.PreprocessingApi(configuration.get_api_client())
option_dict = {key: value for key, value in options} if options else None
for x in data_id:
model = rest.PreprocessingApiModelsRunPreprocessHistoryInputModel(
cpu=cpu,
data_id=x,
gpu=gpu,
memory=memory,
options=option_dict,
partition=partition)
result = api.run_preprocessing(id, model=model)
print('started ', result.preprocess_id, '.', result.data_id, sep='')
def get(id, destination):
"""Get details of a preprocessing"""
api = rest.PreprocessingApi(configuration.get_api_client())
if destination is None:
result = api.get_preprocessing(id)
pprint.pp_dict(util.to_dict(result))
else:
with util.release_conn(
api.get_preprocessing(id, _preload_content=False)) as result:
logging.info('open %s', destination)
with open(destination, 'wb') as f:
logging.info('begin io %s', destination)
f.write(result.data)
logging.info('end io %s', destination)
print('save', id, 'as', destination)
def build_history_files(id, data_id, source, memo, tags):
"""Build file structure for existing history"""
api = rest.PreprocessingApi(configuration.get_api_client())
for entry in os.listdir(source):
if os.path.isdir(os.path.join(source, entry)):
uploaded_files = []
for root, _, files in os.walk(os.path.join(source, entry)):
for file in files:
upload_info = object_storage.upload_file(
api.api_client, os.path.join(root, file), 'Data')
uploaded_files.append(
rest.ComponentsAddFileInputModel(
file_name=upload_info.file_name,
stored_path=upload_info.stored_path))
model = rest.PreprocessingApiModelsAddOutputDataInputModel(
files=uploaded_files, name=entry, memo=memo, tags=list(tags))
api.add_preprocessing_history_files(id, data_id, model=model)
api.complete_preprocessing_history(id, data_id)
def halt_history(id, data_id):
"""Halt a preprocessing of a history"""
api = rest.PreprocessingApi(configuration.get_api_client())
result = api.halt_preprocessing_history(id, data_id)
print('halted ', result.preprocess_id, '.', result.data_id, sep='')
def delete_history(id, data_id):
"""Delete a history of a preprocessing"""
api = rest.PreprocessingApi(configuration.get_api_client())
api.delete_preprocessing_history(id, data_id)
print('deleted ', id, '.', data_id, sep='')
def delete(id):
"""Delete a preprocesssing"""
api = rest.PreprocessingApi(configuration.get_api_client())
api.delete_preprocessing(id)
print('deleted', id)