def execute(self,
spec_path=None,
input_path=None,
workflow_id=None,
cluster_id=None):
spec = None
inputs = None
if spec_path:
if not os.path.exists(spec_path):
raise ApplicationError(
'Source path not found: {}'.format(spec_path))
else:
yaml_spec = open(spec_path, 'r')
spec = yaml.safe_load(yaml_spec)
if input_path:
if not os.path.exists(input_path):
raise ApplicationError(
'Source path not found: {}'.format(input_path))
else:
yaml_inputs = open(input_path, 'r')
inputs = yaml.safe_load(yaml_inputs)
workflow = self.client.run_workflow(spec=spec,
inputs=inputs,
workflow_id=workflow_id,
cluster_id=cluster_id)
return workflow
def assert_supported(self, dataset_id):
dataset_id, _, _ = dataset_id.partition(':')
dataset = self.dataset_client.get(dataset_id)
if dataset.storage_provider.type != 's3':
raise ApplicationError('%s storage type not supported' %
dataset.storage_provider.type)
def _log_table_of_logs(self, id, line, limit):
logs = self.client.logs(id, line, limit)
if not logs:
raise ApplicationError("No logs found")
table_str = self._make_table(logs, id)
if len(table_str.splitlines()) > get_terminal_lines():
pydoc.pager(table_str)
else:
self.logger.log(table_str)
def execute(self, model_id, destination_directory):
model_files_downloader = ModelFilesDownloader(
self.api_key,
logger=self.logger,
ps_client_name=cli_constants.CLI_PS_CLIENT_NAME,
)
try:
model_files_downloader.download(model_id, destination_directory)
except OSError as e:
raise ApplicationError(e)
def _list_files(source_path):
if os.path.isfile(source_path):
yield True, source_path
return
if os.path.isdir(source_path):
for dir_path, _, names in os.walk(source_path):
for name in names:
yield False, os.path.join(dir_path, name)
return
raise ApplicationError('Invalid source path: ' + source_path)
def execute(self, dataset_id, message=None, source_paths=None):
if source_paths:
for source_path in source_paths:
if not os.path.exists(source_path):
raise ApplicationError('Source path not found: {}'.format(source_path))
version = self.client.create(dataset_id=dataset_id, message=message)
dataset_version_id = '{}:{}'.format(dataset_id, version)
self.logger.log('Created dataset version: {}'.format(dataset_version_id))
if source_paths:
create = PutDatasetFilesCommand(api_key=self.api_key, logger=self.logger)
create.execute(dataset_version_id, source_paths=source_paths, target_path='/')
commit = CommitDatasetVersionCommand(api_key=self.api_key, logger=self.logger)
commit.execute(dataset_version_id)
def _get(cls, url, path):
dir_path = os.path.dirname(path)
tmp_path = path + '.tmp-%s' % uuid.uuid4()
if os.path.exists(path) and not os.path.isfile(path):
raise ApplicationError('%s already exists' % path)
os.makedirs(dir_path, exist_ok=True)
try:
with requests.Session() as session:
try:
with session.get(url, stream=True) as r:
cls.validate_s3_response(r)
with open(tmp_path, 'wb') as f:
for chunk in r.iter_content(chunk_size=8192):
f.write(chunk)
except requests.exceptions.ConnectionError as e:
return cls.report_connection_error(e)
os.rename(tmp_path, path)
finally:
if os.path.isfile(tmp_path):
os.remove(tmp_path)
def report_connection_error(exception):
raise ApplicationError(
'Failed to execute request against storage provider: %s' %
exception)
def validate_s3_response(response):
if not response.ok:
raise ApplicationError(
'Failed to execute request against storage provider: %s\n\n%s'
% (response.status_code, response.text))