def create_job(self, namespace_name, job_name, job_type, location=None,
inputs: [DatasetId] = None, outputs: [DatasetId] = None,
description=None, context=None, run_id=None):
Utils.check_name_length(namespace_name, 'namespace_name')
Utils.check_name_length(job_name, 'job_name')
Utils.is_instance_of(job_type, JobType)
payload = {
'type': job_type.value,
'inputs': [
input.__dict__ for input in inputs
] if inputs else [],
'outputs': [
output.__dict__ for output in outputs
] if outputs else []
}
if run_id:
payload['runId'] = run_id
if context:
payload['context'] = context
if location:
payload['location'] = location
if description:
payload['description'] = description
return self._put(
self._url('/namespaces/{0}/jobs/{1}', namespace_name, job_name),
payload=payload
)
def create_job(self,
namespace_name,
job_name,
job_type,
location=None,
input_dataset=None,
output_dataset=None,
description=None,
context=None):
Utils.check_name_length(namespace_name, 'namespace_name')
Utils.check_name_length(job_name, 'job_name')
Utils.is_instance_of(job_type, JobType)
payload = {
'inputs': input_dataset or [],
'outputs': output_dataset or [],
'type': job_type.name
}
if context:
payload['context'] = context
if location:
payload['location'] = location
if description:
payload['description'] = description
return self._put(self._url('/namespaces/{0}/jobs/{1}', namespace_name,
job_name),
payload=payload)
def create_dataset(self,
namespace_name,
dataset_name,
dataset_type,
physical_name,
source_name,
description=None,
run_id=None,
schema_location=None,
fields=None,
tags=None):
Utils.check_name_length(namespace_name, 'namespace_name')
Utils.check_name_length(dataset_name, 'dataset_name')
Utils.is_instance_of(dataset_type, DatasetType)
if dataset_type == DatasetType.STREAM:
MarquezClient._is_none(schema_location, 'schema_location')
Utils.check_name_length(physical_name, 'physical_name')
Utils.check_name_length(source_name, 'source_name')
payload = {
'type': dataset_type.value,
'physicalName': physical_name,
'sourceName': source_name,
}
if description:
payload['description'] = description
if run_id:
payload['runId'] = run_id
if fields:
payload['fields'] = fields
if tags:
payload['tags'] = tags
if schema_location:
payload['schemaLocation'] = schema_location
return self._put(self._url('/namespaces/{0}/datasets/{1}',
namespace_name, dataset_name),
payload=payload)
def test_is_instance_of():
with pytest.raises(ValueError):
Utils.is_instance_of(variable_value=JobType.BATCH,
variable_enum_type=DatasetType)
with pytest.raises(ValueError):
Utils.is_instance_of(variable_value=DatasetType.DB_TABLE,
variable_enum_type=JobType)
with pytest.raises(ValueError):
Utils.is_instance_of(variable_value=JobType.BATCH,
variable_enum_type=RunState)