def schema_validate(schema_basepath: str, schema_name: str,
schema_source_name: Optional[str]) -> None:
"""Validates a schema without doing any other work."""
source: Optional[Schema] = None
if schema_source_name is not None:
source = Schema.load(schema_source_name, schema_basepath)
Schema.load(schema_name, schema_basepath, source_schema=source)
def build(cls, conf_dir, data_dir, name):
"""Build task from yaml, read all input data and create corresponding
objects"""
logging.info("Constructing task execution plan.")
path_locator = PathLocator(conf=conf_dir, data=data_dir)
logging.info(
"Configuration base directory is %s; data base directory is %s." %
(conf_dir, data_dir))
task_path: str = os.path.join(path_locator.tasks_dir, name + '.yaml')
with open(task_path, 'r') as f:
logging.info("Task configuration loaded from %s." % task_path)
spec = yaml.safe_load(f)
resulting_in = spec.get('resulting_in', {})
task = cls(path_locator=path_locator,
origin_data=spec['starting_with']['data'],
origin_schema=Schema.load(path_locator,
spec['starting_with']['schema']),
target_data=resulting_in.get('data'),
target_schema=Schema.load(path_locator,
resulting_in.get('schema')))
task.load_steps(spec['steps'])
# If the last step is a Consume step we don't need target data
assert task.target_data is not None or isinstance(
task.steps[-1], Consume)
return task
def build(cls, context: Context, name: str) -> "Task":
"""Build task from yaml, read all input data and create corresponding
objects"""
logging.info("Constructing task execution plan.")
logging.info(
"Configuration base directory is %s; entities directory is %s." %
(context.conf_dir, context.entities_input_dir))
task_path: str = os.path.join(context.conf_dir, 'tasks',
name + '.yaml')
with open(task_path, 'r') as f:
logging.info("Task configuration loaded from %s." % task_path)
spec = yaml.safe_load(f)
resulting_in = spec.get('resulting_in', {})
origin_schema = Schema.load(spec['starting_with']['schema'],
context.schemas_dir)
assert origin_schema is not None
task = cls(context=context,
origin_data=spec['starting_with']['data'],
origin_schema=origin_schema,
target_data=resulting_in.get('data'),
target_schema=Schema.load(resulting_in.get('schema'),
context.schemas_dir))
task.load_steps(spec['steps'])
# If the last step is a Consume step we don't need target data
assert task.target_data is not None or isinstance(
task.steps[-1], Consume)
return task
def from_files(cls, schema_basepath: str, source_schema: str, target_schema: str, output_file: TextIO) -> None:
source_schema_instance: Optional[Schema] = Schema.load(source_schema, base_path=schema_basepath)
target_schema_instance: Optional[Schema] = Schema.load(target_schema, source_schema=source_schema_instance,
base_path=schema_basepath)
assert target_schema_instance is not None
export: "ExportLinkages" = cls(target_schema_instance, output_file)
export()
output_file.close()
def from_files(cls, schema_basepath: str, source_schema: str, target_schema: str, input_file: TextIO, suffix: str) -> None:
source_schema_instance: Optional[Schema] = Schema.load(source_schema, base_path=schema_basepath)
target_schema_instance: Optional[Schema] = Schema.load(target_schema, source_schema=source_schema_instance,
base_path=schema_basepath)
assert target_schema_instance is not None
do_import: "ImportLinkages" = cls(target_schema_instance, input_file)
do_import()
input_file.close()
output_schema_relpath: str = "%s_%s" % (target_schema, suffix)
output_path: str = os.path.join(schema_basepath, output_schema_relpath)
if not os.path.exists(output_path):
os.mkdir(output_path)
target_schema_instance.serialize(output_path)
def standalone(cls, context: Context, translate_dir: str, trace_dir: str,
source_schema_name: str, target_schema_name: str,
output_filename: str) -> None:
source_schema: Optional[Schema] = Schema.load(source_schema_name,
context.schemas_dir)
assert source_schema is not None
schema: Optional[Schema] = Schema.load(target_schema_name,
context.schemas_dir,
source_schema)
assert schema is not None
coverage: "SourceCoverage" = cls(context, schema, translate_dir,
trace_dir, output_filename)
coverage("dummy", None)
def variable_catalog(schema_basepath: str, schema_name: str,
fh: TextIO) -> None:
schema: Optional[Schema] = Schema.load(schema_name,
base_path=schema_basepath)
assert schema is not None
write_catalog(schema, fh)
fh.close()
def outcomes(example_path) -> FixtureOutcomes:
schema: Schema = Schema.load("conf/schemas/simple", example_path)
fixture_path: str = os.path.join(example_path, "data", "fixtures")
obs_path: str = os.path.join(example_path, "data", "observations")
return FixtureOutcomes(schema, fixture_path, obs_path)
def standalone(cls, context: Context, schema_name: str, output_prefix: str,
t_group: Optional[VariableId], i_group: Optional[VariableId], exclude_trivial: bool = False) -> None:
schema: Optional[Schema] = Schema.load(schema_name, context.schemas_dir)
assert schema is not None
# TODO Refactor so unnecessary arguments aren't required.
coverage: "CoverageFile" = cls(context, schema, output_prefix, t_group, i_group, exclude_trivial)
coverage(context.entities_input_dir, None)
def standalone(cls, schema_basepath: str, schema_name: str, data_path: str, output_prefix: str,
t_group: Optional[VariableId], i_group: Optional[VariableId]) -> None:
schema: Optional[Schema] = Schema.load(schema_name, base_path=schema_basepath)
assert schema is not None
# TODO Refactor so unnecessary arguments aren't required.
coverage: "CoverageFile" = cls(None, schema, output_prefix, t_group, i_group)
coverage(data_path, None)
def build( # type: ignore # Signature of "build" incompatible with supertype "Step"
cls, context: Context, schema: Schema, name: str,
target_schema: str, id_var: str, **kwargs):
target_schema_instance: Optional[Schema] = Schema.load(
target_schema, context.schemas_dir)
aggregations: Dict[str, Type] = load(cls)
return aggregations[name](context=context,
origin_schema=schema,
target_schema=target_schema_instance,
id_var=id_var,
**kwargs)
def build(
cls, path_locator: PathLocator, schema: Schema, name: str, target_schema: str, id_var: str,
input_schema_vars: Dict, output_schema_vars: Dict
):
target_schema_instance: Schema = Schema.load(path_locator, target_schema)
aggregations: Dict[str, Type] = load(cls)
input_variables: Dict[str, Variable] = {
var_name: schema.get(var_id)
for var_name, var_id in input_schema_vars.items()
}
output_variables: Dict[str, Variable] = {
var_name: target_schema_instance.get(var_id)
for var_name, var_id in output_schema_vars.items()
}
return aggregations[name](origin_schema=schema, target_schema=target_schema_instance, id_var=id_var,
**input_variables, **output_variables)
def build(cls, path_locator: "PathLocator", schema: Schema,
target_schema: str):
"""
:param path_locator:
:param schema: The source schema, already instantiated.
:param target_schema: The path to the definition of the target schema.
:return:
"""
target_schema_instance: Schema = Schema.load(path_locator,
target_schema,
source_schema=schema)
translate_immutable: Translator = Translator(
target_schema_instance.immutable)
translate_temporal: Translator = Translator(
target_schema_instance.temporal)
return cls(target_schema_instance, translate_immutable,
translate_temporal)
def build(
cls, path_locator: "PathLocator", schema: Schema, target_schema: str
) -> "Translate": # type: ignore # Signature of "build" incompatible with supertype "Step"
"""
:param path_locator:
:param schema: The source schema, already instantiated.
:param target_schema: The path to the definition of the target schema.
:return:
"""
logging.info("Initializing Translate step.")
target_schema_instance: Optional[Schema] = Schema.load(
target_schema, source_schema=schema, path_locator=path_locator)
assert target_schema_instance is not None
translate_immutable: Translator = Translator(
target_schema_instance.immutable)
translate_temporal: Translator = Translator(
target_schema_instance.temporal)
return cls(target_schema_instance, translate_immutable,
translate_temporal)
def print_from_files(schema_basepath: str, schema_name: str,
hide_ids: bool) -> None:
schema: Optional[Schema] = Schema.load(schema_name, schema_basepath)
assert schema is not None
tree: str = as_ascii(schema, hide_ids)
print(tree)
def print_from_files(schema_basepath: str, schema_name: str) -> None:
schema: Optional[Schema] = Schema.load(schema_name,
base_path=schema_basepath)
assert schema is not None
tree: str = as_ascii(schema)
print(tree)
def schema(basepath) -> Schema:
path: str = os.path.join(basepath, "..", "examples", "s_7_csv", "conf",
"schemas")
return Schema.load("composite", path)
