def schema_validate(schema_basepath: str, schema_name: str, schema_source_name: Optional[str]) -> None: """Validates a schema without doing any other work.""" source: Optional[Schema] = None if schema_source_name is not None: source = Schema.load(schema_source_name, schema_basepath) Schema.load(schema_name, schema_basepath, source_schema=source)
def build(cls, conf_dir, data_dir, name): """Build task from yaml, read all input data and create corresponding objects""" logging.info("Constructing task execution plan.") path_locator = PathLocator(conf=conf_dir, data=data_dir) logging.info( "Configuration base directory is %s; data base directory is %s." % (conf_dir, data_dir)) task_path: str = os.path.join(path_locator.tasks_dir, name + '.yaml') with open(task_path, 'r') as f: logging.info("Task configuration loaded from %s." % task_path) spec = yaml.safe_load(f) resulting_in = spec.get('resulting_in', {}) task = cls(path_locator=path_locator, origin_data=spec['starting_with']['data'], origin_schema=Schema.load(path_locator, spec['starting_with']['schema']), target_data=resulting_in.get('data'), target_schema=Schema.load(path_locator, resulting_in.get('schema'))) task.load_steps(spec['steps']) # If the last step is a Consume step we don't need target data assert task.target_data is not None or isinstance( task.steps[-1], Consume) return task
def build(cls, context: Context, name: str) -> "Task": """Build task from yaml, read all input data and create corresponding objects""" logging.info("Constructing task execution plan.") logging.info( "Configuration base directory is %s; entities directory is %s." % (context.conf_dir, context.entities_input_dir)) task_path: str = os.path.join(context.conf_dir, 'tasks', name + '.yaml') with open(task_path, 'r') as f: logging.info("Task configuration loaded from %s." % task_path) spec = yaml.safe_load(f) resulting_in = spec.get('resulting_in', {}) origin_schema = Schema.load(spec['starting_with']['schema'], context.schemas_dir) assert origin_schema is not None task = cls(context=context, origin_data=spec['starting_with']['data'], origin_schema=origin_schema, target_data=resulting_in.get('data'), target_schema=Schema.load(resulting_in.get('schema'), context.schemas_dir)) task.load_steps(spec['steps']) # If the last step is a Consume step we don't need target data assert task.target_data is not None or isinstance( task.steps[-1], Consume) return task
def from_files(cls, schema_basepath: str, source_schema: str, target_schema: str, output_file: TextIO) -> None: source_schema_instance: Optional[Schema] = Schema.load(source_schema, base_path=schema_basepath) target_schema_instance: Optional[Schema] = Schema.load(target_schema, source_schema=source_schema_instance, base_path=schema_basepath) assert target_schema_instance is not None export: "ExportLinkages" = cls(target_schema_instance, output_file) export() output_file.close()
def from_files(cls, schema_basepath: str, source_schema: str, target_schema: str, input_file: TextIO, suffix: str) -> None: source_schema_instance: Optional[Schema] = Schema.load(source_schema, base_path=schema_basepath) target_schema_instance: Optional[Schema] = Schema.load(target_schema, source_schema=source_schema_instance, base_path=schema_basepath) assert target_schema_instance is not None do_import: "ImportLinkages" = cls(target_schema_instance, input_file) do_import() input_file.close() output_schema_relpath: str = "%s_%s" % (target_schema, suffix) output_path: str = os.path.join(schema_basepath, output_schema_relpath) if not os.path.exists(output_path): os.mkdir(output_path) target_schema_instance.serialize(output_path)
def standalone(cls, context: Context, translate_dir: str, trace_dir: str, source_schema_name: str, target_schema_name: str, output_filename: str) -> None: source_schema: Optional[Schema] = Schema.load(source_schema_name, context.schemas_dir) assert source_schema is not None schema: Optional[Schema] = Schema.load(target_schema_name, context.schemas_dir, source_schema) assert schema is not None coverage: "SourceCoverage" = cls(context, schema, translate_dir, trace_dir, output_filename) coverage("dummy", None)
def variable_catalog(schema_basepath: str, schema_name: str, fh: TextIO) -> None: schema: Optional[Schema] = Schema.load(schema_name, base_path=schema_basepath) assert schema is not None write_catalog(schema, fh) fh.close()
def outcomes(example_path) -> FixtureOutcomes: schema: Schema = Schema.load("conf/schemas/simple", example_path) fixture_path: str = os.path.join(example_path, "data", "fixtures") obs_path: str = os.path.join(example_path, "data", "observations") return FixtureOutcomes(schema, fixture_path, obs_path)
def standalone(cls, context: Context, schema_name: str, output_prefix: str, t_group: Optional[VariableId], i_group: Optional[VariableId], exclude_trivial: bool = False) -> None: schema: Optional[Schema] = Schema.load(schema_name, context.schemas_dir) assert schema is not None # TODO Refactor so unnecessary arguments aren't required. coverage: "CoverageFile" = cls(context, schema, output_prefix, t_group, i_group, exclude_trivial) coverage(context.entities_input_dir, None)
def standalone(cls, schema_basepath: str, schema_name: str, data_path: str, output_prefix: str, t_group: Optional[VariableId], i_group: Optional[VariableId]) -> None: schema: Optional[Schema] = Schema.load(schema_name, base_path=schema_basepath) assert schema is not None # TODO Refactor so unnecessary arguments aren't required. coverage: "CoverageFile" = cls(None, schema, output_prefix, t_group, i_group) coverage(data_path, None)
def build( # type: ignore # Signature of "build" incompatible with supertype "Step" cls, context: Context, schema: Schema, name: str, target_schema: str, id_var: str, **kwargs): target_schema_instance: Optional[Schema] = Schema.load( target_schema, context.schemas_dir) aggregations: Dict[str, Type] = load(cls) return aggregations[name](context=context, origin_schema=schema, target_schema=target_schema_instance, id_var=id_var, **kwargs)
def build( cls, path_locator: PathLocator, schema: Schema, name: str, target_schema: str, id_var: str, input_schema_vars: Dict, output_schema_vars: Dict ): target_schema_instance: Schema = Schema.load(path_locator, target_schema) aggregations: Dict[str, Type] = load(cls) input_variables: Dict[str, Variable] = { var_name: schema.get(var_id) for var_name, var_id in input_schema_vars.items() } output_variables: Dict[str, Variable] = { var_name: target_schema_instance.get(var_id) for var_name, var_id in output_schema_vars.items() } return aggregations[name](origin_schema=schema, target_schema=target_schema_instance, id_var=id_var, **input_variables, **output_variables)
def build(cls, path_locator: "PathLocator", schema: Schema, target_schema: str): """ :param path_locator: :param schema: The source schema, already instantiated. :param target_schema: The path to the definition of the target schema. :return: """ target_schema_instance: Schema = Schema.load(path_locator, target_schema, source_schema=schema) translate_immutable: Translator = Translator( target_schema_instance.immutable) translate_temporal: Translator = Translator( target_schema_instance.temporal) return cls(target_schema_instance, translate_immutable, translate_temporal)
def build( cls, path_locator: "PathLocator", schema: Schema, target_schema: str ) -> "Translate": # type: ignore # Signature of "build" incompatible with supertype "Step" """ :param path_locator: :param schema: The source schema, already instantiated. :param target_schema: The path to the definition of the target schema. :return: """ logging.info("Initializing Translate step.") target_schema_instance: Optional[Schema] = Schema.load( target_schema, source_schema=schema, path_locator=path_locator) assert target_schema_instance is not None translate_immutable: Translator = Translator( target_schema_instance.immutable) translate_temporal: Translator = Translator( target_schema_instance.temporal) return cls(target_schema_instance, translate_immutable, translate_temporal)
def print_from_files(schema_basepath: str, schema_name: str, hide_ids: bool) -> None: schema: Optional[Schema] = Schema.load(schema_name, schema_basepath) assert schema is not None tree: str = as_ascii(schema, hide_ids) print(tree)
def print_from_files(schema_basepath: str, schema_name: str) -> None: schema: Optional[Schema] = Schema.load(schema_name, base_path=schema_basepath) assert schema is not None tree: str = as_ascii(schema) print(tree)
def schema(basepath) -> Schema: path: str = os.path.join(basepath, "..", "examples", "s_7_csv", "conf", "schemas") return Schema.load("composite", path)