def read_xlsx( context: Context, model: Model, *, source=str, dependency: dict, skip: str = None, limit: int = None, ): path = fetch(context, source) rows = _read_xlsx(str(path)) if skip: if isinstance(skip, dict): value = set(skip['value']) if isinstance(skip['value'], list) else {skip['value']} for row in rows: if len(row) > skip['column'] and row[skip['column']] in skip['value']: break else: context.error(f"Can't find header line: {skip!r}") rows = chain([row], rows) else: rows = islice(rows, skip, None) cols = {i: x.strip() for i, x in enumerate(next(rows, []))} if limit: rows = islice(rows, 0, limit) for row in rows: data = {} for i, value in enumerate(row): if i in cols: data[cols[i]] = value yield data
def _check_extends(context: Context, dataset: Dataset, model: Model): if not model.extends: return if model.extends in dataset.objects: return if model.extends in dataset.manifest.objects['model']: return context.error(f"Can't find model {model.name!r} specified in 'extends'.")
def read_xml(context: Context, prop: Property, *, source=str, value: etree.ElementBase): result = value.xpath(source) if len(result) == 1: return result[0] elif len(result) == 0: return None else: context.error(f"More than one value returned for {source}: {value}")
def _dependencies(context: Context, model, deps): if deps: command_calls = {} model_names = set() prop_names = [] prop_name_mapping = {} for name, dep in deps.items(): if isinstance(dep, dict): command_calls[name] = dep continue if '.' not in dep: context.error( f"Dependency must be in 'object/name.property' form, got: {dep}." ) model_name, prop_name = dep.split('.', 1) model_names.add(model_name) prop_names.append(prop_name) prop_name_mapping[prop_name] = name if len(model_names) > 1: names = ', '.join(sorted(model_names)) context.error( f"Dependencies are allowed only from single model, but more than one model found: {names}." ) if len(command_calls) > 1: context.error(f"Only one command call is allowed.") if len(command_calls) > 0: if len(model_names) > 0: context.error( f"Only one command call or one model is allowed in dependencies." ) for name, cmd in command_calls.items(): cmd = load(context, Command(), cmd, parent=model, scope='service') for value in cmd(context): yield {name: value} else: model_name = list(model_names)[0] params = parse_url_path(model_name) depmodel = get_model_from_params(model.manifest, params['path'], params) for row in getall(context, depmodel, depmodel.backend, show=prop_names): yield {prop_name_mapping[k]: v for k, v in row.items()} else: yield {}
def get(context: Context, model: Model, backend: PostgreSQL, id: str): authorize(context, 'getone', model) connection = context.get('transaction').connection table = _get_table(backend, model).main query = (sa.select([table]).where(table.c.id == id)) result = connection.execute(query) result = list(itertools.islice(result, 2)) if len(result) == 1: row = result[0] return _get_data_from_row(model, table, row) elif len(result) == 0: return None else: context.error(f"Multiple rows were found, id={id}.")
def load(context: Context, manifest: Manifest, c: Config): config = context.get('config') ignore = c.get('ignore', default=[], cast=list) # Add all supported node types. for name in config.components['nodes'].keys(): manifest.objects[name] = {} for file in manifest.path.glob('**/*.yml'): if is_ignored(ignore, manifest.path, file): continue try: data = yaml.load(file.read_text()) except (ParserError, ScannerError) as e: context.error(f"{file}: {e}.") if not isinstance(data, dict): context.error(f"{file}: expected dict got {data.__class__.__name__}.") if 'type' not in data: raise Exception(f"'type' is not defined in {file}.") if data['type'] not in manifest.objects: raise Exception(f"Unknown type {data['type']!r} in {file}.") node = config.components['nodes'][data['type']]() data = { 'path': file, 'parent': manifest, 'backend': manifest.backend, **data, } load(context, node, data, manifest) if node.name in manifest.objects[node.type]: raise Exception(f"Object {node.type} with name {node.name} already exist.") manifest.objects[node.type][node.name] = node
def pull(context: Context, dataset: Dataset, *, models: list = None): with context.enter(): tmpdir = context.attach( tempfile.TemporaryDirectory(prefix='spinta-pull-cache-')) context.bind('cache', Cache, path=pathlib.Path(tmpdir)) context.bind('requests', requests.Session) for model in dataset.objects.values(): if model.source is None: continue if models and model.name not in models: continue for dependency in _dependencies(context, model, model.dependencies): for source in model.source.commands: try: yield from _pull(context, model, source, dependency) except Exception as e: context.error( f"Error while pulling model {model.name!r}, with dependency: {dependency!r} and source: {source!r}. Error: {e}" )
def check(context: Context, project: Project): if project.owner and project.owner not in project.manifest.objects['owner']: context.error(f"Unknown owner {project.owner}.")
def check(context: Context, owner: Owner): if owner.logo: path = owner.manifest.path / 'media/owners' / owner.name / owner.logo if not path.exists(): context.error("Can't find media file %s.", path)
def _check_ref(context: Context, dataset: Dataset, prop: Property): if prop.ref and prop.ref not in dataset.objects and prop.ref not in dataset.manifest.objects[ 'model']: context.error( f"{prop.parent.name}.{prop.name} referenced an unknown object {prop.ref!r}." )
def _check_owner(context: Context, dataset: Dataset): if dataset.owner and dataset.owner not in dataset.manifest.objects['owner']: context.error(f"Can't find owner {dataset.owner!r}.")