def read_xlsx(
context: Context, model: Model, *,
source=str,
dependency: dict,
skip: str = None,
limit: int = None,
):
path = fetch(context, source)
rows = _read_xlsx(str(path))
if skip:
if isinstance(skip, dict):
value = set(skip['value']) if isinstance(skip['value'], list) else {skip['value']}
for row in rows:
if len(row) > skip['column'] and row[skip['column']] in skip['value']:
break
else:
context.error(f"Can't find header line: {skip!r}")
rows = chain([row], rows)
else:
rows = islice(rows, skip, None)
cols = {i: x.strip() for i, x in enumerate(next(rows, []))}
if limit:
rows = islice(rows, 0, limit)
for row in rows:
data = {}
for i, value in enumerate(row):
if i in cols:
data[cols[i]] = value
yield data
def _check_extends(context: Context, dataset: Dataset, model: Model):
if not model.extends:
return
if model.extends in dataset.objects:
return
if model.extends in dataset.manifest.objects['model']:
return
context.error(f"Can't find model {model.name!r} specified in 'extends'.")
def read_xml(context: Context,
prop: Property,
*,
source=str,
value: etree.ElementBase):
result = value.xpath(source)
if len(result) == 1:
return result[0]
elif len(result) == 0:
return None
else:
context.error(f"More than one value returned for {source}: {value}")
def _dependencies(context: Context, model, deps):
if deps:
command_calls = {}
model_names = set()
prop_names = []
prop_name_mapping = {}
for name, dep in deps.items():
if isinstance(dep, dict):
command_calls[name] = dep
continue
if '.' not in dep:
context.error(
f"Dependency must be in 'object/name.property' form, got: {dep}."
)
model_name, prop_name = dep.split('.', 1)
model_names.add(model_name)
prop_names.append(prop_name)
prop_name_mapping[prop_name] = name
if len(model_names) > 1:
names = ', '.join(sorted(model_names))
context.error(
f"Dependencies are allowed only from single model, but more than one model found: {names}."
)
if len(command_calls) > 1:
context.error(f"Only one command call is allowed.")
if len(command_calls) > 0:
if len(model_names) > 0:
context.error(
f"Only one command call or one model is allowed in dependencies."
)
for name, cmd in command_calls.items():
cmd = load(context,
Command(),
cmd,
parent=model,
scope='service')
for value in cmd(context):
yield {name: value}
else:
model_name = list(model_names)[0]
params = parse_url_path(model_name)
depmodel = get_model_from_params(model.manifest, params['path'],
params)
for row in getall(context,
depmodel,
depmodel.backend,
show=prop_names):
yield {prop_name_mapping[k]: v for k, v in row.items()}
else:
yield {}
def get(context: Context, model: Model, backend: PostgreSQL, id: str):
authorize(context, 'getone', model)
connection = context.get('transaction').connection
table = _get_table(backend, model).main
query = (sa.select([table]).where(table.c.id == id))
result = connection.execute(query)
result = list(itertools.islice(result, 2))
if len(result) == 1:
row = result[0]
return _get_data_from_row(model, table, row)
elif len(result) == 0:
return None
else:
context.error(f"Multiple rows were found, id={id}.")
def load(context: Context, manifest: Manifest, c: Config):
config = context.get('config')
ignore = c.get('ignore', default=[], cast=list)
# Add all supported node types.
for name in config.components['nodes'].keys():
manifest.objects[name] = {}
for file in manifest.path.glob('**/*.yml'):
if is_ignored(ignore, manifest.path, file):
continue
try:
data = yaml.load(file.read_text())
except (ParserError, ScannerError) as e:
context.error(f"{file}: {e}.")
if not isinstance(data, dict):
context.error(f"{file}: expected dict got {data.__class__.__name__}.")
if 'type' not in data:
raise Exception(f"'type' is not defined in {file}.")
if data['type'] not in manifest.objects:
raise Exception(f"Unknown type {data['type']!r} in {file}.")
node = config.components['nodes'][data['type']]()
data = {
'path': file,
'parent': manifest,
'backend': manifest.backend,
**data,
}
load(context, node, data, manifest)
if node.name in manifest.objects[node.type]:
raise Exception(f"Object {node.type} with name {node.name} already exist.")
manifest.objects[node.type][node.name] = node
def pull(context: Context, dataset: Dataset, *, models: list = None):
with context.enter():
tmpdir = context.attach(
tempfile.TemporaryDirectory(prefix='spinta-pull-cache-'))
context.bind('cache', Cache, path=pathlib.Path(tmpdir))
context.bind('requests', requests.Session)
for model in dataset.objects.values():
if model.source is None:
continue
if models and model.name not in models:
continue
for dependency in _dependencies(context, model,
model.dependencies):
for source in model.source.commands:
try:
yield from _pull(context, model, source, dependency)
except Exception as e:
context.error(
f"Error while pulling model {model.name!r}, with dependency: {dependency!r} and source: {source!r}. Error: {e}"
)
def check(context: Context, project: Project):
if project.owner and project.owner not in project.manifest.objects['owner']:
context.error(f"Unknown owner {project.owner}.")
def check(context: Context, owner: Owner):
if owner.logo:
path = owner.manifest.path / 'media/owners' / owner.name / owner.logo
if not path.exists():
context.error("Can't find media file %s.", path)
def _check_ref(context: Context, dataset: Dataset, prop: Property):
if prop.ref and prop.ref not in dataset.objects and prop.ref not in dataset.manifest.objects[
'model']:
context.error(
f"{prop.parent.name}.{prop.name} referenced an unknown object {prop.ref!r}."
)
def _check_owner(context: Context, dataset: Dataset):
if dataset.owner and dataset.owner not in dataset.manifest.objects['owner']:
context.error(f"Can't find owner {dataset.owner!r}.")
