def analyze_one(source, type_hint=None):
type_hint = type_hint or source.hints.get('type')
entity, metadata, other = parse(source.name, type_hint)
params = source.dict()
params.update({'entity': entity, 'metadata': metadata, 'other': other})
return schema.Source(**params)
def run_analyze(self, app, args):
raw = json.loads(args.input.read())
if isinstance(raw, dict):
raw = [raw]
raw = [schema.Source(**x) for x in raw]
proc = analyze.analyze(*raw, mp=False)
output = json.dumps([x.dict() for x in proc],
indent=2,
default=_json_encode_hook)
args.output.write(output)
def _build_source(self, ctx, item):
item['provider'] = ctx.provider_name
item_hints = {
'type': item.pop('type', None),
'language': item.pop('language', None)
}
ctx_hints = {'type': ctx.type, 'language': ctx.language}
item['hints'] = {
k: v
for (k,
v) in (tuple(item_hints.items()) + tuple(ctx_hints.items()))
if v is not None
}
return schema.Source(**item)
def run_download(self, app, args):
dls = downloads.Downloads()
if args.list:
print(repr(dls.get_active()))
elif args.add:
data = json.loads(args.input.read())
data = [(schema.Entity(**key),
[schema.Source(**src) for src in collection])
for (key, collection) in data]
for (key, collection) in data:
try:
dls.add(collection[0])
except extensions.ExtensionError as e:
print("Add '%s' failed. Extension error: %r" %
(collection[0], e))
else:
raise extensions.CommandUsageError()
def do_query(self, app, args):
def _parse_queryparams(pairs):
for pair in pairs:
key, value = pair.split('=', 1)
if not key or not value:
raise ValueError(pair)
yield (key, value)
if not args.queryparams and not args.querystring:
errmsg = "filter or querystring are requierd"
print(errmsg, file=sys.stderr)
raise extensions.CommandUsageError()
q = {}
if args.querystring:
q.update(query.Query.fromstring(args.querystring))
if args.queryparams:
params = dict(_parse_queryparams(args.queryparams))
q = query.Query(**params)
engine = query.Engine()
try:
ctx = engine.build_filter(q)
except query.MissingFiltersError as e:
errmsg = "Unknow filters: %s"
errmsg = errmsg % ', '.join(e.args[0])
print(errmsg, file=sys.stderr)
raise extensions.CommandUsageError()
data = json.loads(args.input.read())
data = [schema.Source(**x) for x in data]
results = engine.apply(ctx, data)
results = engine.sort(results)
results = [[entity.dict(), [src.dict() for src in sources]]
for (entity, sources) in results]
output = json.dumps(results, indent=2, default=_json_encode_hook)
args.output.write(output)
def sources_for_entity(self, entity):
entity = entity.dict()
return [
schema.Source(**row[self.SOURCE])
for row in self._find(self.ENTITY, entity)
]
def source_for_external(self, external):
row = self._find_one(self.EXTERNAL, external)
srcdata = row[self.SOURCE]
return schema.Source(**srcdata)
def all_states(self):
yield from ((schema.Source(**row[self.SOURCE]), row[self.STATE])
for row in self.data['downloads'].values())