def display_op(op):
timestamp = white(op['date'].strftime(DATE_FORMAT))
label = white(op['type'].title()) + ' '
echo('{label:.<70} [{date}]'.format(label=label, date=timestamp))
format_output(op['output'],
success=op['success'],
traceback=op.get('traceback'))
def list():
'''List all known metrics'''
for cls, metrics in registry.items():
echo(white(cls.__name__))
for metric in metrics.keys():
echo('> {0}'.format(metric))
def format_output(output, success=True):
echo(' │')
for level, msg in output:
echo(' │ {0}'.format(msg))
echo(' │')
echo(' └──[{0}]'.format(green('OK') if success else red('KO')))
echo('')
def list():
'''List all known metrics'''
for cls, metrics in registry.items():
echo(white(cls.__name__))
for metric in metrics.keys():
echo('> {0}'.format(metric))
def scheduled():
'''
List scheduled jobs.
'''
for job in sorted(schedulables(), key=lambda s: s.name):
for task in PeriodicTask.objects(task=job.name):
label = job_label(task.task, task.args, task.kwargs)
echo(
SCHEDULE_LINE.format(name=white(task.name),
label=label,
schedule=task.schedule_display))
def scheduled():
'''
List scheduled jobs.
'''
for job in sorted(schedulables(), key=lambda s: s.name):
for task in PeriodicTask.objects(task=job.name):
label = job_label(task.task, task.args, task.kwargs)
echo(SCHEDULE_LINE.format(
name=white(task.name.encode('utf8')),
label=label,
schedule=task.schedule_display
).encode('utf8'))
def info(plugin_or_specs, filename):
'''
Display detailed info about a migration
'''
migration = migrations.get(plugin_or_specs, filename)
log_status(migration, status_label(migration.record))
try:
echo(migration.module.__doc__)
except migrations.MigrationError:
echo(yellow('Module not found'))
for op in migration.record.get('ops', []):
display_op(op)
def execute_migration(plugin, filename, script, dryrun=False):
'''Execute and record a migration'''
db = get_db()
js = SCRIPT_WRAPPER.format(script)
lines = script.splitlines()
success = True
if not dryrun:
try:
lines = db.eval(js, plugin, filename, script, get_config())
except OperationFailure as e:
log.error(e.details['errmsg'].replace('\n', '\\n'))
success = False
except PyMongoError as e:
log.error('Unable to apply migration: %s', str(e))
success = False
echo('│'.encode('utf8'))
for line in lines:
echo('│ {0}'.format(line).encode('utf8'))
echo('│'.encode('utf8'))
echo('└──[{0}]'.format(green('OK') if success else red('KO')).encode(
'utf8'))
echo('')
return success
def format_output(output, success=True, traceback=None):
echo(' │')
for level, msg in output:
echo(' │ {0}'.format(msg))
echo(' │')
if traceback:
for tb in traceback.split('\n'):
echo(' │ {0}'.format(tb))
echo(' │')
echo(' └──[{0}]'.format(green('OK') if success else red('KO')))
echo('')
def list():
'''List all availables jobs'''
for job in sorted([s.name for s in schedulables()]):
echo(job)
def list():
'''List all availables jobs'''
for job in sorted([s.name for s in schedulables()]):
echo(job)
def parse_url(url, quiet=False, rid=''):
'''Parse the datasets in a DCAT format located at URL (debug)'''
if quiet:
verbose_loggers = ['rdflib', 'udata.core.dataset']
[logging.getLogger(l).setLevel(logging.ERROR) for l in verbose_loggers]
class MockSource:
url = ''
class MockJob:
items = []
class MockDatasetFactory(DatasetFactory):
'''Use DatasetFactory without .save()'''
@classmethod
def _create(cls, model_class, *args, **kwargs):
instance = model_class(*args, **kwargs)
return instance
echo(cyan('Parsing url {}'.format(url)))
source = MockSource()
source.url = url
backend = DcatBackend(source, dryrun=True)
backend.job = MockJob()
format = backend.get_format()
echo(yellow('Detected format: {}'.format(format)))
graph = backend.parse_graph(url, format)
# serialize/unserialize graph like in the job mechanism
_graph = graph.serialize(format=format, indent=None)
graph = Graph(namespace_manager=namespace_manager)
graph.parse(data=_graph, format=format)
for item in backend.job.items:
if not rid or rid in item.remote_id:
echo(magenta('Processing item {}'.format(item.remote_id)))
echo('Item kwargs: {}'.format(yellow(item.kwargs)))
node = backend.get_node_from_item(item)
dataset = MockDatasetFactory()
dataset = dataset_from_rdf(graph, dataset, node=node)
echo('')
echo(green('Dataset found!'))
echo('Title: {}'.format(yellow(dataset)))
echo('License: {}'.format(yellow(dataset.license)))
echo('Description: {}'.format(yellow(dataset.description)))
echo('Tags: {}'.format(yellow(dataset.tags)))
echo('Resources: {}'.format(yellow([(r.title, r.format, r.url) for r in dataset.resources])))
try:
dataset.validate()
except mongoengine.errors.ValidationError as e:
log.error(e, exc_info=True)
else:
echo(green('Dataset is valid ✅'))
echo('')
def list():
'''List all availables jobs'''
for job in sorted(schedulables()):
echo(job.name)
