def display_op(op): timestamp = white(op['date'].strftime(DATE_FORMAT)) label = white(op['type'].title()) + ' ' echo('{label:.<70} [{date}]'.format(label=label, date=timestamp)) format_output(op['output'], success=op['success'], traceback=op.get('traceback'))
def list(): '''List all known metrics''' for cls, metrics in registry.items(): echo(white(cls.__name__)) for metric in metrics.keys(): echo('> {0}'.format(metric))
def format_output(output, success=True): echo(' │') for level, msg in output: echo(' │ {0}'.format(msg)) echo(' │') echo(' └──[{0}]'.format(green('OK') if success else red('KO'))) echo('')
def scheduled(): ''' List scheduled jobs. ''' for job in sorted(schedulables(), key=lambda s: s.name): for task in PeriodicTask.objects(task=job.name): label = job_label(task.task, task.args, task.kwargs) echo( SCHEDULE_LINE.format(name=white(task.name), label=label, schedule=task.schedule_display))
def scheduled(): ''' List scheduled jobs. ''' for job in sorted(schedulables(), key=lambda s: s.name): for task in PeriodicTask.objects(task=job.name): label = job_label(task.task, task.args, task.kwargs) echo(SCHEDULE_LINE.format( name=white(task.name.encode('utf8')), label=label, schedule=task.schedule_display ).encode('utf8'))
def info(plugin_or_specs, filename): ''' Display detailed info about a migration ''' migration = migrations.get(plugin_or_specs, filename) log_status(migration, status_label(migration.record)) try: echo(migration.module.__doc__) except migrations.MigrationError: echo(yellow('Module not found')) for op in migration.record.get('ops', []): display_op(op)
def execute_migration(plugin, filename, script, dryrun=False): '''Execute and record a migration''' db = get_db() js = SCRIPT_WRAPPER.format(script) lines = script.splitlines() success = True if not dryrun: try: lines = db.eval(js, plugin, filename, script, get_config()) except OperationFailure as e: log.error(e.details['errmsg'].replace('\n', '\\n')) success = False except PyMongoError as e: log.error('Unable to apply migration: %s', str(e)) success = False echo('│'.encode('utf8')) for line in lines: echo('│ {0}'.format(line).encode('utf8')) echo('│'.encode('utf8')) echo('└──[{0}]'.format(green('OK') if success else red('KO')).encode( 'utf8')) echo('') return success
def format_output(output, success=True, traceback=None): echo(' │') for level, msg in output: echo(' │ {0}'.format(msg)) echo(' │') if traceback: for tb in traceback.split('\n'): echo(' │ {0}'.format(tb)) echo(' │') echo(' └──[{0}]'.format(green('OK') if success else red('KO'))) echo('')
def list(): '''List all availables jobs''' for job in sorted([s.name for s in schedulables()]): echo(job)
def parse_url(url, quiet=False, rid=''): '''Parse the datasets in a DCAT format located at URL (debug)''' if quiet: verbose_loggers = ['rdflib', 'udata.core.dataset'] [logging.getLogger(l).setLevel(logging.ERROR) for l in verbose_loggers] class MockSource: url = '' class MockJob: items = [] class MockDatasetFactory(DatasetFactory): '''Use DatasetFactory without .save()''' @classmethod def _create(cls, model_class, *args, **kwargs): instance = model_class(*args, **kwargs) return instance echo(cyan('Parsing url {}'.format(url))) source = MockSource() source.url = url backend = DcatBackend(source, dryrun=True) backend.job = MockJob() format = backend.get_format() echo(yellow('Detected format: {}'.format(format))) graph = backend.parse_graph(url, format) # serialize/unserialize graph like in the job mechanism _graph = graph.serialize(format=format, indent=None) graph = Graph(namespace_manager=namespace_manager) graph.parse(data=_graph, format=format) for item in backend.job.items: if not rid or rid in item.remote_id: echo(magenta('Processing item {}'.format(item.remote_id))) echo('Item kwargs: {}'.format(yellow(item.kwargs))) node = backend.get_node_from_item(item) dataset = MockDatasetFactory() dataset = dataset_from_rdf(graph, dataset, node=node) echo('') echo(green('Dataset found!')) echo('Title: {}'.format(yellow(dataset))) echo('License: {}'.format(yellow(dataset.license))) echo('Description: {}'.format(yellow(dataset.description))) echo('Tags: {}'.format(yellow(dataset.tags))) echo('Resources: {}'.format(yellow([(r.title, r.format, r.url) for r in dataset.resources]))) try: dataset.validate() except mongoengine.errors.ValidationError as e: log.error(e, exc_info=True) else: echo(green('Dataset is valid ✅')) echo('')
def list(): '''List all availables jobs''' for job in sorted(schedulables()): echo(job.name)