def parse_url(url, quiet=False, rid=''):
'''Parse the datasets in a DCAT format located at URL (debug)'''
if quiet:
verbose_loggers = ['rdflib', 'udata.core.dataset']
[logging.getLogger(l).setLevel(logging.ERROR) for l in verbose_loggers]
class MockSource:
url = ''
class MockJob:
items = []
class MockDatasetFactory(DatasetFactory):
'''Use DatasetFactory without .save()'''
@classmethod
def _create(cls, model_class, *args, **kwargs):
instance = model_class(*args, **kwargs)
return instance
echo(cyan('Parsing url {}'.format(url)))
source = MockSource()
source.url = url
backend = DcatBackend(source, dryrun=True)
backend.job = MockJob()
format = backend.get_format()
echo(yellow('Detected format: {}'.format(format)))
graph = backend.parse_graph(url, format)
# serialize/unserialize graph like in the job mechanism
_graph = graph.serialize(format=format, indent=None)
graph = Graph(namespace_manager=namespace_manager)
graph.parse(data=_graph, format=format)
for item in backend.job.items:
if not rid or rid in item.remote_id:
echo(magenta('Processing item {}'.format(item.remote_id)))
echo('Item kwargs: {}'.format(yellow(item.kwargs)))
node = backend.get_node_from_item(item)
dataset = MockDatasetFactory()
dataset = dataset_from_rdf(graph, dataset, node=node)
echo('')
echo(green('Dataset found!'))
echo('Title: {}'.format(yellow(dataset)))
echo('License: {}'.format(yellow(dataset.license)))
echo('Description: {}'.format(yellow(dataset.description)))
echo('Tags: {}'.format(yellow(dataset.tags)))
echo('Resources: {}'.format(yellow([(r.title, r.format, r.url) for r in dataset.resources])))
try:
dataset.validate()
except mongoengine.errors.ValidationError as e:
log.error(e, exc_info=True)
else:
echo(green('Dataset is valid ✅'))
echo('')
def status_label(record):
if record.ok:
return green(record.last_date.strftime(DATE_FORMAT))
elif not record.exists():
return yellow('Not applied')
else:
return red(record.status)
def format_output(output, success=True):
echo(' │')
for level, msg in output:
echo(' │ {0}'.format(msg))
echo(' │')
echo(' └──[{0}]'.format(green('OK') if success else red('KO')))
echo('')
def status():
'''Display the database migrations status'''
for plugin, package, filename in available_migrations():
migration = get_migration(plugin, filename)
if migration:
status = green(migration['date'].strftime(DATE_FORMAT))
else:
status = yellow('Not applied')
log_status(plugin, filename, status)
def status():
'''Display the database migrations status'''
for plugin, package, filename in available_migrations():
migration = get_migration(plugin, filename)
if migration:
status = green(migration['date'].strftime(DATE_FORMAT))
else:
status = yellow('Not applied')
log_status(plugin, filename, status)
def execute_migration(plugin, filename, script):
'''Execute and record a migration'''
db = get_db(DEFAULT_CONNECTION_NAME)
js = SCRIPT_WRAPPER.format(script)
print('│')
for line in db.eval(js, plugin, filename, script):
print('│ {0}'.format(line))
print('│')
print('└──[{0}]'.format(green('OK')))
print('')
def format_output(output, success=True, traceback=None):
echo(' │')
for level, msg in output:
echo(' │ {0}'.format(msg))
echo(' │')
if traceback:
for tb in traceback.split('\n'):
echo(' │ {0}'.format(tb))
echo(' │')
echo(' └──[{0}]'.format(green('OK') if success else red('KO')))
echo('')
def execute_migration(plugin, filename, script, dryrun=False):
'''Execute and record a migration'''
db = get_db(DEFAULT_CONNECTION_NAME)
js = SCRIPT_WRAPPER.format(script)
lines = script.splitlines()
success = True
if not dryrun:
try:
lines = db.eval(js, plugin, filename, script)
except OperationFailure as e:
log.error(e.details['errmsg'].replace('\n', '\\n'))
success = False
except PyMongoError as e:
log.error('Unable to apply migration: %s', str(e))
success = False
print('│')
for line in lines:
print('│ {0}'.format(line))
print('│')
print('└──[{0}]'.format(green('OK') if success else red('KO')))
print('')
return success
def render():
'''Force (re)rendering stored images'''
print(cyan('Rendering images'))
count = Counter()
total = Counter()
organizations = Organization.objects(logo__exists=True)
total['orgs'] = organizations.count()
print(white('Processing {0} organizations logos'.format(total['orgs'])))
for org in organizations:
count['orgs'] += render_or_skip(org, 'logo')
users = User.objects(avatar__exists=True)
total['users'] = users.count()
print(white('Processing {0} user avatars'.format(total['users'])))
for user in users:
count['users'] += render_or_skip(user, 'avatar')
posts = Post.objects(image__exists=True)
total['posts'] = posts.count()
print(white('Processing {0} post images'.format(total['posts'])))
for post in posts:
count['posts'] += render_or_skip(post, 'image')
reuses = Reuse.objects(image__exists=True)
total['reuses'] = reuses.count()
print(white('Processing {0} reuse images'.format(total['reuses'])))
for reuse in reuses:
count['reuses'] += render_or_skip(reuse, 'image')
print(white('Summary:'))
print('''
Organization logos: {count[orgs]}/{total[orgs]}
User avatars: {count[users]}/{total[users]}
Post images: {count[posts]}/{total[posts]}
Reuse images: {count[reuses]}/{total[reuses]}
'''.format(count=count, total=total))
print(green('Images rendered'))
def execute_migration(plugin, filename, script, dryrun=False):
'''Execute and record a migration'''
db = get_db(DEFAULT_CONNECTION_NAME)
js = SCRIPT_WRAPPER.format(script)
lines = script.splitlines()
success = True
if not dryrun:
try:
lines = db.eval(js, plugin, filename, script)
except OperationFailure as e:
log.error(e.details['errmsg'].replace('\n', '\\n'))
success = False
except PyMongoError as e:
log.error('Unable to apply migration: %s', str(e))
success = False
print('│')
for line in lines:
print('│ {0}'.format(line))
print('│')
print('└──[{0}]'.format(green('OK') if success else red('KO')))
print('')
return success
def init():
'''Initialize your udata instance (search index, user, sample data...)'''
log.info('Apply DB migrations if needed')
migrate(record=True)
init_search(delete=True, force=not IS_INTERACTIVE)
if IS_INTERACTIVE:
text = _('Do you want to create a superadmin user ?')
if prompt_bool(text, default=True):
user = user_commands.create()
user_commands.set_admin(user.email)
text = _('Do you want to import some data-related license ?')
if prompt_bool(text, default=True):
licenses()
text = _('Do you want to create some sample data ?')
if prompt_bool(text, default=True):
generate_fixtures()
log.info(green(_('Your udata instance is ready !')))
def is_active(ep, actives):
return green(OK) if ep.name in actives else red(KO)
def is_active(ep, actives):
return green(OK) if ep.name in actives else red(KO)