def get_queues(queue):
queues = [q.name for q in current_app.config['CELERY_TASK_QUEUES']]
if queue:
queues = [q for q in queues if q == queue]
if not len(queues):
exit_with_error('Error: no queue found')
return queues
def schedule(cron, name, params):
'''
Schedule the job <name> to run periodically given the <cron> expression.
Jobs args and kwargs are given as parameters without dashes.
Ex:
udata job schedule "* * 0 * *" my-job arg1 arg2 key1=value key2=value
'''
if name not in celery.tasks:
exit_with_error('Job %s not found', name)
args = [p for p in params if '=' not in p]
kwargs = dict(p.split('=') for p in params if '=' in p)
label = 'Job {0}'.format(job_label(name, args, kwargs))
try:
task = PeriodicTask.objects.get(task=name, args=args, kwargs=kwargs)
task.modify(crontab=PeriodicTask.Crontab.parse(cron))
except PeriodicTask.DoesNotExist:
task = PeriodicTask.objects.create(
task=name,
name=label,
description='Periodic {0} job'.format(name),
enabled=True,
args=args,
kwargs=kwargs,
crontab=PeriodicTask.Crontab.parse(cron),
)
msg = 'Scheduled {label} with the following crontab: {cron}'
log.info(msg.format(label=label, cron=task.schedule_display))
def run(name, params, delay):
'''
Run the job <name>
Jobs args and kwargs are given as parameters without dashes.
Ex:
udata job run my-job arg1 arg2 key1=value key2=value
udata job run my-job -- arg1 arg2 key1=value key2=value
'''
args = [p for p in params if '=' not in p]
kwargs = dict(p.split('=') for p in params if '=' in p)
if name not in celery.tasks:
exit_with_error('Job %s not found', name)
job = celery.tasks[name]
label = job_label(name, args, kwargs)
if delay:
log.info('Sending job %s', label)
job.delay(*args, **kwargs)
log.info('Job sended to workers')
else:
log.info('Running job %s', label)
job.run(*args, **kwargs)
log.info('Job %s done', name)
def krbcheck():
'''Check the Kerberos configuration'''
if not manager.kerberos:
exit_with_error('No kerberos support, missing configuration')
if not manager.kerberos.check_keytab():
exit_with_error('Kerberos configuration not working')
def schedule(cron, name, params):
'''
Schedule the job <name> to run periodically given the <cron> expression.
Jobs args and kwargs are given as parameters without dashes.
Ex:
udata job schedule "* * 0 * *" my-job arg1 arg2 key1=value key2=value
'''
if name not in celery.tasks:
exit_with_error('Job %s not found', name)
args = [p for p in params if '=' not in p]
kwargs = dict(p.split('=') for p in params if '=' in p)
label = 'Job {0}'.format(job_label(name, args, kwargs))
try:
task = PeriodicTask.objects.get(task=name, args=args, kwargs=kwargs)
task.modify(crontab=PeriodicTask.Crontab.parse(cron))
except PeriodicTask.DoesNotExist:
task = PeriodicTask.objects.create(
task=name,
name=label,
description='Periodic {0} job'.format(name),
enabled=True,
args=args,
kwargs=kwargs,
crontab=PeriodicTask.Crontab.parse(cron),
)
msg = 'Scheduled {label} with the following crontab: {cron}'
log.info(msg.format(label=label, cron=task.schedule_display))
def get_queues(queue):
queues = [q.name for q in current_app.config['CELERY_TASK_QUEUES']]
if queue:
queues = [q for q in queues if q == queue]
if not len(queues):
exit_with_error('Error: no queue found')
return queues
def run(name, params, delay):
'''
Run the job <name>
Jobs args and kwargs are given as parameters without dashes.
Ex:
udata job run my-job arg1 arg2 key1=value key2=value
udata job run my-job -- arg1 arg2 key1=value key2=value
'''
args = [p for p in params if '=' not in p]
kwargs = dict(p.split('=') for p in params if '=' in p)
if name not in celery.tasks:
exit_with_error('Job %s not found', name)
job = celery.tasks[name]
label = job_label(name, args, kwargs)
if delay:
log.info('Sending job %s', label)
job.delay(*args, **kwargs)
log.info('Job sended to workers')
else:
log.info('Running job %s', label)
job.run(*args, **kwargs)
log.info('Job %s done', name)
def archive_one(dataset_id, comment):
"""Archive one dataset"""
try:
dataset = Dataset.objects.get(id=dataset_id)
except Dataset.DoesNotExist:
exit_with_error('Cannot find a dataset with id %s' % dataset_id)
else:
actions.archive(dataset, comment)
def delete():
'''Delete an existing user'''
email = click.prompt('Email')
user = User.objects(email=email).first()
if not user:
exit_with_error('Invalid user')
user.delete()
success('User deleted successfully')
def delete():
'''Delete an existing user'''
email = click.prompt('Email')
user = User.objects(email=email).first()
if not user:
exit_with_error('Invalid user')
user.delete()
success('User deleted successfully')
def status_print_config(queue):
if not queue:
exit_with_error('--munin-config called without a --queue parameter')
tasks = [n for n, q in get_tasks().items() if q == queue]
print('graph_title Waiting tasks for queue %s' % queue)
print('graph_vlabel Nb of tasks')
print('graph_category celery')
for task in tasks:
print('%s.label %s' % (format_field_for_munin(task), short_name(task)))
def validate():
'''Validate the Swagger/OpenAPI specification with your config'''
with current_app.test_request_context():
schema = json.loads(json.dumps(api.__schema__))
try:
schemas.validate(schema)
success('API specifications are valid')
except schemas.SchemaValidationError as e:
exit_with_error('API specifications are not valid', e)
def status_print_config(queue):
if not queue:
exit_with_error('--munin-config called without a --queue parameter')
tasks = [n for n, q in get_tasks().items() if q == queue]
print('graph_title Waiting tasks for queue %s' % queue)
print('graph_vlabel Nb of tasks')
print('graph_category celery')
for task in tasks:
print('%s.label %s' % (format_field_for_munin(task), short_name(task)))
def detach_zone(organization_id_or_slug):
'''Detach the zone of a given <organization>.'''
organization = Organization.objects.get_by_id_or_slug(
organization_id_or_slug)
if not organization:
exit_with_error(
'No organization found for {0}'.format(organization_id_or_slug))
log.info('Detaching {organization} from {organization.zone}'.format(
organization=organization))
organization.zone = None
organization.save()
log.info('Done')
def activate():
'''Activate an existing user (validate their email confirmation)'''
email = click.prompt('Email')
user = User.objects(email=email).first()
if not user:
exit_with_error('Invalid user')
if user.confirmed_at is not None:
exit_with_error('User email address already confirmed')
return
user.confirmed_at = datetime.utcnow()
user.save()
success('User activated successfully')
def activate():
'''Activate an existing user (validate their email confirmation)'''
email = click.prompt('Email')
user = User.objects(email=email).first()
if not user:
exit_with_error('Invalid user')
if user.confirmed_at is not None:
exit_with_error('User email address already confirmed')
return
user.confirmed_at = datetime.utcnow()
user.save()
success('User activated successfully')
def detach_zone(organization_id_or_slug):
'''Detach the zone of a given <organization>.'''
organization = Organization.objects.get_by_id_or_slug(
organization_id_or_slug)
if not organization:
exit_with_error(
'No organization found for {0}'.format(organization_id_or_slug)
)
log.info('Detaching {organization} from {organization.zone}'.format(
organization=organization))
organization.zone = None
organization.save()
log.info('Done')
def status_print_config(queue):
if not queue:
exit_with_error('--munin-config called without a --queue parameter')
tasks = cache.get(TASKS_LIST_CACHE_KEY) or []
if not tasks:
registered = inspect().registered_tasks()
if registered:
for w, tasks_list in registered.iteritems():
tasks += [t for t in tasks_list if t not in tasks]
cache.set(TASKS_LIST_CACHE_KEY, tasks,
timeout=TASKS_LIST_CACHE_DURATION)
print('graph_title Waiting tasks for queue %s' % queue)
print('graph_vlabel Nb of tasks')
print('graph_category celery')
for task in tasks:
print('%s.label %s' % (format_field_for_munin(task), task))
def status_print_config(queue):
if not queue:
exit_with_error('--munin-config called without a --queue parameter')
tasks = cache.get(TASKS_LIST_CACHE_KEY) or []
if not tasks:
registered = inspect().registered_tasks()
if registered:
for w, tasks_list in registered.iteritems():
tasks += [t for t in tasks_list if t not in tasks]
cache.set(TASKS_LIST_CACHE_KEY,
tasks,
timeout=TASKS_LIST_CACHE_DURATION)
print('graph_title Waiting tasks for queue %s' % queue)
print('graph_vlabel Nb of tasks')
print('graph_category celery')
for task in tasks:
print('%s.label %s' % (format_field_for_munin(task), task))
def create_oauth_client(client_name, user_email, uri, grant_types, scope,
response_types):
'''Creates an OAuth2Client instance in DB'''
user = User.objects(email=user_email).first()
if user is None:
exit_with_error('No matching user to email')
client = OAuth2Client.objects.create(name=client_name,
owner=user,
grant_types=grant_types,
scope=scope,
response_types=response_types,
redirect_uris=uri)
click.echo(f'New OAuth client: {client.name}')
click.echo(f'Client\'s ID {client.id}')
click.echo(f'Client\'s secret {client.secret}')
click.echo(f'Client\'s grant_types {client.grant_types}')
click.echo(f'Client\'s response_types {client.response_types}')
click.echo(f'Client\'s URI {client.redirect_uris}')
def create():
'''Create a new user'''
data = {
'first_name': click.prompt('First name'),
'last_name': click.prompt('Last name'),
'email': click.prompt('Email'),
'password': click.prompt('Password', hide_input=True),
'password_confirm': click.prompt('Confirm Password', hide_input=True),
}
# Until https://github.com/mattupstate/flask-security/issues/672 is fixed
with current_app.test_request_context():
form = RegisterForm(MultiDict(data), meta={'csrf': False})
if form.validate():
data['password'] = encrypt_password(data['password'])
del data['password_confirm']
data['confirmed_at'] = datetime.utcnow()
user = datastore.create_user(**data)
success('User(id={u.id} email={u.email}) created'.format(u=user))
return user
errors = '\n'.join('\n'.join(e) for e in form.errors.values())
exit_with_error('Error creating user', errors)
def create():
'''Create a new user'''
data = {
'first_name': click.prompt('First name'),
'last_name': click.prompt('Last name'),
'email': click.prompt('Email'),
'password': click.prompt('Password', hide_input=True),
'password_confirm': click.prompt('Confirm Password', hide_input=True),
}
# Until https://github.com/mattupstate/flask-security/issues/672 is fixed
with current_app.test_request_context():
form = RegisterForm(MultiDict(data), meta={'csrf': False})
if form.validate():
data['password'] = encrypt_password(data['password'])
del data['password_confirm']
data['confirmed_at'] = datetime.utcnow()
user = datastore.create_user(**data)
success('User(id={u.id} email={u.email}) created'.format(u=user))
return user
errors = '\n'.join('\n'.join(e) for e in form.errors.values())
exit_with_error('Error creating user', errors)
def check():
'''Check the LDAP configuration'''
bind_dn = manager.config.get('LDAP_BIND_USER_DN', None)
if not bind_dn:
exit_with_error('Missing LDAP_BIND_USER_DN setting')
header('Trying to connect with bind user')
try:
who_am_i = manager.connection.extend.standard.who_am_i()
success('Bind DN successfully connected')
echo('Bind DN user is "{}"'.format(white(safe_unicode(who_am_i))))
except Exception as e:
exit_with_error('Unable to connect', e)
header('Trying to authenticate an user')
email = prompt(white('User email'))
password = prompt(white('User password'), hide_input=True)
result = manager.authenticate(email, password)
if result.status == AuthenticationResponseStatus.success:
success('User successfully connected')
echo('Authenticated user is "{email} ({dn})"'.format(
email=white(safe_unicode(result.user_id)),
dn=white(safe_unicode(result.user_dn))
))
echo('User has the following remote attributes:')
for key, value in result.user_info.items():
echo(b'{key}: {value}'.format(key=white(safe_unicode(key)),
value=safe_unicode(value)))
echo('Local user will be created with the following values:')
for key, value in manager.extract_user_infos(result.user_info).items():
echo(b'{key}: {value}'.format(key=white(safe_unicode(key)),
value=safe_unicode(value)))
else:
exit_with_error('Unable to authenticate user "{0}"'.format(safe_unicode(email)))
success('LDAP configuration is working')
def unschedule(name, params):
'''
Unschedule the job <name> with the given parameters.
Jobs args and kwargs are given as parameters without dashes.
Ex:
udata job unschedule my-job arg1 arg2 key1=value key2=value
'''
if name not in celery.tasks:
exit_with_error('Job %s not found', name)
args = [p for p in params if '=' not in p]
kwargs = dict(p.split('=') for p in params if '=' in p)
label = 'Job {0}'.format(job_label(name, args, kwargs))
try:
task = PeriodicTask.objects.get(task=name, args=args, kwargs=kwargs)
except PeriodicTask.DoesNotExist:
exit_with_error('No scheduled job match {0}'.format(label))
task.delete()
msg = 'Unscheduled {label} with the following crontab: {cron}'
log.info(msg.format(label=label, cron=task.schedule_display))
def unschedule(name, params):
'''
Unschedule the job <name> with the given parameters.
Jobs args and kwargs are given as parameters without dashes.
Ex:
udata job unschedule my-job arg1 arg2 key1=value key2=value
'''
if name not in celery.tasks:
exit_with_error('Job %s not found', name)
args = [p for p in params if '=' not in p]
kwargs = dict(p.split('=') for p in params if '=' in p)
label = 'Job {0}'.format(job_label(name, args, kwargs))
try:
task = PeriodicTask.objects.get(task=name, args=args, kwargs=kwargs)
except PeriodicTask.DoesNotExist:
exit_with_error('No scheduled job match {0}'.format(label))
task.delete()
msg = 'Unscheduled {label} with the following crontab: {cron}'
log.info(msg.format(label=label, cron=task.schedule_display))
