async def _get_error(service, request, userdata):
if not userdata:
return web.HTTPFound(deploy_config.external_url(service, '/login'))
app = request.app
k8s = app['k8s_client']
dbpool = app['dbpool']
user_id = str(userdata['id'])
# we just failed a check, so update status from k8s without probe,
# best we can do is 'Initializing'
notebook = await get_user_notebook(dbpool, user_id)
new_status = await k8s_notebook_status_from_notebook(k8s, notebook)
await update_notebook_return_changed(dbpool, user_id, notebook, new_status)
session = await aiohttp_session.get_session(request)
if notebook:
if new_status['state'] == 'Ready':
return web.HTTPFound(deploy_config.external_url(
service,
f'/instance/{notebook["notebook_token"]}/?token={notebook["jupyter_token"]}'))
set_message(session,
'Could not connect to Jupyter instance. Please wait for Jupyter to be ready and try again.',
'error')
else:
set_message(session,
'Jupyter instance not found. Please launch a new instance.',
'error')
return web.HTTPFound(deploy_config.external_url(service, '/notebook'))
async def insert(tx):
row = await tx.execute_and_fetchone(
'''
SELECT billing_projects.name as billing_project, user
FROM billing_projects
LEFT JOIN (SELECT * FROM billing_project_users
WHERE billing_project = %s AND user = %s FOR UPDATE) AS t
ON billing_projects.name = t.billing_project
WHERE billing_projects.name = %s;
''', (billing_project, user, billing_project))
if row is None:
set_message(session, f'No such billing project {billing_project}.',
'error')
raise web.HTTPFound(
deploy_config.external_url('batch', f'/billing_projects'))
if row['user'] is not None:
set_message(
session,
f'User {user} is already member of billing project {billing_project}.',
'info')
raise web.HTTPFound(
deploy_config.external_url('batch', f'/billing_projects'))
await tx.execute_insertone(
'''
INSERT INTO billing_project_users(billing_project, user)
VALUES (%s, %s);
''', (billing_project, user))
async def post_create_billing_projects(request, userdata): # pylint: disable=unused-argument
db = request.app['db']
post = await request.post()
billing_project = post['billing_project']
session = await aiohttp_session.get_session(request)
@transaction(db)
async def insert(tx):
row = await tx.execute_and_fetchone(
'''
SELECT 1 FROM billing_projects
WHERE name = %s
FOR UPDATE;
''', (billing_project))
if row is not None:
set_message(session,
f'Billing project {billing_project} already exists.',
'error')
raise web.HTTPFound(
deploy_config.external_url('batch', f'/billing_projects'))
await tx.execute_insertone(
'''
INSERT INTO billing_projects(name)
VALUES (%s);
''', (billing_project, ))
await insert() # pylint: disable=no-value-for-parameter
set_message(session, f'Added billing project {billing_project}.', 'info')
return web.HTTPFound(
deploy_config.external_url('batch', f'/billing_projects'))
async def create_workshop(request, userdata): # pylint: disable=unused-argument
dbpool = request.app['dbpool']
session = await aiohttp_session.get_session(request)
post = await request.post()
name = post['name']
async with dbpool.acquire() as conn:
async with conn.cursor() as cursor:
try:
active = (post.get('active') == 'on')
if active:
token = secrets.token_urlsafe(32)
else:
token = None
await cursor.execute('''
INSERT INTO workshops (name, image, cpu, memory, password, active, token) VALUES (%s, %s, %s, %s, %s, %s, %s);
''',
(name,
post['image'],
post['cpu'],
post['memory'],
post['password'],
active,
token))
set_message(session, f'Created workshop {name}.', 'info')
except pymysql.err.IntegrityError as e:
if e.args[0] == 1062: # duplicate error
set_message(session,
f'Cannot create workshop {name}: duplicate name.',
'error')
else:
raise
return web.HTTPFound(deploy_config.external_url('notebook', '/workshop-admin'))
async def delete(tx):
row = await tx.execute_and_fetchone(
'''
SELECT billing_projects.name as billing_project, user
FROM billing_projects
LEFT JOIN (SELECT * FROM billing_project_users
WHERE billing_project = %s AND user = %s FOR UPDATE) AS t
ON billing_projects.name = t.billing_project
WHERE billing_projects.name = %s;
''', (billing_project, user, billing_project))
if not row:
set_message(session, f'No such billing project {billing_project}.',
'error')
raise web.HTTPFound(
deploy_config.external_url('batch', f'/billing_projects'))
assert row['billing_project'] == billing_project
if row['user'] is None:
set_message(
session,
f'User {user} is not member of billing project {billing_project}.',
'info')
raise web.HTTPFound(
deploy_config.external_url('batch', f'/billing_projects'))
await tx.just_execute(
'''
DELETE FROM billing_project_users
WHERE billing_project = %s AND user = %s;
''', (billing_project, user))
async def update_workshop(request, userdata): # pylint: disable=unused-argument
app = request.app
dbpool = app['dbpool']
post = await request.post()
name = post['name']
id = post['id']
session = await aiohttp_session.get_session(request)
async with dbpool.acquire() as conn:
async with conn.cursor() as cursor:
active = (post.get('active') == 'on')
# FIXME don't set token unless re-activating
if active:
token = secrets.token_urlsafe(32)
else:
token = None
n = await cursor.execute('''
UPDATE workshops SET name = %s, image = %s, cpu = %s, memory = %s, password = %s, active = %s, token = %s WHERE id = %s;
''',
(name,
post['image'],
post['cpu'],
post['memory'],
post['password'],
active,
token,
id))
if n == 0:
set_message(session,
f'Internal error: cannot update workshop: workshop ID {id} not found.',
'error')
else:
set_message(session, f'Updated workshop {name}.', 'info')
return web.HTTPFound(deploy_config.external_url('notebook', '/workshop-admin'))
async def retry_pr(wb, pr, request):
app = request.app
session = await aiohttp_session.get_session(request)
if pr.batch is None:
log.info(
'retry cannot be requested for PR #{pr.number} because it has no batch'
)
set_message(
session,
f'Retry cannot be requested for PR #{pr.number} because it has no batch.',
'error')
return
batch_id = pr.batch.id
dbpool = app['dbpool']
async with dbpool.acquire() as conn:
async with conn.cursor() as cursor:
await cursor.execute(
'INSERT INTO invalidated_batches (batch_id) VALUES (%s);',
batch_id)
await wb.notify_batch_changed(app)
log.info(f'retry requested for PR: {pr.number}')
set_message(session, f'Retry requested for PR #{pr.number}.', 'info')
async def workshop_post_login(request):
session = await aiohttp_session.get_session(request)
dbpool = request.app['dbpool']
post = await request.post()
name = post['name']
password = post['password']
async with dbpool.acquire() as conn:
async with conn.cursor() as cursor:
await cursor.execute(
'''
SELECT * FROM workshops
WHERE name = %s AND password = %s AND active = 1;
''',
(name, password),
)
workshops = await cursor.fetchall()
if len(workshops) != 1:
assert len(workshops) == 0
set_message(session, 'Workshop Inactive!', 'error')
return web.HTTPFound(location=deploy_config.external_url('workshop', '/login'))
workshop = workshops[0]
# use hex since K8s labels can't start or end with _ or -
user_id = secrets.token_hex(16)
session['workshop_session'] = {'workshop_name': name, 'workshop_token': workshop['token'], 'id': user_id}
set_message(session, f'Welcome to the {name} workshop!', 'info')
return web.HTTPFound(location=deploy_config.external_url('workshop', '/notebook'))
def validate(name, value, predicate, description):
if not predicate(value):
set_message(session,
f'{name} invalid: {value}. Must be {description}.',
'error')
raise web.HTTPFound(deploy_config.external_url('batch-driver', '/'))
return value
def validate(session, name, value, predicate, description):
if not predicate(value):
set_message(session,
f'{name} invalid: {value}. Must be {description}.',
'error')
raise ConfigError()
return value
async def _billing(request):
app = request.app
date_format = '%m/%Y'
now = datetime.datetime.now()
default_time_period = now.strftime(date_format)
time_period_query = request.query.get('time_period', default_time_period)
try:
time_period = datetime.datetime.strptime(time_period_query,
date_format)
except ValueError:
msg = f"Invalid value for time_period '{time_period_query}'; must be in the format of MM/YYYY."
session = await aiohttp_session.get_session(request)
set_message(session, msg, 'error')
return ([], [], [], time_period_query)
db = app['db']
records = db.execute_and_fetchall(
'SELECT * FROM monitoring_billing_data WHERE year = %s AND month = %s;',
(time_period.year, time_period.month))
records = [record async for record in records]
cost_by_service, compute_cost_breakdown, cost_by_sku_source = format_data(
records)
return (cost_by_service, compute_cost_breakdown, cost_by_sku_source,
time_period_query)
async def job_private_config_update(request, userdata): # pylint: disable=unused-argument
app = request.app
inst_coll_manager: InstanceCollectionManager = app['inst_coll_manager']
session = await aiohttp_session.get_session(request)
job_private_inst_manager = inst_coll_manager.job_private_inst_manager
url_path = '/inst_coll/jpim'
post = await request.post()
boot_disk_size_gb = validate_int(
session,
url_path,
'Worker boot disk size',
post['boot_disk_size_gb'],
lambda v: v >= 10,
'a positive integer greater than or equal to 10',
)
max_instances = validate_int(
session, url_path, 'Max instances', post['max_instances'], lambda v: v > 0, 'a positive integer'
)
max_live_instances = validate_int(
session, url_path, 'Max live instances', post['max_live_instances'], lambda v: v > 0, 'a positive integer'
)
await job_private_inst_manager.configure(boot_disk_size_gb, max_instances, max_live_instances)
set_message(session, f'Updated configuration for {job_private_inst_manager}.', 'info')
return web.HTTPFound(deploy_config.external_url('batch-driver', url_path))
def validate_int(session, url_path, name, value, predicate, description):
try:
i = int(value)
except ValueError as e:
set_message(session, f'{name} invalid: {value}. Must be an integer.', 'error')
raise web.HTTPFound(deploy_config.external_url('batch-driver', url_path)) from e
return validate(session, url_path, name, i, predicate, description)
async def get_pool(request, userdata):
app = request.app
inst_coll_manager: InstanceCollectionManager = app[
'driver'].inst_coll_manager
session = await aiohttp_session.get_session(request)
pool_name = request.match_info['pool']
pool = inst_coll_manager.get_inst_coll(pool_name)
if not isinstance(pool, Pool):
set_message(session, f'Unknown pool {pool_name}.', 'error')
return web.HTTPFound(deploy_config.external_url('batch-driver', '/'))
user_resources = await pool.scheduler.compute_fair_share()
user_resources = sorted(
user_resources.values(),
key=lambda record: record['ready_cores_mcpu'] + record[
'running_cores_mcpu'],
reverse=True,
)
ready_cores_mcpu = sum(
[record['ready_cores_mcpu'] for record in user_resources])
page_context = {
'pool': pool,
'instances': pool.name_instance.values(),
'user_resources': user_resources,
'ready_cores_mcpu': ready_cores_mcpu,
}
return await render_template('batch-driver', request, userdata,
'pool.html', page_context)
def validate_int(session, name, value, predicate, description):
try:
i = int(value)
except ValueError as e:
set_message(session, f'{name} invalid: {value}. Must be an integer.',
'error')
raise ConfigError() from e
return validate(session, name, i, predicate, description)
async def ui_cancel_batch(request, userdata):
batch_id = int(request.match_info['batch_id'])
user = userdata['username']
await _cancel_batch(request.app, batch_id, user)
session = await aiohttp_session.get_session(request)
set_message(session, 'Batch {batch_id} cancelled.', 'info')
location = request.app.router['batches'].url_for()
raise web.HTTPFound(location=location)
async def post_authorized_source_sha(request, userdata): # pylint: disable=unused-argument
app = request.app
db: Database = app['db']
post = await request.post()
sha = post['sha'].strip()
await db.execute_insertone(
'INSERT INTO authorized_shas (sha) VALUES (%s);', sha)
log.info(f'authorized sha: {sha}')
session = await aiohttp_session.get_session(request)
set_message(session, f'SHA {sha} authorized.', 'info')
return web.HTTPFound(deploy_config.external_url('ci', '/'))
async def post_authorized_source_sha(request, userdata): # pylint: disable=unused-argument
app = request.app
dbpool = app['dbpool']
post = await request.post()
sha = post['sha'].strip()
async with dbpool.acquire() as conn:
async with conn.cursor() as cursor:
await cursor.execute('INSERT INTO authorized_shas (sha) VALUES (%s);', sha)
log.info(f'authorized sha: {sha}')
session = await aiohttp_session.get_session(request)
set_message(session, f'SHA {sha} authorized.', 'info')
raise web.HTTPFound('/')
async def post_create_role(request, userdata): # pylint: disable=unused-argument
session = await aiohttp_session.get_session(request)
db = request.app['db']
post = await request.post()
name = post['name']
role_id = await db.execute_insertone(
'''
INSERT INTO `roles` (`name`)
VALUES (%s);
''', (name))
set_message(session, f'Created role {role_id} {name}.', 'info')
return web.HTTPFound(deploy_config.external_url('auth', '/roles'))
async def delete_user(request, userdata): # pylint: disable=unused-argument
session = await aiohttp_session.get_session(request)
db = request.app['db']
post = await request.post()
id = post['id']
username = post['username']
try:
await _delete_user(db, username, id)
set_message(session, f'Deleted user {id} {username}.', 'info')
except UnknownUser:
set_message(session, f'Delete failed, no such user {id} {username}.',
'error')
return web.HTTPFound(deploy_config.external_url('auth', '/users'))
async def job_private_config_update(request, userdata): # pylint: disable=unused-argument
app = request.app
jpim: JobPrivateInstanceManager = app['driver'].job_private_inst_manager
session = await aiohttp_session.get_session(request)
url_path = '/inst_coll/jpim'
post = await request.post()
try:
boot_disk_size_gb = validate_int(
session,
'Worker boot disk size',
post['boot_disk_size_gb'],
lambda v: v >= 10,
'a positive integer greater than or equal to 10',
)
if jpim.cloud == 'azure' and boot_disk_size_gb != 30:
set_message(session,
'The boot disk size (GB) must be 30 in azure.',
'error')
raise ConfigError()
max_instances = validate_int(session, 'Max instances',
post['max_instances'], lambda v: v > 0,
'a positive integer')
max_live_instances = validate_int(session, 'Max live instances',
post['max_live_instances'],
lambda v: v > 0,
'a positive integer')
await jpim.configure(boot_disk_size_gb, max_instances,
max_live_instances)
set_message(session, f'Updated configuration for {jpim}.', 'info')
except ConfigError:
pass
except asyncio.CancelledError:
raise
except Exception:
log.exception(f'error while updating pool configuration for {jpim}')
raise
return web.HTTPFound(deploy_config.external_url('batch-driver', url_path))
async def post_create_user(request, userdata): # pylint: disable=unused-argument
session = await aiohttp_session.get_session(request)
db = request.app['db']
post = await request.post()
username = post['username']
email = post['email']
is_developer = post.get('is_developer') == '1'
user_id = await db.execute_insertone(
'''
INSERT INTO users (state, username, email, is_developer)
VALUES (%s, %s, %s, %s);
''', ('creating', username, email, is_developer))
set_message(session, f'Created user {user_id} {username}.', 'info')
return web.HTTPFound(deploy_config.external_url('auth', '/users'))
async def creating_account(request, userdata):
db = request.app['db']
session = await aiohttp_session.get_session(request)
if 'pending' in session:
login_id = session['login_id']
user = await user_from_login_id(db, login_id)
next_url = deploy_config.external_url('auth', '/user')
next_page = session.pop('next', next_url)
cleanup_session(session)
if user is None:
set_message(session,
f'Account does not exist for login id {login_id}.',
'error')
return aiohttp.web.HTTPFound(deploy_config.external_url(
'auth', ''))
page_context = {
'username': user['username'],
'state': user['state'],
'login_id': user['login_id']
}
if user['state'] == 'deleting' or user['state'] == 'deleted':
return await render_template('auth', request, userdata,
'account-error.html', page_context)
if user['state'] == 'active':
session_id = await create_session(db, user['id'])
session['session_id'] = session_id
set_message(session,
f'Account has been created for {user["username"]}.',
'info')
return aiohttp.web.HTTPFound(next_page)
assert user['state'] == 'creating'
session['pending'] = True
session['login_id'] = login_id
session['next'] = next_page
return await render_template('auth', request, userdata,
'account-creating.html', page_context)
return aiohttp.web.HTTPUnauthorized()
async def post_billing_projects_remove_user(request, userdata): # pylint: disable=unused-argument
db = request.app['db']
billing_project = request.match_info['billing_project']
user = request.match_info['user']
session = await aiohttp_session.get_session(request)
@transaction(db)
async def delete(tx):
row = await tx.execute_and_fetchone(
'''
SELECT billing_projects.name as billing_project, user
FROM billing_projects
LEFT JOIN (SELECT * FROM billing_project_users
WHERE billing_project = %s AND user = %s FOR UPDATE) AS t
ON billing_projects.name = t.billing_project
WHERE billing_projects.name = %s;
''', (billing_project, user, billing_project))
if not row:
set_message(session, f'No such billing project {billing_project}.',
'error')
raise web.HTTPFound(
deploy_config.external_url('batch', f'/billing_projects'))
assert row['billing_project'] == billing_project
if row['user'] is None:
set_message(
session,
f'User {user} is not member of billing project {billing_project}.',
'info')
raise web.HTTPFound(
deploy_config.external_url('batch', f'/billing_projects'))
await tx.just_execute(
'''
DELETE FROM billing_project_users
WHERE billing_project = %s AND user = %s;
''', (billing_project, user))
await delete() # pylint: disable=no-value-for-parameter
set_message(
session,
f'Removed user {user} from billing project {billing_project}.', 'info')
return web.HTTPFound(
deploy_config.external_url('batch', f'/billing_projects'))
async def post_billing_projects_add_user(request, userdata): # pylint: disable=unused-argument
db = request.app['db']
post = await request.post()
user = post['user']
billing_project = request.match_info['billing_project']
session = await aiohttp_session.get_session(request)
@transaction(db)
async def insert(tx):
row = await tx.execute_and_fetchone(
'''
SELECT billing_projects.name as billing_project, user
FROM billing_projects
LEFT JOIN (SELECT * FROM billing_project_users
WHERE billing_project = %s AND user = %s FOR UPDATE) AS t
ON billing_projects.name = t.billing_project
WHERE billing_projects.name = %s;
''', (billing_project, user, billing_project))
if row is None:
set_message(session, f'No such billing project {billing_project}.',
'error')
raise web.HTTPFound(
deploy_config.external_url('batch', f'/billing_projects'))
if row['user'] is not None:
set_message(
session,
f'User {user} is already member of billing project {billing_project}.',
'info')
raise web.HTTPFound(
deploy_config.external_url('batch', f'/billing_projects'))
await tx.execute_insertone(
'''
INSERT INTO billing_project_users(billing_project, user)
VALUES (%s, %s);
''', (billing_project, user))
await insert() # pylint: disable=no-value-for-parameter
set_message(session,
f'Added user {user} to billing project {billing_project}.',
'info')
return web.HTTPFound(
deploy_config.external_url('batch', f'/billing_projects'))
async def delete_workshop(request, userdata): # pylint: disable=unused-argument
app = request.app
dbpool = app['dbpool']
post = await request.post()
name = post['name']
async with dbpool.acquire() as conn:
async with conn.cursor() as cursor:
n = await cursor.execute('''
DELETE FROM workshops WHERE name = %s;
''', name)
session = await aiohttp_session.get_session(request)
if n == 1:
set_message(session, f'Deleted workshop {name}.', 'info')
else:
set_message(session, f'Workshop {name} not found.', 'error')
return web.HTTPFound(deploy_config.external_url('notebook', '/workshop-admin'))
async def insert(tx):
row = await tx.execute_and_fetchone(
'''
SELECT 1 FROM billing_projects
WHERE name = %s
FOR UPDATE;
''', (billing_project))
if row is not None:
set_message(session,
f'Billing project {billing_project} already exists.',
'error')
raise web.HTTPFound(
deploy_config.external_url('batch', f'/billing_projects'))
await tx.execute_insertone(
'''
INSERT INTO billing_projects(name)
VALUES (%s);
''', (billing_project, ))
async def unfreeze_batch(request, userdata): # pylint: disable=unused-argument
app = request.app
db: Database = app['db']
session = await aiohttp_session.get_session(request)
if not app['frozen']:
set_message(session, 'Batch is already unfrozen.', 'info')
return web.HTTPFound(deploy_config.external_url('batch-driver', '/'))
await db.execute_update(
'''
UPDATE globals SET frozen = 0;
''')
app['frozen'] = False
set_message(session, 'Unfroze all instance collections and batch submissions.', 'info')
return web.HTTPFound(deploy_config.external_url('batch-driver', '/'))
async def post_create_user(request, userdata): # pylint: disable=unused-argument
session = await aiohttp_session.get_session(request)
dbpool = request.app['dbpool']
post = await request.post()
username = post['username']
email = post['email']
is_developer = post.get('is_developer') == '1'
async with dbpool.acquire() as conn:
async with conn.cursor() as cursor:
await cursor.execute(
'''
INSERT INTO users (state, username, email, is_developer)
VALUES (%s, %s, %s, %s);
''', ('creating', username, email, is_developer))
user_id = cursor.lastrowid
set_message(session, f'Created user {user_id} {username}.', 'info')
return web.HTTPFound(deploy_config.external_url('auth', '/users'))
async def _get_error(service, request, userdata):
if not userdata:
return web.HTTPFound(deploy_config.external_url(service, '/login'))
app = request.app
k8s = app['k8s_client']
dbpool = app['dbpool']
user_id = userdata['id']
# we just failed a check, so update status from k8s without probe,
# best we can do is 'Initializing'
notebook = await get_user_notebook(dbpool, user_id)
new_status = await k8s_notebook_status_from_notebook(k8s, notebook)
await update_notebook_return_changed(dbpool, user_id, notebook, new_status)
session = await aiohttp_session.get_session(request)
set_message(session,
f'Notebook not found. Please create a new notebook.',
'error')
return web.HTTPFound(deploy_config.external_url(service, '/notebook'))