def test_get_close_db(app):
with app.app_context():
db = get_db()
# assert the same db object is returned
assert db is get_db()
with pytest.raises(sqlite3.ProgrammingError) as e:
db.execute('SELECT 1')
assert 'closed' in str(e)
def deploy_project(project_id):
"""Central driving function for deploying projects to AWS.
"""
logging.info('Begining deploy for project {}'.format(project_id))
# get the project from that database
db = get_db()
cursor = db.cursor()
project = cursor.execute(
'SELECT * FROM projects where id=(?);',
(project_id,)).fetchone()
# generate hash if it doesn't already exist, but it should
if project['hash_id'] is None or project['hash_id'] == '':
hash_id = generate_hash_id(project['owner_id'], project['name'])
# write the hash id to the project
cursor.execute(
'UPDATE projects SET hash_id=(?) WHERE id=(?);',
(hash_id, project_id,))
db.commit()
else:
hash_id = project['hash_id']
# first need to build config files
build_config_files(hash_id,
project['id'],
project['min_workers'],
project['secret_key'],
project['deployment_subdomain'])
start_container(hash_id)
# get url
update_status(project_id)
def get_work_page():
"""Returns a work page for a project.
"""
# get all projects that are approved and running
db = get_db()
cursor = db.cursor()
cursor.execute(('SELECT * FROM projects WHERE approval_status=(?) AND '
'health_status LIKE "RUNNING";'),
(ApprovalStatus.APPROVED.value, ))
# algorithm to pick propject
# version 1 is random
projects = cursor.fetchall()
# check that there are projects to work on
if len(projects) == 0:
flash('There are no projects to work on.')
return redirect(url_for('index'))
in_need_projects = list()
for project in projects:
if project['worker_count'] < project['min_workers']:
in_need_projects.append(project)
if len(in_need_projects) > 0:
projects = in_need_projects
project = random.choice(projects)
return redirect(
url_for('work.get_work_page_for_project', project_id=project['id']))
def delete(project_id):
"""Deletes a given project.
Deletes the deploy information and stops the given container.
"""
db = get_db()
# Get the project to verify owner identity
project = db.execute('SELECT * FROM projects where id=(?);',
(project_id, )).fetchone()
if project is None:
abort(404, 'Project does not exist')
# Check that this user can delete the project
if project['owner_id'] != g.user['id'] and g.user['super_user'] == 'false':
# The current user doesn't own this project, don't delete it
flash('You don\'t have permissions to delete this project')
return redirect(url_for('host.detail', project_id=project_id))
# destroy the project if deploy is enabled
if current_app.config['DO_DEPLOY']:
destroy_project(project_id)
# delete the database entry
db.execute('DELETE FROM projects WHERE id=(?);', (project_id, ))
db.commit()
return redirect(url_for('index'))
def serve_file(project_id, filename):
"""Serves the requested file from the project's deployment folder.
Only serves CODE_FILENAME and SECRETS_FILENAME
Query arg: `key` - key used for serving the secrets file
"""
# get the project from the project id
db = get_db()
project = db.execute('SELECT * FROM projects WHERE id=(?);',
(project_id, )).fetchone()
if project is None:
abort(404, 'Project does not exist')
# make sure it's an allowable filename
if filename != CODE_FILENAME and filename != SECRETS_FILENAME:
abort(404, 'File not found.')
# if the filename is SECRETS_FILENAME, make sure they have the key
if filename == SECRETS_FILENAME:
if (request.args.get('key') is None
or request.args.get('key') != project['secret_key']):
# They don't have a valid key, abort with an error
abort(403, 'Key required for accessing secret file.')
# get the project folder path
project_folder_path = get_project_folder(project['hash_id'])
# check that the file exists
if not os.path.isfile(os.path.join(project_folder_path, filename)):
abort(404, 'File does not exist.')
# serve the requested file
return send_from_directory(project_folder_path, filename)
def login():
"""Handles GET and POST requests on the '/login' route."""
if request.method == 'POST':
email = request.form['email']
password = request.form['password']
db = get_db()
error = None
cursor = db.cursor()
cursor.execute('SELECT * FROM users WHERE email = ?', (email, ))
user = cursor.fetchone()
# Check to see that user input required fields and info is valid
if not email:
error = 'Email is required.'
elif not password:
error = 'Password is required.'
elif user is None:
error = 'Incorrect email.'
elif not check_password_hash(user['password'], password):
error = 'Incorrect password.'
if error is None:
session.clear()
session['user_id'] = user['id']
return redirect(url_for('index'))
flash(error)
return render_template('auth/login.html')
def register():
"""Handles GET and POST requests on the '/register' route."""
if request.method == 'POST':
email = request.form['email']
password = request.form['password']
confirm_password = request.form['confirm-password']
db = get_db()
error = None
# input validation
if not email:
error = 'Email is required.'
elif not password:
error = 'Password is required.'
elif not confirm_password:
error = 'Password confirmation is required.'
elif password != confirm_password:
error = 'Passwords do not match.'
# check that the given email isn't already registered
cursor = db.cursor()
cursor.execute('SELECT id FROM users WHERE email = ?', (email, ))
if cursor.fetchone() is not None:
error = 'Account cannot be registered'
if error is None:
cursor.execute('INSERT INTO users (email, password) VALUES (?, ?)',
(email, generate_password_hash(password)))
db.commit()
return redirect(url_for('auth.login'))
flash(error)
return render_template('auth/register.html')
def update_status(project_id):
# get the hash id from the project id from the database
db = get_db()
project = db.execute('SELECT * FROM projects where id=(?);',
(project_id,)).fetchone()
hash_id = project['hash_id']
(docker_compose_path, ecs_params_path) = get_config_file_paths(hash_id)
# build the status command
# TODO - I'm not sure if project-name means something different
# TODO - see if I can just use 'cluster' instead of cluster-config
status_cmd = ('{ecs_cli_path} compose --project-name {hash_id} '
'--ecs-params {ecs_params} '
'--file {docker_compose} '
'ps '
'--cluster-config {cluster_config} '
'--ecs-profile {ecs_profile}')
status_cmd = status_cmd.format(ecs_cli_path=current_app.config['ECS_CLI_PATH'],
hash_id=hash_id,
ecs_params=ecs_params_path,
docker_compose = docker_compose_path,
cluster_config=current_app.config['FLOCK_CLUSTER_CONFIG'],
ecs_profile=current_app.config['FLOCK_ECS_PROFILE'])
# Run the status command and capture the output
logging.info('status command: ' + status_cmd)
proc = subprocess.run(status_cmd.split(' '),
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# decode output and log it
output = proc.stdout.decode('utf-8')
print(output)
logging.info('status output: ' + output)
# find the IP address and health of the project using regex
# Start with hash id, then look for optional colon followed by number
# there will then be a status and eventually an IP
# if the status is a fail condition, this regex won't match
regex = ':?\d*\s*(?P<status>\w*)\s*(?P<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'
regex = hash_id + regex
match = re.search(regex, output)
if match is not None:
db.execute(('UPDATE projects SET deployment_ip=(?), health_status=(?) '
'WHERE id=(?);'),
(match.group('ip'), match.group('status'), project_id,))
db.commit()
else:
logging.error('Did not find ip or status')
if re.search('STOPPED', output) is not None:
print('Container didn\'t start.')
logging.error('Container didn\'t start.')
status = 'STOPPED'
message = 'Container failed to start...'
db.execute(('UPDATE projects SET health_status=(?), '
'health_message=(?) WHERE id=(?);'),
(status, message, project_id,))
db.commit()
def test_delete(client, auth, app):
auth.login()
response = client.get('/host/1/delete')
with app.app_context():
db = get_db()
project = db.execute('SELECT * FROM projects WHERE id=1;').fetchone()
assert project is None
def app():
db_fd, db_path = tempfile.mkstemp()
app = create_app({
'TESTING': True,
'DATABASE': db_path,
'DO_DEPLOY': False,
'LOCALTUNNEL_URL': 'localtunnel.me'
})
with app.app_context():
init_db()
get_db().executescript(_data_sql)
yield app
os.close(db_fd)
os.unlink(db_path)
def get_project_folder_from_id(project_id):
"""Returns the folder for a given project. Creates it if it does not exist.
Safe to call not in a deployment environment.
"""
# get the hash_id
db = get_db()
hash_id = db.execute('SELECT hash_id FROM projects WHERE id=(?);',
(project_id,)).fetchone()
return get_project_folder(hash_id)
def load_logged_in_user():
"""Loads the logged in user if the user id is set on the session."""
user_id = session.get('user_id')
if user_id is None:
g.user = None
else:
cursor = get_db().cursor()
g.user = cursor.execute('SELECT * FROM users WHERE id = ?',
(user_id, )).fetchone()
def test_approve(client, auth, app):
"""Try to approve project id = 1."""
auth.login_super()
response = client.get('/host/1/approve')
assert response.status_code == 302
assert response.headers['Location'].endswith('/host/queue')
with app.app_context():
db = get_db()
project = db.execute('SELECT * FROM projects WHERE id=1;').fetchone()
assert project['approval_status'] == 1
def queue():
"""Shows currently queued projects.
"""
# setup the database
db = get_db()
cursor = db.cursor()
projects = cursor.execute(
'SELECT * FROM projects WHERE approval_status=(?);',
(ApprovalStatus.WAITING.value, )).fetchall()
return render_template('host/queue.html', projects=projects)
def test_serve_file(client, auth, app):
# need to submit some files first
auth.login()
project_name = 'serve-file-test'
code_file_content = b'CODE FILE'
secrets_file_content = b'SECRETS FILE'
client.post(
'/host/submit', buffered=True,
content_type='multipart/form-data',
data = { 'name': project_name,
'source-url': 'https://zacharysang.com',
'min-workers': '1',
'description': 'Creation description',
'code-file': (BytesIO(code_file_content), 'user-code.js'),
'secrets-file': (BytesIO(secrets_file_content), 'secrets.js')
}
)
# project is created, logout because these urls don't require sessions
auth.logout()
# get app context so we can get the project id
with app.app_context():
db = get_db()
project = db.execute('SELECT * FROM projects WHERE name=(?);',
(project_name,)).fetchone()
# make sure the project exists
assert project is not None
# check a couple of different urls to make sure they return content
# check for user-code.js
response = client.get('/host/{}/file/user-code.js'.format(project['id']))
assert response.status_code == 200
assert code_file_content in response.data
# check for secret.js without key
response = client.get('/host/{}/file/secret.js'.format(project['id']))
assert response.status_code == 403
# check for secret.js with key
response = client.get('/host/{}/file/secret.js?key={}'
.format(project['id'], project['secret_key']))
assert response.status_code == 200
assert secrets_file_content in response.data
# check for file that doesn't exist
response = client.get('/host/{}/file/not-real.js'.format(project['id']))
assert response.status_code == 404
def test_node_0_communicate(client, app):
"""Test that node-0-communicate can update the worker_count of a project."""
with app.app_context():
db = get_db()
project = db.execute('SELECT * FROM projects WHERE id=1;').fetchone()
response = client.post(
'/host/{}/node-0-communicate'.format(project['id']),
content_type='application/json',
data = json.dumps({ 'secret_key': project['secret_key'],
'worker_count': '100'
})
)
assert response.status_code == 200
project = db.execute('SELECT * FROM projects WHERE id=1;').fetchone()
assert project['worker_count'] == 100
def approve(project_id):
"""Approves the project associated with a given project_id
"""
db = get_db()
cursor = db.cursor()
cursor.execute('UPDATE projects SET approval_status=(?) WHERE id=(?);', (
ApprovalStatus.APPROVED.value,
project_id,
))
db.commit()
# deploy the project if enabled
if current_app.config['DO_DEPLOY']:
deploy_project(project_id)
return redirect(url_for('host.queue'))
def node_0_communicate(project_id):
"""An endpoint for the node 0 to send information to the master server.
Required values in POST json:
secret_key : secret key for project
Optional values in POST json:
deployment_url : the url the server is deployed at
worker_count : the number of workers currently working on the project
"""
if request.method != 'POST':
abort(405)
# get the project to pull information from
db = get_db()
project = db.execute('SELECT * FROM projects WHERE id=(?);',
(project_id, )).fetchone()
if project is None:
abort(404)
if ('secret_key' not in request.json
or request.json['secret_key'] != project['secret_key']):
abort(403, 'Bad secret key.')
# build the information that this can accept
deployment_url = project['deployment_url']
worker_count = project['worker_count']
if 'deployment_url' in request.json:
deployment_url = request.json['deployment_url']
if 'worker_count' in request.json:
worker_count = request.json['worker_count']
# update the database
db.execute(('UPDATE projects SET deployment_url=(?), '
'worker_count=(?) WHERE id=(?);'), (
deployment_url,
worker_count,
project_id,
))
db.commit()
return '', 200
def test_register(client, app):
# test that register page exists
assert client.get('/register').status_code == 200
# send good register post
user_email = '*****@*****.**'
response = client.post(
'/register', data={'email': user_email,
'password': '******',
'confirm-password': '******'}
)
# redirects to login
assert response.headers['Location'].endswith('/login')
# Check the user was created
with app.app_context():
assert get_db().execute(
'SELECT * FROM users WHERE email = (?)',
('*****@*****.**',)
).fetchone() is not None
def restart(project_id):
"""Endpoint that restarts the container and redirects back to project
details.
"""
db = get_db()
# verify that the project exists
project = db.execute('SELECT * FROM projects WHERE id=(?);',
(project_id, )).fetchone()
if project is None:
abort(404)
# verify user is owner or an admin
if project['owner_id'] != g.user['id'] and g.user['super_user'] == 'false':
abort(403)
restart_container(project['hash_id'])
return redirect(url_for('host.detail', project_id=project_id))
def list():
"""Renders a list of projects separated into my projects and all projects
"""
db = get_db()
# check for a current user
my_projects = None
if g.user is None:
projects = db.execute('SELECT * FROM projects;').fetchall()
else:
# there is a user, render both my projects and all projects
my_projects = db.execute('SELECT * FROM projects WHERE owner_id=(?);',
(g.user['id'], )).fetchall()
projects = db.execute('SELECT * FROM projects WHERE owner_id!=(?);',
(g.user['id'], )).fetchall()
return render_template('host/list.html',
projects=projects,
my_projects=my_projects)
def destroy_project(project_id):
"""Destroys a given project by stopping container and deleting config files.
"""
logging.info('Destroying deployment for project {}'.format(project_id))
# get the project from that database
db = get_db()
project = db.execute(
'SELECT * FROM projects where id=(?);',
(project_id,)).fetchone()
# get hash id from project
hash_id = project['hash_id']
stop_container(hash_id)
# delete the folder with config details
path = get_project_folder(hash_id)
shutil.rmtree(path)
def test_submit(client, auth, app):
"""Tests a successful submit."""
auth.login()
assert client.get('/host/submit').status_code == 200
client.post(
'/host/submit', buffered=True,
content_type='multipart/form-data',
data = { 'name': 'submit-test',
'source-url': 'https://zacharysang.com',
'min-workers': '1',
'description': 'Creation description',
'code-file': (BytesIO(b'CODE_FILE'), 'user-code.js'),
'secrets-file': (BytesIO(b'SECRETS'), 'secrets.js')
}
)
with app.app_context():
db = get_db()
count = db.execute('SELECT COUNT(id) FROM projects;').fetchone()[0]
assert count == 3
def get_work_page_for_project(project_id):
"""Returns the work page for a specific project.
"""
# Get the project from the db
db = get_db()
project = db.execute('SELECT * FROM projects WHERE id=(?);',
(project_id, )).fetchone()
# check that this project exists
if project is None:
abort(404, 'Project not found.')
# check that the project is running and approved
if (project['health_status'] != 'RUNNING'
or project['approval_status'] != ApprovalStatus.APPROVED.value):
abort(400, 'Project not yet running or approved.')
# check for key, which allows us to send secret file
secret = False
secrets_file = ''
if request.args.get('key') == project['secret_key']:
secret = True
# flask makes key a query param:
secrets_file = url_for('host.serve_file',
project_id=project_id,
filename=SECRETS_FILENAME,
key=project['secret_key'])
code_file = url_for('host.serve_file',
project_id=project_id,
filename=CODE_FILENAME)
deployment_url = project['deployment_url']
return render_template('work/index.html',
code_file=code_file,
secret=secret,
secrets_file=secrets_file,
deployment_url=deployment_url)
def detail(project_id):
"""Shows details of project for given project_id.
"""
# setup the database
db = get_db()
cursor = db.cursor()
# Get the project
project = cursor.execute('SELECT * FROM projects WHERE id=(?);',
(project_id, )).fetchone()
if project is None:
abort(404, 'Project does not exist')
# Check that this user can view the project
if project['owner_id'] != g.user['id'] and g.user['super_user'] == 'false':
# The current user doesn't own this project, don't show it to them
flash('You don\'t have permissions to view this project')
return redirect(url_for('index'))
# Set the status variable to string representation
project_approval_status = ApprovalStatus.WAITING.name
project_approved = False
if project['approval_status'] == ApprovalStatus.APPROVED.value:
project_approval_status = ApprovalStatus.APPROVED.name
project_approved = True
# update the status of the project if deploy is enabled
# only worth doing if the project is approved
if current_app.config['DO_DEPLOY']:
update_status(project_id)
return render_template('host/detail.html',
project=project,
project_approval_status=project_approval_status,
project_approved=project_approved)
def submit_project():
"""For submitting of new projects.
"""
if request.method == 'POST':
# pull the values from the request
name = request.form['name']
source_url = request.form['source-url']
description = request.form['description']
min_workers = request.form['min-workers']
# setup the database
db = get_db()
cursor = db.cursor()
error = None
# check that user input exists
if not name:
error = 'Name is required.'
elif not source_url:
error = 'Source URL is required.'
# description is not required
elif not min_workers:
error = 'Minimum number of workers is required.'
elif 'code-file' not in request.files:
error = 'Code file must be uploaded.'
if (error is None and 'code-file' in request.files
and request.files['code-file'].filename != ''
and request.files['code-file'].filename.rsplit(
'.', 1)[1].lower() != 'js'):
error = 'Code file must be a javascript file.'
if (error is None and 'secrets-file' in request.files
and request.files['secrets-file'].filename != ''
and request.files['secrets-file'].filename.rsplit(
'.', 1)[1].lower() != 'js'):
error = 'Secrests file must be a javascript file.'
if error is None:
# good to go forward with input
# generate hash_id from owner id and name
hash_id = generate_hash_id(g.user['id'], name)
deployment_subdomain = hash_id[:25]
deployment_url = deployment_subdomain + '.' + current_app.config[
'LOCALTUNNEL_URL']
# generate a secret key for the project
secret_key = ''.join(
random.SystemRandom().choice(string.ascii_uppercase +
string.ascii_lowercase +
string.digits) for _ in range(10))
cursor.execute(
('INSERT INTO projects (name, source_url, description, '
'min_workers, secret_key, hash_id, deployment_subdomain, '
'deployment_url, worker_count, owner_id) VALUES '
'(?, ?, ?, ?, ?, ?, ?, ?, ?, ?);'),
(name, source_url, description, min_workers, secret_key,
hash_id, deployment_subdomain, deployment_url, 0,
g.user['id']))
db.commit()
# save the code files to the deploy path with predescribed filenames
project_folder = get_project_folder(hash_id)
code_file = request.files['code-file']
code_file.save(os.path.join(project_folder, CODE_FILENAME))
if 'secrets-file' in request.files:
secrets_file = request.files['secrets-file']
secrets_file.save(
os.path.join(project_folder, SECRETS_FILENAME))
return redirect(url_for('index'))
# there was an error, fall through with flash
flash(error)
return render_template('host/submit.html')
