def test_get_close_db(app): with app.app_context(): db = get_db() # assert the same db object is returned assert db is get_db() with pytest.raises(sqlite3.ProgrammingError) as e: db.execute('SELECT 1') assert 'closed' in str(e)
def deploy_project(project_id): """Central driving function for deploying projects to AWS. """ logging.info('Begining deploy for project {}'.format(project_id)) # get the project from that database db = get_db() cursor = db.cursor() project = cursor.execute( 'SELECT * FROM projects where id=(?);', (project_id,)).fetchone() # generate hash if it doesn't already exist, but it should if project['hash_id'] is None or project['hash_id'] == '': hash_id = generate_hash_id(project['owner_id'], project['name']) # write the hash id to the project cursor.execute( 'UPDATE projects SET hash_id=(?) WHERE id=(?);', (hash_id, project_id,)) db.commit() else: hash_id = project['hash_id'] # first need to build config files build_config_files(hash_id, project['id'], project['min_workers'], project['secret_key'], project['deployment_subdomain']) start_container(hash_id) # get url update_status(project_id)
def get_work_page(): """Returns a work page for a project. """ # get all projects that are approved and running db = get_db() cursor = db.cursor() cursor.execute(('SELECT * FROM projects WHERE approval_status=(?) AND ' 'health_status LIKE "RUNNING";'), (ApprovalStatus.APPROVED.value, )) # algorithm to pick propject # version 1 is random projects = cursor.fetchall() # check that there are projects to work on if len(projects) == 0: flash('There are no projects to work on.') return redirect(url_for('index')) in_need_projects = list() for project in projects: if project['worker_count'] < project['min_workers']: in_need_projects.append(project) if len(in_need_projects) > 0: projects = in_need_projects project = random.choice(projects) return redirect( url_for('work.get_work_page_for_project', project_id=project['id']))
def delete(project_id): """Deletes a given project. Deletes the deploy information and stops the given container. """ db = get_db() # Get the project to verify owner identity project = db.execute('SELECT * FROM projects where id=(?);', (project_id, )).fetchone() if project is None: abort(404, 'Project does not exist') # Check that this user can delete the project if project['owner_id'] != g.user['id'] and g.user['super_user'] == 'false': # The current user doesn't own this project, don't delete it flash('You don\'t have permissions to delete this project') return redirect(url_for('host.detail', project_id=project_id)) # destroy the project if deploy is enabled if current_app.config['DO_DEPLOY']: destroy_project(project_id) # delete the database entry db.execute('DELETE FROM projects WHERE id=(?);', (project_id, )) db.commit() return redirect(url_for('index'))
def serve_file(project_id, filename): """Serves the requested file from the project's deployment folder. Only serves CODE_FILENAME and SECRETS_FILENAME Query arg: `key` - key used for serving the secrets file """ # get the project from the project id db = get_db() project = db.execute('SELECT * FROM projects WHERE id=(?);', (project_id, )).fetchone() if project is None: abort(404, 'Project does not exist') # make sure it's an allowable filename if filename != CODE_FILENAME and filename != SECRETS_FILENAME: abort(404, 'File not found.') # if the filename is SECRETS_FILENAME, make sure they have the key if filename == SECRETS_FILENAME: if (request.args.get('key') is None or request.args.get('key') != project['secret_key']): # They don't have a valid key, abort with an error abort(403, 'Key required for accessing secret file.') # get the project folder path project_folder_path = get_project_folder(project['hash_id']) # check that the file exists if not os.path.isfile(os.path.join(project_folder_path, filename)): abort(404, 'File does not exist.') # serve the requested file return send_from_directory(project_folder_path, filename)
def login(): """Handles GET and POST requests on the '/login' route.""" if request.method == 'POST': email = request.form['email'] password = request.form['password'] db = get_db() error = None cursor = db.cursor() cursor.execute('SELECT * FROM users WHERE email = ?', (email, )) user = cursor.fetchone() # Check to see that user input required fields and info is valid if not email: error = 'Email is required.' elif not password: error = 'Password is required.' elif user is None: error = 'Incorrect email.' elif not check_password_hash(user['password'], password): error = 'Incorrect password.' if error is None: session.clear() session['user_id'] = user['id'] return redirect(url_for('index')) flash(error) return render_template('auth/login.html')
def register(): """Handles GET and POST requests on the '/register' route.""" if request.method == 'POST': email = request.form['email'] password = request.form['password'] confirm_password = request.form['confirm-password'] db = get_db() error = None # input validation if not email: error = 'Email is required.' elif not password: error = 'Password is required.' elif not confirm_password: error = 'Password confirmation is required.' elif password != confirm_password: error = 'Passwords do not match.' # check that the given email isn't already registered cursor = db.cursor() cursor.execute('SELECT id FROM users WHERE email = ?', (email, )) if cursor.fetchone() is not None: error = 'Account cannot be registered' if error is None: cursor.execute('INSERT INTO users (email, password) VALUES (?, ?)', (email, generate_password_hash(password))) db.commit() return redirect(url_for('auth.login')) flash(error) return render_template('auth/register.html')
def update_status(project_id): # get the hash id from the project id from the database db = get_db() project = db.execute('SELECT * FROM projects where id=(?);', (project_id,)).fetchone() hash_id = project['hash_id'] (docker_compose_path, ecs_params_path) = get_config_file_paths(hash_id) # build the status command # TODO - I'm not sure if project-name means something different # TODO - see if I can just use 'cluster' instead of cluster-config status_cmd = ('{ecs_cli_path} compose --project-name {hash_id} ' '--ecs-params {ecs_params} ' '--file {docker_compose} ' 'ps ' '--cluster-config {cluster_config} ' '--ecs-profile {ecs_profile}') status_cmd = status_cmd.format(ecs_cli_path=current_app.config['ECS_CLI_PATH'], hash_id=hash_id, ecs_params=ecs_params_path, docker_compose = docker_compose_path, cluster_config=current_app.config['FLOCK_CLUSTER_CONFIG'], ecs_profile=current_app.config['FLOCK_ECS_PROFILE']) # Run the status command and capture the output logging.info('status command: ' + status_cmd) proc = subprocess.run(status_cmd.split(' '), stdout=subprocess.PIPE, stderr=subprocess.PIPE) # decode output and log it output = proc.stdout.decode('utf-8') print(output) logging.info('status output: ' + output) # find the IP address and health of the project using regex # Start with hash id, then look for optional colon followed by number # there will then be a status and eventually an IP # if the status is a fail condition, this regex won't match regex = ':?\d*\s*(?P<status>\w*)\s*(?P<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' regex = hash_id + regex match = re.search(regex, output) if match is not None: db.execute(('UPDATE projects SET deployment_ip=(?), health_status=(?) ' 'WHERE id=(?);'), (match.group('ip'), match.group('status'), project_id,)) db.commit() else: logging.error('Did not find ip or status') if re.search('STOPPED', output) is not None: print('Container didn\'t start.') logging.error('Container didn\'t start.') status = 'STOPPED' message = 'Container failed to start...' db.execute(('UPDATE projects SET health_status=(?), ' 'health_message=(?) WHERE id=(?);'), (status, message, project_id,)) db.commit()
def test_delete(client, auth, app): auth.login() response = client.get('/host/1/delete') with app.app_context(): db = get_db() project = db.execute('SELECT * FROM projects WHERE id=1;').fetchone() assert project is None
def app(): db_fd, db_path = tempfile.mkstemp() app = create_app({ 'TESTING': True, 'DATABASE': db_path, 'DO_DEPLOY': False, 'LOCALTUNNEL_URL': 'localtunnel.me' }) with app.app_context(): init_db() get_db().executescript(_data_sql) yield app os.close(db_fd) os.unlink(db_path)
def get_project_folder_from_id(project_id): """Returns the folder for a given project. Creates it if it does not exist. Safe to call not in a deployment environment. """ # get the hash_id db = get_db() hash_id = db.execute('SELECT hash_id FROM projects WHERE id=(?);', (project_id,)).fetchone() return get_project_folder(hash_id)
def load_logged_in_user(): """Loads the logged in user if the user id is set on the session.""" user_id = session.get('user_id') if user_id is None: g.user = None else: cursor = get_db().cursor() g.user = cursor.execute('SELECT * FROM users WHERE id = ?', (user_id, )).fetchone()
def test_approve(client, auth, app): """Try to approve project id = 1.""" auth.login_super() response = client.get('/host/1/approve') assert response.status_code == 302 assert response.headers['Location'].endswith('/host/queue') with app.app_context(): db = get_db() project = db.execute('SELECT * FROM projects WHERE id=1;').fetchone() assert project['approval_status'] == 1
def queue(): """Shows currently queued projects. """ # setup the database db = get_db() cursor = db.cursor() projects = cursor.execute( 'SELECT * FROM projects WHERE approval_status=(?);', (ApprovalStatus.WAITING.value, )).fetchall() return render_template('host/queue.html', projects=projects)
def test_serve_file(client, auth, app): # need to submit some files first auth.login() project_name = 'serve-file-test' code_file_content = b'CODE FILE' secrets_file_content = b'SECRETS FILE' client.post( '/host/submit', buffered=True, content_type='multipart/form-data', data = { 'name': project_name, 'source-url': 'https://zacharysang.com', 'min-workers': '1', 'description': 'Creation description', 'code-file': (BytesIO(code_file_content), 'user-code.js'), 'secrets-file': (BytesIO(secrets_file_content), 'secrets.js') } ) # project is created, logout because these urls don't require sessions auth.logout() # get app context so we can get the project id with app.app_context(): db = get_db() project = db.execute('SELECT * FROM projects WHERE name=(?);', (project_name,)).fetchone() # make sure the project exists assert project is not None # check a couple of different urls to make sure they return content # check for user-code.js response = client.get('/host/{}/file/user-code.js'.format(project['id'])) assert response.status_code == 200 assert code_file_content in response.data # check for secret.js without key response = client.get('/host/{}/file/secret.js'.format(project['id'])) assert response.status_code == 403 # check for secret.js with key response = client.get('/host/{}/file/secret.js?key={}' .format(project['id'], project['secret_key'])) assert response.status_code == 200 assert secrets_file_content in response.data # check for file that doesn't exist response = client.get('/host/{}/file/not-real.js'.format(project['id'])) assert response.status_code == 404
def test_node_0_communicate(client, app): """Test that node-0-communicate can update the worker_count of a project.""" with app.app_context(): db = get_db() project = db.execute('SELECT * FROM projects WHERE id=1;').fetchone() response = client.post( '/host/{}/node-0-communicate'.format(project['id']), content_type='application/json', data = json.dumps({ 'secret_key': project['secret_key'], 'worker_count': '100' }) ) assert response.status_code == 200 project = db.execute('SELECT * FROM projects WHERE id=1;').fetchone() assert project['worker_count'] == 100
def approve(project_id): """Approves the project associated with a given project_id """ db = get_db() cursor = db.cursor() cursor.execute('UPDATE projects SET approval_status=(?) WHERE id=(?);', ( ApprovalStatus.APPROVED.value, project_id, )) db.commit() # deploy the project if enabled if current_app.config['DO_DEPLOY']: deploy_project(project_id) return redirect(url_for('host.queue'))
def node_0_communicate(project_id): """An endpoint for the node 0 to send information to the master server. Required values in POST json: secret_key : secret key for project Optional values in POST json: deployment_url : the url the server is deployed at worker_count : the number of workers currently working on the project """ if request.method != 'POST': abort(405) # get the project to pull information from db = get_db() project = db.execute('SELECT * FROM projects WHERE id=(?);', (project_id, )).fetchone() if project is None: abort(404) if ('secret_key' not in request.json or request.json['secret_key'] != project['secret_key']): abort(403, 'Bad secret key.') # build the information that this can accept deployment_url = project['deployment_url'] worker_count = project['worker_count'] if 'deployment_url' in request.json: deployment_url = request.json['deployment_url'] if 'worker_count' in request.json: worker_count = request.json['worker_count'] # update the database db.execute(('UPDATE projects SET deployment_url=(?), ' 'worker_count=(?) WHERE id=(?);'), ( deployment_url, worker_count, project_id, )) db.commit() return '', 200
def test_register(client, app): # test that register page exists assert client.get('/register').status_code == 200 # send good register post user_email = '*****@*****.**' response = client.post( '/register', data={'email': user_email, 'password': '******', 'confirm-password': '******'} ) # redirects to login assert response.headers['Location'].endswith('/login') # Check the user was created with app.app_context(): assert get_db().execute( 'SELECT * FROM users WHERE email = (?)', ('*****@*****.**',) ).fetchone() is not None
def restart(project_id): """Endpoint that restarts the container and redirects back to project details. """ db = get_db() # verify that the project exists project = db.execute('SELECT * FROM projects WHERE id=(?);', (project_id, )).fetchone() if project is None: abort(404) # verify user is owner or an admin if project['owner_id'] != g.user['id'] and g.user['super_user'] == 'false': abort(403) restart_container(project['hash_id']) return redirect(url_for('host.detail', project_id=project_id))
def list(): """Renders a list of projects separated into my projects and all projects """ db = get_db() # check for a current user my_projects = None if g.user is None: projects = db.execute('SELECT * FROM projects;').fetchall() else: # there is a user, render both my projects and all projects my_projects = db.execute('SELECT * FROM projects WHERE owner_id=(?);', (g.user['id'], )).fetchall() projects = db.execute('SELECT * FROM projects WHERE owner_id!=(?);', (g.user['id'], )).fetchall() return render_template('host/list.html', projects=projects, my_projects=my_projects)
def destroy_project(project_id): """Destroys a given project by stopping container and deleting config files. """ logging.info('Destroying deployment for project {}'.format(project_id)) # get the project from that database db = get_db() project = db.execute( 'SELECT * FROM projects where id=(?);', (project_id,)).fetchone() # get hash id from project hash_id = project['hash_id'] stop_container(hash_id) # delete the folder with config details path = get_project_folder(hash_id) shutil.rmtree(path)
def test_submit(client, auth, app): """Tests a successful submit.""" auth.login() assert client.get('/host/submit').status_code == 200 client.post( '/host/submit', buffered=True, content_type='multipart/form-data', data = { 'name': 'submit-test', 'source-url': 'https://zacharysang.com', 'min-workers': '1', 'description': 'Creation description', 'code-file': (BytesIO(b'CODE_FILE'), 'user-code.js'), 'secrets-file': (BytesIO(b'SECRETS'), 'secrets.js') } ) with app.app_context(): db = get_db() count = db.execute('SELECT COUNT(id) FROM projects;').fetchone()[0] assert count == 3
def get_work_page_for_project(project_id): """Returns the work page for a specific project. """ # Get the project from the db db = get_db() project = db.execute('SELECT * FROM projects WHERE id=(?);', (project_id, )).fetchone() # check that this project exists if project is None: abort(404, 'Project not found.') # check that the project is running and approved if (project['health_status'] != 'RUNNING' or project['approval_status'] != ApprovalStatus.APPROVED.value): abort(400, 'Project not yet running or approved.') # check for key, which allows us to send secret file secret = False secrets_file = '' if request.args.get('key') == project['secret_key']: secret = True # flask makes key a query param: secrets_file = url_for('host.serve_file', project_id=project_id, filename=SECRETS_FILENAME, key=project['secret_key']) code_file = url_for('host.serve_file', project_id=project_id, filename=CODE_FILENAME) deployment_url = project['deployment_url'] return render_template('work/index.html', code_file=code_file, secret=secret, secrets_file=secrets_file, deployment_url=deployment_url)
def detail(project_id): """Shows details of project for given project_id. """ # setup the database db = get_db() cursor = db.cursor() # Get the project project = cursor.execute('SELECT * FROM projects WHERE id=(?);', (project_id, )).fetchone() if project is None: abort(404, 'Project does not exist') # Check that this user can view the project if project['owner_id'] != g.user['id'] and g.user['super_user'] == 'false': # The current user doesn't own this project, don't show it to them flash('You don\'t have permissions to view this project') return redirect(url_for('index')) # Set the status variable to string representation project_approval_status = ApprovalStatus.WAITING.name project_approved = False if project['approval_status'] == ApprovalStatus.APPROVED.value: project_approval_status = ApprovalStatus.APPROVED.name project_approved = True # update the status of the project if deploy is enabled # only worth doing if the project is approved if current_app.config['DO_DEPLOY']: update_status(project_id) return render_template('host/detail.html', project=project, project_approval_status=project_approval_status, project_approved=project_approved)
def submit_project(): """For submitting of new projects. """ if request.method == 'POST': # pull the values from the request name = request.form['name'] source_url = request.form['source-url'] description = request.form['description'] min_workers = request.form['min-workers'] # setup the database db = get_db() cursor = db.cursor() error = None # check that user input exists if not name: error = 'Name is required.' elif not source_url: error = 'Source URL is required.' # description is not required elif not min_workers: error = 'Minimum number of workers is required.' elif 'code-file' not in request.files: error = 'Code file must be uploaded.' if (error is None and 'code-file' in request.files and request.files['code-file'].filename != '' and request.files['code-file'].filename.rsplit( '.', 1)[1].lower() != 'js'): error = 'Code file must be a javascript file.' if (error is None and 'secrets-file' in request.files and request.files['secrets-file'].filename != '' and request.files['secrets-file'].filename.rsplit( '.', 1)[1].lower() != 'js'): error = 'Secrests file must be a javascript file.' if error is None: # good to go forward with input # generate hash_id from owner id and name hash_id = generate_hash_id(g.user['id'], name) deployment_subdomain = hash_id[:25] deployment_url = deployment_subdomain + '.' + current_app.config[ 'LOCALTUNNEL_URL'] # generate a secret key for the project secret_key = ''.join( random.SystemRandom().choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(10)) cursor.execute( ('INSERT INTO projects (name, source_url, description, ' 'min_workers, secret_key, hash_id, deployment_subdomain, ' 'deployment_url, worker_count, owner_id) VALUES ' '(?, ?, ?, ?, ?, ?, ?, ?, ?, ?);'), (name, source_url, description, min_workers, secret_key, hash_id, deployment_subdomain, deployment_url, 0, g.user['id'])) db.commit() # save the code files to the deploy path with predescribed filenames project_folder = get_project_folder(hash_id) code_file = request.files['code-file'] code_file.save(os.path.join(project_folder, CODE_FILENAME)) if 'secrets-file' in request.files: secrets_file = request.files['secrets-file'] secrets_file.save( os.path.join(project_folder, SECRETS_FILENAME)) return redirect(url_for('index')) # there was an error, fall through with flash flash(error) return render_template('host/submit.html')