def send_daily_stats(): day_ago = datetime.utcnow() - timedelta(days=1) new_registrations = get_db_session().query(User).filter( User.created_at >= day_ago).count() new_jobs = get_db_session().query(Job).filter( Job.created_at >= day_ago).count() new_job_runs = get_db_session().query(JobRun).filter( JobRun.created_at >= day_ago).count() _send_message(f"Last 24h update:\n" f"{new_registrations} registrations\n" f"{new_jobs} Jobs published\n" f"{new_job_runs} Job Runs (run button + scheduled)")
def test_execution_flow(get_path, send_update, session_id, user_id, workspace_id): job_runs_count_initial = get_db_session().query(JobRun).count() # create job we want to execute later job = Job(user_id=user_id, name=f'another_test_name_{session_id}', entrypoint='main_module.read_news', requirements='custom_requirements.txt', workspace_id=workspace_id) get_db_session().add(job) db_commit() job_service.execute(job.id, JobRunType.RunButton.value, user_id, workspace_id) assert send_update.called job_runs_count_new = get_db_session().query(JobRun).count() assert job_runs_count_initial + 1 == job_runs_count_new, "New JobRun should be created"
def update_templates(templates_package): try: _unpack_templates_archive(templates_package, EXTRACTED_PACKAGE_FOLDER_NAME) templates_list = _get_templates_list_from_config( EXTRACTED_PACKAGE_FOLDER_NAME, TEMPLATES_CONFIG_FILE) for template_config in templates_list: existing_template = JobTemplate.get_template_from_name( template_config['name'], get_db_session()) if existing_template: # Update existing record existing_template.short_description = template_config[ 'short_description'] existing_template.long_description_url = template_config[ 'long_description_url'] existing_template.tags = ','.join(template_config['tags']) existing_template.parameters = ','.join( template_config['parameters']) template = existing_template else: # create new record new_template = JobTemplate( name=template_config['name'], short_description=template_config['short_description'], long_description_url=template_config[ 'long_description_url'], tags=','.join(template_config['tags']), parameters=','.join(template_config['parameters'])) get_db_session().add(new_template) template = new_template db_commit() path_to_template_files = os.path.join( EXTRACTED_PACKAGE_FOLDER_NAME, template_config['path_to_files']) archive_location = _archive_template_files(str(template.id), path_to_template_files) with open(archive_location, "rb") as f: storage.save(io.BytesIO(f.read()), Type.Template, str(template.id)) # Notice that we do not delete templates automatically if we cannot find them in config # Deleting a template is a complex workflow (because of its dependencies) and also pretty risky # So for now you need to delete templates manually from the db if you need to finally: try: shutil.rmtree(EXTRACTED_PACKAGE_FOLDER_NAME) except OSError: pass
def get_by_api_key(api_key: str): try: user = User.get_user_from_api_key(api_key, get_db_session()) except NoResultFound: raise UserNotFoundException('API Key is wrong, please go to our account https://app.seamlesscloud.io/account,' ' copy the API Key field and run "smls init <api key>"') return user
def get_default_workspace(user_id: int): session = get_db_session() # default workspace will be personal, until we figure out how to setup preferences workspace = session.query(Workspace).filter_by(owner_id=user_id, personal=True).one() return workspace
def get_template(template_id): job_template = get_db_session().query(JobTemplate).filter_by( id=template_id).one_or_none() if job_template is None: raise JobTemplateNotFoundException( f"Job Template id:{template_id} Not Found") return job_template
def create(user_id: int, name: str, personal: bool): session = get_db_session() workspace = Workspace(owner_id=user_id, name=name, personal=personal) session.add(workspace) db_commit() return workspace.id
def delete(user_id: int): session = get_db_session() user = session.query(User).filter_by(id=user_id).one() personal_workspace = user.workspaces.filter_by(personal=True).one() session.delete(personal_workspace) session.delete(user) db_commit()
def create(email: str, api_key: str = None): session = get_db_session() user = User(email=email, api_key=api_key or generate_api_key()) session.add(user) db_commit() workspace = Workspace(owner_id=user.id) session.add(workspace) db_commit() return user.id
def test_marketplace_flow(web_client, automation_client, archived_templates_repo, user_id): resp = web_client.get('/api/v1/templates') assert resp.status_code == 200 assert len(resp.json) == 0 # There are no templates in the database yet resp = automation_client.post('/api/v1/marketplace', headers=_basic_auth_headers(), data={'templates': (archived_templates_repo, "templates.tar.gz")}, content_type="multipart/form-data") assert resp.status_code == 200 # Good, now let's make sure the data is in database templates_in_db = get_db_session().query(JobTemplate).all() assert len(templates_in_db) == 1 # We have only 1 template template = templates_in_db[0] # Data in db has to match the data in the marketplace_templates_files/table_of_contents.yml file assert template.name == 'template1' assert template.short_description == 'template1' assert template.long_description_url == 'template1' assert template.tags == 'tag_1,tag_2' assert template.parameters == 'PARAM_1,PARAM_2' file_on_s3 = storage.get_archive(Type.Template, str(template.id)) assert file_on_s3.getbuffer().nbytes > 0 # There is a real file that was saved to s3 # TODO assert that the file actually corresponds to files of the template resp = web_client.get('/api/v1/templates') assert resp.status_code == 200 assert len(resp.json) == 1 # There is one template that we've just uploaded template_from_json = resp.json[0] assert template_from_json['name'] == 'template1' assert template_from_json['short_description'] == 'template1' assert template_from_json['long_description_url'] == 'template1' assert template_from_json['tags'] == 'tag_1,tag_2' assert template_from_json['parameters'] == 'PARAM_1,PARAM_2' template_id = template_from_json["id"] # Now let's test if we can create a job from this template resp = web_client.post(f'/api/v1/templates/{template_id}/create_job') assert resp.status_code == 200 job_from_template_id = resp.json['job_id'] job = job_service.get(job_from_template_id, user_id) # The job we've just created is linked to the template assert str(job.job_template_id) == template_id # And the parameters are created based on template assert job.parameters[0].key == 'PARAM_1' assert job.parameters[1].key == 'PARAM_2' assert job.parameters[0].value is None assert job.parameters[1].value is None
def _run_job_and_assert_logs(web_client, job_id, string_to_find_in_logs): rv = web_client.post(f'/api/v1/jobs/{job_id}/run') assert rv.status_code == 200 # We've just triggered a Job run job_run = get_db_session().query(JobRun).filter( JobRun.job_id == job_id).one() logs = [] for i in range(5): sleep(1) logs = list(job_run.logs) if len(logs) == 1: # We are waiting for a single logs line to appear break if len(logs) == 0: assert False, "Running the job with parameters did not produce logs in the database" else: assert len(logs) == 1 assert string_to_find_in_logs in logs[0].message
def test_code_update(web_client, published_job_default_requirements, user_id): job_id = published_job_default_requirements res = web_client.put(f'/api/v1/jobs/{job_id}/source-code', json={ 'filename': DEFAULT_ENTRYPOINT, 'contents': NEW_CODE_CONTENTS }) assert res.status_code == 200 job_service.execute(job_id, JobRunType.RunButton.value, user_id) session = get_db_session() job_run = session.query(JobRun).filter_by(job_id=job_id).one() logs = [log.message for log in job_run.logs] assert f"{PRINT_MSG}\n" in logs, "There should be custom message provided in the test"
def test_parameters_flow(web_client, published_job_no_requirements): test_parameter_1 = {"key": "key1", "value": "value1"} test_parameter_2 = {"key": "key2", "value": "value2"} rv = web_client.get( f'/api/v1/jobs/{published_job_no_requirements}/parameters') assert rv.status_code == 200 assert len( rv.json) == 0 # There are no parameters attached to this job yet rv = web_client.post( f'/api/v1/jobs/{published_job_no_requirements}/parameters', json=test_parameter_1) assert rv.status_code == 200 # We've added one parameter rv = web_client.get( f'/api/v1/jobs/{published_job_no_requirements}/parameters') assert rv.status_code == 200 assert len(rv.json) == 1 # There is one parameter that we've added parameter = rv.json[0] assert parameter['key'] == test_parameter_1['key'] assert parameter['value'] == test_parameter_1['value'] rv = web_client.post( f'/api/v1/jobs/{published_job_no_requirements}/parameters', json=test_parameter_2) assert rv.status_code == 200 # We've added a second parameter rv = web_client.get( f'/api/v1/jobs/{published_job_no_requirements}/parameters') assert rv.status_code == 200 assert len(rv.json) == 2 # There are two parameters that we've added parameter_1 = rv.json[0] parameter_2 = rv.json[1] assert parameter_1['key'] == test_parameter_1['key'] assert parameter_1['value'] == test_parameter_1['value'] assert parameter_2['key'] == test_parameter_2['key'] assert parameter_2['value'] == test_parameter_2['value'] rv = web_client.delete( f'/api/v1/jobs/{published_job_no_requirements}/parameters/{parameter_2["id"]}' ) assert rv.status_code == 200 # We've just deleted a second parameter rv = web_client.get( f'/api/v1/jobs/{published_job_no_requirements}/parameters') assert rv.status_code == 200 assert len(rv.json) == 1 # We're back to the one parameter we added first parameter = rv.json[0] assert parameter['key'] == test_parameter_1['key'] assert parameter['value'] == test_parameter_1['value'] rv = web_client.post(f'/api/v1/jobs/{published_job_no_requirements}/run') assert rv.status_code == 200 # We've just triggered a Job run job_run = get_db_session().query(JobRun).filter( JobRun.job_id == published_job_no_requirements).one() logs = [] for i in range(5): sleep(1) logs = list(job_run.logs) if len(logs) == 1: # We are waiting for a single logs line to appear break if len(logs) == 0: assert False, "Running the job with parameters did not produce logs in the database" else: assert len(logs) == 1 assert test_parameter_1['value'] in logs[ 0].message # Logs have the value of a parameter
def get_by_id(user_id: str): try: user = User.get_user_from_id(user_id, get_db_session()) except NoResultFound: raise UserNotFoundException(f'Cannot find a user with id: {user_id}') return user
def get(workspace_id: int): session = get_db_session() return session.query(Workspace).filter_by(id=workspace_id).one()
def delete(workspace_id: int): session = get_db_session() session.query(Workspace).filter_by(id=workspace_id).delete() db_commit()
def get_templates(): return get_db_session().query(JobTemplate).all()