def send_daily_stats():
day_ago = datetime.utcnow() - timedelta(days=1)
new_registrations = get_db_session().query(User).filter(
User.created_at >= day_ago).count()
new_jobs = get_db_session().query(Job).filter(
Job.created_at >= day_ago).count()
new_job_runs = get_db_session().query(JobRun).filter(
JobRun.created_at >= day_ago).count()
_send_message(f"Last 24h update:\n"
f"{new_registrations} registrations\n"
f"{new_jobs} Jobs published\n"
f"{new_job_runs} Job Runs (run button + scheduled)")
def test_execution_flow(get_path, send_update, session_id, user_id, workspace_id):
job_runs_count_initial = get_db_session().query(JobRun).count()
# create job we want to execute later
job = Job(user_id=user_id, name=f'another_test_name_{session_id}',
entrypoint='main_module.read_news', requirements='custom_requirements.txt', workspace_id=workspace_id)
get_db_session().add(job)
db_commit()
job_service.execute(job.id, JobRunType.RunButton.value, user_id, workspace_id)
assert send_update.called
job_runs_count_new = get_db_session().query(JobRun).count()
assert job_runs_count_initial + 1 == job_runs_count_new, "New JobRun should be created"
def update_templates(templates_package):
try:
_unpack_templates_archive(templates_package,
EXTRACTED_PACKAGE_FOLDER_NAME)
templates_list = _get_templates_list_from_config(
EXTRACTED_PACKAGE_FOLDER_NAME, TEMPLATES_CONFIG_FILE)
for template_config in templates_list:
existing_template = JobTemplate.get_template_from_name(
template_config['name'], get_db_session())
if existing_template:
# Update existing record
existing_template.short_description = template_config[
'short_description']
existing_template.long_description_url = template_config[
'long_description_url']
existing_template.tags = ','.join(template_config['tags'])
existing_template.parameters = ','.join(
template_config['parameters'])
template = existing_template
else:
# create new record
new_template = JobTemplate(
name=template_config['name'],
short_description=template_config['short_description'],
long_description_url=template_config[
'long_description_url'],
tags=','.join(template_config['tags']),
parameters=','.join(template_config['parameters']))
get_db_session().add(new_template)
template = new_template
db_commit()
path_to_template_files = os.path.join(
EXTRACTED_PACKAGE_FOLDER_NAME,
template_config['path_to_files'])
archive_location = _archive_template_files(str(template.id),
path_to_template_files)
with open(archive_location, "rb") as f:
storage.save(io.BytesIO(f.read()), Type.Template,
str(template.id))
# Notice that we do not delete templates automatically if we cannot find them in config
# Deleting a template is a complex workflow (because of its dependencies) and also pretty risky
# So for now you need to delete templates manually from the db if you need to
finally:
try:
shutil.rmtree(EXTRACTED_PACKAGE_FOLDER_NAME)
except OSError:
pass
def get_by_api_key(api_key: str):
try:
user = User.get_user_from_api_key(api_key, get_db_session())
except NoResultFound:
raise UserNotFoundException('API Key is wrong, please go to our account https://app.seamlesscloud.io/account,'
' copy the API Key field and run "smls init <api key>"')
return user
def get_default_workspace(user_id: int):
session = get_db_session()
# default workspace will be personal, until we figure out how to setup preferences
workspace = session.query(Workspace).filter_by(owner_id=user_id,
personal=True).one()
return workspace
def get_template(template_id):
job_template = get_db_session().query(JobTemplate).filter_by(
id=template_id).one_or_none()
if job_template is None:
raise JobTemplateNotFoundException(
f"Job Template id:{template_id} Not Found")
return job_template
def create(user_id: int, name: str, personal: bool):
session = get_db_session()
workspace = Workspace(owner_id=user_id, name=name, personal=personal)
session.add(workspace)
db_commit()
return workspace.id
def delete(user_id: int):
session = get_db_session()
user = session.query(User).filter_by(id=user_id).one()
personal_workspace = user.workspaces.filter_by(personal=True).one()
session.delete(personal_workspace)
session.delete(user)
db_commit()
def create(email: str, api_key: str = None):
session = get_db_session()
user = User(email=email,
api_key=api_key or generate_api_key())
session.add(user)
db_commit()
workspace = Workspace(owner_id=user.id)
session.add(workspace)
db_commit()
return user.id
def test_marketplace_flow(web_client, automation_client, archived_templates_repo, user_id):
resp = web_client.get('/api/v1/templates')
assert resp.status_code == 200
assert len(resp.json) == 0 # There are no templates in the database yet
resp = automation_client.post('/api/v1/marketplace', headers=_basic_auth_headers(),
data={'templates': (archived_templates_repo, "templates.tar.gz")},
content_type="multipart/form-data")
assert resp.status_code == 200 # Good, now let's make sure the data is in database
templates_in_db = get_db_session().query(JobTemplate).all()
assert len(templates_in_db) == 1 # We have only 1 template
template = templates_in_db[0]
# Data in db has to match the data in the marketplace_templates_files/table_of_contents.yml file
assert template.name == 'template1'
assert template.short_description == 'template1'
assert template.long_description_url == 'template1'
assert template.tags == 'tag_1,tag_2'
assert template.parameters == 'PARAM_1,PARAM_2'
file_on_s3 = storage.get_archive(Type.Template, str(template.id))
assert file_on_s3.getbuffer().nbytes > 0 # There is a real file that was saved to s3
# TODO assert that the file actually corresponds to files of the template
resp = web_client.get('/api/v1/templates')
assert resp.status_code == 200
assert len(resp.json) == 1 # There is one template that we've just uploaded
template_from_json = resp.json[0]
assert template_from_json['name'] == 'template1'
assert template_from_json['short_description'] == 'template1'
assert template_from_json['long_description_url'] == 'template1'
assert template_from_json['tags'] == 'tag_1,tag_2'
assert template_from_json['parameters'] == 'PARAM_1,PARAM_2'
template_id = template_from_json["id"]
# Now let's test if we can create a job from this template
resp = web_client.post(f'/api/v1/templates/{template_id}/create_job')
assert resp.status_code == 200
job_from_template_id = resp.json['job_id']
job = job_service.get(job_from_template_id, user_id)
# The job we've just created is linked to the template
assert str(job.job_template_id) == template_id
# And the parameters are created based on template
assert job.parameters[0].key == 'PARAM_1'
assert job.parameters[1].key == 'PARAM_2'
assert job.parameters[0].value is None
assert job.parameters[1].value is None
def _run_job_and_assert_logs(web_client, job_id, string_to_find_in_logs):
rv = web_client.post(f'/api/v1/jobs/{job_id}/run')
assert rv.status_code == 200 # We've just triggered a Job run
job_run = get_db_session().query(JobRun).filter(
JobRun.job_id == job_id).one()
logs = []
for i in range(5):
sleep(1)
logs = list(job_run.logs)
if len(logs) == 1: # We are waiting for a single logs line to appear
break
if len(logs) == 0:
assert False, "Running the job with parameters did not produce logs in the database"
else:
assert len(logs) == 1
assert string_to_find_in_logs in logs[0].message
def test_code_update(web_client, published_job_default_requirements, user_id):
job_id = published_job_default_requirements
res = web_client.put(f'/api/v1/jobs/{job_id}/source-code',
json={
'filename': DEFAULT_ENTRYPOINT,
'contents': NEW_CODE_CONTENTS
})
assert res.status_code == 200
job_service.execute(job_id, JobRunType.RunButton.value, user_id)
session = get_db_session()
job_run = session.query(JobRun).filter_by(job_id=job_id).one()
logs = [log.message for log in job_run.logs]
assert f"{PRINT_MSG}\n" in logs, "There should be custom message provided in the test"
def test_parameters_flow(web_client, published_job_no_requirements):
test_parameter_1 = {"key": "key1", "value": "value1"}
test_parameter_2 = {"key": "key2", "value": "value2"}
rv = web_client.get(
f'/api/v1/jobs/{published_job_no_requirements}/parameters')
assert rv.status_code == 200
assert len(
rv.json) == 0 # There are no parameters attached to this job yet
rv = web_client.post(
f'/api/v1/jobs/{published_job_no_requirements}/parameters',
json=test_parameter_1)
assert rv.status_code == 200 # We've added one parameter
rv = web_client.get(
f'/api/v1/jobs/{published_job_no_requirements}/parameters')
assert rv.status_code == 200
assert len(rv.json) == 1 # There is one parameter that we've added
parameter = rv.json[0]
assert parameter['key'] == test_parameter_1['key']
assert parameter['value'] == test_parameter_1['value']
rv = web_client.post(
f'/api/v1/jobs/{published_job_no_requirements}/parameters',
json=test_parameter_2)
assert rv.status_code == 200 # We've added a second parameter
rv = web_client.get(
f'/api/v1/jobs/{published_job_no_requirements}/parameters')
assert rv.status_code == 200
assert len(rv.json) == 2 # There are two parameters that we've added
parameter_1 = rv.json[0]
parameter_2 = rv.json[1]
assert parameter_1['key'] == test_parameter_1['key']
assert parameter_1['value'] == test_parameter_1['value']
assert parameter_2['key'] == test_parameter_2['key']
assert parameter_2['value'] == test_parameter_2['value']
rv = web_client.delete(
f'/api/v1/jobs/{published_job_no_requirements}/parameters/{parameter_2["id"]}'
)
assert rv.status_code == 200 # We've just deleted a second parameter
rv = web_client.get(
f'/api/v1/jobs/{published_job_no_requirements}/parameters')
assert rv.status_code == 200
assert len(rv.json) == 1 # We're back to the one parameter we added first
parameter = rv.json[0]
assert parameter['key'] == test_parameter_1['key']
assert parameter['value'] == test_parameter_1['value']
rv = web_client.post(f'/api/v1/jobs/{published_job_no_requirements}/run')
assert rv.status_code == 200 # We've just triggered a Job run
job_run = get_db_session().query(JobRun).filter(
JobRun.job_id == published_job_no_requirements).one()
logs = []
for i in range(5):
sleep(1)
logs = list(job_run.logs)
if len(logs) == 1: # We are waiting for a single logs line to appear
break
if len(logs) == 0:
assert False, "Running the job with parameters did not produce logs in the database"
else:
assert len(logs) == 1
assert test_parameter_1['value'] in logs[
0].message # Logs have the value of a parameter
def get_by_id(user_id: str):
try:
user = User.get_user_from_id(user_id, get_db_session())
except NoResultFound:
raise UserNotFoundException(f'Cannot find a user with id: {user_id}')
return user
def get(workspace_id: int):
session = get_db_session()
return session.query(Workspace).filter_by(id=workspace_id).one()
def delete(workspace_id: int):
session = get_db_session()
session.query(Workspace).filter_by(id=workspace_id).delete()
db_commit()
def get_templates():
return get_db_session().query(JobTemplate).all()
