def delete_job_parameter(job_id: int, user_id: int, parameter_id: int):
job = get(job_id, user_id)
affected_rows = job.parameters.filter_by(id=parameter_id).delete()
if affected_rows == 0:
raise ParameterNotFoundException(
f'Cannot delete parameter {parameter_id}: Not Found')
db_commit()
def delete(job_id: int, user_id: int, workspace_id: Optional[int] = None):
job = get(job_id, user_id, workspace_id)
remove_job_schedule(job_id)
storage.delete(storage.Type.Job, job_id)
get_db_session().delete(job)
db_commit()
return job_id
def create(user_id: int, name: str, personal: bool):
session = get_db_session()
workspace = Workspace(owner_id=user_id, name=name, personal=personal)
session.add(workspace)
db_commit()
return workspace.id
def update_job_parameter(job_id: int, user_id: int, parameter_id: int,
param_key: str, param_value: str):
job = get(job_id, user_id)
found_parameter = job.parameters.filter_by(id=parameter_id).one_or_none()
if not found_parameter:
raise ParameterNotFoundException(
f'Cannot update parameter {parameter_id}: Not Found')
found_parameter.key = param_key
found_parameter.value = param_value
db_commit()
def delete(user_id: int):
session = get_db_session()
user = session.query(User).filter_by(id=user_id).one()
personal_workspace = user.workspaces.filter_by(personal=True).one()
session.delete(personal_workspace)
session.delete(user)
db_commit()
def update_schedule(job_id: int, user_id: int, cron: str):
job = get(job_id, user_id)
aws_cron, human_cron = parse_cron(cron)
job.cron = cron
job.aws_cron = aws_cron
job.human_cron = human_cron
db_commit()
job.schedule_job()
def create(email: str, api_key: str = None):
session = get_db_session()
user = User(email=email,
api_key=api_key or generate_api_key())
session.add(user)
db_commit()
workspace = Workspace(owner_id=user.id)
session.add(workspace)
db_commit()
return user.id
def test_execution_flow(get_path, send_update, session_id, user_id, workspace_id):
job_runs_count_initial = get_db_session().query(JobRun).count()
# create job we want to execute later
job = Job(user_id=user_id, name=f'another_test_name_{session_id}',
entrypoint='main_module.read_news', requirements='custom_requirements.txt', workspace_id=workspace_id)
get_db_session().add(job)
db_commit()
job_service.execute(job.id, JobRunType.RunButton.value, user_id, workspace_id)
assert send_update.called
job_runs_count_new = get_db_session().query(JobRun).count()
assert job_runs_count_initial + 1 == job_runs_count_new, "New JobRun should be created"
def update_templates(templates_package):
try:
_unpack_templates_archive(templates_package,
EXTRACTED_PACKAGE_FOLDER_NAME)
templates_list = _get_templates_list_from_config(
EXTRACTED_PACKAGE_FOLDER_NAME, TEMPLATES_CONFIG_FILE)
for template_config in templates_list:
existing_template = JobTemplate.get_template_from_name(
template_config['name'], get_db_session())
if existing_template:
# Update existing record
existing_template.short_description = template_config[
'short_description']
existing_template.long_description_url = template_config[
'long_description_url']
existing_template.tags = ','.join(template_config['tags'])
existing_template.parameters = ','.join(
template_config['parameters'])
template = existing_template
else:
# create new record
new_template = JobTemplate(
name=template_config['name'],
short_description=template_config['short_description'],
long_description_url=template_config[
'long_description_url'],
tags=','.join(template_config['tags']),
parameters=','.join(template_config['parameters']))
get_db_session().add(new_template)
template = new_template
db_commit()
path_to_template_files = os.path.join(
EXTRACTED_PACKAGE_FOLDER_NAME,
template_config['path_to_files'])
archive_location = _archive_template_files(str(template.id),
path_to_template_files)
with open(archive_location, "rb") as f:
storage.save(io.BytesIO(f.read()), Type.Template,
str(template.id))
# Notice that we do not delete templates automatically if we cannot find them in config
# Deleting a template is a complex workflow (because of its dependencies) and also pretty risky
# So for now you need to delete templates manually from the db if you need to
finally:
try:
shutil.rmtree(EXTRACTED_PACKAGE_FOLDER_NAME)
except OSError:
pass
def _trigger_job_run(job: Job, trigger_type: str,
user_id: int) -> Optional[int]:
job_run = JobRun(job_id=job.id, type=trigger_type)
get_db_session().add(job_run)
db_commit() # we need to have an id generated before we start writing logs
send_update(
'status', {
'job_id': str(job.id),
'job_run_id': str(job_run.id),
'status': job_run.status
}, user_id)
job_entrypoint = job.entrypoint or constants.DEFAULT_ENTRYPOINT
job_requirements = job.requirements or constants.DEFAULT_REQUIREMENTS
path_to_job_files = storage.get_path_to_files(storage.Type.Job, job.id)
try:
with executor.execute(path_to_job_files, job_entrypoint,
job.get_parameters_as_dict(), job_requirements,
_generate_container_name(str(
job.id), user_id)) as executor_result:
logs, get_exit_code = executor_result.output, executor_result.get_exit_code
for line in logs:
_create_log_entry(line, job.id, job_run.id, user_id)
exit_code = get_exit_code()
except ExecutorBuildException as exc:
logs, get_exit_code = (el for el in [str(exc)]), lambda: 1
for line in logs:
_create_log_entry(line, job.id, job_run.id, user_id)
exit_code = get_exit_code()
if exit_code == 0:
job_run.status = JobRunStatus.Ok.value
else:
job_run.status = JobRunStatus.Failed.value
db_commit()
send_update(
'status', {
'job_id': str(job.id),
'job_run_id': str(job_run.id),
'status': job_run.status
}, user_id)
return exit_code
def execute(job_id: int,
trigger_type: str,
user_id: int,
workspace_id: Optional[int] = None):
"""
Implementing the logic of Job execution
"""
job = get(job_id, user_id, workspace_id)
job.status = JobStatus.Executing.value
exit_code = _trigger_job_run(job, trigger_type, user_id)
if exit_code == 0:
job.status = JobStatus.Ok.value
else:
job.status = JobStatus.Failed.value
db_commit()
def add_parameters_to_job(job_id: int, user_id: int,
parameters: List[Tuple[str, Optional[str]]]):
job = get(job_id, user_id)
if len(list(job.parameters)) + len(parameters) > PARAMETERS_LIMIT_PER_JOB:
raise JobsParametersLimitExceededException(
f"You cannot have more than {PARAMETERS_LIMIT_PER_JOB} "
f"Parameters per single Job.")
for key, value in parameters:
parameter = JobParameter(job_id=job.id, key=key, value=value)
get_db_session().add(parameter)
try:
db_commit()
except IntegrityError as e:
# Potential duplicate Key value. Let's check.
existing_keys = set([parameter.key for parameter in job.parameters])
new_keys = set([key for key, value in parameters])
duplicate_keys = set.intersection(existing_keys, new_keys)
if len(duplicate_keys) > 0:
raise DuplicateParameterKeyException(
"Parameter with the same Key already exists.")
else:
raise e
def publish(name: str,
cron: str,
entrypoint: str,
requirements: str,
user: User,
project_file: io.BytesIO,
workspace_id: str,
schedule_is_active=True):
existing_job = get_db_session().query(Job).filter_by(
name=name, user_id=user.id).one_or_none()
if existing_job:
job = _update_job(existing_job, cron, entrypoint, requirements)
else:
_check_user_quotas_for_job_creation(user)
job = _create_job_in_db(name, cron, entrypoint, requirements, user.id,
schedule_is_active, workspace_id)
db_commit()
job.schedule_job()
storage.save(project_file, storage.Type.Job, job.id)
return job, bool(existing_job)
def link_job_to_template(job: Job, template_id: int):
job.job_template_id = template_id
db_commit()
def delete(workspace_id: int):
session = get_db_session()
session.query(Workspace).filter_by(id=workspace_id).delete()
db_commit()
def disable_schedule(job_id: int, user_id: int):
job = get(job_id, user_id)
disable_job_schedule(job_id)
job.schedule_is_active = False
db_commit()
def enable_schedule(job_id: int, user_id: int):
job = get(job_id, user_id)
enable_job_schedule(job_id)
job.schedule_is_active = True
db_commit()
