def delete_job_parameter(job_id: int, user_id: int, parameter_id: int): job = get(job_id, user_id) affected_rows = job.parameters.filter_by(id=parameter_id).delete() if affected_rows == 0: raise ParameterNotFoundException( f'Cannot delete parameter {parameter_id}: Not Found') db_commit()
def delete(job_id: int, user_id: int, workspace_id: Optional[int] = None): job = get(job_id, user_id, workspace_id) remove_job_schedule(job_id) storage.delete(storage.Type.Job, job_id) get_db_session().delete(job) db_commit() return job_id
def create(user_id: int, name: str, personal: bool): session = get_db_session() workspace = Workspace(owner_id=user_id, name=name, personal=personal) session.add(workspace) db_commit() return workspace.id
def update_job_parameter(job_id: int, user_id: int, parameter_id: int, param_key: str, param_value: str): job = get(job_id, user_id) found_parameter = job.parameters.filter_by(id=parameter_id).one_or_none() if not found_parameter: raise ParameterNotFoundException( f'Cannot update parameter {parameter_id}: Not Found') found_parameter.key = param_key found_parameter.value = param_value db_commit()
def delete(user_id: int): session = get_db_session() user = session.query(User).filter_by(id=user_id).one() personal_workspace = user.workspaces.filter_by(personal=True).one() session.delete(personal_workspace) session.delete(user) db_commit()
def update_schedule(job_id: int, user_id: int, cron: str): job = get(job_id, user_id) aws_cron, human_cron = parse_cron(cron) job.cron = cron job.aws_cron = aws_cron job.human_cron = human_cron db_commit() job.schedule_job()
def create(email: str, api_key: str = None): session = get_db_session() user = User(email=email, api_key=api_key or generate_api_key()) session.add(user) db_commit() workspace = Workspace(owner_id=user.id) session.add(workspace) db_commit() return user.id
def test_execution_flow(get_path, send_update, session_id, user_id, workspace_id): job_runs_count_initial = get_db_session().query(JobRun).count() # create job we want to execute later job = Job(user_id=user_id, name=f'another_test_name_{session_id}', entrypoint='main_module.read_news', requirements='custom_requirements.txt', workspace_id=workspace_id) get_db_session().add(job) db_commit() job_service.execute(job.id, JobRunType.RunButton.value, user_id, workspace_id) assert send_update.called job_runs_count_new = get_db_session().query(JobRun).count() assert job_runs_count_initial + 1 == job_runs_count_new, "New JobRun should be created"
def update_templates(templates_package): try: _unpack_templates_archive(templates_package, EXTRACTED_PACKAGE_FOLDER_NAME) templates_list = _get_templates_list_from_config( EXTRACTED_PACKAGE_FOLDER_NAME, TEMPLATES_CONFIG_FILE) for template_config in templates_list: existing_template = JobTemplate.get_template_from_name( template_config['name'], get_db_session()) if existing_template: # Update existing record existing_template.short_description = template_config[ 'short_description'] existing_template.long_description_url = template_config[ 'long_description_url'] existing_template.tags = ','.join(template_config['tags']) existing_template.parameters = ','.join( template_config['parameters']) template = existing_template else: # create new record new_template = JobTemplate( name=template_config['name'], short_description=template_config['short_description'], long_description_url=template_config[ 'long_description_url'], tags=','.join(template_config['tags']), parameters=','.join(template_config['parameters'])) get_db_session().add(new_template) template = new_template db_commit() path_to_template_files = os.path.join( EXTRACTED_PACKAGE_FOLDER_NAME, template_config['path_to_files']) archive_location = _archive_template_files(str(template.id), path_to_template_files) with open(archive_location, "rb") as f: storage.save(io.BytesIO(f.read()), Type.Template, str(template.id)) # Notice that we do not delete templates automatically if we cannot find them in config # Deleting a template is a complex workflow (because of its dependencies) and also pretty risky # So for now you need to delete templates manually from the db if you need to finally: try: shutil.rmtree(EXTRACTED_PACKAGE_FOLDER_NAME) except OSError: pass
def _trigger_job_run(job: Job, trigger_type: str, user_id: int) -> Optional[int]: job_run = JobRun(job_id=job.id, type=trigger_type) get_db_session().add(job_run) db_commit() # we need to have an id generated before we start writing logs send_update( 'status', { 'job_id': str(job.id), 'job_run_id': str(job_run.id), 'status': job_run.status }, user_id) job_entrypoint = job.entrypoint or constants.DEFAULT_ENTRYPOINT job_requirements = job.requirements or constants.DEFAULT_REQUIREMENTS path_to_job_files = storage.get_path_to_files(storage.Type.Job, job.id) try: with executor.execute(path_to_job_files, job_entrypoint, job.get_parameters_as_dict(), job_requirements, _generate_container_name(str( job.id), user_id)) as executor_result: logs, get_exit_code = executor_result.output, executor_result.get_exit_code for line in logs: _create_log_entry(line, job.id, job_run.id, user_id) exit_code = get_exit_code() except ExecutorBuildException as exc: logs, get_exit_code = (el for el in [str(exc)]), lambda: 1 for line in logs: _create_log_entry(line, job.id, job_run.id, user_id) exit_code = get_exit_code() if exit_code == 0: job_run.status = JobRunStatus.Ok.value else: job_run.status = JobRunStatus.Failed.value db_commit() send_update( 'status', { 'job_id': str(job.id), 'job_run_id': str(job_run.id), 'status': job_run.status }, user_id) return exit_code
def execute(job_id: int, trigger_type: str, user_id: int, workspace_id: Optional[int] = None): """ Implementing the logic of Job execution """ job = get(job_id, user_id, workspace_id) job.status = JobStatus.Executing.value exit_code = _trigger_job_run(job, trigger_type, user_id) if exit_code == 0: job.status = JobStatus.Ok.value else: job.status = JobStatus.Failed.value db_commit()
def add_parameters_to_job(job_id: int, user_id: int, parameters: List[Tuple[str, Optional[str]]]): job = get(job_id, user_id) if len(list(job.parameters)) + len(parameters) > PARAMETERS_LIMIT_PER_JOB: raise JobsParametersLimitExceededException( f"You cannot have more than {PARAMETERS_LIMIT_PER_JOB} " f"Parameters per single Job.") for key, value in parameters: parameter = JobParameter(job_id=job.id, key=key, value=value) get_db_session().add(parameter) try: db_commit() except IntegrityError as e: # Potential duplicate Key value. Let's check. existing_keys = set([parameter.key for parameter in job.parameters]) new_keys = set([key for key, value in parameters]) duplicate_keys = set.intersection(existing_keys, new_keys) if len(duplicate_keys) > 0: raise DuplicateParameterKeyException( "Parameter with the same Key already exists.") else: raise e
def publish(name: str, cron: str, entrypoint: str, requirements: str, user: User, project_file: io.BytesIO, workspace_id: str, schedule_is_active=True): existing_job = get_db_session().query(Job).filter_by( name=name, user_id=user.id).one_or_none() if existing_job: job = _update_job(existing_job, cron, entrypoint, requirements) else: _check_user_quotas_for_job_creation(user) job = _create_job_in_db(name, cron, entrypoint, requirements, user.id, schedule_is_active, workspace_id) db_commit() job.schedule_job() storage.save(project_file, storage.Type.Job, job.id) return job, bool(existing_job)
def link_job_to_template(job: Job, template_id: int): job.job_template_id = template_id db_commit()
def delete(workspace_id: int): session = get_db_session() session.query(Workspace).filter_by(id=workspace_id).delete() db_commit()
def disable_schedule(job_id: int, user_id: int): job = get(job_id, user_id) disable_job_schedule(job_id) job.schedule_is_active = False db_commit()
def enable_schedule(job_id: int, user_id: int): job = get(job_id, user_id) enable_job_schedule(job_id) job.schedule_is_active = True db_commit()