def update_job(job_id, job):
validator.validate_job(job)
rd = _get_redis()
# assert that the key is in redis
# if it isn't raise NotFoundError
# validate the job
# upsert job into redis at hash/key
# update the crontab
return job
def __init__(self, environment, shared_properties, job_list, properties_path, git_repo = None):
'''
Take configured jobs as arguments, validate configuration, generate xml output.
Orchestrate generation of overall workflow with subworkflows if applicable.
Change oozie parameters based on environment specified, read from yaml file.
'''
# Set environment, cluster properties, shared properties, passed jobs,
# and email flag as instance variables. Cluster properties and environment
# are validated in custom validation method.
self.environment = environment
self.cluster_properties = validator.validate_properties(properties_path, self.environment)
self.job_properties = shared_properties
self.jobs = {}
self.conclusion_email = 'email' in self.job_properties
self.submitted = False
# Check that job_list is a list.
if type(job_list) != list:
raise validator.ListError('job_list')
# Check that emails are passed as a list, if applicable.
if 'email' in self.job_properties and type(self.job_properties['email']) != list:
raise validator.ListError('shared_properties.email')
# Git sync on files if a repository is passed.
# This assumes credentials to the repository are available.
# May be better to replace this with some sync using the GitHub API.
self.git_sync(git_repo)
# Assign each job dictionary a key-value pair in the main jobs dictionary.
# Also call job validation functions.
for job in job_list:
validator.validate_keys(job, self.job_properties)
self.jobs[job['jobType'] + '-' + str(hash(job['jobName']) % 10000)] = job
job['jobKey'] = job['jobType'] + '-' + str(hash(job['jobName']) % 10000)
for prop in self.job_properties:
job[prop] = self.job_properties[prop]
validator.validate_job(job, self.job_properties)
# Instantiate job layout as a graph, with nodes bucketed by job order.
self._generateDAG()
def add_jobs(jobs):
""" Adds one or more scheduled jobs to the database.
Args:
jobs (list): list of Jobs to be added
Returns:
job_ids (list): list of the resulting job_ids in added order.
Raises:
api_error.BadRequestError
"""
job_ids = []
rd = _get_redis()
for job in jobs:
validator.validate_job(job)
job_id = hashlib.sha1(job['name']).hexdigest()
if rd.exists(job_id):
raise api_error.BadRequestError("Job named {} already exists".format(job['name']))
rd.set(job_id, json.dumps(job))
crontabs.add_job(job)
job_ids.append(job_id)
return job_ids