def set_data_transferred(job_id, is_transferred):
"""
Sets the transferred flag on a Job.
Args:
db (SQLClient): a SQLAlchemy database write connection
job_id (str): a job id, must exist in the database or the method
will throw a NepheleError.NepheleRowNotFound exception
is_transferred (bool): True if data successfully transferred to S3,
False otherwise
Raises:
"""
if is_transferred:
is_transferred = 1
else:
is_transferred = 0
with nephele2.db_write() as db:
try:
job = db.query(Job).filter_by(job_id=job_id).one()
job.transferred = is_transferred
db.add(job)
except sqlalchemy.orm.exc.NoResultFound:
raise RuntimeError(
'Cannot find job, unable to set data transferred', job_id)
def set_confirm_email(email_addr, is_confirmed):
"""
Sets the confirmed flag on a UserEmail.
Args:
db (SQLClient): a SQLAlchemy database write connection
email_addr (str): an email address, must exist in the database or
the method will throw a NepheleError.NepheleRowNotFound exception
is_confirmed (bool): True if email_addr is confirmed, False
otherwise
"""
# Booleans are represented as ints in the DB, so we need to convert
if is_confirmed:
is_confirmed = 1
else:
is_confirmed = 0
# TODO : possibly want to raise error if no such email_addr
with nephele2.db_write() as db:
try:
user_email = db.query(UserEmail).filter_by(
address=email_addr).one()
user_email.is_confirmed = is_confirmed
db.add(user_email)
except sqlalchemy.orm.exc.NoResultFound:
return
def create_user(email_addr,
fname,
lname,
affiliation,
affiliation_category,
ref,
subscribe,
analysis=None):
"""
Creates a new user, adding rows to the users, user_email, and user_info
tables.
Args:
db (SQLClient): a SQLAlchemy database write connection
email_addr (str): a validly formed email address
fname (str): the user's first name
lname (str): the user's last name
affiliation (str): name of the user's employer
affiliation_category (str): one of: NIH, Government (non-NIH),
University, Research Institute (Private), Citizen Scientist
(no-affiliation), Other ref (str): who referred the user. One of:
NA, NIH Bioinformatics listserve, NIH Microbiome listserve,
Internet, Colleague, Other subscribe (bool): indicates whether the
user wants to subscribe to the newsletter analysis (str): the type
of analysis the user intends to use the system for
"""
# Apparently the database considers an empty string to not be NULL,
# but we don't want to allow blank strings in the address field in
# the DB, so make sure "empty" strings are considered NULL.
# subscribe will come in as a boolean, but needs to be stored as an int
if subscribe:
subscribe = 1
else:
subscribe = 0
user_data = {
'first_name': fname,
'last_name': lname,
'user_address': {
'address': email_addr,
'subscribed': subscribe
},
'user_info': {
'affiliation': affiliation,
'affiliation_category': affiliation_category,
'referrer': ref,
'analysis_type': analysis
}
}
with nephele2.db_write() as db:
db.add(User(user_data))
def set_job_description(job_id, desc):
"""
Sets the job description field.
Args:
job_id (str): a unique job id
desc (str): string describing the job
Raises:
"""
with nephele2.db_write() as db:
try:
job = db.query(Job).filter_by(job_id=job_id).one()
job.description = desc
db.add(job)
except sqlalchemy.orm.exc.NoResultFound:
raise RuntimeError('Unable to set job desc', job_id)
def load_job_types():
""" Loads the JobType table, which will be considered read-only by the
system. This data must be present in order for the application to
function.
Args:
db (SQLClient): a SQLAlchemy database connection with write
permissions.
"""
with open('rds/conf.yaml') as yaml_file:
conf = yaml.safe_load(yaml_file)
with nephele2.db_write() as db:
for entry in conf['job_types']:
try:
db.query(JobType).filter_by(name=entry['name']).one()
db.query(JobType).filter_by(
name=entry['name']).update(entry)
except sqlalchemy.orm.exc.NoResultFound:
db.add(JobType(entry))
def set_job_status(job_id, status, stack_trace=None, error=None):
"""
Sets the job status to one of the enum values allowed by the database.
Args:
db (SQLClient): a SQLAlchemy database write connection
job_id (str): a valid job ID
status (str): must be one of
['Pending', 'Pre-Processing', 'Running', 'Failed', 'Succeeded']
Raises:
ValueError, RuntimeError
"""
# TODO this is could be cleaned up.
valid_statuses = ['Pending', 'Pre-Processing', 'Running', 'Failed',
'Succeeded']
if status not in valid_statuses:
raise ValueError('Bad Status on job:{}'.format(job_id),
status)
with nephele2.db_write() as db:
try:
job = db.query(Job).filter_by(job_id=job_id).one()
job.status = status
if status == 'Pending':
job.submitted = datetime.now()
elif status == 'Running':
job.started = datetime.now()
elif status in ['Failed', 'Succeeded']:
job.completed = datetime.now()
if error:
if job.error_msg:
# if there's already an error don't loose it
job.error_msg = job.error_msg + ", " + error
else:
job.error_msg = error
if stack_trace:
if job.stack_trace:
job.stack_trace = job.stack_trace + ', ' + stack_trace
else:
job.stack_trace = stack_trace
db.add(job)
except sqlalchemy.orm.exc.NoResultFound:
raise RuntimeError('Unable to set job status', job_id)
def create_job(user_email, job_id):
"""
Verifies that the user is valid then creates new row in jobs table.
Args:
db (SQLClient): a SQLAlchemy database write connection
user_email (str): a validly formatted email address. Must exist in
the database and be marked as confirmed.
job_id (str): a valid job id
Raises:
"""
with nephele2.db_write() as db:
try:
user = db.query(User).filter_by(email_address=user_email).one()
job = Job(job_id=job_id, user=user, status="Initializing")
db.add(job)
except sqlalchemy.orm.exc.NoResultFound:
return
def set_bad_email(email_addr):
"""
Sets the is_bad flag on a UserEmail.
called by lambda
Args:
db (SQLClient): a SQLAlchemy database write connection
email_addr (str): an email address, must exist in the database or
the method will throw a NepheleError.NepheleRowNotFound exception
is_bad (bool): True if the email address is bad, False otherwise
"""
with nephele2.db_write() as db:
try:
user_email = db.query(UserEmail).filter_by(
address=email_addr).one()
user_email.is_bad = 1
db.add(user_email)
except sqlalchemy.orm.exc.NoResultFound:
return
def set_job_arguments(job_id, args):
"""
Adds the arguments to the Job as a JSON string.
Args:
job_id (str): a valid job ID
args (str): a JSON string of job parameters
Raises:
"""
args = json.loads(args)
with nephele2.db_write() as db:
try:
job = db.query(Job).filter_by(job_id=job_id).one()
job.args = args
db.add(job)
except sqlalchemy.orm.exc.NoResultFound:
raise RuntimeError(
'Unable to find job, failed to set job arguments',
job_id)
def set_job_type(job_id, job_name):
""""
Sets the job type for the requested job.
Args:
db (SQLClient): a SQLAlchemy database write connection
job_id (str): a valid job ID
job_name (str): one of the job types in job_type.name
Raises:
RuntimeError
"""
with nephele2.db_write() as db:
try:
job = db.query(Job).filter_by(job_id=job_id).one()
job.job_type = DBUtils.get_job_type_by_name(job_name)
db.add(job)
except sqlalchemy.orm.exc.NoResultFound:
raise RuntimeError(
'Unable to find job, failed to set job type', job_id)
def set_machine_info(job_id, machine_id, ami_id, instance_type):
"""
Creates a machine_info entry and adds it to the job identified by
job_id. If, for some reason, AWS returns the same machine ID twice
(which is theoretically probable, but highly unlikely) this method will
fail to create the second entry, since the machine ID is considered a
primary key and must be unique.
Args:
db (SQLClient): a SQLAlchemy database write connection
job_id (str): a valid job ID
machine_id (str): an AWS instance identifier
ami_id (str): an AWS AMI ID
instance_type (str): the type of EC2 spun up
"""
machine = MachineInfo(job_id=job_id,
instance_id=machine_id,
instance_type=instance_type,
ami=ami_id)
with nephele2.db_write() as db:
db.add(machine)
def delete_job(job_id):
job = DBUtils.get_job(job_id)
if job:
with nephele2.db_write() as db:
db.delete(job)