def post_completion(transfer_coordinator, originator_emails):
'''
transfer_coordinator is a TransferCoordinator instance
originator_emails is a list of email addresses for the originator(s) of
the transfers
'''
all_transfers = Transfer.objects.filter(coordinator=transfer_coordinator)
failed_transfers = []
for t in all_transfers:
if not t.success:
resource = t.resource
failed_transfers.append(resource.name)
if settings.EMAIL_ENABLED:
current_site = Site.objects.get_current()
domain = current_site.domain
template_dir = os.path.join(settings.MAIN_TEMPLATE_DIR, 'transfer_app')
email_subject = open(
os.path.join(template_dir,
'transfer_complete_subject.txt')).read().strip()
# get the templates and fill them out:
env = Environment(loader=FileSystemLoader(template_dir))
plaintext_template = env.get_template('transfer_complete_message.txt')
html_template = env.get_template('transfer_complete_message.html')
params = {'domain': domain, 'failed_transfers': failed_transfers}
plaintext_msg = plaintext_template.render(params)
html_msg = html_template.render(params)
for email in originator_emails:
send_email(plaintext_msg, html_msg, email, email_subject)
def send_reminder(user, data):
'''
Constructs the email message and sends the reminder
'''
email_address = user.email
current_site = Site.objects.get_current()
domain = current_site.domain
url = 'https://%s' % domain
context = {'site': url, 'user_email': email_address, 'data': data}
email_template = get_jinja_template(
'email_templates/expiration_reminder.html')
email_html = email_template.render(context)
email_plaintxt_template = get_jinja_template(
'email_templates/expiration_reminder.txt')
email_plaintxt = email_plaintxt_template.render(context)
email_subject = open(
'email_templates/expiration_reminder_subject.txt').readline().strip()
send_email(email_plaintxt, email_html, email_address, email_subject)
def handle_failure(job):
'''
This is executed when a WDL job has completed and Cromwell has indicated a failure has occurred
`job` is an instance of SubmittedJob
'''
project = job.project
cj = CompletedJob(project=project,
job_id=job.job_id,
job_status=job.job_status,
success=False,
job_staging_dir=job.job_staging_dir)
cj.save()
job.delete()
# update the AnalysisProject instance to reflect the failure:
project.completed = False
project.success = False
project.error = True
project.status = 'The job submission has failed. An administrator has been notified.'
project.finish_time = datetime.datetime.now()
project.restart_allowed = False # do not allow restarts for runtime failures
project.save()
# inform client (if desired):
if not settings.SILENT_CLIENTSIDE_FAILURE:
recipient = project.owner.email
email_html = open('email_templates/analysis_fail.html').read()
email_plaintext = open('email_templates/analysis_fail.txt').read()
email_subject = open(
'email_templates/analysis_fail_subject.txt').readline().strip()
send_email(email_plaintext, email_html, recipient, email_subject)
# notify admins:
message = 'Job (%s) experienced failure.' % cj.job_id
subject = 'Cromwell job failure'
notify_admins(message, subject)
def post(self, request, *args, **kwargs):
if request.user.is_staff:
analysis_uuid = kwargs['analysis_uuid']
try:
project = AnalysisProject.objects.get(analysis_uuid=analysis_uuid)
except:
return HttpResponseBadRequest('Invalid project ID')
# remove the older constraints, if any
existing_constraints = ProjectConstraint.objects.filter(project=project)
if len(existing_constraints) > 0:
for c in existing_constraints:
c.delete()
# now go on to apply new constraints, if any
workflow = project.workflow
constraints = WorkflowConstraint.objects.filter(workflow=workflow)
real_error = False
all_forms = []
for c in constraints:
constraint_class = c.implementation_class
clazz = getattr(analysis.models, constraint_class)
label_dict = {'value': c.description}
modelform = modelform_factory(clazz, fields=['value'], labels=label_dict)
f = modelform(request.POST, prefix=c.name)
all_forms.append(f)
if f.is_valid():
constraint_obj = f.save(commit=False)
constraint_obj.workflow_constraint = c
constraint_obj.save()
proj_constraint = ProjectConstraint(project=project, constraint=constraint_obj)
proj_constraint.save()
else:
was_required = c.required
if was_required:
real_error = True
if real_error:
return render(request, 'analysis/constraints.html', {'forms': all_forms})
try:
do_email = request.POST['send_email']
do_email = True
except MultiValueDictKeyError:
do_email = False
if do_email and settings.EMAIL_ENABLED:
email_address = project.owner.email
current_site = Site.objects.get_current()
domain = current_site.domain
url = 'https://%s' % domain
context = {'site': url, 'user_email': email_address}
email_template = get_jinja_template('email_templates/new_project.html')
email_html = email_template.render(context)
email_plaintxt_template = get_jinja_template('email_templates/new_project.txt')
email_plaintxt = email_plaintxt_template.render(context)
email_subject = open('email_templates/new_project_subject.txt').readline().strip()
send_email(email_plaintxt, \
email_html, \
email_address, \
email_subject \
)
return HttpResponse('Project created and constraints applied.')
else:
return HttpResponseForbidden()
def post(self, request, format=None):
if request.user.is_staff:
# to create a project we need to know:
# - client (email)
# - the workflow
# - whether to allow restarts
# - constraints (i.e. how many analyses were purchased)
# check whether the client exists. If not, create them
# including some password, which we will send them
# the data as a dict:
payload = request.data
client_email = payload['client_email']
try:
user = get_user_model().objects.get(email=client_email)
except get_user_model().DoesNotExist:
user = get_user_model().objects.create(email=client_email)
random_pwd = ''.join([random.choice(string.ascii_lowercase + string.ascii_uppercase + string.digits) for _ in range(15)])
user.set_password(random_pwd)
user.save()
# now have a client. Can create a project based on the workflow
workflow_pk = payload['workflow_pk']
workflow_obj = Workflow.objects.get(pk=workflow_pk)
# if the workflow allows restarts (given presence of a pre-check WDL)
if workflow_obj.restartable:
allow_restart = True
else:
allow_restart = False
# create the project:
project = AnalysisProject(workflow=workflow_obj, owner=user, restart_allowed=allow_restart)
project.save()
# now that we have a project, we can apply constraints (i.e. how many analyses did they order?)
# here the only type of constraint we apply are AnalysisUnitConstraints
units_ordered = int(payload['number_ordered'])
constraint = AnalysisUnitConstraint.objects.create(value=units_ordered)
project_constraint = ProjectConstraint.objects.create(
project=project,
constraint = constraint
)
if settings.EMAIL_ENABLED:
email_address = project.owner.email
current_site = Site.objects.get_current()
domain = current_site.domain
url = 'https://%s' % domain
context = {'site': url, 'user_email': email_address}
email_template = get_jinja_template('email_templates/new_project.html')
email_html = email_template.render(context)
email_plaintxt_template = get_jinja_template('email_templates/new_project.txt')
email_plaintxt = email_plaintxt_template.render(context)
email_subject = open('email_templates/new_project_subject.txt').readline().strip()
send_email(email_plaintxt, \
email_html, \
email_address, \
email_subject \
)
else:
return HttpResponseForbidden()
def handle_success(job):
'''
This is executed when a WDL job has completed and Cromwell has indicated success
`job` is an instance of SubmittedJob
'''
try:
# if everything goes well, we set the AnalysisProject to a completed state,
# notify the client, and delete the SubmittedJob. Since there is a 1:1
# between AnalysisProject and a complete job, that's enough to track history
register_outputs(job)
copy_pipeline_components(job)
# update the AnalysisProject instance to reflect the success:
project = job.project
project.completed = True
project.success = True
project.error = False
project.status = 'Successful completion'
project.finish_time = datetime.datetime.now()
project.save()
# inform client:
email_address = project.owner.email
current_site = Site.objects.get_current()
domain = current_site.domain
url = 'https://%s' % domain
context = {'site': url, 'user_email': email_address}
email_template = get_jinja_template(
'email_templates/analysis_success.html')
email_html = email_template.render(context)
email_plaintxt_template = get_jinja_template(
'email_templates/analysis_success.txt')
email_plaintxt = email_plaintxt_template.render(context)
email_subject = open(
'email_templates/analysis_success_subject.txt').readline().strip()
send_email(email_plaintxt, email_html, email_address, email_subject)
# delete the staging dir where the files were:
staging_dir = job.job_staging_dir
shutil.rmtree(staging_dir)
except Exception as ex:
# Set the project parameters so that clients will know what is going on:
project = job.project
project.status = 'Analysis completed. Error encountered when preparing final output. An administrator has been notified'
project.error = True
project.success = False
project.completed = False
project.save()
if type(ex) == JobOutputsException:
message = str(ex)
else:
message = 'Some other exception was raised following wrap-up from a completed job.'
handle_exception(ex, message=message)
finally:
# regardless of what happened, save a CompletedJob instance
project = job.project
cj = CompletedJob(project=project,
job_id=job.job_id,
job_status=job.job_status,
success=True,
job_staging_dir=job.job_staging_dir)
cj.save()
job.delete()
def handle_precheck_failure(job):
'''
If a pre-check job failed, something was wrong with the inputs.
We query the cromwell metadata to get the error so the user can correct it
'''
config_path = os.path.join(THIS_DIR, 'wdl_job_config.cfg')
config_dict = utils.load_config(config_path)
# pull together the components of the request to the Cromwell server
metadata_endpoint = config_dict['metadata_endpoint']
metadata_url_template = Template(settings.CROMWELL_SERVER_URL +
metadata_endpoint)
metadata_url = metadata_url_template.render({'job_id': job.job_id})
try:
response = requests.get(metadata_url)
response_json = response.json()
stderr_file_list = walk_response('', response_json, 'stderr')
error_obj_list = log_client_errors(job, stderr_file_list)
# update the AnalysisProject instance:
project = job.project
project.completed = False
project.success = False
project.error = True
project.status = 'Issue encountered with inputs.'
project.message = ''
project.finish_time = datetime.datetime.now()
project.save()
# inform the client of this problem so they can fix it (if allowed):
email_address = project.owner.email
current_site = Site.objects.get_current()
domain = current_site.domain
project_url = reverse('analysis-project-execute',
args=[
project.analysis_uuid,
])
url = 'https://%s%s' % (domain, project_url)
context = {'site': url, 'user_email': email_address}
if project.restart_allowed:
email_template_path = 'email_templates/analysis_fail_with_recovery.html'
email_plaintxt_path = 'email_templates/analysis_fail_with_recovery.txt'
email_subject = 'email_templates/analysis_fail_subject.txt'
else:
email_template_path = 'email_templates/analysis_fail.html'
email_plaintxt_path = 'email_templates/analysis_fail.txt'
email_subject = 'email_templates/analysis_fail_subject.txt'
email_template = get_jinja_template(email_template_path)
email_html = email_template.render(context)
email_plaintxt_template = get_jinja_template(email_plaintxt_path)
email_plaintxt = email_plaintxt_template.render(context)
email_subject = open(email_subject).readline().strip()
send_email(email_plaintxt, email_html, email_address, email_subject)
if not project.restart_allowed:
# a project that had a pre-check failed, but a restart was NOT allowed.
# need to inform admins:
message = 'Job (%s) experienced failure during pre-check. No restart was allowed. Staging dir was %s' % (
job.job_id, job.job_staging_dir)
subject = 'Cromwell job failure on pre-check'
notify_admins(message, subject)
# delete the failed job:
job.delete()
except Exception as ex:
print('An exception was raised when requesting metadata '
'from cromwell server following a pre-check failure')
print(ex)
message = 'An exception occurred when trying to query metadata. \n'
message += 'Job ID was: %s' % job.job_id
message += 'Project ID was: %s' % job.project.analysis_uuid
message += str(ex)
try:
warnings_sent = Warning.objects.get(job=job)
print(
'Error when querying cromwell for metadata. Notification suppressed'
)
except analysis.models.Warning.DoesNotExist:
handle_exception(ex, message=message)
# add a 'Warning' object in the database so that we don't
# overwhelm the admin email boxes.
warn = Warning(message=message, job=job)
warn.save()
raise ex
def transfer_google_bucket(admin_pk, bucket_user_pk, client_bucket_name):
'''
Copies files from the provided bucket into the user's bucket.
Then adds the copied files to the CNAP database
Used in situations where a user has files already in a Google bucket. This
gets copied to our own storage and auto-added to the user's Resources.
Note that this function assumes the bucket can already be accessed.
'''
storage_client = storage.Client()
# get the user object. This is the person who will eventually
# 'own' the files. It was previously verified to be a valid PK
user = get_user_model().objects.get(pk=bucket_user_pk)
# get the destination bucket (to where we are moving the files)
destination_bucket_prefix = settings.CONFIG_PARAMS[ \
'storage_bucket_prefix' \
][len(settings.CONFIG_PARAMS['google_storage_gs_prefix']):]
destination_bucket_name = '%s-%s' % (destination_bucket_prefix, str(user.user_uuid)) # <prefix>-<uuid>
# typically this bucket would already exist due to a previous upload, but
# we create the bucket if it does not exist
try:
destination_bucket = storage_client.get_bucket(destination_bucket_name)
print('Destination bucket at %s existed.' % destination_bucket_name)
except google.api_core.exceptions.NotFound:
b = storage.Bucket(destination_bucket_name)
b.name = destination_bucket_name
zone_str = get_zone_as_string() # if the zone is (somehow) not set, this will be None
# if zone_str was None, b.location=None, which is the default (and the created bucket is multi-regional)
if zone_str:
b.location = '-'.join(zone_str.split('-')[:-1]) # e.g. makes 'us-east4-c' into 'us-east4'
destination_bucket = storage_client.create_bucket(b)
# get a list of the names of the files within the destination bucket.
# This is how we will check against overwriting.
destination_bucket_objects = [x.name for x in list(destination_bucket.list_blobs())]
# Now iterate through the files we are transferring:
blobs = storage_client.list_blobs(client_bucket_name)
failed_filepaths = []
for source_blob in blobs:
basename = os.path.basename(source_blob.name)
new_blob_name = os.path.join(settings.CONFIG_PARAMS['uploads_folder_name'], basename)
target_path = settings.CONFIG_PARAMS['google_storage_gs_prefix'] + os.path.join(destination_bucket.name, new_blob_name)
# we do NOT want to overwrite. We check the destination bucket to see if the file is there.
# If we find it, we consider that a "failure" and will report it to the admins
if new_blob_name in destination_bucket_objects:
failed_filepaths.append(target_path)
else:
p = do_google_copy(source_blob, destination_bucket, new_blob_name)
# now register the resources to this user:
try:
Resource.objects.create(
source = 'google_bucket',
path=target_path,
size=source_blob.size,
name = basename,
owner=user,
)
except IntegrityError as ex:
# OK to pass, because regardless of whether the file was in the bucket
# previously, we acknowledge it now and need it to be in the db.
pass
# done. Inform the admin user:
admin_user = get_user_model().objects.get(pk=admin_pk)
email_address = admin_user.email
context = {'original_bucket': client_bucket_name, 'failed_paths': failed_filepaths}
email_template = get_jinja_template('email_templates/bucket_transfer_success.html')
email_html = email_template.render(context)
email_plaintxt_template = get_jinja_template('email_templates/bucket_transfer_success.txt')
email_plaintxt = email_plaintxt_template.render(context)
email_subject = open('email_templates/bucket_transfer_success_subject.txt').readline().strip()
send_email(email_plaintxt, email_html, email_address, email_subject)
