def handle(self, *args, **options):
template_name = options['template']
group_name = options['name']
subject = options['subject']
from_email = options['from'] or settings.ADMINS[0][1]
group = models.EmailGroup.objects.filter(name=group_name).first()
# Sender requires a list.
if not group:
logger.error(f"group.name={group_name} does not exist.")
return
# Get the recipients.
recipients = [
person.address.email for person in group.subscription_set.all()
]
logger.info(f"Email list size: {len(recipients)}")
auth.notify(template_name=template_name,
email_list=recipients,
from_email=from_email,
subject=subject,
send=True)
def handle(self, *args, **options):
from_email = settings.DEFAULT_FROM_EMAIL
subject = "[biostar] test email "
recipient_list = [settings.ADMINS[0][1]]
logger.info("sending to %s" % recipient_list)
auth.notify(template_name="test_email.html",
email_list=recipient_list,
from_email=from_email,
subject=subject,
send=False)
def send_verification_email(user):
from_email = settings.DEFAULT_FROM_EMAIL
userid = urlsafe_base64_encode(force_bytes(user.pk)).decode()
token = account_verification_token.make_token(user)
template = "accounts/email_verify.html"
email_list = [user.email]
context = dict(token=token, userid=userid, user=user)
# Send the verification email
notify(template_name=template, email_list=email_list,
extra_context=context, from_email=from_email,
subject="Verify your email", send=True)
return True
def handle(self, *args, **options):
to_emails = options["emails"]
send = options["send"]
if to_emails:
to_emails = to_emails.split(",")
from_email = settings.ADMINS[0][1]
subject = "test email"
recipient_list = to_emails or [settings.ADMINS[0][1]]
logger.info("sending to %s" % recipient_list)
auth.notify(template_name="test_email.html",
email_list=recipient_list,
from_email=from_email,
subject=subject,
send=send)
def test_send_mail(self):
"Test email sending using auth."
context = dict(target_email="*****@*****.**")
from_mail = "*****@*****.**"
template_name = "test_email.html"
successful = auth.notify(email_list=["*****@*****.**"],
extra_context=context,
template_name=template_name,
from_email=from_mail)
self.assertTrue(successful, "Error sending mail")
def run(job, options={}):
"""
Runs a job
"""
# Options that cause early termination.
show_json = options.get('show_json')
show_template = options.get('show_template')
show_script = options.get('show_script')
show_command = options.get('show_command')
use_template = options.get('use_template')
use_json = options.get('use_json')
verbosity = options.get('verbosity', 0)
# Defined in case we bail on errors before setting it.
script = command = proc = None
stdout_log = []
stderr_log = []
try:
# Find the json and the template.
json_data = hjson.loads(job.json_text)
template = job.template
# This is the work directory.
work_dir = job.path
# The bade URL of the site.
url_base = f'{settings.PROTOCOL}://{settings.SITE_DOMAIN}{settings.HTTP_PORT}'
# Populate extra context
def extra_context(job):
extras = dict(
media_root=settings.MEDIA_ROOT,
media_url=settings.MEDIA_URL,
work_dir=work_dir, local_root=settings.LOCAL_ROOT,
user_id=job.owner.id, user_email=job.owner.email,
job_id=job.id, job_name=job.name,
job_url=f'{url_base}{settings.MEDIA_URL}{job.get_url()}'.rstrip("/"),
project_id=job.project.id, project_name=job.project.name,
analyis_name=job.analysis.name,
analysis_id=job.analysis.id,
domain=settings.SITE_DOMAIN, protocol=settings.PROTOCOL,
)
return extras
# Add the runtime context.
json_data['runtime'] = extra_context(job)
# Override template.
if use_template:
template = open(use_template).read()
# Override json.
if use_json:
json_data = hjson.loads(open(use_json).read())
# Print the json.
if show_json:
print(hjson.dumps(json_data, indent=4))
return
# Print the template.
if show_template:
print(template)
return
# Extract the execute commands from the spec.
settings_dict = json_data.get("settings", {})
# Specifies the command that gets executed.
execute = settings_dict.get('execute', {})
# The name of the file that contain the commands.
script_name = execute.get("filename", "recipe.sh")
# Make the log directory that stores sdout, stderr.
LOG_DIR = 'runlog'
log_dir = os.path.join(work_dir, f"{LOG_DIR}")
if not os.path.isdir(log_dir):
os.mkdir(log_dir)
# Runtime information will be saved in the log files.
json_fname = f"{log_dir}/input.json"
stdout_fname = f"{log_dir}/stdout.txt"
stderr_fname = f"{log_dir}/stderr.txt"
# Build the command line
command = execute.get("command", "bash recipe.sh")
# The commands can be substituted as well.
context = Context(json_data)
command_template = Template(command)
command = command_template.render(context)
# This is the full command that will be executed.
full_command = f'(cd {work_dir} && {command})'
if show_command:
print(full_command)
return
# Script template.
context = Context(json_data)
script_template = Template(template)
script = script_template.render(context)
# Show the script.
if show_script:
print(f'{script}')
return
# Logging should start after the early returns.
logger.info(f'Job id={job.id} name={job.name}')
# Make the output directory
logger.info(f'Job id={job.id} work_dir: {work_dir}')
if not os.path.isdir(work_dir):
os.mkdir(work_dir)
# Create the script in the output directory.
with open(os.path.join(work_dir, script_name), 'wt') as fp:
fp.write(script)
# Create a file that stores the json data for reference.
with open(json_fname, 'wt') as fp:
fp.write(hjson.dumps(json_data, indent=4))
# Initial create each of the stdout, stderr file placeholders.
for path in [stdout_fname, stderr_fname]:
with open(path, 'wt') as fp:
pass
# Show the command that is executed.
logger.info(f'Job id={job.id} executing: {full_command}')
# Job must be authorized to run.
if job.security != Job.AUTHORIZED:
raise Exception(f"Job security error: {job.get_security_display()}")
# Switch the job state to RUNNING and save the script field.
Job.objects.filter(pk=job.pk).update(state=Job.RUNNING,
start_date=timezone.now(),
script=script)
# Run the command.
proc = subprocess.run(command, cwd=work_dir, shell=True,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# Raise an error if returncode is anything but 0.
proc.check_returncode()
# If we made it this far the job has finished.
logger.info(f"uid={job.uid}, name={job.name}")
Job.objects.filter(pk=job.pk).update(state=Job.COMPLETED)
except Exception as exc:
# Handle all errors here.
Job.objects.filter(pk=job.pk).update(state=Job.ERROR)
stderr_log.append(f'{exc}')
logger.error(f'job id={job.pk} error {exc}')
# Collect the output.
if proc:
stdout_log.extend(force_text(proc.stdout).splitlines())
stderr_log.extend(force_text(proc.stderr).splitlines())
# Save the logs and end time
Job.objects.filter(pk=job.pk).update(end_date=timezone.now(),
stdout_log="\n".join(stdout_log),
stderr_log="\n".join(stderr_log))
# Reselect the job to get refresh fields.
job = Job.objects.filter(pk=job.pk).first()
# Create a log script in the output directory as well.
with open(stdout_fname, 'wt') as fp:
fp.write(job.stdout_log)
# Create a log script in the output directory as well.
with open(stderr_fname, 'wt') as fp:
fp.write(job.stderr_log)
# Log job status.
logger.info(f'Job id={job.id} finished, status={job.get_state_display()}')
# Use -v 2 to see the output of the command.
if verbosity > 1:
print("-" * 40)
print(job.stdout_log)
print("-" * 40)
print(job.stderr_log)
if job.owner.profile.notify:
context = dict(subject=job.project.name, job=job)
# Send notification emails
notify(template_name="emailer/job_finished.html", email_list=[job.owner.email], send=True,
extra_context=context)