def notify_watched_tags(post):
"""
Send emails to users watching this tags
"""
if not post.is_toplevel:
return
# Get all users that have watched tags with this post
tags = post.tags.all()
# Iterate over tags and get users that are watching them
users = [
User.objects.filter(profile__watched_tags__contains=tag)
for tag in tags
]
# Get the emails to send notifications to.
emails = [u.first().email for u in users for u.email in u]
emails = set(emails)
context = dict(post=post)
template = 'messages/watched_tags.html'
send_email(template_name=template,
extra_context=context,
recipient_list=emails)
return
def notify_watched_tags(post, extra_context):
"""
Notify users watching a given tag found in post.
"""
from biostar.accounts.models import User
from django.conf import settings
users = [
User.objects.filter(
profile__watched_tags__iregex=tpatt(tag.name)).distinct()
for tag in post.root.tags.all()
]
# Flatten nested users queryset and get email.
emails = set(u.email for o in users for u in o)
from_email = settings.DEFAULT_NOREPLY_EMAIL
send_email(template_name='messages/watched_tags.html',
extra_context=extra_context,
name=post.author.profile.name,
recipient_list=emails,
from_email=from_email,
mass=True)
return
def notify_followers(subs, author, extra_context={}):
"""
Generate notification to users subscribed to a post, excluding author, a message/email.
"""
from biostar.forum.models import Subscription
# Template used to send local messages
local_template = "messages/subscription_message.md"
# Template used to send emails with
email_template = "messages/subscription_email.html"
# Does not have subscriptions.
if not subs:
return
# Select users that should be notified.
users = [sub.user for sub in subs]
# Every subscribed user gets local messages with any subscription type.
create_messages(template=local_template, extra_context=extra_context, rec_list=users, sender=author)
# Select users with email subscriptions.
email_subs = subs.filter(type=Subscription.EMAIL_MESSAGE)
# No email subscriptions
if not email_subs:
return
recipient_list = [sub.user.email for sub in email_subs]
send_email(template_name=email_template, extra_context=extra_context, recipient_list=recipient_list)
def handle(self, *args, **options):
template_name = options['template']
group_name = options['name']
subject = options['subject']
from_email = options['from'] or settings.ADMINS[0][1]
group = models.EmailGroup.objects.filter(name=group_name).first()
# Sender requires a list.
if not group:
logger.error(f"group.name={group_name} does not exist.")
return
# Get the recipients.
recipients = [
person.address.email for person in group.subscription_set.all()
]
logger.info(f"Email list size: {len(recipients)}")
tasks.send_email(template_name=template_name,
email_list=recipients,
from_email=from_email,
subject=subject,
send=True)
def notify_watched_tags(uid, extra_context):
"""
Notify users watching a given tag found in post.
"""
from biostar.accounts.models import User
from biostar.forum.models import Post
from django.conf import settings
post = Post.objects.filter(uid=uid).first()
# Update template context with post
extra_context.update(dict(post=post))
users = [
User.objects.filter(
profile__watched__name__iexact=tag.name).distinct()
for tag in post.root.tags.all()
]
# Flatten nested users queryset and get email.
emails = set(u.email for o in users for u in o)
from_email = settings.DEFAULT_NOREPLY_EMAIL
if emails:
send_email(template_name='messages/watched_tags.html',
extra_context=extra_context,
name=post.author.profile.name,
recipient_list=emails,
from_email=from_email,
mass=True)
def send_digests(days=1, subject=""):
'''
Send digest emails to users
'''
prefs_map = {
1: models.Profile.DAILY_DIGEST,
7: models.Profile.WEEKLY_DIGEST,
30: models.Profile.MONTHLY_DIGEST
}
# Send digests for posts within the last x days.
delta = util.now() - timedelta(days=days)
# Get users with the appropriate digest prefs.
digest_prefs = prefs_map.get(days, models.Profile.DAILY_DIGEST)
users = models.User.objects.filter(profile__digest_prefs=digest_prefs)
# Fetch posts within the last x amount of days
posts = Post.objects.filter(lastedit_date__gt=delta)
email_template = loader.get_template("messages/digest.html")
context = dict(subject=subject, posts=posts)
# Queue and send digest emails.
emails = users.values_list('email', flat=True)
send_email(template_name=email_template,
extra_context=context,
recipient_list=emails)
return
def mailing_list(uid, extra_context={}):
"""
Generate notification for mailing list users.
"""
from django.conf import settings
from biostar.forum.models import Post
from biostar.accounts.models import User, Profile
# Get the post and users that have this enabled.
post = Post.objects.filter(uid=uid).first()
users = User.objects.filter(profile__digest_prefs=Profile.ALL_MESSAGES)
emails = [user.email for user in users]
# Update template context with post
extra_context.update(dict(post=post))
# Prepare the templates and emails
email_template = "messages/mailing_list.html"
author = post.author.profile.name
from_email = settings.DEFAULT_NOREPLY_EMAIL
if emails:
send_email(template_name=email_template,
extra_context=extra_context,
name=author,
from_email=from_email,
recipient_list=emails,
mass=True)
def notify_followers(sub_ids, author_id, uid, extra_context={}):
"""
Generate notification to users subscribed to a post, excluding author, a message/email.
"""
from biostar.forum.models import Subscription
from biostar.accounts.models import Profile, User
from biostar.forum.models import Post
from django.conf import settings
# Template used to send local messages
local_template = "messages/subscription_message.md"
# Template used to send emails with
email_template = "messages/subscription_email.html"
# Does not have subscriptions.
if not sub_ids:
return
post = Post.objects.filter(uid=uid).first()
author = User.objects.filter(id=author_id).first()
subs = Subscription.objects.filter(uid__in=sub_ids)
users = [sub.user for sub in subs]
user_ids = [u.pk for u in users]
# Update template context with post
extra_context.update(dict(post=post))
# Every subscribed user gets local messages with any subscription type.
create_messages(template=local_template,
extra_context=extra_context,
user_ids=user_ids,
sender=author)
# Select users with email subscriptions.
# Exclude mailing list users to avoid duplicate emails.
email_subs = subs.filter(type=Subscription.EMAIL_MESSAGE)
email_subs = email_subs.exclude(
user__profile__digest_prefs=Profile.ALL_MESSAGES)
# No email subscriptions
if not email_subs:
return
recipient_list = [sub.user.email for sub in email_subs]
from_email = settings.DEFAULT_NOREPLY_EMAIL
send_email(template_name=email_template,
extra_context=extra_context,
name=author.profile.name,
from_email=from_email,
recipient_list=recipient_list,
mass=True)
def send_verification_email(user):
from_email = settings.DEFAULT_FROM_EMAIL
userid = urlsafe_base64_encode(force_bytes(user.pk))
token = account_verification_token.make_token(user)
template = "accounts/email_verify.html"
email_list = [user.email]
context = dict(token=token, userid=userid, user=user)
subject = "Welcome to Bioinformatics Recipes!"
# Send the verification email
send_email(template_name=template, recipient_list=email_list,
extra_context=context, from_email=from_email, subject=subject)
return True
def handle(self, *args, **options):
from_email = options["from"]
recipient_list = options["to"]
recipient_list = recipient_list.split(",")
subject = "Test email"
logger.info(f"settings.EMAIL_BACKEND={settings.EMAIL_BACKEND}")
logger.info(f"sending test email from {from_email} to {recipient_list}")
mass = len(recipient_list) > 1
tasks.send_email(template_name="test_email.html", recipient_list=recipient_list,
from_email=from_email, subject=subject, name="Testing", mass=mass)
# Triggers send if the backend is queued.
tasks.send_all()
def mailing_list(users, post, extra_context={}):
"""
Generate notification for mailing list users.
"""
from django.conf import settings
# Prepare the templates and emails
email_template = "messages/mailing_list.html"
emails = [user.email for user in users]
author = post.author.profile.name
from_email = settings.DEFAULT_NOREPLY_EMAIL
send_email(template_name=email_template,
extra_context=extra_context,
name=author,
from_email=from_email,
recipient_list=emails,
mass=True)
def send_verification_email(user):
from_email = settings.DEFAULT_FROM_EMAIL
userid = urlsafe_base64_encode(force_bytes(user.pk))
token = account_verification_token.make_token(user)
template = "accounts/email_verify.html"
email_list = [user.email]
context = dict(token=token, userid=userid, user=user)
# Send the verification email
send_email(template_name=template,
email_list=email_list,
extra_context=context,
from_email=from_email,
subject="Verify your email",
send=True)
return True
def notify_followers(subs, author, extra_context={}):
"""
Generate notification to users subscribed to a post, excluding author, a message/email.
"""
from biostar.forum.models import Subscription
from biostar.accounts.models import Profile
from django.conf import settings
message('Notifying users')
# Template used to send local messages
local_template = "messages/subscription_message.md"
# Template used to send emails with
email_template = "messages/subscription_email.html"
# Does not have subscriptions.
if not subs:
return
users = [sub.user for sub in subs]
# Every subscribed user gets local messages with any subscription type.
create_messages(template=local_template,
extra_context=extra_context,
rec_list=users,
sender=author)
# Select users with email subscriptions.
# Exclude mailing list users to avoid duplicate emails.
email_subs = subs.filter(type=Subscription.EMAIL_MESSAGE)
email_subs = email_subs.exclude(
user__profile__digest_prefs=Profile.ALL_MESSAGES)
# No email subscriptions
if not email_subs:
return
recipient_list = [sub.user.email for sub in email_subs]
from_email = settings.DEFAULT_NOREPLY_EMAIL
send_email(template_name=email_template,
extra_context=extra_context,
name=author.profile.name,
from_email=from_email,
recipient_list=recipient_list,
mass=True)
def test_send_mail(self):
"Test email sending using auth."
context = dict(target_email="*****@*****.**")
from_mail= "*****@*****.**"
template_name = "test_email.html"
successful = tasks.send_email(email_list=["*****@*****.**"], extra_context=context,
template_name=template_name, from_email=from_mail)
self.assertTrue(successful, "Error sending mail")
def handle(self, *args, **options):
to_emails = options["emails"]
send = options["send"]
if to_emails:
to_emails = to_emails.split(",")
from_email = settings.ADMINS[0][1]
subject = "tested email"
recipient_list = to_emails or [settings.ADMINS[0][1]]
logger.info("sending to %s" % recipient_list)
tasks.send_email(template_name="test_email.html",
email_list=recipient_list,
from_email=from_email,
subject=subject,
send=send)
def herald_emails(uid):
"""
Send emails to herald subscribers
"""
from biostar.emailer.models import EmailSubscription, EmailGroup
from biostar.forum.models import Post
post = Post.objects.filter(uid=uid).first()
group = EmailGroup.objects.filter(uid='herald').first()
# Get active subscriptions to herald.
subs = EmailSubscription.objects.filter(group=group,
state=EmailSubscription.ACTIVE)
if not subs:
return
emails = subs.values_list('email', flat=True)
context = dict(post=post)
# Prepare the templates and emails
email_template = "herald/herald_email.html"
author = post.author.profile.name
from_email = settings.DEFAULT_NOREPLY_EMAIL
# Total number of recipients allowed per open connection,
# Amazon SES has limit of 50
batch_size = 40
# Iterate through recipients and send emails in batches.
for idx in range(0, len(emails), batch_size):
# Get the next set of emails
end = idx + batch_size
rec_list = emails[idx:end]
send_email(template_name=email_template,
extra_context=context,
name=author,
from_email=from_email,
recipient_list=rec_list,
mass=True)
return
def notify_followers(post, author):
"""
Send subscribed users, excluding author, a message/email.
"""
# Template used to send local messages
local_template = "messages/subscription_message.html"
# Template used to send emails with
email_template = "messages/subscription_email.html"
context = dict(post=post)
# Everyone subscribed gets a local message.
subs = Subscription.objects.filter(post=post.root).exclude(
type=models.Profile.NO_MESSAGES)
# Send local messages
users = set(sub.user for sub in subs if sub.user != author)
create_messages(template=local_template,
extra_context=context,
rec_list=users,
sender=author)
# Send emails to users that specified so
subs = subs.filter(type=models.Profile.EMAIL_MESSAGE)
emails = [
sub.user.email for sub in subs
if (sub.user != author and sub.type == models.Profile.EMAIL_MESSAGE)
]
from_email = settings.ADMIN_EMAIL
send_email(
template_name=email_template,
extra_context=context,
subject="Subscription",
email_list=emails,
from_email=from_email,
send=True,
)
def mailing_list(emails, uid, extra_context={}):
"""
Generate notification for mailing list users.
"""
from django.conf import settings
from biostar.forum.models import Post
post = Post.objects.filter(uid=uid).first()
# Update template context with post
extra_context.update(dict(post=post))
# Prepare the templates and emails
email_template = "messages/mailing_list.html"
author = post.author.profile.name
from_email = settings.DEFAULT_NOREPLY_EMAIL
send_email(template_name=email_template,
extra_context=extra_context,
name=author,
from_email=from_email,
recipient_list=emails,
mass=True)
def send_digests(days=1, subject=""):
'''
Send digest emails to users
'''
mapper = {1: models.Profile.DAILY_DIGEST,
7: models.Profile.WEEKLY_DIGEST,
30: models.Profile.MONTHLY_DIGEST}
# Get posts made within the given time range.
trange = util.now() - timedelta(days=days)
posts = Post.objects.filter(lastedit_date__gt=trange, is_toplevel=True).order_by('-lastedit_date')
if not posts:
logger.info(f'No new posts found in the last {days} days.')
return
# Total number of recipients allowed per batch,
# AWS has limit of 50
batch_size = 40
# Get users with the appropriate digest preference.
pref = mapper.get(days, models.Profile.DAILY_DIGEST)
context = dict(subject=subject, posts=posts)
users = models.User.objects.filter(profile__digest_prefs=pref)
emails = users.values_list('email', flat=True)
# Iterate through recipients and send emails in batches.
for idx in range(0, len(emails), batch_size):
# Get the next set of emails
end = idx + batch_size
rec_list = emails[idx:end]
send_email(template_name="messages/digest.html", extra_context=context, recipient_list=rec_list)
return
def run(job, options={}):
"""
Runs a job
"""
# Options that cause early termination.
show_json = options.get('show_json')
show_template = options.get('show_template')
show_script = options.get('show_script')
show_command = options.get('show_command')
use_template = options.get('use_template')
use_json = options.get('use_json')
verbosity = options.get('verbosity', 0)
# Create log directories and files
stdout_fname, stderr_fname = create_logs(job)
try:
# Find the json and the template.
json_data = hjson.loads(job.json_text)
template = job.template
# This is the work directory.
work_dir = job.path
# The bade URL of the site.
url_base = f'{settings.PROTOCOL}://{settings.SITE_DOMAIN}{settings.HTTP_PORT}'
# Populate extra context
def extra_context(job):
extras = dict(
media_root=settings.MEDIA_ROOT,
media_url=settings.MEDIA_URL,
work_dir=work_dir,
local_root=settings.LOCAL_ROOT,
user_id=job.owner.id,
user_email=job.owner.email,
job_id=job.id,
job_name=job.name,
job_url=f'{url_base}{settings.MEDIA_URL}{job.get_url()}'.
rstrip("/"),
project_id=job.project.id,
project_name=job.project.name,
analyis_name=job.analysis.name,
analysis_id=job.analysis.id,
domain=settings.SITE_DOMAIN,
protocol=settings.PROTOCOL,
)
return extras
# Add the runtime context.
json_data['runtime'] = extra_context(job)
# Override template.
if use_template:
template = open(use_template).read()
# Override json.
if use_json:
json_data = hjson.loads(open(use_json).read())
# Print the json.
if show_json:
print(hjson.dumps(json_data, indent=4))
return
# Print the template.
if show_template:
print(template)
return
# Extract the execute commands from the spec.
settings_dict = json_data.get("settings", {})
# Specifies the command that gets executed.
execute = settings_dict.get('execute', {})
# The name of the file that contain the commands.
script_name = execute.get("script_name", "recipe.sh")
# Runtime information will be saved in the log files.
json_fname = os.path.join(job.path, f"{settings.JOB_LOGDIR}",
"input.json")
# Build the command line
command = execute.get("command", f"bash {script_name}")
# The commands can be substituted as well.
context = Context(json_data)
command_template = Template(command)
command = command_template.render(context)
# This is the full command that will be executed.
full_command = f'(cd {work_dir} && {command})'
if show_command:
print(full_command)
return
# Script template.
context = Context(json_data)
script_template = Template(template)
script = script_template.render(context)
# Show the script.
if show_script:
print(f'{script}')
return
# Logging should start after the early returns.
logger.info(f'Job id={job.id} name={job.name}')
# Make the output directory
logger.info(f'Job id={job.id} work_dir: {work_dir}')
os.makedirs(work_dir, exist_ok=True)
# Create the script in the output directory.
with open(os.path.join(work_dir, script_name), 'wt') as fp:
fp.write(script)
# Create a file that stores the json data for reference.
with open(json_fname, 'wt') as fp:
fp.write(hjson.dumps(json_data, indent=4))
# Show the command that is executed.
logger.info(f'Job id={job.id} executing: {full_command}')
# Job must be authorized to run.
if job.security != Job.AUTHORIZED:
raise Exception(
f"Job security error: {job.get_security_display()}. Recipe security : {job.analysis.get_security_display()}"
)
# Switch the job state to RUNNING and save the script field.
Job.objects.filter(pk=job.pk).update(state=Job.RUNNING,
start_date=timezone.now(),
script=script)
# Run the command.
proc = subprocess.run(command,
cwd=work_dir,
shell=True,
stdout=open(stdout_fname, "w"),
stderr=open(stderr_fname, "w"))
# Raise an error if returncode is anything but 0.
proc.check_returncode()
# If we made it this far the job has finished.
logger.info(f"uid={job.uid}, name={job.name}")
Job.objects.filter(pk=job.pk).update(state=Job.COMPLETED)
except Exception as exc:
# Write error to log file
open(stderr_fname, "a").write(f"\n{exc}")
# Handle all errors here.
Job.objects.filter(pk=job.pk).update(state=Job.ERROR)
logger.error(f'job id={job.pk} error {exc}')
stdout_log = open(stdout_fname, "r").read()
stderr_log = open(stderr_fname, "r").read()
# Save the logs and end time
Job.objects.filter(pk=job.pk).update(end_date=timezone.now(),
stdout_log=stdout_log,
stderr_log=stderr_log)
# Reselect the job to get refresh fields.
job = Job.objects.filter(pk=job.pk).first()
# Log job status.
logger.info(f'Job id={job.id} finished, status={job.get_state_display()}')
# Use -v 2 to see the output of the command.
if verbosity > 1:
print("-" * 40)
print(job.stdout_log)
print("-" * 40)
print(job.stderr_log)
if job.owner.profile.notify:
context = dict(subject=job.project.name, job=job)
# Send notification emails
send_email(template_name="emailer/job_finished.html",
recipient_list=[job.owner.email],
extra_context=context)
