def main(self):
try:
self.get_id()
self.parse_link()
except:
logger.error('mtime crawl bug!!! %s' % traceback.format_exc())
send_mail('mtime crawl bug', traceback.format_exc(), '*****@*****.**')
def message_builder_and_forwarder(message_properties_dictionary):
for key in message_properties_dictionary:
msg = '''Hello\n\n\n'''
job_dict = message_properties_dictionary[key]
count = 0
for ky in job_dict:
msg = msg + '----------------------------------\n'
msg = msg + '----------------------------------\n'
count = count + 1
msg = msg + str(count) + ' Site: ' + job_dict[ky][
3] + ' Keyword: ' + job_dict[ky][1] + '\n\n'
msg = msg + 'Other info: ' + job_dict[ky][8] + '\n\n'
msg = msg + 'Job Link: ' + job_dict[ky][6] + '\n\n\n'
msg = msg + '. ' + job_dict[ky][4] + '\n\n\n'
if (count > 0):
logger.info("Sending jobs")
msg = msg + 'Kind Regards'
send_mail(msg, '*****@*****.**', job_dict[ky][5],
'Scrapped Data')
else:
pass
if (len(message_properties_dictionary) > 0):
update_sent_jobs(message_properties_dictionary)
else:
logger.info("No Jobs to send")
def add(request):
'''
新增反馈信息
'''
if request.POST:
title = request.POST.get('title', None)
content = request.POST.get('content', None)
feedbacks = Feedback.objects.filter(title__iexact=title)
if feedbacks:
return HttpResponse(simplejson.dumps({"statusCode": 400,
"message": u'此主题的反馈信息已存在，请查看'}),
mimetype='application/json')
feedback = Feedback(title=title, content=content,
reporter=request.user.userprofile.realname)
feedback.save()
# 发送邮件安
email_content = render_to_string('feedback/to_IT_email.html',
{'feedback': feedback})
send_mail(u'IT总监绿色通道', settings.IT_HEAD_EMAIL, u'反馈信息提醒', email_content)
Log(username=request.user.username,
content=u'新增反馈信息：‘' + title + u'’。',
level=1).save()
return HttpResponse(simplejson.dumps({'statusCode': 200,
'url': '/feedback/index',
'message': u'反馈信息‘' + title + u'’添加成功'}),
mimetype='application/json')
return render_to_response('feedback/add.html',
context_instance=RequestContext(request))
def post(self):
try:
visitor_info = request.get_json()
provider = os.getenv('GEOIP_PROVIDER')
visitor_info['location']['data'] = [
fetch_provider(provider=provider, query=x) for x in request.access_route
]
print(visitor_info)
record_id = self.db.visitors.insert_one(dict(visitor_info)).inserted_id
visitor_info['mongodb_id'] = str(record_id)
self.logger.debug(visitor_info)
send_mail(
from_email=os.getenv('EMAIL_USER', ''),
to_email=os.getenv('EMAIL_TO', ''),
subject='Visita al sitio!',
content=style_mail(title='Registro de visita', track_obj=visitor_info),
receiver_name='Luis Esteban Rodriguez',
)
return jsonify({
'success': True,
'data': visitor_info,
})
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print(exc_type, fname, exc_tb.tb_lineno)
self.logger.debug(str(e))
return jsonify({
'success': False,
'reason': str(e),
'data': [],
})
def send_notification(self):
from utils.mail import send_mail, render_template
from settings.settings import EMAIL_ADMIN, PROJECT_PATH
sa = 0
out =u''
for o in self.mshopbasketpositions_set.all():
s = o.ammount*o.position.cost
out += u'<tr><td>'+unicode(o.position.good)+u'</td>'
out += u'<td>'+unicode(o.position.cost)+u'руб/'+unicode(o.position.good.masure)+u'</td>'
out += u'<td>'+unicode(o.ammount)+u' '+unicode(o.position.good.masure)+u'</td>'
out += u'<td>'+unicode(s)+u'</td></tr>'
sa += s
t = render_template('order.txt',{
'email':self.email,
'name':self.name,
'address':self.address,
'desc':self.description,
'city':self.city,
'phone': self.phone,
'order': out,
'total': sa
})
send_mail(EMAIL_ADMIN,EMAIL_NOREPLY,u'Поступил новый заказ!',t)
def handle(self, *args, **options):
date_today_minus_notification_timedelta = datetime.datetime.now() - settings.NOTIFICATION_TIMEDELTA_PERIOD
# Get all the users that have notifications active
# and exclude the ones that have the last email sent for less than settings.NOTIFICATION_TIMEDELTA_PERIOD
# (because they have been sent an email already)
users_enabled_notifications = Profile.objects.filter(enabled_stream_emails=True).exclude(last_stream_email_sent__gt=date_today_minus_notification_timedelta).order_by("last_stream_email_sent")[:settings.MAX_EMAILS_PER_COMMAND_RUN]
logger.info("Sending stream updates notification for %i potential users" % len(users_enabled_notifications))
email_tuples = ()
n_emails_sent = 0
for profile in users_enabled_notifications:
username = profile.user.username
email_to = profile.user.email
# Variable names use the terminology "week" because settings.NOTIFICATION_TIMEDELTA_PERIOD defaults to a
# week, but a more generic terminology could be used
week_first_day = profile.last_stream_email_sent
week_last_day = datetime.datetime.now()
week_first_day_str = week_first_day.strftime("%d %b").lstrip("0")
week_last_day_str = week_last_day.strftime("%d %b").lstrip("0")
subject_str = u'new sounds from users and tags you are following ('
subject_str += unicode(week_first_day_str) + u' - ' + unicode(week_last_day_str) + u')'
# Set date range from which to get upload notifications
time_lapse = follow_utils.build_time_lapse(week_first_day, week_last_day)
# construct message
user = User.objects.get(username=username)
try:
users_sounds, tags_sounds = follow_utils.get_stream_sounds(user, time_lapse)
except Exception, e:
# If error occur do not send the email
print "could not get new sounds data for", username
continue
if not users_sounds and not tags_sounds:
print "no news sounds for", username
continue
# print users_sound_ids
# print tags_sound_ids
text_content = render_mail_template('follow/email_stream.txt', locals())
email_tuples += (subject_str, text_content, settings.DEFAULT_FROM_EMAIL, [email_to]),
# send email
try:
send_mail(subject_str, text_content, email_from=settings.DEFAULT_FROM_EMAIL, email_to=[email_to], reply_to=None)
except Exception, e:
logger.info("An error occurred sending notification stream email to %s (%s)" % (str(email_to),str(e)) )
# Do not send the email and do not update the profile
continue
def main(self):
try:
self.paser_data()
except:
logger.error('%s%s%s' %
('have no pageing!!!', '\t', traceback.format_exc()))
send_mail('zhihu crawl bug!', traceback.format_exc(),
'*****@*****.**')
def alert_app_mail(appname, host_ip, notify_type, state, info, email_list):
subject = gen_subject(appname, host_ip, notify_type, state)
content = gen_mail_content(appname, host_ip, notify_type, state, info)
# 发送邮件
content = content.replace('\\n','\n')
if email_list:
send_mail(u"技术支持中心--运维监控中心", email_list, subject, content)
def send_activation_email(self):
tvars = {
'user': self,
'username': self.username,
'hash': create_hash(self.id)
}
send_mail(self.email,
subject='Activate your Audio Commons user account',
template='emails/account_activation.txt',
context=tvars)
def get_comment(self):
url = job_redis.spop(self.redis_idname)
while url:
try:
self.worker(url)
except:
logger.error('crawl maotu bug %s' % traceback.format_exc())
send_mail('maotu error', traceback.format_exc(),
'*****@*****.**')
job_redis.sadd(self.redis_idname)
url = job_redis.spop(self.redis_idname)
job_redis.delete(self.redis_idname)
def get_comments(self):
id = job_redis.spop(self.redis_idname)
while id:
try:
self.coms_worker(id)
except:
logger.error('crawl mafengwo bug %s' % traceback.format_exc())
send_mail('mafengwo error', traceback.format_exc(),
'*****@*****.**')
job_redis.sadd(self.redis_idname, id)
id = job_redis.spop(self.redis_idname)
job_redis.delete(self.redis_idname)
logger.info('景点点评爬取完毕')
def sendOTP(self, user_data, skypedata):
"""
Sends OTP, based in the number of digits set in environment
"""
try:
digits = int(os.environ.get('OTP_DIGITS_LIMIT'))
otp_special_char = os.environ.get('OTP_SPECIAL_CHARACTERS')
otp = ''.join(random.sample(otp_special_char, 2)) + str(random.randint(10**(digits - 1), 10**digits)) + ''.join(random.sample(otp_special_char, 2))
if self.createUserOTP(user_data, skypedata, otp):
send_mail(user_data['email'], user_data['name'], otp)
return True
return False
except Exception as e:
print('send OTP except', e)
def run():
print 'fork before'
try:
res = os.fork()
except Exception as e:
print e
res = None
sys.exit(1)
if res == 0:
print 'mail start'
send_mail(['*****@*****.**'], subject=u'测试', content=u'fork 进程发邮件test')
print 'success'
else:
print u'父进程ok'
def parse_comment(self):
for ix, url in enumerate(self.get_all_links()):
try:
self.worker(url)
except Exception as e:
if isinstance(e, requests.exceptions.ConnectionError):
continue
else:
logger.error('crawl Douban bug %s' %
traceback.format_exc())
send_mail('Douban crawl error', traceback.format_exc(),
'*****@*****.**')
finally:
if ix % 20 == 0:
logger.info('has ben download %s' % ix)
def send_invitation(self, request, tpl):
"""Send an email with the invitation."""
subject = _("Welcome to {app_name}!").format(app_name=app_name)
link = request.build_absolute_uri(self.invitation_link)
content = re.sub('{{ *link *}}', link, tpl)
return send_mail(request, subject, self.email, content=content)
def api_users_password_post():
is_json_ok, data, json_error = parse_json(request.data,
["email", "action"])
if not is_json_ok:
return Response(json_error, 400)
try:
user = User(email=data["email"])
except:
return Response("invalid email", 404)
if data["action"] == "reset":
# generate new code with 16 uppercase letters or numbers
code = ''.join(
random.choices(string.ascii_uppercase + string.digits, k=16))
user.password_reset_code = code
user.update()
mail.send_mail(
current_app.config['MAIL_USERNAME'], [user.email],
"ChoiceMD password change",
"Click the following link: <a href='http://choicemd.com/reset-password?code={0}'>Reset password</a>"
.format(code))
return Response("password reset link sent", 200)
elif data["action"] == "new":
if not "code" in data:
return Response("missing password reset code", 400)
if not "password" in data:
return Response("missing new password", 400)
if data["code"] == "":
return Response("invalid password reset code", 400)
if data["code"] != user.password_reset_code:
return Response("invalid password reset code", 400)
user.password_reset_code = ""
user.password = User.encrypt_password(data["password"])
user.update()
return Response("password changed", 200)
def get_otc_data(params):
headers = {
'content-type':
'application/json',
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:22.0) Gecko/20100101 Firefox/22.0'
}
try:
r = requests.get(config.URL, params=params, headers=headers)
res = json.loads(r.text)
return res.get('datas')
except Exception as e:
logging.error(e)
email = config.get('Owner_email')
if email is not None:
mail.send_mail(email, '请求异常', str(e))
def get_travel_notes(self):
logger.info('抓取游记: %s%s', self.old_dest, '\n')
url = job_redis.spop(self.redis_urlname)
count = 0
while url:
try:
self.note_worker(url)
except:
logger.error('crawl maotu bug %s' % traceback.format_exc())
send_mail('maotu error', traceback.format_exc(),
'*****@*****.**')
job_redis.sadd(self.redis_urlname, url)
url = job_redis.spop(self.redis_urlname)
count += 1
if count % 20 == 0:
logger.info('has been download %s travel_notes' % count)
job_redis.delete(self.redis_urlname)
def send_reset(self, request):
"""Send an email with the password reset link."""
subject = _("{app_name} password reset").format(app_name=app_name)
tpl = 'mail/password_reset.html'
ctx = {
'app_name': app_name,
'link': self.reset_link,
}
return send_mail(request, subject, self.email, tpl, ctx)
def email_reset(request):
if request.method == "POST":
form = EmailResetForm(request.POST, user = request.user)
if form.is_valid():
# save new email info to DB (temporal)
try:
rer = ResetEmailRequest.objects.get(user=request.user)
rer.email = form.cleaned_data['email']
except ResetEmailRequest.DoesNotExist:
rer = ResetEmailRequest(user=request.user, email=form.cleaned_data['email'])
rer.save()
# send email to the new address
user = request.user
email = form.cleaned_data["email"]
current_site = get_current_site(request)
site_name = current_site.name
domain = current_site.domain
c = {
'email': email,
'domain': domain,
'site_name': site_name,
'uid': int_to_base36(user.id),
'user': user,
'token': default_token_generator.make_token(user),
'protocol': 'http',
}
subject = loader.render_to_string('accounts/email_reset_subject.txt', c)
subject = ''.join(subject.splitlines())
email_body = loader.render_to_string('accounts/email_reset_email.html', c)
send_mail(subject=subject, email_body=email_body, email_to=[email])
return HttpResponseRedirect(reverse('accounts.views.email_reset_done'))
else:
form = EmailResetForm(user = request.user)
return render_to_response('accounts/email_reset_form.html',locals(),context_instance=RequestContext(request))
def pet_detail(request, slug):
pet = get_object_or_404(
Pet.objects.prefetch_related('pictures'),
slug=slug,
)
owner = False
if request.user.is_authenticated:
if pet in Pet.objects.filter(user=request.user):
owner = True
form = None
if pet.announcement:
form = CommentForm()
if request.method == 'POST':
form = CommentForm(request.POST)
if not form.is_valid():
messages.add_message(request, messages.ERROR,
'Ops, ocorreu um erro!')
else:
comment = form.save(commit=False)
comment.announcement = pet.announcement
comment.user = request.user
comment.save()
messages.add_message(request, messages.SUCCESS,
'Comentário postado com sucesso!')
if request.user.is_authenticated and request.user != pet.user:
logger.info('Trying to send email')
context = dict(
to=pet.user,
pet=pet,
absolute_url=request.META['HTTP_HOST'],
)
response = send_mail(
to=[pet.user.email],
subject='Novo comentário em seu anúncio!',
template='comment',
bcc=[DEFAULT_EMAIL],
context=context,
)
try:
response.raise_for_status()
except RequestException:
logger.error(response.text)
else:
logger.info(response.text)
else:
logger.warning('Email will not be sent!')
logger.warning(str(request.user.is_authenticated))
logger.warning(str(request.user != pet.user))
return render(request, 'web/pet_detail.html', {
'pet': pet,
'owner': owner,
'form': form,
})
def task():
global tt, try_count
if tt is not None:
try:
html = get_content()
tt.write_article(html)
# tt.close()
try_count = 0
print('发送成功')
except Exception as e:
try_count += 1
if try_count < 5:
task()
else:
logging.error(e)
try_count = 0
send_mail(config.Mail_user, '发送文章失败', '发送文章失败')
else:
# 浏览器实例不存在
send_mail(config.Mail_user, '浏览器实例不存在', '浏览器实例不存在，需要重启服务')
def form_valid(self, request, form):
new_user = self.register(request, **form.cleaned_data)
messages.success(request, "Спасибо. Вы успешно зарегистрированы на нашем сайте. Выбирайте что Вам по душе и мы с радостью поделимся.")
#import pdb; pdb.set_trace()
new_user.backend = 'django.contrib.auth.backends.ModelBackend'
login(request,new_user)
from settings.settings import EMAIL_ADMIN
from utils.mail import send_mail, render_template
c = render_template('new_user.txt',{'email':new_user.email, 'name': new_user.username})
send_mail(EMAIL_ADMIN,EMAIL_NOREPLY,u'На сайте зарегистрировался новый пользователь',c)
success_url = 'products_list'
# success_url may be a simple string, or a tuple providing the
# full argument set for redirect(). Attempting to unpack it
# tells us which one it is.
try:
to, args, kwargs = success_url
return redirect(to, *args, **kwargs)
except ValueError:
return redirect(success_url)
def send_invitation(self, request):
"""Send an email with the invitation."""
subject = _("Welcome to {app_name}!") \
.format(app_name=console_app_name)
tpl = 'mail/invite.html'
url_name = 'invitation'
ctx = {
'app_name': console_app_name,
'link': reverse(url_name, args=[self.token])
}
return send_mail(request, subject, self.admin.email, tpl, ctx)
def send_invitation(self, request):
"""Send an email with the invitation."""
subject = _("Welcome to {app_name}!") \
.format(app_name=console_app_name)
tpl = 'mail/invite.html'
url_name = 'invitation'
ctx = {
'app_name': console_app_name,
'link': reverse(url_name, args=[self.token])
}
return send_mail(request, subject, self.admin.email, tpl, ctx)
def send_reset(self, request):
"""Send an email with the password reset link."""
subject = _("{app_name} password reset") \
.format(app_name=console_app_name)
tpl = 'mail/password_reset.html'
url_name = 'password_reset'
ctx = {
'app_name': console_app_name,
'link': reverse(url_name, args=[self.token])
}
return send_mail(request, subject, self.admin.email, tpl, ctx)
def alert_host4app(appname, host_ip, ntype, state):
message = "%s Host Alert: IP: %s for %s is %s" % (ntype, host_ip, appname, state)
try:
app = AppService.objects.get(app_name = appname)
except BaseException:
return
email_list = change(app.email_list)
mobile_list = change(app.mobile_list)
# 当前逻辑, 子应用会继承父应用的报警人信息
# 获取自己的父应用--单继承
rel_list = AppRelation.objects.filter(child_app=app)
for item in rel_list:
email_list.extend(change(item.parent_app.email_list))
mobile_list.extend(change(item.parent_app.mobile_list))
if email_list:
send_mail(u"技术支持中心--运维监控中心", email_list, message, message)
if mobile_list:
sms(mobile_list, message)
def send_reset(self, request):
"""Send an email with the password reset link."""
subject = _("{app_name} password reset") \
.format(app_name=console_app_name)
tpl = 'mail/password_reset.html'
url_name = 'password_reset'
ctx = {
'app_name': console_app_name,
'link': reverse(url_name, args=[self.token])
}
return send_mail(request, subject, self.admin.email, tpl, ctx)
def email_reset(request):
if request.method == "POST":
form = EmailResetForm(request.POST, user=request.user)
if form.is_valid():
# Save new email info to DB (temporal)
try:
rer = ResetEmailRequest.objects.get(user=request.user)
rer.email = form.cleaned_data['email']
except ResetEmailRequest.DoesNotExist:
rer = ResetEmailRequest(user=request.user,
email=form.cleaned_data['email'])
rer.save()
# Send email to the new address
user = request.user
email = form.cleaned_data["email"]
current_site = get_current_site(request)
site_name = current_site.name
domain = current_site.domain
c = {
'email': email,
'domain': domain,
'site_name': site_name,
'uid': int_to_base36(user.id),
'user': user,
'token': default_token_generator.make_token(user),
'protocol': 'http',
}
subject = loader.render_to_string(
'accounts/email_reset_subject.txt', c)
subject = ''.join(subject.splitlines())
email_body = loader.render_to_string(
'accounts/email_reset_email.html', c)
send_mail(subject=subject, email_body=email_body, email_to=[email])
return HttpResponseRedirect(
reverse('accounts.views.email_reset_done'))
else:
form = EmailResetForm(user=request.user)
tvars = {'form': form}
return render(request, 'accounts/email_reset_form.html', tvars)
def write_article(self, content):
self.driver.get('https://mp.toutiao.com/profile_v3/graphic/publish')
time.sleep(3)
if 'https://mp.toutiao.com/auth/page/login/' in self.driver.current_url:
send_mail(config.Mail_user, '需要重新登录', '需要更新 cookie')
else:
self.send_keys_by_xpath(
'//*[@id="graphic"]/div/div[2]/div/div[1]/div[3]/div/div/div/textarea',
'搞笑动图GIF')
self.wait_visible(
'//*[@id="graphic"]/div/div[2]/div/div[1]/div[4]/div/div')
js = 'document.querySelector(".ProseMirror").innerHTML="{}"'.format(
content)
self.driver.execute_script(js)
self.driver.execute_script(
'window.scrollTo(0, document.body.scrollHeight)')
# 选择无图
self.click_by_xpath(
'//*[@id="graphic"]/div/div[2]/div/div[2]/div[1]/div/div[2]/div/div/label[3]/div'
)
time.sleep(10)
# 发表
self.click_by_xpath('//*[@id="publish"]')
def make_pdf_report(host, taskid):
'''
生成pdf报告
'''
data = []
result_text = "{0}/{1}_report.txt".format(TMEP_REPORT_PATH, taskid)
result = open(result_text,"r")
for line in result.readlines():
data.append(line.strip())
html = ''.join(data)
if "漏洞名称" in html:
pdfname = '{0}/{1}.pdf'.format(TMEP_REPORT_PATH,taskid)
story=[]
stylesheet=getSampleStyleSheet()
normalStyle = stylesheet['Normal']
story.append(Paragraph(html,normalStyle))
doc = SimpleDocTemplate(pdfname)
doc.build(story)
send_mail(host, pdfname)
return True
else:
return False
def email_reset(request):
if request.method == "POST":
form = EmailResetForm(request.POST, user=request.user)
if form.is_valid():
# Save new email info to DB (temporal)
try:
rer = ResetEmailRequest.objects.get(user=request.user)
rer.email = form.cleaned_data["email"]
except ResetEmailRequest.DoesNotExist:
rer = ResetEmailRequest(user=request.user, email=form.cleaned_data["email"])
rer.save()
# Send email to the new address
user = request.user
email = form.cleaned_data["email"]
current_site = get_current_site(request)
site_name = current_site.name
domain = current_site.domain
c = {
"email": email,
"domain": domain,
"site_name": site_name,
"uid": int_to_base36(user.id),
"user": user,
"token": default_token_generator.make_token(user),
"protocol": "http",
}
subject = loader.render_to_string("accounts/email_reset_subject.txt", c)
subject = "".join(subject.splitlines())
email_body = loader.render_to_string("accounts/email_reset_email.html", c)
send_mail(subject=subject, email_body=email_body, email_to=[email])
return HttpResponseRedirect(reverse("accounts.views.email_reset_done"))
else:
form = EmailResetForm(user=request.user)
tvars = {"form": form}
return render(request, "accounts/email_reset_form.html", tvars)
def notify_host(criterion, host, state, now_time):
# 1. 产生通知信息
dd = {}
dd['time'] = now_time.strftime('%Y-%m-%d %H:%M:%S')
dd['host'] = host
if state == 'UP':
dd['type'] = 'RECOVERY'
dd['information'] = 'OK - The host is up now.'
else:
dd['type'] = 'PROBLEM'
dd['information'] = 'CRITICAL - The host may be down.'
dd['state'] = state
settings.REDIS_DB.rpush('host_event', json.dumps(dd))
# 2. 触发报警
host_ip = settings.REDIS_DB.hget('host_ip', host)
message = "%s Host Alert: IP: %s is %s" % (dd['type'], host_ip, state)
if criterion.email_list:
send_mail(u'技术支持中心--运维监控中心', criterion.email_list.split(','), message, message)
if criterion.mobile_list:
sms(criterion.mobile_list.split(','), message)
def save(self):
data = self.cleaned_data
subject, from_email, to = u'Поступило сообщение из контактной формы', data['email'], EMAIL_ADMIN
html_content = render_template('message.txt',{'email':data['email'], 'name': data['name'], 'message':data['body']})
send_mail(to,from_email,subject,html_content)
def handle(self, *args, **options):
self.log_start()
date_today_minus_notification_timedelta = datetime.datetime.now(
) - settings.NOTIFICATION_TIMEDELTA_PERIOD
# Get all the users that have notifications active
# and exclude the ones that have the last email sent for less than settings.NOTIFICATION_TIMEDELTA_PERIOD
# (because they have been sent an email already)
email_type = EmailPreferenceType.objects.get(name="stream_emails")
user_ids = email_type.useremailsetting_set.values_list('user_id')
users_enabled_notifications = Profile.objects.filter(
user_id__in=user_ids
).exclude(
last_stream_email_sent__gt=date_today_minus_notification_timedelta
).order_by("-last_attempt_of_sending_stream_email"
)[:settings.MAX_EMAILS_PER_COMMAND_RUN]
n_emails_sent = 0
for profile in users_enabled_notifications:
username = profile.user.username
profile.last_attempt_of_sending_stream_email = datetime.datetime.now(
)
# Variable names use the terminology "week" because settings.NOTIFICATION_TIMEDELTA_PERIOD defaults to a
# week, but a more generic terminology could be used
week_first_day = profile.last_stream_email_sent
week_last_day = datetime.datetime.now()
week_first_day_str = week_first_day.strftime("%d %b").lstrip("0")
week_last_day_str = week_last_day.strftime("%d %b").lstrip("0")
extra_email_subject = unicode(
week_first_day_str) + u' to ' + unicode(week_last_day_str)
# Set date range from which to get upload notifications
time_lapse = follow_utils.build_time_lapse(week_first_day,
week_last_day)
# construct message
user = User.objects.get(username=username)
try:
users_sounds, tags_sounds = follow_utils.get_stream_sounds(
user, time_lapse)
except Exception as e:
# If error occur do not send the email
console_logger.info(
"could not get new sounds data for {0}".format(username))
profile.save() # Save last_attempt_of_sending_stream_email
continue
if not users_sounds and not tags_sounds:
console_logger.info("no news sounds for {0}".format(username))
profile.save() # Save last_attempt_of_sending_stream_email
continue
tvars = {
'username': username,
'users_sounds': users_sounds,
'tags_sounds': tags_sounds
}
text_content = render_mail_template('follow/email_stream.txt',
tvars)
# Send email
try:
send_mail(settings.EMAIL_SUBJECT_STREAM_EMAILS,
text_content,
extra_subject=extra_email_subject,
user_to=user)
except Exception as e:
# Do not send the email and do not update the last email sent field in the profile
profile.save() # Save last_attempt_of_sending_stream_email
commands_logger.error(
"Unexpected error while sending stream notification email (%s)"
% json.dumps({
'email_to': profile.get_email_for_delivery(),
'username': profile.user.username,
'error': str(e)
}))
continue
n_emails_sent += 1
# update last stream email sent date
profile.last_stream_email_sent = datetime.datetime.now()
profile.save()
self.log_end({'n_users_notified': n_emails_sent})
def _send_mail(user_to):
return send_mail('Test subject', 'Test body', user_to=user_to)
def handle(self, *args, **options):
date_today_minus_notification_timedelta = datetime.datetime.now(
) - settings.NOTIFICATION_TIMEDELTA_PERIOD
# Get all the users that have notifications active
# and exclude the ones that have the last email sent for less than settings.NOTIFICATION_TIMEDELTA_PERIOD
# (because they have been sent an email already)
email_type = EmailPreferenceType.objects.get(name="stream_emails")
user_ids = email_type.useremailsetting_set.values_list('user_id')
users_enabled_notifications = Profile.objects.filter(
user_id__in=user_ids
).exclude(
last_stream_email_sent__gt=date_today_minus_notification_timedelta
).order_by("-last_attempt_of_sending_stream_email"
)[:settings.MAX_EMAILS_PER_COMMAND_RUN]
logger.info(
"Sending stream updates notification for %i potential users" %
len(users_enabled_notifications))
n_emails_sent = 0
for profile in users_enabled_notifications:
username = profile.user.username
profile.last_attempt_of_sending_stream_email = datetime.datetime.now(
)
# Variable names use the terminology "week" because settings.NOTIFICATION_TIMEDELTA_PERIOD defaults to a
# week, but a more generic terminology could be used
week_first_day = profile.last_stream_email_sent
week_last_day = datetime.datetime.now()
week_first_day_str = week_first_day.strftime("%d %b").lstrip("0")
week_last_day_str = week_last_day.strftime("%d %b").lstrip("0")
subject_str = u'new sounds from users and tags you are following ('
subject_str += unicode(week_first_day_str) + u' - ' + unicode(
week_last_day_str) + u')'
# Set date range from which to get upload notifications
time_lapse = follow_utils.build_time_lapse(week_first_day,
week_last_day)
# construct message
user = User.objects.get(username=username)
try:
users_sounds, tags_sounds = follow_utils.get_stream_sounds(
user, time_lapse)
except Exception as e:
# If error occur do not send the email
print "could not get new sounds data for", username.encode(
'utf-8')
profile.save() # Save last_attempt_of_sending_stream_email
continue
if not users_sounds and not tags_sounds:
print "no news sounds for", username.encode('utf-8')
profile.save() # Save last_attempt_of_sending_stream_email
continue
text_content = render_mail_template('follow/email_stream.txt',
locals())
tvars = {
'username': username,
'users_sounds': users_sounds,
'tags_sounds': tags_sounds
}
text_content = render_mail_template('follow/email_stream.txt',
tvars)
# Send email
try:
send_mail(subject_str, text_content, user_to=user)
except Exception as e:
# Do not send the email and do not update the last email sent field in the profile
profile.save() # Save last_attempt_of_sending_stream_email
logger.info(
"An error occurred sending notification stream email to %s (%s)"
% (profile.get_email_for_delivery(), str(e)))
continue
n_emails_sent += 1
# update last stream email sent date
profile.last_stream_email_sent = datetime.datetime.now()
profile.save()
logger.info(
"Sent stream updates notification to %i users (others had no updates)"
% n_emails_sent)
save_thread(thread_id, thread_content_now)
continue
# We'll get the content of the document from last run in our state folder
thread_content_state = get_thread_content_from_state(thread_id)
diff_output = diff(thread_content_state, thread_content_now)
# Print to stdout if any changes
if diff_output:
# Store the new content for next round
save_thread(thread_id, thread_content_now)
if config.output == "gmailsmtp":
# Sending one mail per thread with diff
username = config.smtp_username
password = config.smtp_password
smtp_recipient = config.smtp_recipient
subject = "Quipdiff in {}".format(thread_title)
body_plain = " ***** {} ***** \n\n {}".format(
thread_link, diff_output)
body_html = html_enclose(html_colorize_diff(body_plain))
send_mail(username, password, smtp_recipient, subject, body_plain,
body_html)
else:
# stderr
print(" ***** {} ***** ".format(thread_title))
print(" ***** {} ***** ".format(thread_link))
print("\n" + diff_output + "\n")
def handle(self, *args, **options):
date_today_minus_notification_timedelta = datetime.datetime.now() - settings.NOTIFICATION_TIMEDELTA_PERIOD
# Get all the users that have notifications active
# and exclude the ones that have the last email sent for less than settings.NOTIFICATION_TIMEDELTA_PERIOD
# (because they have been sent an email already)
email_type = EmailPreferenceType.objects.get(name="stream_emails")
user_ids = email_type.useremailsetting_set.values_list('user_id')
users_enabled_notifications = Profile.objects.filter(user_id__in=user_ids).exclude(
last_stream_email_sent__gt=date_today_minus_notification_timedelta).order_by(
"-last_attempt_of_sending_stream_email")[:settings.MAX_EMAILS_PER_COMMAND_RUN]
logger.info("Sending stream updates notification for %i potential users" % len(users_enabled_notifications))
n_emails_sent = 0
for profile in users_enabled_notifications:
username = profile.user.username
profile.last_attempt_of_sending_stream_email = datetime.datetime.now()
# Variable names use the terminology "week" because settings.NOTIFICATION_TIMEDELTA_PERIOD defaults to a
# week, but a more generic terminology could be used
week_first_day = profile.last_stream_email_sent
week_last_day = datetime.datetime.now()
week_first_day_str = week_first_day.strftime("%d %b").lstrip("0")
week_last_day_str = week_last_day.strftime("%d %b").lstrip("0")
subject_str = u'new sounds from users and tags you are following ('
subject_str += unicode(week_first_day_str) + u' - ' + unicode(week_last_day_str) + u')'
# Set date range from which to get upload notifications
time_lapse = follow_utils.build_time_lapse(week_first_day, week_last_day)
# construct message
user = User.objects.get(username=username)
try:
users_sounds, tags_sounds = follow_utils.get_stream_sounds(user, time_lapse)
except Exception as e:
# If error occur do not send the email
print "could not get new sounds data for", username.encode('utf-8')
profile.save() # Save last_attempt_of_sending_stream_email
continue
if not users_sounds and not tags_sounds:
print "no news sounds for", username.encode('utf-8')
profile.save() # Save last_attempt_of_sending_stream_email
continue
text_content = render_mail_template('follow/email_stream.txt', locals())
tvars = {'username': username,
'users_sounds': users_sounds,
'tags_sounds': tags_sounds}
text_content = render_mail_template('follow/email_stream.txt', tvars)
# Send email
try:
send_mail(subject_str, text_content, user_to=user)
except Exception as e:
# Do not send the email and do not update the last email sent field in the profile
profile.save() # Save last_attempt_of_sending_stream_email
logger.info("An error occurred sending notification stream email to %s (%s)"
% (profile.get_email_for_delivery(), str(e)))
continue
n_emails_sent += 1
# update last stream email sent date
profile.last_stream_email_sent = datetime.datetime.now()
profile.save()
logger.info("Sent stream updates notification to %i users (others had no updates)" % n_emails_sent)
def handle(self, *args, **options):
date_today_minus_notification_timedelta = datetime.datetime.now(
) - settings.NOTIFICATION_TIMEDELTA_PERIOD
# Get all the users that have notifications active
# and exclude the ones that have the last email sent for less than settings.NOTIFICATION_TIMEDELTA_PERIOD
# (because they have been sent an email already)
users_enabled_notifications = Profile.objects.filter(
enabled_stream_emails=True
).exclude(
last_stream_email_sent__gt=date_today_minus_notification_timedelta
).order_by("-last_attempt_of_sending_stream_email"
)[:settings.MAX_EMAILS_PER_COMMAND_RUN]
logger.info(
"Sending stream updates notification for %i potential users" %
len(users_enabled_notifications))
email_tuples = ()
n_emails_sent = 0
for profile in users_enabled_notifications:
username = profile.user.username
email_to = profile.user.email
profile.last_attempt_of_sending_stream_email = datetime.datetime.now(
)
# Variable names use the terminology "week" because settings.NOTIFICATION_TIMEDELTA_PERIOD defaults to a
# week, but a more generic terminology could be used
week_first_day = profile.last_stream_email_sent
week_last_day = datetime.datetime.now()
week_first_day_str = week_first_day.strftime("%d %b").lstrip("0")
week_last_day_str = week_last_day.strftime("%d %b").lstrip("0")
subject_str = u'new sounds from users and tags you are following ('
subject_str += unicode(week_first_day_str) + u' - ' + unicode(
week_last_day_str) + u')'
# Set date range from which to get upload notifications
time_lapse = follow_utils.build_time_lapse(week_first_day,
week_last_day)
# construct message
user = User.objects.get(username=username)
try:
users_sounds, tags_sounds = follow_utils.get_stream_sounds(
user, time_lapse)
except Exception, e:
# If error occur do not send the email
print "could not get new sounds data for", username
profile.save() # Save last_attempt_of_sending_stream_email
continue
if not users_sounds and not tags_sounds:
print "no news sounds for", username
profile.save() # Save last_attempt_of_sending_stream_email
continue
text_content = render_mail_template('follow/email_stream.txt',
locals())
email_tuples += (subject_str, text_content,
settings.DEFAULT_FROM_EMAIL, [email_to]),
# Send email
try:
send_mail(subject_str,
text_content,
email_from=settings.DEFAULT_FROM_EMAIL,
email_to=[email_to],
reply_to=None)
except Exception, e:
logger.info(
"An error occurred sending notification stream email to %s (%s)"
% (str(email_to), str(e)))
# Do not send the email and do not update the last email sent field in the profile
profile.save() # Save last_attempt_of_sending_stream_email
continue
def train(data_path,
encoder_vocab_path,
decoder_vocab_path,
model_path,
embedding='fasttext'):
print('Training...')
if embedding == 'word2vec':
init_word2vec()
encoder_vocab, embeddings = load_word2vec(encoder_vocab_path)
global WORD2VEC_MODEL
del WORD2VEC_MODEL
else:
encoder_vocab, embeddings = load_fasttext(encoder_vocab_path)
decoder_vocab = load_decoder_vocab(decoder_vocab_path)
global ENC_VOCAB_SIZE
ENC_VOCAB_SIZE = len(encoder_vocab)
global DEC_VOCAB_SIZE
DEC_VOCAB_SIZE = len(decoder_vocab)
global EMBEDDING_SIZE
EMBEDDING_SIZE = len(embeddings[0])
gen = generate_batch_data(data_path,
encoder_vocab=encoder_vocab,
decoder_vocab=decoder_vocab)
model = BeliefRnn(decoder_vocab)
model.build_graph()
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(model.embedding_init,
feed_dict={model.embedding_placeholder: embeddings})
_check_restore_parameters(sess, saver, model_path)
max_loss = 0.5
i = 0
for enci, deci, lab, encil, decil in gen:
# print(enci[0])
# print(deci[0])
# print(enci_emb.shape)
# print(deci_emb.shape)
# print(lab[0])
model.optimizer.run(
feed_dict={
model.encoder_inputs.name: enci,
model.encoder_inputs_length.name: encil,
model.decoder_inputs.name: deci,
model.decoder_inputs_length.name: decil,
model.labels_.name: lab
})
if (i + 1) % 1 == 0:
loss, predictions, logits, c = sess.run(
[
model.loss, model.predictions_, model.training_logits_,
model.labels_
],
feed_dict={
model.encoder_inputs.name: enci,
model.encoder_inputs_length.name: encil,
model.decoder_inputs.name: deci,
model.decoder_inputs_length.name: decil,
model.labels_.name: lab
})
print('---------------------------------------')
print("step", i)
print('question: >',
recover_source(enci[0], encoder_vocab))
print('answer: >', recover_label(deci[0], decoder_vocab))
# print('prediction: >', predictions[0])
print('prediction: >',
recover_label(predictions[0], decoder_vocab))
print('loss: >', loss)
if loss < max_loss:
max_loss = loss * 0.7
print('saving model...', i, loss)
saver.save(sess, model_path, global_step=i)
if i % 1000 == 0 and i > 100:
print('safe_mode saving model...', i, loss)
loss, predictions, logits, c = sess.run(
[
model.loss, model.predictions_,
model.training_logits_, model.labels_
],
feed_dict={
model.encoder_inputs.name: enci,
model.encoder_inputs_length.name: encil,
model.decoder_inputs.name: deci,
model.decoder_inputs_length.name: decil,
model.labels_.name: lab
})
sends = 'step ' + str(i) + '\n' \
+ 'question: >' + recover_source(enci[0], encoder_vocab) + '\n' \
+ 'answer: >' + recover_label(deci[0], decoder_vocab) + '\n' \
+ 'prediction: >' + recover_label(predictions[0], decoder_vocab) + '\n' \
+ 'loss: >' + str(loss)
send_mail(sends)
saver.save(sess, model_path, global_step=i)
i = i + 1
