def send_email(self,
subject="Submitted",
reason="Job submission successful"):
try:
dir_path = "/tmp"
email_list = self.params["email"].replace(" ", "")
email_to = "{0}@cisco.com".format(self.params["submitter"])
for email in email_list.split(','):
if email == "":
continue
if not email.endswith("@cisco.com"):
email += "@cisco.com"
email_to = "{0},{1}".format(email_to, email)
input_dict = {}
input_dict["submitter"] = self.params["submitter"]
input_dict["submit_id"] = self.params["submit_id"]
input_dict["status"] = subject
input_dict["reason"] = reason
subject = "N9k Sanity Job {0} Submission: {1}".format(
self.params["submit_id"], subject)
j2env = J2Env()
template_file = 'job_submission_email.html'
out_file = "{0}/{1}".format(dir_path, "req_creation_mail.html")
# print(out_file)
j2env.create_file(template_file, input_dict, out_file)
mailer = Mailer(email_to)
mailer.send_html_mail(subject, out_file)
except BaseException as e:
log.error("Exception in send mail:" + repr(e))
def create_resource(request):
if request.method == 'POST':
form = ResourceForm(request.POST, request.FILES)
files = request.FILES.getlist('images')
if form.is_valid():
with transaction.atomic():
res = form.save(commit=True)
res.author = request.user
res.save()
for file in files:
file = ResourceImage(image=file)
file.resource = res
file.save()
mailer = Mailer()
mailer._send_email(
CDCRC_MEDIA_EMAIL,
'Approval Request for New Resource {}'.format(res.title),
'{} has added a resource titled {}. Please take a look.'.
format(res.author.username, res.title))
return render_message(
request, 'Approval Pending',
'Thanks for sharing the resource...it will be posted on the website after being approved by our team!'
)
else:
return render_message(
request, 'Error',
'There was an error in creating the resource')
else:
form = ResourceForm()
return render(request,
'info/resource_create.html',
context={'form': form})
def __init__(self, config):
self.name = 'ip_manager'
self.config = config
self.sql_helper = SqlIpManager()
self._mailer = Mailer(config)
self.vendors = []
self._vendor_initialized = True
for vendor_name in config.get('VENDORS', []):
logger.info('vendor[{0}] with config[{1}].'.format(
vendor_name, config['VENDORS'][vendor_name]))
if config['VENDORS'][vendor_name]['enabled']:
if vendor_name == 'zmhttp':
self.vendors.append(
ZmHttp(config['VENDORS'][vendor_name], self.sql_helper,
vendor_name))
elif vendor_name == 'horocn':
self.vendors.append(
Horocn(config['VENDORS'][vendor_name], self.sql_helper,
vendor_name))
else:
logger.warning(
'unknown vendor[{0}] with config[{1}].'.format(
vendor_name, config['VENDORS'][vendor_name]))
self.seeker_info = None
self.update_time = datetime.datetime.now()
today = datetime.datetime.now()
report_time_str = '{0}-{1}-{2} {3}'.format(
today.year, today.month, today.day,
config['ROUTINE'].get('report_time', '23:55:00'))
self._next_report_time = datetime.datetime.strptime(
report_time_str, "%Y-%m-%d %H:%M:%S")
if datetime.datetime.now() > self._next_report_time:
self._next_report_time = self._next_report_time + datetime.timedelta(
days=1)
def contact_us_form(request):
if request.method == 'POST':
form = ContactUsForm(request.POST)
if form.is_valid():
contact_us_obj = form.save(commit=True)
mailer = Mailer()
mailer._send_email(
PRIMARY_ALERT_EMAILS,
'[ALERT] New Registration on Website: {}'.format(
contact_us_obj.name),
'Please reach out to {} from {} who has registered on the CDCRC Website.\n Contact Info - {}, {}.\nMessage- {}'
.format(contact_us_obj.name, contact_us_obj.organization,
contact_us_obj.email, contact_us_obj.phone,
contact_us_obj.message),
)
return render_message(
request, 'Thanks!',
'Our team will reach out to you shortly. You can also call us at {} whenever you want!'
.format(PRIMARY_PHONE))
else:
return render_message(request, 'Error',
'There was an error in filling the form..')
else:
title = 'Contact Us Form'
form = ContactUsForm()
return render(request,
'info/contact_us_form.html',
context={
'title': title,
'form': form
})
def __init__(self, config):
self.name = 'cdfbj_subscriber'
self._config = config
self._sql_helper = SqlCdfBj()
self._message_queue = []
self._workers = []
self._mailers = []
self._mailer = Mailer(config)
def __init__(self, name, config, user_id_set):
threading.Thread.__init__(self)
self.name = name
self._running = True
self._config = config
self._sql_helper = SqlCdfBj()
self._mailer = Mailer(config)
self._user_id_set = user_id_set
self._mutex = threading.Lock()
self._mail_tasks = []
def signup(request):
if request.method=='POST':
form = SignupForm(request.POST)
if form.is_valid():
to_email = form.cleaned_data.get('email')
print(to_email)
if not to_email.endswith('@iitrpr.ac.in'):
return render_message(request, 'Access Denied', 'Sorry ... only IIT Ropar domain emails can be used to sign up!')
mailer = Mailer()
user = form.save(commit=False)
user.is_active = False
user.username = to_email
try:
user.save()
current_site = get_current_site_from_env()
message = render_to_string('accounts/activate_account_email.html', {
'user': user,
'domain': current_site,
'uid':urlsafe_base64_encode(force_bytes(user.pk)),
'token':account_activation_token.make_token(user),
})
response=mailer.send_email(to_email, 'CDCRC User Registration Activation', message )
print("Mail Response: ", response)
if(response=='fail'):
raise Exception('Mail Service is down! 😢')
return render_message(request,
'Email Confirmation',
'To activate your account, please see your email!'
)
except IntegrityError as e:
if 'UNIQUE constraint failed' in str(e):
return render_message(request,
'User Exists',
'An account with this user already exists!'
)
else:
raise Exception('DB Integrity Error Occurred')
except Exception as e:
# Deleting the user
user.delete()
return render_message(request,
'An error occurred while creating your account',
'Trace: '+ str(e)
)
else:
form = SignupForm()
return render(request, 'accounts/signup.html', {'form': form})
def signup(request):
if request.method == 'POST':
form = SignupForm(request.POST)
if form.is_valid():
to_email = form.cleaned_data.get('email')
if not to_email.endswith('@iitrpr.ac.in'):
return render_message(
request, 'Access Denied',
'Sorry ... only IIT Ropar domain emails can be used to sign up!'
)
mailer = Mailer()
user = form.save(commit=False)
user.is_active = False
user.username = to_email
try:
user.save()
current_site = get_current_site_from_env()
message = render_to_string(
'accounts/activate_account_email.html', {
'user': user,
'domain': current_site,
'uid': urlsafe_base64_encode(force_bytes(user.pk)),
'token': account_activation_token.make_token(user),
})
response = mailer._send_email(
to_email, 'CDCRC User Registration Activation', message)
print("Mail Response: ", response)
if (response == 'fail'):
raise Exception('Mail Service is down! 😢')
return render_message(
request, 'Email Confirmation',
'Thank you for joining CDCRC Internal Network. We have sent an activation link to your email. Once verified, you can access plethora of resources on the webpage. Hope to see you onboard soon!'
)
except Exception as e:
# Deleting the user
if user.id != None:
print("Deleting user: ", user.id)
user.delete()
return render_message(
request, 'An error occurred while creating your account',
'Trace: ' + str(e))
else:
form = SignupForm()
return render(request, 'accounts/signup.html', {'form': form})
def post(self, request):
form = SignupForm(request.POST, request.FILES)
if form.is_valid():
# Creating user and profile
try:
with transaction.atomic():
new_user = User(username=form.cleaned_data['login'],
password=make_password(
form.cleaned_data['password']),
email=form.cleaned_data['email'],
avatar=form.cleaned_data['avatar'],
reg_date=datetime.datetime.now())
new_user.save()
Mailer().send(new_user.email,
'sign_up',
context={"login": new_user.username})
return HttpResponseRedirect('/signup/done/')
except Exception as error:
message = 'Error while adding new user: '******'Error while adding new user, check fields '
return render(request, "index/signup.html", {
"form": form,
"message": message,
"trends": Trend.get_trends()
})
def open_issue(request):
if request.method == 'POST':
pk = request.POST['issue_id']
issue = Issue.objects.get(pk=pk)
issue.status = 'open'
issue.save()
mailer = Mailer()
#send open alert to creator
mailer.send_issue_open_alert_to_creator(
[issue.creator.user.email],
issue.creator.user.first_name + " " + issue.creator.user.last_name,
request.user.first_name + " " + request.user.last_name,
'CDCRC System Alert: ' + issue.title, issue.title,
issue.get_detail_url(request))
return HttpResponseRedirect(
reverse('internal:issue_detail', kwargs={'pk': pk}))
class MWorker(threading.Thread):
def __init__(self, name, config, user_id_set):
threading.Thread.__init__(self)
self.name = name
self._running = True
self._config = config
self._sql_helper = SqlCdfBj()
self._mailer = Mailer(config)
self._user_id_set = user_id_set
self._mutex = threading.Lock()
self._mail_tasks = []
def stop(self):
self._running = False
def add_user_id(self, user_id):
self._mutex.acquire()
self._user_id_set.add(user_id)
self._mutex.release()
def add_mail_tasks(self, tasks):
self._mail_tasks.extend(tasks)
def user_id_in_set(self, user_id):
return user_id in self._user_id_set
def run(self):
while self._running:
if len(self._mail_tasks) != 0:
session = self._sql_helper.create_session()
try:
while len(self._mail_tasks) != 0:
task = self._mail_tasks.pop()
user_id, mail_title, goods_info = task[0], task[
1], task[2]
query = session.query(User).filter(
User.id == user_id).filter(User.email.isnot(None))
user_data = query.first()
if not user_data:
continue
user_data.email_code = user_data.email_code.strip()
response = self._mailer.send_subscriber_mail(
user_data, mail_title, goods_info)
if response['mailer'] == user_data.email:
if response['code'] != 0:
user_data.email_status = user_data.email_status + 1
else:
user_data.email_status = 0
except Exception as e:
logger.exception('mailer worker exception[{0}].'.format(e))
finally:
self._sql_helper.close_session(session)
else:
time.sleep(3)
def create_issue_followup(request, pk):
if request.method == 'POST':
form = IssueFollowupForm(request.POST)
if form.is_valid():
next_reminder = datetime.datetime.strptime(
request.POST['next_reminder'],
'%m/%d/%Y %H:%M').strftime('%Y-%m-%d %H:%M')
print(next_reminder)
followup_obj = form.save(commit=False)
issue_obj = Issue.objects.get(pk=pk)
followup_obj.issue = issue_obj
issue_obj.next_reminder = next_reminder
issue_obj.save()
followup_obj.save()
for assignee in request.POST.getlist('assignees'):
followup_obj.assignees.add(
TeamMemberProfile.objects.get(pk=int(assignee)))
followup_obj.save()
mailer = Mailer()
#alert the assignees
for assignee in followup_obj.assignees.all():
#send alert via mail to the assignees
mailer.send_issue_followup_alert_to_assignee(
[assignee.user.email],
assignee.user.first_name + " " + assignee.user.last_name,
'CDCRC System Alert: ' + followup_obj.issue.title,
followup_obj.issue.title, followup_obj.comment,
followup_obj.get_detail_url(request))
#alert the creator
mailer.send_issue_followup_alert_to_creator(
[followup_obj.issue.creator.user.email],
request.user.first_name + " " + request.user.last_name,
'CDCRC System Alert: ' + followup_obj.issue.title,
followup_obj.issue.title, followup_obj.comment,
followup_obj.get_detail_url(request))
mailer.send_issue_followup_alert_to_follower(
[request.user.email],
request.user.first_name + " " + request.user.last_name,
'CDCRC System Alert: ' + followup_obj.issue.title,
followup_obj.issue.title, followup_obj.comment,
followup_obj.get_detail_url(request))
return HttpResponseRedirect(
reverse('internal:issue_detail', kwargs={'pk': pk}))
else:
form = IssueFollowupForm()
return render(request,
'internal/create_issue_followup.html',
context={'form': form})
def post(self, request, id):
if request.user.is_authenticated:
form = AnswerForm(request.POST)
if form.is_valid():
quest_author_email = Answer.add(id, form.cleaned_data,
request.user)
url, link = make_question_url(id)
Mailer.send(quest_author_email,
'new_answer',
context={"link": link})
return HttpResponseRedirect(url)
else:
message = 'Error while adding'
return render(
request, "question/question.html", {
"form": form,
"id": id,
"message": message,
"trends": Trend.get_trends()
})
def __init__(self, name, config, goods_id_list, message_queue):
threading.Thread.__init__(self)
self.name = name
self._running = True
self._config = config
self._sql_helper = SqlCdfBj()
self._http_util = HttpUtil(config)
self._mailer = Mailer(config)
self._goods_id_list = goods_id_list # 负责查询的产品id
self._update_time = time.time()
self._ip_util = IpUtil(config)
self._mutex = threading.Lock()
self._message_queue = message_queue
def send_email(self, subject="Finished", reason="Sanity Job Completed"):
try:
dir_path = "/tmp"
email_to = self.email
input_dict = {}
input_dict["submitter"] = self.job.submitter
input_dict["submit_id"] = self.job.submit_id
input_dict["status"] = subject
input_dict["reason"] = "{0} on testbed {1}".format(
reason, self.job.testbed)
subject = "N9k Sanity Job {0} Status: {1}".format(
self.job.submit_id, subject)
j2env = J2Env()
template_file = 'job_submission_email.html'
out_file = "{0}/{1}".format(dir_path, "req_creation_mail.html")
# print(out_file)
j2env.create_file(template_file, input_dict, out_file)
mailer = Mailer(email_to)
mailer.send_html_mail(subject, out_file)
except BaseException as e:
log.error("Exception in send mail:" + repr(e))
def post(self, request):
form = LotCreditForm(request.POST)
if form.is_valid():
lot_id = int(form.cleaned_data['lot_id'])
lot = Lot.objects.get(id=lot_id)
message_context = {
'name': form.cleaned_data['name'],
'phone': form.cleaned_data['phone'],
'email': form.cleaned_data['email']
}
subject = 'Заявка на покупку в кредит, лот: ' + lot.name
result = Mailer().send_template_message([settings.EMAIL_TO],
'credit_message', subject,
message_context)
return JsonResponse({'result': result}, safe=False)
def post(self, request):
form = BrandForm(request.POST)
if form.is_valid():
brand_id = int(form.cleaned_data['id'])
brand = Brand.objects.get(id=brand_id)
message_context = {
'name': form.cleaned_data['name'],
'phone': form.cleaned_data['phone'],
'email': form.cleaned_data['email'],
'message': form.cleaned_data['message']
}
subject = 'Помощь в подборе ' + brand.name
result = Mailer().send_template_message([settings.EMAIL_TO],
'brand_message', subject,
message_context)
return JsonResponse({'result': result}, safe=False)
def post(self, request):
form = SupplierContactForm(request.POST)
if form.is_valid():
supplier_id = int(form.cleaned_data['id'])
supplier = Supplier.objects.get(id=supplier_id)
message_context = {
'name': form.cleaned_data['name'],
'phone': form.cleaned_data['phone'],
'email': form.cleaned_data['email'],
'message': form.cleaned_data['message']
}
subject = 'Cообщение для постащика ' + supplier.full_name
result = Mailer().send_template_message([settings.EMAIL_TO],
'supplier_message',
subject, message_context)
return JsonResponse({'result': result}, safe=False)
def post(self, request):
form = RegisterForm(request.POST, request.FILES)
if form.is_valid():
new_user = User().add_user(form.cleaned_data)
if new_user:
Mailer().send(new_user.email,
'sign_up',
context={"login": new_user.username})
return HttpResponseRedirect('/register/done/')
else:
message = 'Ошибка при регистрации пользователя: ' + str(
form.cleaned_data)
else:
message = 'Ошибка при регистрации пользователя, проверьте поля '
return render(request, "user/register.html", {
"form": form,
"message": message,
"context": Context.get(request)
})
def create_issue(request):
if request.method == 'POST':
form = IssueForm(request.POST)
if form.is_valid():
issue_obj = form.save(commit=False)
issue_obj.status = 'open'
team_member_profile = TeamMemberProfile.objects.get(
user=request.user)
issue_obj.creator = team_member_profile
issue_obj.save()
for assignee in request.POST.getlist('assignees'):
issue_obj.assignees.add(
TeamMemberProfile.objects.get(pk=int(assignee)))
issue_obj.save()
mailer = Mailer()
#send mail alert to the creator of this issue
mailer.send_issue_create_alert_to_creator(
[request.user.email],
request.user.first_name + " " + request.user.last_name,
'CDCRC System Alert: ' + issue_obj.title, issue_obj.title,
issue_obj.priority.upper(), issue_obj.get_detail_url(request))
#send mail alert to the assignees
for assignee in issue_obj.assignees.all():
mailer.send_issue_create_alert_to_assignee(
[assignee.user.email],
assignee.user.first_name + " " + assignee.user.last_name,
'CDCRC System Alert: ' + issue_obj.title, issue_obj.title,
issue_obj.priority.upper(),
issue_obj.get_detail_url(request))
return HttpResponseRedirect(
reverse('internal:issue_detail', kwargs={'pk': issue_obj.pk}))
else:
form = IssueForm()
return render(request,
'internal/create_issue.html',
context={'form': form})
class IpManager(object):
def __init__(self, config):
self.name = 'ip_manager'
self.config = config
self.sql_helper = SqlIpManager()
self._mailer = Mailer(config)
self.vendors = []
self._vendor_initialized = True
for vendor_name in config.get('VENDORS', []):
logger.info('vendor[{0}] with config[{1}].'.format(
vendor_name, config['VENDORS'][vendor_name]))
if config['VENDORS'][vendor_name]['enabled']:
if vendor_name == 'zmhttp':
self.vendors.append(
ZmHttp(config['VENDORS'][vendor_name], self.sql_helper,
vendor_name))
elif vendor_name == 'horocn':
self.vendors.append(
Horocn(config['VENDORS'][vendor_name], self.sql_helper,
vendor_name))
else:
logger.warning(
'unknown vendor[{0}] with config[{1}].'.format(
vendor_name, config['VENDORS'][vendor_name]))
self.seeker_info = None
self.update_time = datetime.datetime.now()
today = datetime.datetime.now()
report_time_str = '{0}-{1}-{2} {3}'.format(
today.year, today.month, today.day,
config['ROUTINE'].get('report_time', '23:55:00'))
self._next_report_time = datetime.datetime.strptime(
report_time_str, "%Y-%m-%d %H:%M:%S")
if datetime.datetime.now() > self._next_report_time:
self._next_report_time = self._next_report_time + datetime.timedelta(
days=1)
def __get_local_ip(self):
url = 'https://ip.tool.lu/'
try:
r = requests.get(url, timeout=10, verify=False)
if r.status_code == requests.codes.ok:
logger.info('get local ip response[{0}].'.format(r.text))
local_ip = r.text.split('\r\n')[0].split(':')[-1].strip()
return local_ip
logger.info('[{0}] get local ip from url[{1}] failed[{2}].'.format(
self.name, url, r.text))
except Exception as e:
logger.exception(
'[{0}] get local ip from url[{1}] exception[{2}].'.format(
self.name, url, e))
def __init(self):
# 开启会话
self.sql_helper.begin_session()
self.seeker_info = self.sql_helper.query_seeker(self.name)
ip = self.__get_local_ip()
# 第一次使用ip manager，需要在seeker中注册ip manager
if ip is not None and self.seeker_info is None:
self.seeker_info = {
'id': self.name,
'tag': None,
'ip': ip,
'register_time': datetime.datetime.now()
}
self.sql_helper.update_seeker(self.seeker_info)
# 本地ip发生变更，更新seeker中ip manager数据
if ip is not None and ip != self.seeker_info['ip']:
logger.info('local ip changed from [{0}] to [{1}].'.format(
self.seeker_info['ip'], ip))
self.seeker_info['ip'] = ip
# 启动后初始化，需要更新seeker中ip manager数据，同时更新各ip vendor
self.sql_helper.update_seeker(self.seeker_info)
# 更新各ip vendor
for vendor in self.vendors:
vendor.init_vendor(self.seeker_info['ip'])
self._vendor_initialized = reduce(
lambda x, y: x and y,
[vendor.vendor_initialized for vendor in self.vendors])
# 结束会话
self.sql_helper.close_session()
def __update(self):
# 如果有vendor初始化没成功，则尝试再次初始化
if not self._vendor_initialized:
ip = self.__get_local_ip()
if ip is not None:
self._vendor_initialized = reduce(lambda x, y: x and y, [
vendor.init_vendor(ip)
for vendor in self.vendors if not vendor.vendor_initialized
])
time_span = (datetime.datetime.now() -
self.update_time).total_seconds()
if time_span < 300:
return
# 日期改变，则清除过时的数据
if datetime.datetime.now().day != self.update_time.day:
self.sql_helper.delete_stale_data(days=30)
self.update_time = datetime.datetime.now()
# 获取本地ip
ip = self.__get_local_ip()
if ip is None:
return
# 更新register time
self.seeker_info['register_time'] = datetime.datetime.now()
# 本地ip发生变更，更新seeker info中的ip
if self.seeker_info['ip'] != ip:
logger.info('local ip changed from [{0}] to [{1}].'.format(
self.seeker_info['ip'], ip))
self.seeker_info['ip'] = ip
# 更新各vendor
for vendor in self.vendors:
vendor.vendor_initialized = False
vendor.init_vendor(self.seeker_info['ip'])
# 本地ip没有发生变化，只更新没有初始化成功的vendor
else:
for vendor in self.vendors:
# if not vendor.vendor_initialized:
vendor.init_vendor(ip)
# 更新ip manager到seeker表
self.sql_helper.update_seeker(self.seeker_info)
def __push_ips(self):
for vendor in self.vendors:
vendor.check_activated_ips()
def __report(self):
if datetime.datetime.now() < self._next_report_time:
return
session = self.sql_helper.get_session()
try:
# 统计当天Ip的使用情况
ip_statistics = {}
create_time = self._next_report_time - datetime.timedelta(days=1)
query = session.query(IpPool.vendor, func.count('1')).filter(IpPool.create_time > create_time)\
.filter(or_(IpPool.failed_num <= 30, IpPool.success_num / (IpPool.success_num + IpPool.failed_num) >= 0.85))\
.group_by(IpPool.vendor)
for vendor, num in query.all():
ip_statistics[vendor] = {
'good_ip': num,
'bad_ip': 0,
'total_ip': num
}
query = session.query(IpPool.vendor, func.count('1')).filter(IpPool.create_time > create_time)\
.filter(IpPool.failed_num > 30).\
filter(IpPool.success_num / (IpPool.success_num + IpPool.failed_num) < 0.85).group_by(IpPool.vendor)
for vendor, num in query.all():
ip_statistics[vendor].update({'bad_ip': num})
ip_statistics[vendor]['total_ip'] = ip_statistics[vendor][
'good_ip'] + ip_statistics[vendor]['bad_ip']
query = session.query(IpPool.vendor, func.sum(IpPool.failed_num), func.sum(IpPool.success_num))\
.filter(IpPool.create_time > create_time).group_by(IpPool.vendor)
for vendor, failed, success in query.all():
ip_statistics[vendor].update({
'failed_num': int(failed),
'success_num': int(success),
'total_num': int(failed + success)
})
# 统计当前用户情况
total_user = session.query(func.count(User.id)).first()[0]
query = session.query(func.count(func.distinct(CdfBjSubscriberInfo.user_id)))\
.filter(or_(CdfBjSubscriberInfo.replenishment_switch == 1, CdfBjSubscriberInfo.discount_switch == 1))
subscriber = query.first()[0]
query = session.query(
User.id, User.email,
User.email_code).filter(User.email_status >= 10)
users = [user for user in query.all()]
user_statistics = {
'total': total_user,
'subscriber': subscriber,
'email_error': len(users),
'email_error_users': users
}
# 统计产品情况
query = session.query(func.count(func.distinct(CdfBjSubscriberInfo.goods_id)))\
.filter(or_(CdfBjSubscriberInfo.replenishment_switch == 1, CdfBjSubscriberInfo.discount_switch == 1))
subscribe_goods = query.first()[0]
total_goods = session.query(func.count(
CdfBjGoodsInfo.goods_id)).first()[0]
goods_statistics = {
'total': total_goods,
'subscribe': subscribe_goods
}
# 发送邮件
mail_title = '{0} daily report'.format(
self._next_report_time.strftime('%Y-%m-%d %H:%M:%S'))
self._mailer.send_report_mail(mail_title, ip_statistics,
user_statistics, goods_statistics)
# 更新下次统计的日期
self._next_report_time = self._next_report_time + datetime.timedelta(
days=1)
except Exception as e:
logger.info('report exception[{0}].'.format(e))
def execute(self):
# 初始化
self.__init()
while True:
try:
time.sleep(10)
self.sql_helper.begin_session()
self.__push_ips()
self.__update()
self.__report()
self.sql_helper.close_session()
except Exception as e:
logger.exception('execute [{0}] exception[{1}].'.format(
self.name, e))
class Subscriber(object):
def __init__(self, config):
self.name = 'cdfbj_subscriber'
self._config = config
self._sql_helper = SqlCdfBj()
self._message_queue = []
self._workers = []
self._mailers = []
self._mailer = Mailer(config)
def init_sync_db(self):
"""初始化同步数据库"""
sys_goods_list = reader.load_sys_goods_file(
self._config['SYS_GOODS_FILE'])
sys_user_dict = reader.load_sys_user_file(
self._config['SYS_USER_FILE'])
# 开启会话
session = self._sql_helper.create_session()
for user_id in sys_user_dict.keys():
# 更新db中的user表[用户存在则更新，不存在则插入新用户]
self._sql_helper.insert_update_user(session,
sys_user_dict[user_id])
# 更新db中的cdfbj_subscriber_info表，为用户更新或添加对应的订阅信息
for goods_id in sys_goods_list:
subscriber_info_item = {
'goods_id': goods_id,
'user_id': user_id,
'replenishment_switch': 1,
'discount_switch': 1
} # 其他参数使用默认值
self._sql_helper.insert_update_cdfbj_subscriber_info(
session, subscriber_info_item)
# 结束会话
self._sql_helper.close_session(session)
def __get_goods_id_subscribe_info(self):
"""获取cdf北京产品的订阅情况"""
session = self._sql_helper.create_session()
subscribe_info = self._sql_helper.get_cdfbj_goods_id_subscribe_info(
session)
self._sql_helper.close_session(session)
return subscribe_info
def __distribute_subscribe_tasks(self):
"""将要订阅的产品分配到不同的Worker"""
# 获取worker（线程数量）
worker_num = self._config.get('WORKER_NUM', 1)
# 获取订阅产品的订阅情况
subscribe_info = self.__get_goods_id_subscribe_info()
# 根据订阅人数比例重组产品id，生成goods_id_list
goods_id_list = []
for goods_id, subscribe_num in subscribe_info:
# goods_id_list.extend([goods_id for i in range(0, subscribe_num)])
goods_id_list.extend([goods_id])
random.shuffle(goods_id_list)
# 计算每个worker需要查询的产品id数量
goods_per_thread = int(len(goods_id_list) / worker_num)
# 将要查询的产品id平均分配到各个worker
distributed_goods_id_list = []
for i in range(0, worker_num):
if i == worker_num - 1:
goods_id_slice = goods_id_list[i * goods_per_thread:]
else:
goods_id_slice = goods_id_list[i * goods_per_thread:(i + 1) *
goods_per_thread]
distributed_goods_id_list.append(goods_id_slice)
return distributed_goods_id_list
def __get_user_id_list(self):
"""获取当前使用订阅功能的所有用户的id"""
session = self._sql_helper.create_session()
try:
sql = "SELECT distinct(user_id) FROM cdfbj_subscriber_info where " \
"discount_switch != 0 or replenishment_switch !=0"
cursor = session.execute(sql)
return [user_id[0] for user_id in cursor.fetchall()]
except Exception as e:
logger.exception('get user id list exception[{0}].'.format(e))
finally:
self._sql_helper.close_session(session)
def __distribute_mail_tasks(self):
"""将不同User Id的邮件发送任务分配给不同的mailer"""
# 获取worker（线程数量）
mailer_num = self._config.get('MAILER_NUM', 1)
# 获取所有开启了订阅服务用户的User Id
user_id_list = self.__get_user_id_list()
random.shuffle(user_id_list)
user_per_thread = int(len(user_id_list) / mailer_num)
distributed_user_id_list = []
for i in range(0, mailer_num):
if i == mailer_num - 1:
user_id_slice = user_id_list[i * user_per_thread:]
else:
user_id_slice = user_id_list[i * user_per_thread:(i + 1) *
user_per_thread]
distributed_user_id_list.append(user_id_slice)
return distributed_user_id_list
def __do_monitoring(self):
while True:
if len(self._message_queue) != 0:
session = self._sql_helper.create_session()
try:
while len(self._message_queue) != 0:
message = self._message_queue.pop()
goods_info, subscriber_user_id_list = message[
0], message[1]
self.__mail_subscribers(session, goods_info,
subscriber_user_id_list)
session.commit()
except Exception as e:
logger.exception('do monitoring exception[{0}].'.format(e))
session.rollback()
finally:
self._sql_helper.close_session(session)
else:
time.sleep(3)
def __do_monitoring2(self):
while True:
if len(self._message_queue) != 0:
while len(self._message_queue) != 0:
message = self._message_queue.pop()
goods_info, subscriber_user_id_list = message[0], message[
1]
self.__mail_subscribers2(goods_info,
subscriber_user_id_list)
else:
time.sleep(3)
def __find_matching_mailer(self, user_id):
for mailer in self._mailers:
if mailer.user_id_in_set(user_id):
return self._mailers.index(mailer)
mailer = random.choice(self._mailers)
mailer.add_user_id(user_id)
return self._mailers.index(mailer)
def __mail_subscribers2(self, goods_info, subscriber_user_id_list):
user_id_both = subscriber_user_id_list[0].intersection(
subscriber_user_id_list[1])
user_id_replenishment = subscriber_user_id_list[
0] - subscriber_user_id_list[1]
user_id_discount = subscriber_user_id_list[
1] - subscriber_user_id_list[0]
user_id_list = list(subscriber_user_id_list[0].union(
subscriber_user_id_list[1]))
if not user_id_list:
return
logger.info('goods[{0}] send mail to users[{1}].'.format(
goods_info, subscriber_user_id_list))
replenishment_mail_title = '补货提醒 {0}'.format(goods_info['title'])
discount_mail_title = '折扣/价格变动 {0}'.format(goods_info['title'])
distributed_tasks = [[] for i in range(0, len(self._mailers))]
for user_id in user_id_list:
# if user_id != 'JingleBell200201':
# continue
if user_id in user_id_both or user_id in user_id_discount:
mail_title = discount_mail_title
else:
mail_title = replenishment_mail_title
index = self.__find_matching_mailer(user_id)
distributed_tasks[index].append((user_id, mail_title, goods_info))
for i in range(0, len(self._mailers)):
self._mailers[i].add_mail_tasks(distributed_tasks[i])
if user_id_both or user_id_discount:
mail_title = discount_mail_title
else:
mail_title = replenishment_mail_title
self._mailer.send_sys_subscriber_mail(mail_title, goods_info,
user_id_list)
def __mail_subscribers(self, session, goods_info, subscriber_user_id_list):
user_id_both = subscriber_user_id_list[0].intersection(
subscriber_user_id_list[1])
user_id_replenishment = subscriber_user_id_list[
0] - subscriber_user_id_list[1]
user_id_discount = subscriber_user_id_list[
1] - subscriber_user_id_list[0]
user_id_list = list(subscriber_user_id_list[0].union(
subscriber_user_id_list[1]))
if not user_id_list:
return
logger.info('goods[{0}] send mail to users[{1}].'.format(
goods_info, subscriber_user_id_list))
if user_id_both or user_id_discount:
mail_title = '折扣/价格变动 {0}'.format(goods_info['title'])
else:
mail_title = '补货提醒 {0}'.format(goods_info['title'])
self._mailer.send_sys_subscriber_mail(mail_title, goods_info,
user_id_list)
try:
query = session.query(User).filter(
User.id.in_(user_id_list)).filter(User.email.isnot(None))
user_all_list = [user for user in query.all()]
random.shuffle(user_all_list)
for user_data in user_all_list:
user_data.email_code = user_data.email_code.strip()
# if user_data.id != 'JingleBell200201':
# continue
if user_data.id in user_id_both:
mail_title = '折扣/价格变动 {0}'.format(goods_info['title'])
elif user_data.id in user_id_replenishment:
mail_title = '补货提醒 {0}'.format(goods_info['title'])
else:
mail_title = '折扣/价格变动 {0}'.format(goods_info['title'])
response = self._mailer.send_subscriber_mail(
user_data, mail_title, goods_info)
if response['mailer'] == user_data.email:
if response['code'] != 0:
user_data.email_status = user_data.email_status + 1
else:
user_data.email_status = 0
except Exception as e:
logger.exception(
'goods[{0}] mail subscribers[{1}] exception[{2}].'.format(
goods_info, subscriber_user_id_list, e))
def execute(self):
# self.init_sync_db()
distributed_user_id_list = self.__distribute_mail_tasks()
for i in range(0, len(distributed_user_id_list)):
mailer = MWorker('mailer[{0}]'.format(i + 1), self._config,
set(distributed_user_id_list[i]))
self._mailers.append(mailer)
mailer.start()
distributed_goods_id_list = self.__distribute_subscribe_tasks()
for i in range(0, len(distributed_goods_id_list)):
worker = SWorker('worker[{0}]'.format(i + 1), self._config,
distributed_goods_id_list[i], self._message_queue)
self._workers.append(worker)
worker.start()
self.__do_monitoring2()