def get_candidates_from_smartlist(list_id,
candidate_ids_only=False,
user_id=None):
"""
Calls inter services method and retrieves the candidates of a smart or dumb list.
:param list_id: smartlist id.
:param candidate_ids_only: Whether or not to get only ids of candidates
:param user_id: Id of user.
:type list_id: int | long
:type candidate_ids_only: bool
:type user_id: int | long | None
:rtype: list
"""
raise_if_not_positive_int_or_long(list_id)
raise_if_not_instance_of(candidate_ids_only, bool)
raise_if_not_positive_int_or_long(user_id)
candidates = []
try:
candidates = get_candidates_of_smartlist(
list_id=list_id,
candidate_ids_only=candidate_ids_only,
access_token=None,
user_id=user_id)
except Exception as error:
logger.error(
'Error occurred while getting candidates of smartlist(id:%s).'
'.Reason: %s' % (list_id, error.message))
logger.info("There are %s candidates in smartlist(id:%s)" %
(len(candidates), list_id))
return candidates
def patch(self, campaign_id):
"""
This endpoint updates an existing campaign
:param int|long campaign_id: Id of campaign
"""
raise_if_dict_values_are_not_int_or_long(dict(campaign_id=campaign_id))
# Get and validate request data
data = request.get_json(silent=True)
if not data:
raise InvalidUsage("Received empty request body",
error_code=INVALID_REQUEST_BODY[1])
is_hidden = data.get('is_hidden', False)
if is_hidden not in (True, False, 1, 0):
raise InvalidUsage(
"is_hidden field should be a boolean, given: %s" % is_hidden,
error_code=INVALID_INPUT[1])
email_campaign = EmailCampaignBase.get_campaign_if_domain_is_valid(
campaign_id, request.user)
# Unschedule task from scheduler_service
if email_campaign.scheduler_task_id:
headers = {'Authorization': request.oauth_token}
# campaign was scheduled, remove task from scheduler_service
if CampaignUtils.delete_scheduled_task(
email_campaign.scheduler_task_id, headers):
email_campaign.update(
scheduler_task_id='') # Delete scheduler task id
email_campaign.update(is_hidden=is_hidden)
logger.info("Email campaign(id:%s) has been archived successfully" %
campaign_id)
return dict(message="Email campaign (id: %s) updated successfully" %
campaign_id), codes.OK
def do_mergetag_replacements(texts,
current_user,
requested_object=None,
candidate_address=None):
"""
Here we do the replacements of merge tags with required values. This serves for candidate and user.
If no candidate or user is provided, name is set to "John Doe".
It replaces MergeTags with candidate's or user's first name, last name.
It also replaces preferences URL only for candidate.
:param list[> 0](string) texts: List of e.g. subject, body_text and body_html
:rtype: list[> 0](string)
"""
if not isinstance(current_user, User):
raise InternalServerError('Invalid object passed for user')
if requested_object and not isinstance(requested_object,
(Candidate, User)):
raise InternalServerError('Invalid object passed')
first_name = "John"
last_name = "Doe"
if requested_object:
first_name = requested_object.first_name if requested_object.first_name else first_name
last_name = requested_object.last_name if requested_object.last_name else last_name
new_texts = []
merge_tag_replacement_dict = {
DEFAULT_FIRST_NAME_MERGETAG: first_name,
DEFAULT_LAST_NAME_MERGETAG: last_name,
DEFAULT_USER_NAME_MERGETAG: current_user.name
}
logger.info('Replacing merge tags for campaign of user(id:%s, name:%s)' %
(current_user.id, current_user.name))
for text in texts:
if text:
logger.info('Replacing merge tags for string:%s' % text)
for key, value in merge_tag_replacement_dict.iteritems():
if key in text:
# Do first_name, last_name and username replacements
logger.info('Merge tag:%s found and replaced with:%s' %
(key, value))
text = text.replace(key, value)
else:
logger.info('Merge tag:%s was not found. Value:%s' %
(key, value))
# Do 'Unsubscribe' link replacements
if isinstance(
requested_object,
Candidate) and DEFAULT_PREFERENCES_URL_MERGETAG in text:
text = do_prefs_url_replacement(text, requested_object,
candidate_address)
elif isinstance(requested_object,
User) and DEFAULT_PREFERENCES_URL_MERGETAG in text:
text = text.replace(DEFAULT_PREFERENCES_URL_MERGETAG,
TEST_PREFERENCE_URL)
new_texts.append(text)
logger.info('Converted body_html, body_text and subject are:%s' %
new_texts)
return new_texts
def post(self):
"""
This will save an entry in database table email_client_credentials.
.. Request Body::
{
"host": "Host Name",
"port": 123,
"name": "Server Name",
"email": "*****@*****.**",
"password": "******",
}
.. Response::
{
"id": 347
}
.. Status:: 201 (Resource created)
400 (Bad request)
401 (Unauthorized to access getTalent)
500 (Internal server error)
"""
data = get_json_data_if_validated(request, EMAIL_CLIENTS_SCHEMA)
data = format_email_client_data(data)
data['user_id'] = request.user.id
client_in_db = EmailClientCredentials.get_by_user_id_host_and_email(
data['user_id'], data['host'], data['email'])
if client_in_db:
raise InvalidUsage(
'Email client with given data already present in database')
client = EmailClientBase.get_client(data['host'])
client = client(data['host'], data['port'], data['email'],
data['password'])
logger.info('Connecting with given email-client')
client.connect()
client.authenticate()
logger.info(
'Successfully connected and authenticated with given email-client')
# Encrypt password
ciphered_password = encrypt(
app.config[TalentConfigKeys.ENCRYPTION_KEY], data['password'])
b64_password = b64encode(ciphered_password)
data['password'] = b64_password
email_client = EmailClientCredentials(**data)
EmailClientCredentials.save(email_client)
headers = {
'Location':
EmailCampaignApiUrl.EMAIL_CLIENT_WITH_ID % email_client.id
}
return ApiResponse(dict(id=email_client.id),
status=requests.codes.CREATED,
headers=headers)
def post(self, campaign_id):
"""
Sends campaign emails to the candidates present in smartlists of campaign.
Scheduler service will call this to send emails to candidates.
:param int|long campaign_id: Campaign id
"""
raise_if_dict_values_are_not_int_or_long(dict(campaign_id=campaign_id))
email_campaign = EmailCampaignBase.get_campaign_if_domain_is_valid(
campaign_id, request.user)
if email_campaign.is_hidden:
logger.info(
"Email campaign(id:%s) is archived, it cannot be sent." %
campaign_id)
# Unschedule task from scheduler_service
if email_campaign.scheduler_task_id:
headers = {'Authorization': request.oauth_token}
# campaign was scheduled, remove task from scheduler_service
if CampaignUtils.delete_scheduled_task(
email_campaign.scheduler_task_id, headers):
email_campaign.update(
scheduler_task_id='') # Delete scheduler task id
raise ResourceNotFound("Email campaign(id:%s) has been deleted." %
campaign_id,
error_code=EMAIL_CAMPAIGN_NOT_FOUND[1])
email_client_id = email_campaign.email_client_id
results_send = send_email_campaign(request.user,
email_campaign,
new_candidates_only=False)
if email_client_id:
if not isinstance(results_send, list):
raise InternalServerError(
error_message="Something went wrong, response is not list")
data = {
'email_campaign_sends': [{
'email_campaign_id':
email_campaign.id,
'new_html':
new_email_html_or_text.get('new_html'),
'new_text':
new_email_html_or_text.get('new_text'),
'candidate_email_address':
new_email_html_or_text.get('email')
} for new_email_html_or_text in results_send]
}
return jsonify(data)
return dict(
message='email_campaign(id:%s) is being sent to candidates.' %
campaign_id), codes.OK
def send_email(self, to_address, subject, body):
"""
This connects and authenticate with SMTP server and sends email to given email-address
:param string to_address: Recipient's email address
:param string subject: Subject of email
:param string body: Body of email
"""
self.connect()
self.authenticate(connection_quit=False)
msg = "From: %s\r\nTo: %s\r\nSubject: %s\n%s\n" % (
self.email, to_address, subject, body)
self.connection.sendmail(self.email, to_address, msg)
logger.info('Email has been sent from:%s, to:%s via SMTP server.' %
(self.email, to_address))
self.connection.quit()
def schedule_job_for_email_conversations():
"""
Schedule general job that hits /v1/email-conversations endpoint every hour.
"""
url = EmailCampaignApiUrl.EMAIL_CONVERSATIONS
task_name = 'get_email_conversations'
start_datetime = datetime.utcnow() + timedelta(seconds=15)
# Schedule for next 100 years
end_datetime = datetime.utcnow() + timedelta(weeks=52 * 100)
frequency = 3600
access_token = User.generate_jw_token()
headers = {
'Content-Type': 'application/json',
'Authorization': access_token
}
data = {
'start_datetime':
start_datetime.strftime(DatetimeUtils.ISO8601_FORMAT),
'end_datetime': end_datetime.strftime(DatetimeUtils.ISO8601_FORMAT),
'frequency': frequency,
'is_jwt_request': True
}
logger.info('Checking if `{}` task already running...'.format(task_name))
response = requests.get(SchedulerApiUrl.TASK_NAME % task_name,
headers=headers)
# If job is not scheduled then schedule it
if response.status_code == requests.codes.NOT_FOUND:
logger.info('Task `{}` not scheduled. Scheduling `{}` task.'.format(
task_name, task_name))
data.update({'url': url})
data.update({'task_name': task_name, 'task_type': 'periodic'})
response = requests.post(SchedulerApiUrl.TASKS,
headers=headers,
data=json.dumps(data))
if response.status_code == codes.CREATED:
logger.info('Task `{}` has been scheduled.'.format(task_name))
elif response.json()['error']['code'] == TASK_ALREADY_SCHEDULED:
logger.info('Job already scheduled. `{}`'.format(task_name))
else:
logger.error(response.text)
raise InternalServerError(
error_message=
'Unable to schedule job for getting email-conversations')
elif response.status_code == requests.codes.ok:
logger.info('Job already scheduled. `{}`'.format(response.text))
else:
logger.error(response.text)
def import_emails(self, candidate_id, candidate_email):
"""
This will import emails of user's account to getTalent database table email-conversations.
:param positive candidate_id: Id of candidate
:param string candidate_email: Email of candidate
"""
search_criteria = '(FROM "%s")' % candidate_email
typ, [searched_data] = self.connection.search(None, search_criteria)
msg_ids = [msg_id for msg_id in searched_data.split()]
logger.info("Account:%s, %s email(s) found from %s." %
(self.email, len(msg_ids), candidate_email))
for num in msg_ids:
body = ''
typ, data = self.connection.fetch(num, '(RFC822)')
raw_email = data[0][1]
raw_email_string = raw_email.decode('utf-8')
# converts byte literal to string removing b''
email_message = email.message_from_string(raw_email_string)
for header in ['subject', 'to', 'from', 'date']:
logger.info('%s: %s' % (header.title(), email_message[header]))
# convert string date to datetime object
email_received_datetime = parser.parse(email_message['date'])
# this will loop through all the available multiparts in mail
for part in email_message.walk():
if part.get_content_type(
) == "text/plain": # ignore attachments/html
body = part.get_payload(decode=True)
logger.info('Body: %s' % body)
self.save_email_conversation(candidate_id,
email_message['subject'].strip(),
body.strip(), email_received_datetime)
def import_email_conversations(queue_name):
"""
This gets all the records for incoming clients from database table email_client_credentials.
It then calls "import_email_conversations_per_account" to imports email-conversations for selected email-client.
:type queue_name: string
"""
email_client_credentials = \
EmailClientCredentials.get_by_client_type(EmailClientCredentials.CLIENT_TYPES['incoming'])
if not email_client_credentials:
logger.info('No IMAP/POP email-client found in database')
for email_client_credential in email_client_credentials:
logger.info(
'Importing email-conversations from host:%s, account:%s, user_id:%s'
% (email_client_credential.host, email_client_credential.email,
email_client_credential.user.id))
import_email_conversations_per_client.apply_async(
[
email_client_credential.host, email_client_credential.port,
email_client_credential.email,
email_client_credential.password,
email_client_credential.user.id, email_client_credential.id
],
queue_name=queue_name)
def create_email_campaign_url_conversion(destination_url,
email_campaign_send_id,
type_,
destination_url_custom_params=None):
"""
Creates url_conversion in DB and returns source url
"""
# Insert url_conversion
if destination_url_custom_params:
destination_url = set_query_parameters(destination_url,
destination_url_custom_params)
url_conversion = UrlConversion(destination_url=destination_url,
source_url='')
UrlConversion.save(url_conversion)
# source_url = current.HOST_NAME + str(URL(a='web', c='default', f='url_redirect',
# args=url_conversion_id, hmac_key=current.HMAC_KEY))
logger.info('create_email_campaign_url_conversion: url_conversion_id:%s' %
url_conversion.id)
signed_source_url = CampaignUtils.sign_redirect_url(
EmailCampaignApiUrl.URL_REDIRECT % url_conversion.id,
datetime.utcnow() + relativedelta(years=+1))
# In case of prod, do not save source URL
if CampaignUtils.IS_DEV:
# Update source url
url_conversion.update(source_url=signed_source_url)
# Insert email_campaign_send_url_conversion
email_campaign_send_url_conversion = EmailCampaignSendUrlConversion(
email_campaign_send_id=email_campaign_send_id,
url_conversion_id=url_conversion.id,
type=type_)
EmailCampaignSendUrlConversion.save(email_campaign_send_url_conversion)
return signed_source_url
def email_conversation_importer(self):
"""
This imports email-conversations from the candidates of user.
"""
logger.info('POP email-conversations importer needs to be implemented')
def amazon_sns_endpoint():
"""
This endpoint handles email bounces and complaints using Amazon Simple Email Service (SES)
and Simple Notification Service (SNS).
To get email bounces and complaints callback hits, we have to setup two SNS topics,
one for email bounces `email_bounces` and other one for email complains `email_complaints`.
We have subscribed this HTTP endpoint (/amazon_sns_endpoint) for any notifications on `email_bounces` topic.
When an email is bounced or someone complains about email, SES sends a JSON message with email
information and cause of failure to our subscribed topics on SNS service which, in turn, sends an HTTP request to
our subscribed endpoint with JSON message that can be processed to handle bounces and complaints.
When a bounce occurs, we mark that email address as bounced and no further emails will be sent to this
email address.
Here is a wiki article explaining how to setup this.
https://github.com/gettalent/talent-flask-services/wiki/Email-Bounces
"""
data = json.loads(request.data)
headers = request.headers
logger.info('SNS Callback: Headers: %s\nRequest Data: %s', headers, data)
# SNS first sends a confirmation request to this endpoint, we then confirm our subscription by sending a
# GET request to given url in subscription request body.
if request.headers.get(aws.HEADER_KEY) == aws.SUBSCRIBE:
response = requests.get(data[aws.SUBSCRIBE_URL])
if data[aws.TOPIC_ARN] not in response.text:
logger.info(
'Could not verify topic subscription. TopicArn: %s, RequestData: %s',
data[aws.TOPIC_ARN], request.data)
return 'Not verified', requests.codes.INTERNAL_SERVER_ERROR
logger.info(
'Aws SNS topic subscription for email notifications was successful.'
'\nTopicArn: %s\nRequestData: %s', data[aws.TOPIC_ARN],
request.data)
elif request.headers.get(aws.HEADER_KEY) == aws.NOTIFICATION:
# In case of notification, check its type (bounce or complaint) and process accordingly.
data = json.loads(request.data)
message = data[aws.MESSAGE]
message = json.loads(message)
message_id = message[aws.MAIL][aws.MESSAGE_ID]
if message[aws.NOTIFICATION_TYPE] == aws.BOUNCE_NOTIFICATION:
bounce = message[aws.BOUNCE]
emails = [
recipient[aws.EMAIL_ADDRESSES]
for recipient in bounce[aws.BOUNCE_RECIPIENTS]
]
handle_email_bounce(message_id, bounce, emails)
elif message[aws.NOTIFICATION_TYPE] == aws.COMPLAINT_NOTIFICATION:
pass # TODO: Add implementation for complaints
elif request.headers.get(aws.HEADER_KEY) == aws.UNSUBSCRIBE:
logger.info(
'SNS notifications for email campaign has been unsubscribed.'
'\nRequestData: %s', request.data)
else:
logger.info('Invalid request. Request data %s', request.data)
return 'Thanks SNS for notification'
"""
Run Celery Worker
For Celery to run from command line, script runs as separate process with celery command
Usage: open terminal cd to talent-flask-services directory
Run the following command to start celery worker:
$ celery -A email_campaign_service.email_campaign_app.celery_app worker --concurrency=4 --loglevel=info
"""
# Service Specific
from email_campaign_service.common.talent_celery import CELERY_WORKER_ARGS
from email_campaign_service.email_campaign_app import celery_app, logger, app
from email_campaign_service.common.talent_config_manager import TalentConfigKeys
from email_campaign_service.common.campaign_services.campaign_utils import CampaignUtils
try:
logger.info("Starting Celery worker for:%s" % app.name)
celery_app.start(argv=CELERY_WORKER_ARGS + [CampaignUtils.EMAIL] +
['-n', CampaignUtils.EMAIL])
except Exception as e:
logger.exception(
"Couldn't start Celery worker for email_campaign_service in "
"%s environment." % (app.config[TalentConfigKeys.ENV_KEY]))
def get_priority_emails(user, candidate_ids):
"""
This returns tuple (candidate_id, email) choosing priority email form all the emails of candidate.
:type user: User
:type candidate_ids: list
:rtype: list[tuple]
"""
# Get candidate emails sorted by updated time and then by candidate_id
candidate_email_rows = CandidateEmail.get_emails_by_updated_time_candidate_id_desc(
candidate_ids)
# list of tuples (candidate id, email address)
group_id_and_email_and_labels = []
# ids_and_email_and_labels will be [(1, '*****@*****.**', 1), (2, '*****@*****.**', 3), ...]
# id_email_label: (id, email, label)
ids_and_email_and_labels = [(row.candidate_id, row.address,
row.email_label_id)
for row in candidate_email_rows]
# Again sorting on the basis of candidate_id
ids_and_email_and_labels = sorted(ids_and_email_and_labels,
key=itemgetter(0))
"""
After running groupby clause, the data will look like
group_id_and_email_and_labels = [[(candidate_id1, email_address1, email_label1),
(candidate_id2, email_address2, email_label2)],... ]
"""
for key, group_id_email_label in itertools.groupby(
ids_and_email_and_labels,
lambda id_email_label: id_email_label[0]):
group_id_and_email_and_labels.append(list(group_id_email_label))
filtered_email_rows = []
# Check if primary EmailLabel exist in db
if not EmailLabel.get_primary_label_description(
) == EmailLabel.PRIMARY_DESCRIPTION:
raise InternalServerError(
"get_email_campaign_candidate_ids_and_emails: Email label with primary description not found in db."
)
# We don't know email_label id of primary email. So, get that from db
email_label_id_desc_tuples = [(email_label.id, email_label.description)
for email_label in EmailLabel.query.all()]
# If there are multiple emails of a single candidate, then get the primary email if it exist, otherwise get any
# other email
for id_and_email_and_label in group_id_and_email_and_labels:
_id, email = get_candidate_id_email_by_priority(
id_and_email_and_label, email_label_id_desc_tuples)
search_result = CandidateEmail.search_email_in_user_domain(
User, user, email)
if CandidateEmail.is_bounced_email(email):
logger.info(
'Skipping this email because this email address is marked as bounced.'
'CandidateId : %s, Email: %s.' % (_id, email))
continue
# If there is only one candidate for an email-address in user's domain, we are good to go,
# otherwise log error and send campaign email to that email id only once.
if len(search_result) == 1:
filtered_email_rows.append((_id, email))
else:
# Check if this email is already present in list of addresses to which campaign would be sent.
# If so, omit the entry and continue.
if any(email in emails for emails in filtered_email_rows):
continue
else:
logger.error(
'%s candidates found for email address %s in user(id:%s)`s domain(id:%s). '
'Candidate ids are: %s' %
(len(search_result), email, user.id, user.domain_id, [
candidate_email.candidate_id
for candidate_email in search_result
]))
filtered_email_rows.append((_id, email))
return filtered_email_rows
def create_email_campaign_url_conversions(new_html, new_text,
is_track_text_clicks,
is_track_html_clicks,
custom_url_params_json,
is_email_open_tracking, custom_html,
email_campaign_send_id):
soup = None
# HTML open tracking
logger.info(
'create_email_campaign_url_conversions: email_campaign_send_id: %s' %
email_campaign_send_id)
if new_html and is_email_open_tracking:
soup = BeautifulSoup(new_html, "lxml")
num_conversions = convert_html_tag_attributes(
soup,
lambda url: create_email_campaign_url_conversion(
url, email_campaign_send_id, TRACKING_URL_TYPE),
tag="img",
attribute="src",
convert_first_only=True)
# If no images found, add a tracking pixel
if not num_conversions:
image_url = TRACKING_PIXEL_URL
new_image_url = create_email_campaign_url_conversion(
image_url, email_campaign_send_id, TRACKING_URL_TYPE)
new_image_tag = soup.new_tag("img", src=new_image_url)
soup.insert(0, new_image_tag)
# HTML click tracking
if new_html and is_track_html_clicks:
soup = soup or BeautifulSoup(new_html)
# Fetch the custom URL params dict, if any
if custom_url_params_json:
destination_url_custom_params = json.loads(custom_url_params_json)
else:
destination_url_custom_params = dict()
# Convert all of soup's <a href=> attributes
convert_html_tag_attributes(
soup,
lambda url: create_email_campaign_url_conversion(
url, email_campaign_send_id, HTML_CLICK_URL_TYPE,
destination_url_custom_params),
tag="a",
attribute="href")
# Add custom HTML. Doesn't technically belong in this function, but since we have access to the BeautifulSoup
# object, let's do it here.
if new_html and custom_html:
soup = soup or BeautifulSoup(new_html)
body_tag = soup.find(name="body") or soup.find(name="html")
"""
:type: Tag | None
"""
if body_tag:
custom_html_soup = BeautifulSoup(custom_html)
body_tag.insert(0, custom_html_soup)
else:
logger.error(
"Email campaign HTML did not have a body or html tag, "
"so couldn't insert custom_html! email_campaign_send_id=%s",
email_campaign_send_id)
# Convert soup object into new HTML
if new_html and soup:
new_html = soup.prettify()
new_html = HTMLParser.HTMLParser().unescape(new_html)
return new_text, new_html
