def test_send_email_with_required_args(self):
try:
gmail_configuration = self.configuration.get_email_apps()[0]
gmail_app = GmailEmailApp(config=gmail_configuration)
gmail_app.send_email(subject='test_send_email_with_required_args',
to=[gmail_configuration['email_address']])
except Exception as e:
logger.error("Test failed with exception: %s" % e)
self.fail("Test failed with exception: %s" % e)
def test_send_email_with_html(self):
try:
gmail_configuration = self.configuration.get_email_apps()[0]
gmail_app = GmailEmailApp(config=gmail_configuration)
gmail_app.send_email(subject='test_send_email_with_html',
to=[gmail_configuration['email_address']],
html='<h1>Html only</h1>')
except Exception as e:
logger.error("Test failed with exception: %s" % e)
self.fail("Test failed with exception: %s" % e)
def test_connect(self):
# Test the connection with the correct api key
try:
gmail_configuration = self.configuration.get_email_apps()[0]
GmailEmailApp(config=gmail_configuration)
except SMTPAuthenticationError as e:
logger.error('Error connecting with the correct credentials: %s',
e)
self.fail('Error connecting with the correct credentials')
else:
logger.info('Connected with the correct credentials successfully.')
# Test that the connection is failed with the wrong credentials
with self.assertRaises(SMTPAuthenticationError):
gmail_wrong_configuration = copy.deepcopy(gmail_configuration)
gmail_wrong_configuration['api_key'] = 'wrong_key'
GmailEmailApp(config=gmail_wrong_configuration)
logger.info(
"Loading Dropbox with wrong credentials failed successfully.")
def test_send_email_with_all_args(self):
try:
gmail_configuration = self.configuration.get_email_apps()[0]
gmail_app = GmailEmailApp(config=gmail_configuration)
gmail_app.send_email(subject='test_send_email_with_all_args',
to=[gmail_configuration['email_address']],
cc=[gmail_configuration['email_address']],
bcc=[gmail_configuration['email_address']],
text='Test plain/text body',
html='<h1>Test html body</h1>',
attachments=[
os.path.join(self.test_data_path,
'sample_data.txt')
],
sender=gmail_configuration['email_address'],
reply_to=gmail_configuration['email_address'])
except Exception as e:
logger.error("Test failed with exception: %s" % e)
self.fail("Test failed with exception: %s" % e)
def main():
"""
:Example:
python main.py [-m crawl_and_send]
-c confs/template_conf.yml
-l logs/output.log
"""
# Initializing
args, configuration = init_main()
# Start in the specified mode
if args.run_mode == 'list_emails':
data_store = JobBotMySqlDatastore(
config=configuration.get_datastores()[0])
show_ads_checked(ads=data_store.get_applications_sent())
elif args.run_mode == 'remove_email':
data_store = JobBotMySqlDatastore(
config=configuration.get_datastores()[0])
data_store.remove_ad(email_id=args.email_id)
elif args.run_mode == 'upload_files':
upload_files_to_cloudstore(cloud_store=JobBotDropboxCloudstore(
config=configuration.get_cloudstores()[0]))
elif args.run_mode == 'create_table':
data_store = JobBotMySqlDatastore(
config=configuration.get_datastores()[0])
data_store.create_applications_sent_table()
elif args.run_mode == 'crawl_and_send':
crawl_and_send_loop(lookup_url=configuration.lookup_url,
check_interval=configuration.check_interval,
crawl_interval=configuration.crawl_interval,
anchor_class_name=configuration.anchor_class_name,
data_store=JobBotMySqlDatastore(
config=configuration.get_datastores()[0]),
cloud_store=JobBotDropboxCloudstore(
config=configuration.get_cloudstores()[0]),
email_app=GmailEmailApp(
config=configuration.get_email_apps()[0],
test_mode=configuration.test_mode))
else:
logger.error('Incorrect run_mode specified!')
raise argparse.ArgumentTypeError('Incorrect run_mode specified!')
def main():
"""
:Example:
python main.py -m run_mode_1
-c confs/template_conf.yml
-l logs/output.log
"""
# Initializing
args = _argparser()
_setup_log(args.log, args.debug)
logger.info("Starting in run mode: {0}".format(args.run_mode))
# Load the configuration
configuration = Configuration(config_src=args.config_file)
# Init the Cloudstore
cloud_store = DropboxCloudstore(config=configuration.get_cloudstores()[0])
# Init the Datastore
data_store = MySqlDatastore(**configuration.get_datastores()[0])
# Init the Email App
gmail_configuration = configuration.get_email_apps()[0]
gmail_app = GmailEmailApp(config=configuration.get_email_apps()[0])
# Mysql examples
logger.info("\n\nMYSQL EXAMPLE\n-------------------------")
logger.info("\n\nTables in current DB: {0}".format(
list(data_store.show_tables())))
logger.info("Creating Table: orders")
table_schema = """ order_id INT(6) PRIMARY KEY,
order_type VARCHAR(30) NOT NULL,
location VARCHAR(30) NOT NULL """
data_store.create_table(table='orders', schema=table_schema)
logger.info("Tables in current DB:\n{0}".format(
list(data_store.show_tables())))
logger.info("Inserting into orders the values:\n(1 simple newyork)..")
insert_data = {
"order_id": 1,
"order_type": "plain",
"location": "new_york"
}
data_store.insert_into_table(table='orders', data=insert_data)
logger.info("SELECT * FROM orders;\n{0}".format(
data_store.select_from_table(table='orders')))
logger.info("Deleting the inserted row from table orders..")
data_store.delete_from_table(table='orders', where='order_id=1')
logger.info("SELECT * FROM orders;\n{0}".format(
data_store.select_from_table(table='orders')))
logger.info("Dropping Table: orders")
data_store.drop_table(table='orders')
logger.info("Tables in current DB:\n{0}".format(
list(data_store.show_tables())))
# Dropbox examples
logger.info("\n\nDROPBOX EXAMPLE\n-------------------------")
logger.info("List of files in Dropbox /python_template:\n{0}".format(
list(cloud_store.ls(path='/python_template').keys())))
upload_path = "/python_template/file1.txt"
file_content = "test file content"
logger.info("Uploading file {file} with content:\n{content}".format(
file=upload_path, content=file_content))
cloud_store.upload_file(file_bytes=file_content.encode(),
upload_path=upload_path)
logger.info("List of files in Dropbox /python_template:\n{0}".format(
list(cloud_store.ls(path='/python_template').keys())))
downloaded_file = cloud_store.download_file(frompath=upload_path)
logger.info(
"Downloaded file and its content is:\n{0}".format(downloaded_file))
cloud_store.delete_file(file_path=upload_path)
logger.info("Deleting file {file}..".format(file=upload_path))
logger.info("List of files in Dropbox /python_template:\n{0}".format(
list(cloud_store.ls(path='/python_template').keys())))
# Gmail examples
logger.info("\n\nGMAIL EXAMPLE\n-------------------------")
subject = "Email example"
body = "<h1>This is an html body example</h1><br><b>This goes to the html argument. " \
"You can use the text argument for plain text.</b>"
emails_list = [gmail_configuration['email_address']]
attachments_paths = [os.path.join('data', 'sample_data.txt')]
logger.info(
"Sending email with `subject` = `{subject}`, `from,to,cc,bcc,reply_to` = `{email_addr}`, "
"`html` = `{body}` and `attachments` = `{attachments}`".format(
subject=subject,
email_addr=emails_list[0],
body=body,
attachments=attachments_paths))
gmail_app.send_email(subject=subject,
to=emails_list,
cc=emails_list,
bcc=emails_list,
html=body,
attachments=attachments_paths,
sender=emails_list[0],
reply_to=emails_list[0])
def test_is_connected_and_exit(self):
gmail_configuration = self.configuration.get_email_apps()[0]
gmail_app = GmailEmailApp(config=gmail_configuration)
self.assertEqual(True, gmail_app.is_connected())
gmail_app.__exit__()
self.assertEqual(False, gmail_app.is_connected())
def crawl_and_send_loop(lookup_url: str, check_interval: int,
crawl_interval: int, anchor_class_name: str,
data_store: JobBotMySqlDatastore,
cloud_store: JobBotDropboxCloudstore,
email_app: GmailEmailApp) -> None:
"""
The main loop.
Crawls the ad site for new ads and sends emails where applicable and informs the applicant.
:params lookup_url:
:params check_interval:
:params data_store:
:params cloud_store:
:params gmail_app:
"""
ad_site_crawler = XeGrAdSiteCrawler(
stop_words=cloud_store.get_stop_words_data(),
anchor_class_name=anchor_class_name)
attachments_local_paths = [
os.path.join(cloud_store.local_files_folder, attachment_name)
for attachment_name in cloud_store.attachments_names
]
# Get the email_data, the attachments and the stop_words list from the cloudstore
cloud_store.download_attachments()
application_to_send_subject, application_to_send_html = cloud_store.get_application_to_send_email_data(
)
inform_should_call_subject, inform_should_call_html = cloud_store.get_inform_should_call_email_data(
)
inform_success_subject, inform_success_html = cloud_store.get_inform_success_email_data(
)
links_checked = [
row[0] for row in data_store.get_applications_sent(columns='link')
]
logger.info("Waiting for new ads..")
while True:
new_ads = list(
ad_site_crawler.get_new_ads(lookup_url=lookup_url,
ads_checked=links_checked,
crawl_interval=crawl_interval))
if len(new_ads) > 0:
links_checked = [
row[0]
for row in data_store.get_applications_sent(columns='link')
]
emails_checked = [
row[0]
for row in data_store.get_applications_sent(columns='email')
]
for link, email in new_ads:
if link not in links_checked and (email not in emails_checked
or email is None):
if email is None:
# Email applicant to inform him that he should call manually
logger.info(
"Link ({}) has no email. Inform the applicant.".
format(link))
email_app.send_email(
subject=inform_should_call_subject,
html=inform_should_call_html.format(link=link),
to=[email_app.get_self_email()])
else:
# Send application after 1 minute (don't be too cocky)
time.sleep(60)
logger.info("Sending email to: {}. Ad Link: {}".format(
email, link))
email_app.send_email(
subject=application_to_send_subject,
html=application_to_send_html.format(link),
to=[email],
attachments=attachments_local_paths)
# Inform applicant that an application has been sent successfully
email_app.send_email(subject=inform_success_subject,
html=inform_success_html.format(
email=email, link=link),
to=[email_app.get_self_email()])
email_info = {
"link": link,
"email": email,
"sent_on": datetime.datetime.utcnow().isoformat()
}
data_store.save_sent_application(email_info)
logger.info("Waiting for new ads..")
# Look for new ads every 2 minutes
logger.debug("Sleeping for {check_interval} seconds..".format(
check_interval=check_interval))
time.sleep(check_interval)