def invoice_send_email(entity, receiver, attachment):
# Send email
scrapymail = Email()
if (attachment is None) or attachment.empty:
subject = '[PAM Tax Checking] - {0} 发票无异常 {1}'.format(
TIME['TODAY'], entity)
content = 'Hi All,\r\n\r\n{}的发票无异常记录。\r\n\r\nThanks.'.format(entity)
scrapymail.send(subject=subject,
content=content,
receivers=receiver,
attachment=None)
else:
subject = '[PAM Tax Checking] - {0} 发票异常清单 {1}'.format(
TIME['TODAY'], entity)
content = 'Hi All,\r\n\r\n请查看附件关于{}的发票异常记录。\r\n\r\nThanks.'.format(
entity)
entity_path = df_to_excel(df=attachment,
path=PATH['FILE_DIR'],
file_name=PATH['ATTACHMENT_FILE'].format(
TIME['TODAY'], entity),
sheet_name=entity)
scrapymail.send(subject=subject,
content=content,
receivers=receiver,
attachment=entity_path)
logger.info('Delete attachment file.')
os.remove(entity_path)
scrapymail.close()
def main(argv):
"""
Get tag manager account ID, Site Name and Google Analytics Tracking ID from command line.
Account ID as a string input to create a Tag for. Site name for creating container to get javascript code snippet.
Google Analytics tracking id, where you want get all type of tracking
"""
sys.tracebacklimit = settings.DUBUG
args_help = """
Site Name, Site URL and Google Analytics Tracking ID from command line.
Site name and Site URL for creating container to get javascript code snippet.
Google Analytics tracking id, where you want get all type of tracking
"""
parser = argparse.ArgumentParser(description=args_help)
parser.add_argument('--site_name',
type=str,
help='Your site name',
required=True)
parser.add_argument('--site_url',
type=str,
help='Your site URL',
required=True)
args = parser.parse_args()
container_name = str(args.site_name)
container_site = str(args.site_url)
if not validators.url(container_site):
raise Exception('invalid site URL')
secret_key_file = settings.GOOGLE_DEVELOPER_SECRET_KEY
if not os.path.isfile(secret_key_file):
raise Exception(
'Missing required secret key JSON file. Please check file name in settings.py or create one '
'here https://console.developers.google.com/permissions/serviceaccounts'
)
# Define the auth scopes to request.
analytics_scope = ['https://www.googleapis.com/auth/analytics.edit']
# Authenticate and construct service.
analytics_service = GetAnalyticsService('analytics', 'v3', analytics_scope,
secret_key_file)
tracking_id = GetOrCreateTrackingId(analytics_service, container_name,
container_site)
# Define the auth scopes to request.
tag_manager_scope = [
'https://www.googleapis.com/auth/tagmanager.edit.containers',
'https://www.googleapis.com/auth/tagmanager.edit.containerversions',
'https://www.googleapis.com/auth/tagmanager.publish'
]
# Authenticate and construct service.
tag_manager_service = GetService('tagmanager', 'v1', tag_manager_scope,
secret_key_file)
account_id = GetAccountID(tag_manager_service)
# get container id to create tag
container_id = CreateOrGetContainer(tag_manager_service, account_id,
container_name, container_site)
# Create the hello world tag for tracking id
CreateOrGetTag(tag_manager_service, account_id, container_id, tracking_id)
container_version_id = CreateContainerVersion(tag_manager_service,
account_id, container_id)
PublishContainerVersion(tag_manager_service, account_id, container_id,
container_version_id)
container_public_id = CreateOrGetContainer(tag_manager_service, account_id,
container_name, container_site,
'public_id')
print('Preparing javascript code snippet...')
with open(os.path.join('code_snippet', 'gtm_backup.txt'), 'r') as gtm:
gtm_snippet = gtm.read()
gtm.close()
with open(os.path.join('code_snippet', 'gtm.txt'), 'w') as gtm:
gtm_snippet = re.sub(r'XXXXXXXX', container_public_id, gtm_snippet)
gtm.write(gtm_snippet)
gtm.close()
print(gtm_snippet)
if settings.SEND_CODE_SNIPPET_EMAIL:
Email.send()
log.debug("=== data ===")
log.debug(data)
log.debug("=== template ===")
log.debug(tpl.render(data))
log.debug("=== end ===")
if CFG['email_users']:
to = c
if '@' not in c:
to += '@you.com'
email = Email('[TSM] Expiration warning for contact %s' % c,
tpl, [to],
bcc=CFG['bcc_address'],
replyto=CFG['support_address'])
email.send()
log.info("email sent to %s" % to)
# Now send a summary email to the admins
expire = [x for x in alert.events if x['retire'] == 0 and x['days_to_expire']]
data = {
'retire': retire,
'expire': [x for x in expire if int(x['days_to_expire']) < 1],
'expire_soon': [x for x in expire if int(x['days_to_expire']) >= 1],
}
tpl = Template('%s_escalation' % alert.search_name)
log.debug("=== data ===")
log.debug(data)
datas = ExcelReader(self.excel).data
for d in datas:
with self.subTest(data=d):
self.sub_setUp()
self.driver.find_element(*self.locator_kw).send_keys(
d['search'])
self.driver.find_element(*self.locator_su).click()
time.sleep(2)
links = self.driver.find_elements(*self.locator_result)
for link in links:
logger.info(link.text)
self.sub_tearDown()
if __name__ == '__main__':
report = REPORT_PATH + '\\report.html'
with open(report, 'wb') as f:
runner = HTMLTestRunner(f,
verbosity=2,
title='从0搭建测试框架 灰蓝',
description='修改html报告')
runner.run(TestBaiDu('test_search'))
e = Email(title='百度搜素测试报告',
message='这是今天的测试报告，请查收！',
receiver='*****@*****.**',
server='...',
sender='...',
password='******',
path=report)
e.send()
condition_done = '[Timestamp] >= {0} AND [Source] = {1}'\
.format(get_sql_list(TIME['TODAY']), get_sql_list(PATH['SITE']))
entities_run = exist_db.get_to_runs(table_name=DB['LOG_TABLE_NAME'],
condition=condition_done,
source=PATH['SITE'])
for entity in entities_run:
entity_object = Haozu(entity)
entity_object.run()
with ODBC(config=keys.dbconfig_mkt) as entity_db:
entity_db.upload(df=entity_object.df['df'],
table_name=DB['DETAIL_TABLE'],
new_id=PATH['SITE'])
entity_db.upload(df=entity_object.df['info'],
table_name=DB['INFO_TABLE'],
new_id=PATH['SITE'],
dedupe_col='Source_ID')
entity_db.log(Entity=entity,
Timestamp=TIME['TODAY'],
Source=PATH['SITE'],
start=1,
end=len(entity_object.df['info']))
scrapyemail = Email()
scrapyemail.send(subject='[Scrapy] ' + DB['DETAIL_TABLE'],
content='Done',
attachment=PATH['LOG_PATH'])
scrapyemail.close()
exit(0)
# Upload to database
entity_db = ODBC(keys.dbconfig)
entity_db.upload(df=tax_df, table_name=DB['TAX_INFO_TABLE'])
entity_db.upload(df=tax_detail_df, table_name=DB['TAX_DETAIL_TABLE'])
entity_db.log(start=TIME['PRE3MONTH'], end=TIME['TODAY'], Timestamp=TIME['TIMESTAMP'], Source=PATH['SITE'],
Entity=row['Entity_Name'])
entity_db.close()
# Ensure failure of scraping process do not interrupt email and sp execution
with ODBC(keys.dbconfig) as execute_db:
# Update Irregular_Ind by executing stored procedure
execute_db.call_sp(sp='CHN.Irregular_Tax_Refresh', table_name=DB['TAX_DETAIL_TABLE'],
table_name2=DB['TAX_DETAIL_TABLE'])
for index, row in access.iterrows():
# Get irregular record
att = execute_db.call_sp(sp='CHN.Irregular_Tax_ETL', output=True, table_name=DB['TAX_DETAIL_TABLE'],
entity_name=row['Entity_Name'])
numeric_col = ['金额', '单价', '税率', '税额']
if att is not None:
att[numeric_col] = att[numeric_col].apply(pd.to_numeric)
invoice_send_email(entity=row['Entity_Name'], receiver=row['Email_List'], attachment=att)
# Send email summary
scrapyemail_summary = Email()
scrapyemail_summary.send('[Scrapy]' + PATH['SITE'], 'Done', PATH['LOG_PATH'],
receivers='[email protected];[email protected]')
scrapyemail_summary.close()
exit(0)
