def main():
"""Run each morning after data processing has occured
- This is triggered to run after every_night.py
"""
parser = argparse.ArgumentParser()
parser.add_argument("-a")
args = parser.parse_args()
try:
account_id = args.a
if (not account_id):
print('Please specify an account id with -a')
return
if not Helpers().isActiveAccount(account_id):
Log("info", "this account isn't active. Exiting", '', account_id)
return
NotifyViaEmail(account_id).main()
MonthlyStop(account_id)
ControlSpend(account_id).main()
# TODO: catch proper exception
except:
Log("error",
"error starting run_budget_commander.py from command line",
traceback.format_exc())
# TODO: return proper exception
raise
def main(self):
"""Populate the adverts and advert_performance tables"""
Log("info", "populating the adverts and advert_performance tables", "",
self.account_id)
settings = Settings()
for date_range in settings.date_ranges:
df_chunks = self.dataFrameFromAdPerformanceReports(date_range)
self.deleteExisting(date_range)
while True:
try:
df = next(df_chunks)
except StopIteration:
break
if functions.dfIsEmpty(df):
continue
try:
df = self.processDf(df, date_range)
except Exception as exception:
Log("error", str(exception), traceback.format_exc())
raise
df = self.addInfo(df)
self.writeToAdvertPerformanceTable(df, date_range)
def addUserAccounts(self, user_id, first_run):
if not Helpers().isActiveUser(user_id):
Log("info", "this user isn't active. Exiting",
'user_id: %s' % (user_id))
return
try:
Log("info", "adding accounts for user id '%s'" % user_id)
self.user_id = user_id
accounts_df = self.getAccountsDf()
if functions.dfIsEmpty(accounts_df):
return
accounts_df = accounts_df.drop_duplicates('google_id')
accounts_df = self.dropDuplicates(accounts_df, first_run)
if (accounts_df.shape[0] == 0 and first_run):
Log('warning',
"no unique google accounts were found for this user",
"user_id (%s)" % (user_id), "")
accounts_df.to_sql("accounts",
Database().createEngine(),
index=False,
if_exists="append")
except Exception as exception:
Log("error",
str(exception) + " (User id: %s)" % (user_id),
traceback.format_exc())
Log("info", "finished adding account meta data")
def main(self):
if self.user_settings["pause_campaigns"]:
print("paused campaigns is enabled. Exiting...")
return # they'll get an email when the campaigns pause so there's no need to send one here
print("running email notifier")
if self.budget is None:
return
print("under_budget: " + str(self.under_budget))
print("email_sent: " + str(self.user_settings["email_sent"]))
if not self.user_settings["notify_via_email"]:
Log("info", "email notifications turned off", '', self.account_id)
return
if self.under_budget and self.user_settings["email_sent"]:
self.markAsNotSent()
return
if not self.user_settings["email_sent"] and not self.under_budget:
self.sendEmail()
return
Log("info", "no action", "NotifyViaEmail", self.account_id)
def main():
"""Download data from the api
Process the data ready for the app"""
Log("info", "process_account running", "from process_account.py")
parser = argparse.ArgumentParser()
parser.add_argument("-a")
args = parser.parse_args()
try:
account_id = args.a
if (not account_id):
Log('info', 'Please specify an account id with -a')
return
if not Helpers().isActiveAccount(account_id):
Log("info", "this account isn't active. Exiting", '', account_id)
return
download.main(account_id)
except:
Log("error", "error starting every night from command line",
traceback.format_exc())
raise
Log("info", "firing run_budget_commander command", '', account_id)
myProcess().main("run_budget_commander.py", account_id)
def createNotification(account_id):
#only when running for the first time...
def isFirstRun(account_id):
query = "select ad_performance_report_processed_at from accounts where id = '%s'" % (
account_id)
results = Database().executeQuery(query)
for result in results:
return result[0] is None
if not isFirstRun(account_id):
Log('info',
"the notification won't be created as this isn't the first run",
'', account_id)
return
Log('info', "creating successful sync notification", '', account_id)
user_id = Helpers().getUserIdFromAccountId(account_id)
account_name = Helpers().getAccountNameFromId(account_id)
account_google_id = Helpers().getAccountGoogleIdFromId(account_id)
username = Helpers().getUsername(account_id)
query = r"""
insert into notifications
(id, type, notifiable_id, notifiable_type, data, created_at, updated_at)
values
('%s', 'App\\Notifications\\AccountSynced', '%s','App\\User',
'{"message":"%s was synced successfully! Refesh the page to access the account."}', now(), now())
""" % (str(uuid.uuid4()), user_id, account_name)
Database().executeQuery(query)
sendEmail(account_name, account_google_id, username, account_id)
def main(self):
self.store_excess_budget()
if not self.budget_commander.user_settings["pause_campaigns"]:
Log("info", "pause_campaigns is disabled", '', self.account_id)
return
campaigns_are_enabled_month = self.budget_commander.campaignsAreEnabledMonth(
)
spend_is_over_budget = self.spendIsOverBudget()
if spend_is_over_budget and campaigns_are_enabled_month:
(PauseEnableCampaigns(self.account_id)).pauseForMonth()
Log(
"info",
"Budget commander monthly stop: campaigns paused for the month",
"", self.account_id)
self.sendEmail('Paused')
return
campaigns_are_paused_month = self.budget_commander.campaignsArePausedMonth(
)
spend_is_under_budget = self.spendIsUnderBudget()
if spend_is_under_budget and campaigns_are_paused_month and self.budget_commander.user_settings[
"enable_campaigns"]:
(PauseEnableCampaigns(self.account_id)).enableForMonth()
Log(
"info",
"Budget commander monthly stop: campaigns enabled for the month",
"", self.account_id)
self.sendEmail('Enabled')
return
Log("info", "Budget commander monthly stop: no actions", "",
self.account_id)
def __init__(self):
super(sendEmail, self).__init__()
self.L = Log("snedEmail", 'DEBUG').logger
self.sender_email = 'EMAIL_SENDER'
self.password = '******'
self.smtpHost = 'smtpHost'
self.receiver = 'receiver'
def startGame():
global state, timer
Log.info('START', language)
timer = None
state = WAITING_FOR_PHOTO
isGameRunning = False
pygame.mixer.music.load(sounds[language][0])
pygame.mixer.music.play()
def loop(self):
try:
isGameRunning = True
clock = pygame.time.Clock()
lastTime = pygame.time.get_ticks()
font = pygame.font.Font(None, 30)
while isGameRunning:
for event in pygame.event.get():
if event.type == MOUSEBUTTONDOWN:
if not self.config.isTouch():
self.onMouseDown(event.pos)
elif event.type == MOUSEBUTTONUP:
if not self.config.isTouch():
self.onMouseUp(event.pos)
elif event.type == KEYDOWN:
if event.key == K_ESCAPE:
isGameRunning = False
if self.config.isTouch():
event = self.touchScreen.readUpDownEvent()
while event is not None:
if event['type'] == TouchScreen.DOWN_EVENT:
self.onMouseDown(event['pos'])
elif event['type'] == TouchScreen.UP_EVENT:
self.onMouseUp(event['pos'])
event = self.touchScreen.readUpDownEvent()
if not self.config.isTouch():
self.onMouseMove(pygame.mouse.get_pos())
else:
pos = self.touchScreen.getPosition()
self.onMouseMove(pos)
self.screen.fill([0, 0, 0])
currTime = pygame.time.get_ticks()
dt = currTime - lastTime
lastTime = currTime
self.draw(dt / 1000)
if not self.config.isTouch() and self.blitCursor:
self.screen.blit(self.cursor, (pygame.mouse.get_pos()))
if self.config.showFPS():
fps = font.render(str(int(clock.get_fps())), True,
Color('white'))
self.screen.blit(fps, (50, 50))
pygame.display.flip()
clock.tick(60)
pygame.quit()
except Exception as e:
Log.getLogger().exception('ERROR,Error occured!')
def __init__(self, path: str):
self.path = path
self.log = Log().get_logger()
if not os.path.exists(self.path):
self.log.error(f"指定文件路径不存在：{self.path}")
self.yml = None
with open(path, 'r', encoding='utf-8') as f:
try:
self.yml = yaml.safe_load(f.read())
except Exception as e:
print(e)
class DataBase:
_conn = None
_cursor = None
def __init__(self, conf, database):
self.log = Log()
self.host = conf['host']
self.user = conf['user']
self.passwd = conf['passwd']
self.database = database
# 连接数据库
def connectDatabase(self):
try:
self._conn = pymysql.connect(self.host, self.user, self.passwd)
self._conn.select_db(self.database)
self._cursor = self._conn.cursor()
self.log.info("Connect DB successfully!")
except ConnectionError as ex:
self.log.error(str(ex))
# 执行sql语句 ---查询
def execute(self, sql):
self.connectDatabase()
self._cursor.execute(sql)
data = self._cursor.fetchall()
return data
# def executeSQL(self, sql, params):
# self.connectDB()
# # executing sql
# self.cursor.execute(sql, params)
# # executing by committing to DB
# self.db.commit()
# return self.cursor
# def get_all(self, cursor):
# # value = cursor.fetchall()
# # return value
# #
# # def get_one(self, cursor):
# # value = cursor.fetchone()
# # return value
# 关闭数据库
def close(self):
if self._conn and self._cursor:
self._cursor.close()
self._conn.close()
print("Database closed!")
def __init__(self):
global shcme, baseUrl, ip, username, password, port, dataname
shcme = local_Read_Config.get_HTTP("schme")
baseUrl = local_Read_Config.get_HTTP("baseUrl")
ip = local_Read_Config.get_DATA("ip")
username = local_Read_Config.get_DATA("username")
password = local_Read_Config.get_DATA("password")
port = local_Read_Config.get_DATA("port")
dataname = local_Read_Config.get_DATA("dataname")
self.logger = Log().logger
self.header = {}
self.param = {}
self.data = {}
self.url = None
self.file = {}
class Testbuild3(Testtodolist):
log = Log()
def test_01(self):
"""
步骤：
1、美购首页
2、品类聚合
3、搜索框
4、点击热词
:return:
"""
begin_date = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print(begin_date)
testbuild = TodolistBuild(self.driver)
testbuild. click_alert()
testbuild.click_meigou()
time.sleep(5)
testbuild.click_czsl()
testbuild.click_search()
time.sleep(2)
testbuild.click_hot_search()
time.sleep(5)
self.driver.background_app(5)
time.sleep(5)
# 美购搜索 埋点
result = mysql_test.query(action='search_result_open', event_time=begin_date)
assert result != ' ', 'search_result_open事件为空！'
class Testbuild(Testtodolist):
log = Log()
def test_01(self):
"""
美购首页->品类聚合除皱廋脸
美购列表pv事件
:return:
"""
begin_date = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
testbuild = TodolistBuild(self.driver)
testbuild.click_alert()
testbuild.click_meigou()
testbuild.click_czsl()
time.sleep(10)
self.driver.background_app(5)
time.sleep(10)
result = mysql_test.query(action='page_view',
event_time=begin_date,
page_name='welfare_list')
print(result)
end_page_view2 = result[0]['params']
referrer = end_page_view2['referrer']
page_name = end_page_view2['page_name']
assert referrer == 'welfare_home', 'referrer获取错误！'
assert page_name == 'welfare_list', 'page_name获取错误！'
assert len(result) == 1, f'埋点数量错误，预期为1个，实际为{len(result)}'
class Testbuild3(Testtodolist):
log = Log()
def test_01(self):
"""
步骤：
1、打开搜索结果页
2、下拉刷新一次，退出搜索结果页
3、首页浏览事件记一次
:return:
"""
begin_date = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print(begin_date)
testbuild = TodolistBuild(self.driver)
testbuild.click_alert()
testbuild.click_meigou()
time.sleep(5)
testbuild.click_search()
time.sleep(2)
testbuild.switch_android_up()
time.sleep(5)
self.driver.background_app(5)
time.sleep(5)
def get_log():
if obj.run is not None:
#只有一个线程能成功地获取锁
obj.mutex.acquire()
MyLog.log = Log()
#解锁
MyLog.mutex.release()
return MyLog.log
'''
finally:
logger.info("*********TEST END*********")
# send test report by email
if Email_on_off == 'on':
self.email.send_email()
elif Email_on_off == 'off':
logger.info("Doesn't send report email to developer.")
else:
logger.info("Unknow state.")
# # send test report by DingTalk
# if DingTalk_on_off == 'on':
# #self.Dingtalk.send_message_to_robot()
# elif DingTalk_on_off == 'off':
# logger.info("Doesn't send report DingTalk to developer.")
# else:
# logger.info("Unknow state.")
''' '''
def main(account_id):
Log("info", "getting keyword performance from the api", "", account_id)
report = Report(account_id, "", options)
df = report.createDfWithAllDateRanges(account_id)
if functions.dfIsEmpty(df):
return
# print df[df.google_id=="309491001346"].cpc_bid
# remember column headers are as per the download here
df["keyword_id"] = df.apply(lambda row: functions.addIdToDf(
account_id, row["Keyword ID"], row["Ad group ID"]),
axis=1)
df["id"] = df["keyword_id"]
df = addParentId(df, account_id) # our UUID from the keywords table
df = report.basicProcessing(df)
df = reportSpecificProcessing(df)
report.writeToEntitiesTable(df, report, account_id)
report.writeToPerformanceTable(df, report, account_id)
def __init__(self,
configurationName=None,
logName='dataProcessing',
logLevel='DEBUG'):
super(dataProcessing, self).__init__()
AssemblyConfig()
self.configurationName = configurationName
self.sender_email = None
self.password = None
self.mysql_dict = None
self.conf = None
self.smtpHost = None
self.L = Log(logName, logLevel).logger
self.abs_path = os.path.dirname(os.path.abspath(__file__))
self.initialize_parameter()
self.db = DataBaseOperate()
class Testbuild4(Testtodolist):
log = Log()
def testbuild(self):
"""
步骤：
1、美购首页
2、搜索框
4、点击热词
5、切换医生tab
:return:
"""
begin_date = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
testbuild = TodolistBuild(self.driver)
testbuild.click_alert()
testbuild.click_meigou()
testbuild.click_welfare_home_search() # 偶现失败
testbuild.click_doctor()
testbuild.click_hot_search() # 偶现失败
time.sleep(5)
self.driver.background_app(5)
time.sleep(5)
# 搜索框 埋点
result = mysql_test.query(action='search_result_open',
event_time=begin_date)
assert result != ' ', 'search_result_open事件为空！'
class Testbuild(Testtodolist):
log = Log()
def test_01(self):
"""
首页品类聚合->点击玻尿酸
打开首页进入任意二级页面，首页的浏览事件记一次
"""
begin_date = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print(begin_date)
testbuild = TodolistBuild(self.driver)
testbuild.click_alert()
time.sleep(10)
testbuild.click_bns()
time.sleep(3)
self.driver.background_app(5)
time.sleep(10)
result = mysql_test.query(action='page_view', event_time=begin_date, page_name='home')
print(result)
end_page_view2 = result[0]['params']
referrer_page_name = end_page_view2['page_name']
assert referrer_page_name == 'home', 'page_name获取错误！'
assert len(result) == 1, f'埋点数量错误，预期为1个，实际为{len(result)}'
class Testbuild(Testtodolist):
log = Log()
def test_01(self):
"""
首页, 刷新
:return:
"""
begin_date = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print(begin_date)
testbuild = TodolistBuild(self.driver)
testbuild.click_alert()
time.sleep(3)
testbuild.switch_android_up()
time.sleep(2)
self.driver.background_app(5)
time.sleep(10)
result = mysql_test.query(action='page_view',
event_time=begin_date,
page_name='home')
print(result)
end_page_view2 = result[0]['params']
referrer2 = end_page_view2['page_name']
assert referrer2 == 'home', 'page_name获取错误！'
assert len(result) == 1, f'埋点数量错误，预期为1个，实际为{len(result)}'
def store_excess_budget(self):
"""Only run on the 1st of the month
* - Take the budget
* - Take away last month's spend
* - Any remaining budget is stored as the excess
"""
def update_excess_budget(excess_budget):
Log('info', 'Storing excess budget',
"excess_budget: %s" % (excess_budget), self.account_id)
Database().setValue('budget_commander', 'excess_budget',
excess_budget,
'where account_id = "%s"' % (self.account_id))
if not self.budget_commander.user_settings['rollover_spend']:
Log('info', 'rollover_spend is disabled. Setting excess to 0', '',
self.account_id)
update_excess_budget(0)
return
if not self.local_dates.is_first_of_month:
return
if self.budget_commander.budget_group_id: #no rollover for budget groups
return
remaining = float(self.budget) - float(
self.budget_commander.last_month_spend)
if remaining < 0:
return 0
update_excess_budget(remaining)
class Testbuild(Testtodolist):
log = Log()
def test_01(self):
"""
首页----点击日记
:return:
"""
begin_date = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print(begin_date)
testbuild = TodolistBuild(self.driver)
testbuild.click_alert()
time.sleep(10)
testbuild.click_home_diary()
time.sleep(10)
self.driver.background_app(5)
time.sleep(10)
result = mysql_test.query(action='page_view',
event_time=begin_date,
page_name='diary_detail')
print(result)
end_page_view2 = result[0]['params']
referrer2 = end_page_view2['referrer']
referrer_tab_name2 = end_page_view2['referrer_tab_name']
assert referrer2 == 'home', 'referrer获取错误！'
assert referrer_tab_name2 == '精选', 'referrer_tab_name获取错误！'
assert len(result) == 1, f'埋点数量错误，预期为1个，实际为{len(result)}'
print("page_view: %s" % end_page_view2)
def __init__(self, account_id):
self.account_id = account_id
self.budget_commander = BudgetCommander(account_id)
self.local_dates = LocalDates(account_id)
if not self.budget_commander.user_settings['emergency_stop']:
Log("info", "Emergency stop is disabled.", "", self.account_id)
return
self.budget = self.budget_commander.budget
if self.budget_commander.this_month_spend >= self.budget:
Log("info", "this month spend (%s) is over this month's budget (%s). Exiting." %(self.budget_commander.this_month_spend, self.budget), "", self.account_id)
return
self.costs = GetCostFromApi(account_id)
self.today_cost = self.costs.today_cost
self.day_budget_percentage = self.costs.day_budget_percentage
self.day_limit = self.getDayLimit()
self.main()
def main(account_id):
Log("info", "getting account performance reports from the api", "",
account_id)
report = Report(account_id, "last_30_days", options)
report.createAccountDirectory()
report.createReportDirectory()
report.downloadReport(account_id, options["where_string"])
df = report.convertCsvToDataframe()
df = report.basicProcessing(df)
df = reportSpecificProcessing(df, account_id)
# for col in df.columns:
# print col
# return
deleteExitingData(account_id, options["performance_table_name"])
report.writeDataframeToTable(df, options["performance_table_name"])
class Testbuild3(Testtodolist):
log = Log()
def test_01(self):
"""
步骤：
1、美购首页
2、刷新
3、美购首页浏览事件记一次
:return:
"""
begin_date = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
testbuild = TodolistBuild(self.driver)
testbuild.click_alert()
# 点击美购首页
testbuild.click_meigou()
testbuild.switch_android_up()
time.sleep(2)
self.driver.background_app(5) # 将app置于后台5秒钟，再唤起到前台
result = mysql_test.query(action='page_view',
event_time=begin_date,
page_name='welfare_home')
print(result)
end_page_view2 = result[0]['params']
referrer3 = end_page_view2['referrer']
referrer_page_name = end_page_view2['page_name']
assert referrer3 == 'home', 'referrer获取错误！'
assert referrer_page_name == 'welfare_home', 'page_name获取错误！'
assert len(result) == 1, f'埋点数量错误，预期为1个，实际为{len(result)}'
def showPictures():
global image1, image2, imageSurface1, imageSurface2, timer
imageSurface1 = getSurfaceFromFrame(image1)
imageSurface2 = getSurfaceFromFrame(image2)
Log.info('SHOW')
# Save both images with timestamp
timeString = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
image = np.concatenate((image1, image2), axis=1)
cv2.imwrite('images/' + timeString + '-image.png', image)
Log.info('SAVED')
timer = Timer(DELAY_BETWEEN_SOUNDS, moveNext)
def addNew(self):
"""Add accounts for new users - users without any accounts"""
first_run = True
self.user_ids = self.getUserIdsToProcess(first_run)
if not self.user_ids:
Log('info', 'no new users to process')
for user_id in self.user_ids:
self.addUserAccounts(user_id, first_run)
def addAll(self):
"""Add accounts for all users - to run daily. Used for getting new accounts"""
first_run = False
self.user_ids = self.getUserIdsToProcess(first_run)
for user_id in self.user_ids:
self.addUserAccounts(user_id, first_run)
Log("info", "finished adding account meta data")
