def main_loop(task_type, sql_file_name, start_time, stop_time):
global app
start_time = int(start_time)
stop_time = int(stop_time)
task_config = config.facebook_task_list[task_type]
table_name = task_config["table_name"]
field_list = task_config["field_list"]
unique_key = task_config["unique_key"]
appname_list = config.facebook_appid_config.keys()
for appname in appname_list:
appid = config.facebook_appid_config[appname]['appid']
secret = config.facebook_appid_config[appname]['secret']
data = FacebookApi.FacebookApi(appid,
secret).fetch(start_time, stop_time,
field_list)
# print "FacebookApi.FacebookApi(%s, secret).fetch(%s, %s, %s)" % (appid, start_time, stop_time, field_list)
if not data:
logging.error(
"main:FacebookApi.FacebookApi.No_Data:"
"[task:%s,appid:%s,start_time:%s,stop_time:%s,field_list:%s]" %
(task_type, appid, start_time, stop_time, field_list))
continue
sql_list = []
for model in data:
# insert some info
model["appname"] = appname
model["platform"] = "facebook"
model_field_list = field_list + ["date", "appname", "platform"]
model_unique_key = unique_key + ["date", "appname", "platform"]
sql = toolkit.gen_save_sql(table_name,
model=model,
field_list=model_field_list,
unique_key=model_unique_key)
if sql:
sql_list.append(sql)
else:
logging.error(
"main:gen_save_sql.Fail:"
"[table_name:%s, model=%s, field_list=%s, unique_key=%s]" %
(task_type, model, field_list, unique_key))
# print sql_list
app["total_count"] += len(sql_list)
with open(sql_file_name, 'a') as f:
if not sql_list:
return
big_sql = "\n".join(sql_list)
# print "big_sql:", big_sql
f.write(big_sql)
def save_to_db_new(conn, model):
table_name = config.googleplay_task["table_name"]
field_list = ["date", "platform", "revenue"]
unique_key = [
"date",
"platform",
]
sql = toolkit.gen_save_sql(table_name, model, field_list, unique_key)
status = False
try:
toolkit.mysql_execute(conn, sql)
status = True
except Exception as e:
logging.error("save_to_db_new():FAIL:[%s][%s]" % (sql, e))
status = False
finally:
return status
def main_loop(task_type, start_date, stop_date, sql_file_name):
pingstart_appid_list = config.pingstart_appid_config
pingstart_task_config = config.pingstart_task_list[task_type]
table_name = pingstart_task_config["table_name"]
field_list = pingstart_task_config["field_list"]
unique_key = pingstart_task_config["unique_key"]
date_or_country = pingstart_task_config["date_or_country"]
field_list = field_list + ["date", "appname", "platform"]
unique_key = unique_key + ["date", "appname", "platform"]
slot_list = pingstart_task_config["slot"]
for app_name in pingstart_appid_list.keys():
for slot in slot_list:
pingstart_app = pingstart_appid_list[app_name]
Cookie = pingstart_app["Cookie"]
ssl_cert = pingstart_app["ssl_crt"]
pingstart = PingstartAdApi.PingstartAdApi(Cookie, ssl_cert)
model_list = pingstart.fetch(start_date,
stop_date,
date_or_country,
slot=slot)
if not model_list or not isinstance(model_list, (list, tuple)):
__msg = "main_loop():pingstart.fetch:get None:(status:%s)(message:%s)" % (
pingstart.status, pingstart.message)
logging.error(__msg)
continue
sql_list = []
for model in model_list:
model["appname"] = app_name
model["platform"] = "pingstart"
if model["date"] == "" and start_date == stop_date:
model["date"] = start_date
sql = toolkit.gen_save_sql(table_name, model, field_list,
unique_key)
if not sql:
logging.error("main_loop():gen_save_sql():error")
else:
sql_list.append(sql)
app["total_count"] += len(sql_list)
with open(sql_file_name, "a") as f:
big_sql = "\n".join(sql_list)
f.write(big_sql)
def main_loop(task_type, date_str, sql_file_name):
baidu_appid_list = config.baidu_appid_config
baidu_task_config = config.baidu_task_list[task_type]
table_name = baidu_task_config["table_name"]
field_list = baidu_task_config["field_list"]
unique_key = baidu_task_config["unique_key"]
field_list = field_list + ["date", "appname", "platform"]
unique_key = unique_key + ["date", "appname", "platform"]
for app_name in baidu_appid_list.keys():
baidu_app = baidu_appid_list[app_name]
username = baidu_app["username"]
appid = baidu_app["appid"]
source = baidu_app["source"]
baidu = BaiduApi.BaiduApi()
print "baidu.fetch(%s, %s, %s, %s)" % (date_str, username, appid,
source)
model_list = baidu.fetch(date_str, username, appid, source)
if not model_list or not isinstance(model_list, (list, tuple)):
__msg = "main_loop():baidu.fetch:get None:(status:%s)(message:%s)" % (
baidu.status, baidu.message)
logging.error(__msg)
continue
sql_list = []
for model in model_list:
model["appname"] = app_name
model["platform"] = "baidu"
sql = toolkit.gen_save_sql(table_name, model, field_list,
unique_key)
if not sql:
logging.error("main_loop():gen_save_sql():error")
else:
sql_list.append(sql)
app["total_count"] += len(sql_list)
with open(sql_file_name, "a") as f:
big_sql = "\n".join(sql_list)
f.write(big_sql)
def main_loop(task_type, start_date, stop_date, sql_file_name):
admob_appid_list = config.admob_appid_list
admob_task_config = config.admob_task_list[task_type]
table_name = admob_task_config["table_name"]
field_list = admob_task_config["field_list"]
unique_key = admob_task_config["unique_key"]
field_list = list(set(field_list + ["date", "platform"]))
unique_key = list(set(unique_key + ["date", "platform"]))
for app_name in admob_appid_list.keys():
admob_app = admob_appid_list[app_name]
client_id = admob_app["CLIENT_ID"]
client_secret = admob_app["CLIENT_SECRET"]
refresh_token = admob_app["REFRESH_TOKEN"]
admob = AdmobApi.AdmobApi(client_id, client_secret, refresh_token)
model_list = admob.fetch(start_date, stop_date, field_list)
if not model_list or not isinstance(model_list, (list, tuple)):
__msg = "main_loop():admob.fetch:get None:(status:%s)(message:%s)" % (
admob.status, admob.message)
logging.error(__msg)
continue
sql_list = []
for model in model_list:
model["appname"] = model.get('appname', 'unnamed')
model["platform"] = "admob"
if model["date"] == "" and start_date == stop_date:
model["date"] = start_date
sql = toolkit.gen_save_sql(table_name, model, field_list,
unique_key)
if not sql:
logging.error("main_loop():gen_save_sql():error")
else:
sql_list.append(sql)
app["total_count"] += len(sql_list)
with open(sql_file_name, "a") as f:
big_sql = "\n".join(sql_list)
f.write(big_sql)
def main_loop(task_type, sql_file_name, start_time, stop_time):
global app
_start_time = time.strftime('%Y%m%d', time.gmtime(start_time))
_stop_time = time.strftime('%Y%m%d', time.gmtime(stop_time))
task_config = config.mobvista_task_list[task_type]
table_name = task_config["table_name"]
field_list = task_config["field_list"]
unique_key = task_config["unique_key"]
account_list = config.mobvista_account_list.keys()
for account in account_list:
skey = config.mobvista_account_list[account]['skey']
secret = config.mobvista_account_list[account]['secret']
data = MobvistaApi(skey, secret).fetch_report(_start_time, _stop_time,
field_list)
if not data:
logging.error(
"main:MobvistaApi.fetch_report.No_Data:"
"[task:%s,appid:%s,start_time:%s,stop_time:%s,field_list:%s]" %
(task_type, account, _start_time, _stop_time, field_list))
continue
sql_list = []
for model in data:
# insert some info
model["account"] = account
model['appname'] = model.get('app_id', 'unnamed')
model['placement'] = model.get('unit_id', '')
model["platform"] = "mobvista"
model['revenue'] = model.get('est_revenue', 0)
model['filled_rate'] = model.get('fill_rate', 0) / 100.0
model['filled'] = model.get('filled', 0)
model['impression'] = model.get('impression', 0)
model['click'] = model.get('click', 0)
model['country'] = model.get('country', 0)
filled = model.get('filled', 0)
fill_rate = model.get('filled_rate', 0.0)
request = 0
if toolkit.isfloat(filled) and toolkit.isfloat(
fill_rate) and filled > 0 and fill_rate > 0:
request = int(float(filled) / float(fill_rate))
else:
logging.info("main():main_loop:Cannot calculate Request:%s" %
model)
model['request'] = request
# print model
model_field_list = field_list + ["date", "appname", "platform"]
model_unique_key = unique_key + ["date", "appname", "platform"]
uniq_model_field_list = list(set(model_field_list))
uniq_model_unique_key = list(set(model_unique_key))
sql = toolkit.gen_save_sql(table_name,
model=model,
field_list=uniq_model_field_list,
unique_key=uniq_model_unique_key)
if sql:
sql_list.append(sql)
else:
logging.error(
"main:gen_save_sql.Fail:"
"[table_name:%s, model=%s, field_list=%s, unique_key=%s]" %
(task_type, model, field_list, unique_key))
# print sql_list
app["total_count"] += len(sql_list)
with open(sql_file_name, 'a') as f:
if not sql_list:
return
big_sql = "\n".join(sql_list)
# print "big_sql:", big_sql
f.write(big_sql)
def main_loop(task_type, sql_file_name, start_time):
global app
time_str = time.strftime('%Y-%m-%d', time.gmtime(start_time))
task_config = config.mopub_task_list[task_type]
table_name = task_config["table_name"]
field_list = task_config["field_list"]
unique_list = task_config['unique_list']
account_list = config.mopub_account_list.keys()
for account in account_list:
api_key = config.mopub_account_list[account]['api_key']
report_key = config.mopub_account_list[account]['report_key']
data = MopubApi(api_key, report_key).fetch_report(time_str)
if not data:
logging.error("main:mopub.fetch_report.No_Data:"
"[task:%s,appid:%s,time:%sfield_list:%s]" % (
task_type, account, time_str, field_list)
)
continue
sql_list = []
for model in data:
# insert some info
model["account"] = account
model["platform"] = "mopub"
model['country'] = safe_type_str(model.get('country', '00'), 2)
model['appname'] = safe_type_str(model['appname'], 100)
model['placement'] = safe_type_str(model['placement'], 255)
model['request'] = safe_type_int(model['request'])
model['filled'] = 0
model['impression'] = safe_type_int(model['impression'])
model['click'] = safe_type_int(model['click'])
model['filled_rate'] = 0
model['ctr'] = safe_type_float(model['ctr'])
model['ecpm'] = 0
model['revenue'] = safe_type_float(model['revenue'])
for key in ['app_id', 'creative_id', 'creative', 'priority', 'order_id', 'line_item',
'line_item_type', 'device', 'segment', 'adunit_id', 'segment_id',
'os', 'order', 'conversions']:
model[key] = safe_type_str(model.get(key, '00'), 32)
hash_key = compute_hash(model, unique_list)
model['hash'] = hash_key
model_field_list = field_list + ["date", "appname", "platform"]
unique_model_field_list = list(set(model_field_list))
sql = toolkit.gen_save_sql(table_name, model=model, field_list=unique_model_field_list,
unique_key=model_field_list)
if sql:
sql_list.append(sql)
else:
logging.error("main:gen_save_sql.Fail:"
"[table_name:%s, model=%s, field_list=%s]" % (
task_type, model, field_list)
)
# print sql_list
app["total_count"] += len(sql_list)
with open(sql_file_name, 'a') as f:
if not sql_list:
return
big_sql = "\n".join(sql_list)
# print "big_sql:", big_sql
f.write(big_sql)