def commonStoreByMinute(datatype, timestamp, *args, **kwargs):
global mode_writer_contrast
global tasks
ct = time.strftime("%Y-%m-%d+%H%M", time.localtime(timestamp))
yyyymmddhhmm = ct.replace("-", "").replace("+", "")
yyyy_mm_dd, hhmm = ct.split("+")[0], ct.split("+")[1]
apptype = kwargs["apptype"]
for modename in tasks["".join(["minute", "_%s"%apptype])]:
# print(modename, datatype, apptype)
try:
mode_class, store_class = mode_writer_contrast[modename]
modecreator = mode_class(datatype, yyyy_mm_dd, hhmm, last=1) if mode_class else None
result = modecreator.dataCollect() if modecreator else None
# 获取datatype对应入库mongo编号
m_client = MysqlClient("saas_server")
mongo_ids = m_client.get_mongoid(datatype)
print __name__, datatype, mongo_ids
m_client.closeMysql()
for mongo_id in mongo_ids:
try:
# print datatype, store_class, mode_class
storer = store_class(mongo_id)
if yyyymmddhhmm.endswith("0000"):
storer.remove(datatype, modename, yyyymmddhhmm[:8])
print("invoking remove interface, remove data or Nothing to do! ", datatype, modename, yyyymmddhhmm[:8])
storer.write(result, datatype, modename, modecreator, today=yyyy_mm_dd)
except:
import traceback
print(traceback.print_exc())
except:
import traceback
print(traceback.print_exc())
def get_datatype_saas():
client = MysqlClient("saas_meta")
result = [[item["cdkey"], item["appkey"], item["plat"]]
for item in client.getAppkey_kwargs(
plat=["android", "ios", "all", "feeling"])]
client.closeMysql()
return result
def create_client(self, appkey):
m_client = MysqlClient("saas_server")
mongo_ids = m_client.get_mongoid(appkey)
m_client.closeMysql()
self.client = PyMongoClient(mongo_id=mongo_ids[0])
def GetAppkeys(**kwargs):
client = MysqlClient("saas_server")
data = client.getAppkey_kwargs(**kwargs)
result = {}
for item in data:
appkey = item["appkey"]
dbname = item["cdkey"]
plat = item["plat"]
if (dbname, plat) not in result.get(appkey, []):
result.setdefault(appkey, []).append((dbname, plat))
client.closeMysql()
return result
def GetAppkey_h5():
plats = ["h5"]
client = MysqlClient("saas_server")
# result = client.getAppkey_h5()
result = client.getAppkey_kwargs(plat=plats,
mongo_id=mongo_id,
filter_keys=["appkey", "cdkey"])
data = {}
for item in result:
dbname, appkey, plat = item["cdkey"], item["appkey"], item["plat"]
if (dbname, plat) not in data.get(appkey, []):
data.setdefault(appkey, []).append((dbname, plat))
client.closeMysql()
return data
def GetAppkey_saas():
plats = ["ios", "android", "all", "feeling"]
client = MysqlClient("saas_server")
result = client.getAppkey_kwargs(plat=plats,
mongo_id=mongo_id,
filter_keys=["appkey", "cdkey"])
data = {}
for item in result:
print item
dbname, appkey, plat = item["cdkey"], item["appkey"], item["plat"]
if (dbname, plat) not in data.get(appkey, []):
data.setdefault(appkey, []).append((dbname, plat))
client.closeMysql()
return data
def transformresult(self, analysisresult, *args, **kwargs):
client = MysqlClient(self.dbname)
data = client.select(
"select id, name, isdel from %(dbname)s.%(datatype)s_%(plat)s_eventid_udf"
% {
"dbname": self.dbname,
"datatype": self.datatype,
"plat": self.plat,
})
result = analysisresult.result
for item in data:
_id, name, isdel = item[0], item[1], item[2]
result.setdefault(_id, [name, isdel])
# result = analysisresult.result
analysisresult.transformresult = result
client.closeMysql()
def appointModeStoreByDaily(datatype, num, modename, *args, **kwargs):
global mode_writer_contrast
ct = time.strftime("%Y-%m-%d+%H%M", time.localtime(time.time() - 86400 * num))
yyyy_mm_dd, hhmm = ct.split("+")[0], "2359"
mode_class, storer_class = mode_writer_contrast[modename]
modecreator = mode_class(datatype, yyyy_mm_dd) if mode_class else None
result = modecreator.dataCollect() if modecreator else None
# 获取datatype对应入库mongo编号
m_client = MysqlClient("saas_server")
mongo_ids = m_client.get_mongoid(datatype)
m_client.closeMysql()
for mongo_id in mongo_ids:
try:
storer = storer_class(mongo_id)
storer.write(result, datatype, modename, modecreator, num=num, today=yyyy_mm_dd)
except:
import traceback
print(traceback.print_exc())
def DevUsers(datatype='all', iscache=False):
global result
global myclock
if ((time.time() - myclock) > 3600 or (not result)) or iscache:
client = MysqlClient("saas_meta")
con, cur = client.connection
sql = "SELECT a.appkey, b.userkey FROM saas_exclude_appkey a LEFT JOIN saas_dev_userkey b on a.appkey = b.appkey WHERE a.`enable` = 1"
cur.execute(sql)
for item in cur.fetchall():
appkey, userkey = item[0], item[1]
result.setdefault(appkey, set()).add(userkey)
myclock = time.time()
client.closeMysql()
else:
pass
tmp = {}
for key in result:
if "all" in datatype:
tmp.setdefault(key, list(result[key]))
elif key in datatype:
tmp.setdefault(key, list(result[key]))
return tmp
def appointModeStoreByMinute(datatype, timestamp, modename, *args, **kwargs):
global mode_writer_contrast
ct = time.strftime("%Y-%m-%d+%H%M", time.localtime(timestamp))
yyyy_mm_dd, hhmm = ct.split("+")[0], ct.split("+")[1]
mode_class, storer_class = mode_writer_contrast[modename]
modecreator = mode_class(datatype, yyyy_mm_dd, hhmm, last=1) if mode_class else None
result = modecreator.dataCollect() if modecreator else None
m_client = MysqlClient("saas_server")
mongo_ids = m_client.get_mongoid(datatype)
m_client.closeMysql()
for mongo_id in mongo_ids:
try:
storer = storer_class(mongo_id)
ct = time.strftime("%Y-%m-%d+%H%M", time.localtime(timestamp))
yyyymmddhhmm = ct.replace("-", "").replace("+", "")
yyyy_mm_dd, hhmm = ct.split("+")[0], ct.split("+")[1]
if yyyymmddhhmm.endswith("0000"):
storer.remove(datatype, modename, yyyymmddhhmm[:8])
print("invoking remove interface, remove data or Nothing to do! ", datatype, modename, yyyymmddhhmm[:8])
storer.write(result, datatype, modename, modecreator, today=yyyy_mm_dd)
except:
import traceback
print(traceback.print_exc())
def getTopics(self, once_sleep=60):
'''
获取需要消费的topic，可通过数据库表控制
:param once_sleep: topics更新频率
:return:
'''
while True:
if self.debug:
debug_topic = self.configures.get("debugconf", "debug_topic")
self.topics = [(item, "debug")
for item in debug_topic.split(",")]
else:
kafka_topics = set()
saas_appkey = set()
appkey_logpath = {}
try:
from kafka import SimpleClient
hostname = self.configures.get("kafka", "hostname")
client = SimpleClient(hosts=hostname)
for topic in client.topics:
kafka_topics.add(topic)
client.close()
log.info("get kafka topics: %s" %
json.dumps(list(kafka_topics)))
except:
logging.error(sys.exc_info())
continue
try:
client = MysqlClient("saas_server")
topics = client.getTopics(group_id=self.group_id)
for topic, logpath in topics:
saas_appkey.add(topic)
appkey_logpath.setdefault(topic, set()).add(logpath)
client.closeMysql()
log.info("get mysql appkeys: %s" %
json.dumps(list(saas_appkey)))
except:
logging.error(sys.exc_info())
continue
self.topics = [(topic, logpath)
for topic in list(kafka_topics & saas_appkey)
for logpath in appkey_logpath[topic]]
log.info("current topics: %s" % json.dumps(self.topics))
time.sleep(once_sleep)
def get_mongoid(self, appkey):
m_client = MysqlClient()
self.mongo_id = m_client.get_mongoid(appkey)[0]
m_client.closeMysql()
def get_datatype_saas_h5():
client = MysqlClient("saas_meta")
result = [item for item in client.getAppkey_h5()]
client.closeMysql()
return result
def __init__(self):
self.client = MysqlClient("information_schema")
self.con, self.cur = self.client.connection
class CombineTable(CombineTableBasic):
def __init__(self):
self.client = MysqlClient("information_schema")
self.con, self.cur = self.client.connection
def createtablelike(self, dbname, tablename, liketablename):
sql_format = "CREATE TABLE IF NOT EXISTS %(dbname)s.%(tablename)s LIKE %(dbname)s.%(liketablename)s"
sql = sql_format % {
"dbname": dbname,
"tablename": tablename,
"liketablename": liketablename
}
self.cur.execute(sql)
self.con.commit()
def combinetable_fromappkey(self, num, dbname, appkey_plat_pairs,
tablename, combineappkey):
tablename1 = "%(appkey)s_%(plat)s_%(tablename)s" % {
"appkey": appkey_plat_pairs[0][0],
"plat": appkey_plat_pairs[0][1],
"tablename": tablename
}
tablename2 = "%(appkey)s_%(plat)s_%(tablename)s" % {
"appkey": appkey_plat_pairs[1][0],
"plat": appkey_plat_pairs[1][1],
"tablename": tablename
}
combinetablename = "%(appkey)s_%(plat)s_%(tablename)s" % {
"appkey": combineappkey,
"plat": "all",
"tablename": tablename
}
if 'week' in tablename1:
yyyymmdd = time.strftime("%Y%m%d",
time.localtime(time.time() - 86400 * num))
week_day = datetime.datetime.strptime(yyyymmdd, "%Y%m%d").weekday()
if week_day == 6:
tm = time.strftime("%Y-%m-%d", time.localtime(time.time()))
num = getWeekFirstDay(1, tm).values()[0] + num - 1
self.createtablelike(dbname, combinetablename, tablename1)
if "remain" in tablename1:
self.combinetable_all("all", dbname, tablename1, tablename2,
combinetablename)
else:
self.combinetable(num, dbname, tablename1, tablename2,
combinetablename)
self.con.commit()
def cleartable(self, num, dbname, tablename, tm_columnname):
if num == "all":
delete_sql_format = "truncate table %(dbname)s.%(tablename)s"
delete_sql = delete_sql_format % {
"dbname": dbname,
"tablename": tablename
}
else:
tm = time.strftime("%Y-%m-%d",
time.localtime(time.time() - 86400 * num))
delete_sql_format = "DELETE FROM %(dbname)s.%(tablename)s WHERE %(tm_columnname)s = '%(tm)s'"
delete_sql = delete_sql_format % {
"dbname": dbname,
"tablename": tablename,
"tm_columnname": tm_columnname,
"tm": tm
}
self.cur.execute(delete_sql)
self.con.commit()
def combinetable_all(self, num, dbname, tablename1, tablename2,
combinetablename):
combine_columns, tm_columnname = self.combinecolumns(
dbname, tablename1, tablename2)
self.cleartable("all", dbname, combinetablename, tm_columnname)
result = {}
select_sql_format = "SELECT %(combinekeys)s, -666, %(combinevalues)s FROM %(dbname)s.%(tablename)s"
insert_sql_format = "INSERT INTO %(dbname)s.%(tablename)s (%(combinekeys)s, %(combinevalues)s) VALUES (%(colvalues)s)"
tables = [tablename1, tablename2]
combinekeys = self.get_combinekey(combine_columns)
combinevalues = self.get_combinevalue(combine_columns)
for sql in [
select_sql_format % {
"combinekeys": ", ".join(["tm"] + combinekeys),
"combinevalues": ", ".join(combinevalues),
"dbname": dbname,
"tablename": tablename,
"tm_columnname": tm_columnname,
} for tablename in tables
]:
for item in self.client.select(sql):
item = list(item)
key = tuple([key for key in item[:item.index(-666)]])
defaultvalue = [
None for i in range(0, len(item[item.index(-666) + 1:]))
]
value = [i for i in item[item.index(-666) + 1:]]
# 初始化
result.setdefault(key, defaultvalue)
result[key] = [((i if i else 0.0) + (j if j else 0.0))
if not (i is None and j is None) else None
for i, j in zip(result[key], value)]
for key_items in result:
try:
insert_sql = insert_sql_format % {
"dbname":
dbname,
"tablename":
combinetablename,
"combinekeys":
", ".join([tm_columnname] + combinekeys),
"combinevalues":
", ".join(combinevalues),
"colvalues":
", ".join([
"'" + "', '".join(map(str, key_items)) + "'",
", ".join(
map(str, [
'null' if item is None else item
for item in result[key_items]
])),
]),
}
self.cur.execute(insert_sql)
except:
import traceback
print(traceback.print_exc())
print("insert faild", key_items, result[key_items])
def combinetable(self, num, dbname, tablename1, tablename2,
combinetablename):
combine_columns, tm_columnname = self.combinecolumns(
dbname, tablename1, tablename2)
tm = time.strftime("%Y-%m-%d",
time.localtime(time.time() - 86400 * num))
self.cleartable(num, dbname, combinetablename, tm_columnname)
result = {}
select_sql_format = "SELECT %(combinekeys)s, -666, %(combinevalues)s FROM %(dbname)s.%(tablename)s WHERE %(tm_columnname)s = '%(tm)s'"
insert_sql_format = "INSERT INTO %(dbname)s.%(tablename)s (%(combinekeys)s, %(combinevalues)s) VALUES (%(colvalues)s)"
tables = [tablename1, tablename2]
combinekeys = self.get_combinekey(combine_columns)
combinevalues = self.get_combinevalue(combine_columns)
for sql in [
select_sql_format % {
"combinekeys": ", ".join(combinekeys),
"combinevalues": ", ".join(combinevalues),
"dbname": dbname,
"tablename": tablename,
"tm_columnname": tm_columnname,
"tm": tm
} for tablename in tables
]:
for item in self.client.select(sql):
item = list(item)
key = tuple([key for key in item[:item.index(-666)]])
defaultvalue = [
None for i in range(0, len(item[item.index(-666) + 1:]))
]
value = [i for i in item[item.index(-666) + 1:]]
# 初始化
result.setdefault(key, defaultvalue)
result[key] = [((i if i else 0.0) + (j if j else 0.0))
if not (i is None and j is None) else None
for i, j in zip(result[key], value)]
for key_items in result:
try:
insert_sql = insert_sql_format % {
"dbname":
dbname,
"tablename":
combinetablename,
"combinekeys":
", ".join([tm_columnname] + combinekeys),
"combinevalues":
", ".join(combinevalues),
"colvalues":
", ".join([
"'" + tm + "'",
"'" + "', '".join(map(str, key_items)) + "'",
", ".join(
map(str, [
'null' if item is None else item
for item in result[key_items]
])),
]),
}
self.cur.execute(insert_sql)
except:
import traceback
print(traceback.print_exc())
print("insert faild", key_items, result[key_items])
def get_combinekey(self, combine_columns):
result = set()
for col_name, data_type in combine_columns.items():
if data_type.upper() in ("VARCHAR", "CHAR", "TEXT"):
result.add(col_name)
result = list(result)
result.sort()
return result
def get_combinevalue(self, combine_columns):
result = set()
for col_name, data_type in combine_columns.items():
if data_type.upper() in ("TINYINT", "SMALLINT", "MEDIUMINT", "INT",
"INTEGER", "BIGINT", "FLOAT", "DOUBLE",
"DECIMAL"):
result.add(col_name)
result = list(result)
result.sort()
return result
def combinecolumns(self, dbname, tablename1, tablename2):
sql = "SELECT COLUMN_NAME, DATA_TYPE FROM information_schema.COLUMNS \
WHERE table_name = '%(tablename)s' and table_schema = '%(dbname)s' and extra != 'auto_increment'"
tm_columnname = None
columns_list = set()
columns = OrderedDict()
for tablename in [tablename1, tablename2]:
for col_name, data_type in self.client.select(
sql % {
"dbname": dbname,
"tablename": tablename
}):
if data_type in ["date", "datetime"]:
tm_columnname = col_name
continue
columns_list.add((col_name, data_type))
columns_list = list(columns_list)
columns_list.sort()
for col_name, data_type in columns_list:
columns.setdefault(col_name, data_type)
return columns, tm_columnname
def GetAppkeys(**kwargs):
client = MysqlClient("saas_server")
result = client.getAppkey_kwargs(kwargs)
return result
def get_datatype_saas_h5():
client = MysqlClient("saas_meta")
result = [[item["cdkey"], item["appkey"], item["plat"]]
for item in client.getAppkey_kwargs(plat=["h5"])]
client.closeMysql()
return result
def get_mongo_conn(appkey):
m_client = MysqlClient()
mongo_id = m_client.get_mongoid(appkey)[0]
m_client.closeMysql()
conn = PyMongoClient(mongo_id=mongo_id)
return conn.getConn()