def get_datatype_saas():
client = MysqlClient("saas_meta")
result = [[item["cdkey"], item["appkey"], item["plat"]]
for item in client.getAppkey_kwargs(
plat=["android", "ios", "all", "feeling"])]
client.closeMysql()
return result
def commonStoreByMinute(datatype, timestamp, *args, **kwargs):
global mode_writer_contrast
global tasks
ct = time.strftime("%Y-%m-%d+%H%M", time.localtime(timestamp))
yyyymmddhhmm = ct.replace("-", "").replace("+", "")
yyyy_mm_dd, hhmm = ct.split("+")[0], ct.split("+")[1]
apptype = kwargs["apptype"]
for modename in tasks["".join(["minute", "_%s"%apptype])]:
# print(modename, datatype, apptype)
try:
mode_class, store_class = mode_writer_contrast[modename]
modecreator = mode_class(datatype, yyyy_mm_dd, hhmm, last=1) if mode_class else None
result = modecreator.dataCollect() if modecreator else None
# 获取datatype对应入库mongo编号
m_client = MysqlClient("saas_server")
mongo_ids = m_client.get_mongoid(datatype)
print __name__, datatype, mongo_ids
m_client.closeMysql()
for mongo_id in mongo_ids:
try:
# print datatype, store_class, mode_class
storer = store_class(mongo_id)
if yyyymmddhhmm.endswith("0000"):
storer.remove(datatype, modename, yyyymmddhhmm[:8])
print("invoking remove interface, remove data or Nothing to do! ", datatype, modename, yyyymmddhhmm[:8])
storer.write(result, datatype, modename, modecreator, today=yyyy_mm_dd)
except:
import traceback
print(traceback.print_exc())
except:
import traceback
print(traceback.print_exc())
def create_client(self, appkey):
m_client = MysqlClient("saas_server")
mongo_ids = m_client.get_mongoid(appkey)
m_client.closeMysql()
self.client = PyMongoClient(mongo_id=mongo_ids[0])
def GetAppkeys(**kwargs):
client = MysqlClient("saas_server")
data = client.getAppkey_kwargs(**kwargs)
result = {}
for item in data:
appkey = item["appkey"]
dbname = item["cdkey"]
plat = item["plat"]
if (dbname, plat) not in result.get(appkey, []):
result.setdefault(appkey, []).append((dbname, plat))
client.closeMysql()
return result
def GetAppkey_h5():
plats = ["h5"]
client = MysqlClient("saas_server")
# result = client.getAppkey_h5()
result = client.getAppkey_kwargs(plat=plats,
mongo_id=mongo_id,
filter_keys=["appkey", "cdkey"])
data = {}
for item in result:
dbname, appkey, plat = item["cdkey"], item["appkey"], item["plat"]
if (dbname, plat) not in data.get(appkey, []):
data.setdefault(appkey, []).append((dbname, plat))
client.closeMysql()
return data
def GetAppkey_saas():
plats = ["ios", "android", "all", "feeling"]
client = MysqlClient("saas_server")
result = client.getAppkey_kwargs(plat=plats,
mongo_id=mongo_id,
filter_keys=["appkey", "cdkey"])
data = {}
for item in result:
print item
dbname, appkey, plat = item["cdkey"], item["appkey"], item["plat"]
if (dbname, plat) not in data.get(appkey, []):
data.setdefault(appkey, []).append((dbname, plat))
client.closeMysql()
return data
def transformresult(self, analysisresult, *args, **kwargs):
client = MysqlClient(self.dbname)
data = client.select(
"select id, name, isdel from %(dbname)s.%(datatype)s_%(plat)s_eventid_udf"
% {
"dbname": self.dbname,
"datatype": self.datatype,
"plat": self.plat,
})
result = analysisresult.result
for item in data:
_id, name, isdel = item[0], item[1], item[2]
result.setdefault(_id, [name, isdel])
# result = analysisresult.result
analysisresult.transformresult = result
client.closeMysql()
def appointModeStoreByDaily(datatype, num, modename, *args, **kwargs):
global mode_writer_contrast
ct = time.strftime("%Y-%m-%d+%H%M", time.localtime(time.time() - 86400 * num))
yyyy_mm_dd, hhmm = ct.split("+")[0], "2359"
mode_class, storer_class = mode_writer_contrast[modename]
modecreator = mode_class(datatype, yyyy_mm_dd) if mode_class else None
result = modecreator.dataCollect() if modecreator else None
# 获取datatype对应入库mongo编号
m_client = MysqlClient("saas_server")
mongo_ids = m_client.get_mongoid(datatype)
m_client.closeMysql()
for mongo_id in mongo_ids:
try:
storer = storer_class(mongo_id)
storer.write(result, datatype, modename, modecreator, num=num, today=yyyy_mm_dd)
except:
import traceback
print(traceback.print_exc())
def DevUsers(datatype='all', iscache=False):
global result
global myclock
if ((time.time() - myclock) > 3600 or (not result)) or iscache:
client = MysqlClient("saas_meta")
con, cur = client.connection
sql = "SELECT a.appkey, b.userkey FROM saas_exclude_appkey a LEFT JOIN saas_dev_userkey b on a.appkey = b.appkey WHERE a.`enable` = 1"
cur.execute(sql)
for item in cur.fetchall():
appkey, userkey = item[0], item[1]
result.setdefault(appkey, set()).add(userkey)
myclock = time.time()
client.closeMysql()
else:
pass
tmp = {}
for key in result:
if "all" in datatype:
tmp.setdefault(key, list(result[key]))
elif key in datatype:
tmp.setdefault(key, list(result[key]))
return tmp
def appointModeStoreByMinute(datatype, timestamp, modename, *args, **kwargs):
global mode_writer_contrast
ct = time.strftime("%Y-%m-%d+%H%M", time.localtime(timestamp))
yyyy_mm_dd, hhmm = ct.split("+")[0], ct.split("+")[1]
mode_class, storer_class = mode_writer_contrast[modename]
modecreator = mode_class(datatype, yyyy_mm_dd, hhmm, last=1) if mode_class else None
result = modecreator.dataCollect() if modecreator else None
m_client = MysqlClient("saas_server")
mongo_ids = m_client.get_mongoid(datatype)
m_client.closeMysql()
for mongo_id in mongo_ids:
try:
storer = storer_class(mongo_id)
ct = time.strftime("%Y-%m-%d+%H%M", time.localtime(timestamp))
yyyymmddhhmm = ct.replace("-", "").replace("+", "")
yyyy_mm_dd, hhmm = ct.split("+")[0], ct.split("+")[1]
if yyyymmddhhmm.endswith("0000"):
storer.remove(datatype, modename, yyyymmddhhmm[:8])
print("invoking remove interface, remove data or Nothing to do! ", datatype, modename, yyyymmddhhmm[:8])
storer.write(result, datatype, modename, modecreator, today=yyyy_mm_dd)
except:
import traceback
print(traceback.print_exc())
def get_mongoid(self, appkey):
m_client = MysqlClient()
self.mongo_id = m_client.get_mongoid(appkey)[0]
m_client.closeMysql()
def get_mongo_conn(appkey):
m_client = MysqlClient()
mongo_id = m_client.get_mongoid(appkey)[0]
m_client.closeMysql()
conn = PyMongoClient(mongo_id=mongo_id)
return conn.getConn()
def get_datatype_saas_h5():
client = MysqlClient("saas_meta")
result = [[item["cdkey"], item["appkey"], item["plat"]]
for item in client.getAppkey_kwargs(plat=["h5"])]
client.closeMysql()
return result
def GetAppkeys(**kwargs):
client = MysqlClient("saas_server")
result = client.getAppkey_kwargs(**kwargs)
client.closeMysql()
return result
def get_datatype_saas_h5():
client = MysqlClient("saas_meta")
result = [item for item in client.getAppkey_h5()]
client.closeMysql()
return result