def commonStoreByMinute(datatype, timestamp, *args, **kwargs):
global mode_writer_contrast
global tasks
ct = time.strftime("%Y-%m-%d+%H%M", time.localtime(timestamp))
yyyymmddhhmm = ct.replace("-", "").replace("+", "")
yyyy_mm_dd, hhmm = ct.split("+")[0], ct.split("+")[1]
apptype = kwargs["apptype"]
for modename in tasks["".join(["minute", "_%s"%apptype])]:
# print(modename, datatype, apptype)
try:
mode_class, store_class = mode_writer_contrast[modename]
modecreator = mode_class(datatype, yyyy_mm_dd, hhmm, last=1) if mode_class else None
result = modecreator.dataCollect() if modecreator else None
# 获取datatype对应入库mongo编号
m_client = MysqlClient("saas_server")
mongo_ids = m_client.get_mongoid(datatype)
print __name__, datatype, mongo_ids
m_client.closeMysql()
for mongo_id in mongo_ids:
try:
# print datatype, store_class, mode_class
storer = store_class(mongo_id)
if yyyymmddhhmm.endswith("0000"):
storer.remove(datatype, modename, yyyymmddhhmm[:8])
print("invoking remove interface, remove data or Nothing to do! ", datatype, modename, yyyymmddhhmm[:8])
storer.write(result, datatype, modename, modecreator, today=yyyy_mm_dd)
except:
import traceback
print(traceback.print_exc())
except:
import traceback
print(traceback.print_exc())
def get_datatype_saas():
client = MysqlClient("saas_meta")
result = [[item["cdkey"], item["appkey"], item["plat"]]
for item in client.getAppkey_kwargs(
plat=["android", "ios", "all", "feeling"])]
client.closeMysql()
return result
def create_client(self, appkey):
m_client = MysqlClient("saas_server")
mongo_ids = m_client.get_mongoid(appkey)
m_client.closeMysql()
self.client = PyMongoClient(mongo_id=mongo_ids[0])
def GetAppkeys(**kwargs):
client = MysqlClient("saas_server")
data = client.getAppkey_kwargs(**kwargs)
result = {}
for item in data:
appkey = item["appkey"]
dbname = item["cdkey"]
plat = item["plat"]
if (dbname, plat) not in result.get(appkey, []):
result.setdefault(appkey, []).append((dbname, plat))
client.closeMysql()
return result
def GetAppkey_h5():
plats = ["h5"]
client = MysqlClient("saas_server")
# result = client.getAppkey_h5()
result = client.getAppkey_kwargs(plat=plats,
mongo_id=mongo_id,
filter_keys=["appkey", "cdkey"])
data = {}
for item in result:
dbname, appkey, plat = item["cdkey"], item["appkey"], item["plat"]
if (dbname, plat) not in data.get(appkey, []):
data.setdefault(appkey, []).append((dbname, plat))
client.closeMysql()
return data
def GetAppkey_saas():
plats = ["ios", "android", "all", "feeling"]
client = MysqlClient("saas_server")
result = client.getAppkey_kwargs(plat=plats,
mongo_id=mongo_id,
filter_keys=["appkey", "cdkey"])
data = {}
for item in result:
print item
dbname, appkey, plat = item["cdkey"], item["appkey"], item["plat"]
if (dbname, plat) not in data.get(appkey, []):
data.setdefault(appkey, []).append((dbname, plat))
client.closeMysql()
return data
def transformresult(self, analysisresult, *args, **kwargs):
client = MysqlClient(self.dbname)
data = client.select(
"select id, name, isdel from %(dbname)s.%(datatype)s_%(plat)s_eventid_udf"
% {
"dbname": self.dbname,
"datatype": self.datatype,
"plat": self.plat,
})
result = analysisresult.result
for item in data:
_id, name, isdel = item[0], item[1], item[2]
result.setdefault(_id, [name, isdel])
# result = analysisresult.result
analysisresult.transformresult = result
client.closeMysql()
def getTopics(self, once_sleep=60):
'''
获取需要消费的topic，可通过数据库表控制
:param once_sleep: topics更新频率
:return:
'''
while True:
if self.debug:
debug_topic = self.configures.get("debugconf", "debug_topic")
self.topics = [(item, "debug")
for item in debug_topic.split(",")]
else:
kafka_topics = set()
saas_appkey = set()
appkey_logpath = {}
try:
from kafka import SimpleClient
hostname = self.configures.get("kafka", "hostname")
client = SimpleClient(hosts=hostname)
for topic in client.topics:
kafka_topics.add(topic)
client.close()
log.info("get kafka topics: %s" %
json.dumps(list(kafka_topics)))
except:
logging.error(sys.exc_info())
continue
try:
client = MysqlClient("saas_server")
topics = client.getTopics(group_id=self.group_id)
for topic, logpath in topics:
saas_appkey.add(topic)
appkey_logpath.setdefault(topic, set()).add(logpath)
client.closeMysql()
log.info("get mysql appkeys: %s" %
json.dumps(list(saas_appkey)))
except:
logging.error(sys.exc_info())
continue
self.topics = [(topic, logpath)
for topic in list(kafka_topics & saas_appkey)
for logpath in appkey_logpath[topic]]
log.info("current topics: %s" % json.dumps(self.topics))
time.sleep(once_sleep)
def appointModeStoreByDaily(datatype, num, modename, *args, **kwargs):
global mode_writer_contrast
ct = time.strftime("%Y-%m-%d+%H%M", time.localtime(time.time() - 86400 * num))
yyyy_mm_dd, hhmm = ct.split("+")[0], "2359"
mode_class, storer_class = mode_writer_contrast[modename]
modecreator = mode_class(datatype, yyyy_mm_dd) if mode_class else None
result = modecreator.dataCollect() if modecreator else None
# 获取datatype对应入库mongo编号
m_client = MysqlClient("saas_server")
mongo_ids = m_client.get_mongoid(datatype)
m_client.closeMysql()
for mongo_id in mongo_ids:
try:
storer = storer_class(mongo_id)
storer.write(result, datatype, modename, modecreator, num=num, today=yyyy_mm_dd)
except:
import traceback
print(traceback.print_exc())
def DevUsers(datatype='all', iscache=False):
global result
global myclock
if ((time.time() - myclock) > 3600 or (not result)) or iscache:
client = MysqlClient("saas_meta")
con, cur = client.connection
sql = "SELECT a.appkey, b.userkey FROM saas_exclude_appkey a LEFT JOIN saas_dev_userkey b on a.appkey = b.appkey WHERE a.`enable` = 1"
cur.execute(sql)
for item in cur.fetchall():
appkey, userkey = item[0], item[1]
result.setdefault(appkey, set()).add(userkey)
myclock = time.time()
client.closeMysql()
else:
pass
tmp = {}
for key in result:
if "all" in datatype:
tmp.setdefault(key, list(result[key]))
elif key in datatype:
tmp.setdefault(key, list(result[key]))
return tmp
def appointModeStoreByMinute(datatype, timestamp, modename, *args, **kwargs):
global mode_writer_contrast
ct = time.strftime("%Y-%m-%d+%H%M", time.localtime(timestamp))
yyyy_mm_dd, hhmm = ct.split("+")[0], ct.split("+")[1]
mode_class, storer_class = mode_writer_contrast[modename]
modecreator = mode_class(datatype, yyyy_mm_dd, hhmm, last=1) if mode_class else None
result = modecreator.dataCollect() if modecreator else None
m_client = MysqlClient("saas_server")
mongo_ids = m_client.get_mongoid(datatype)
m_client.closeMysql()
for mongo_id in mongo_ids:
try:
storer = storer_class(mongo_id)
ct = time.strftime("%Y-%m-%d+%H%M", time.localtime(timestamp))
yyyymmddhhmm = ct.replace("-", "").replace("+", "")
yyyy_mm_dd, hhmm = ct.split("+")[0], ct.split("+")[1]
if yyyymmddhhmm.endswith("0000"):
storer.remove(datatype, modename, yyyymmddhhmm[:8])
print("invoking remove interface, remove data or Nothing to do! ", datatype, modename, yyyymmddhhmm[:8])
storer.write(result, datatype, modename, modecreator, today=yyyy_mm_dd)
except:
import traceback
print(traceback.print_exc())
def __init__(self):
self.client = MysqlClient("information_schema")
self.con, self.cur = self.client.connection
def GetAppkeys(**kwargs):
client = MysqlClient("saas_server")
result = client.getAppkey_kwargs(kwargs)
return result
def get_datatype_saas_h5():
client = MysqlClient("saas_meta")
result = [[item["cdkey"], item["appkey"], item["plat"]]
for item in client.getAppkey_kwargs(plat=["h5"])]
client.closeMysql()
return result
def get_datatype_saas_h5():
client = MysqlClient("saas_meta")
result = [item for item in client.getAppkey_h5()]
client.closeMysql()
return result
def get_mongoid(self, appkey):
m_client = MysqlClient()
self.mongo_id = m_client.get_mongoid(appkey)[0]
m_client.closeMysql()
def get_mongo_conn(appkey):
m_client = MysqlClient()
mongo_id = m_client.get_mongoid(appkey)[0]
m_client.closeMysql()
conn = PyMongoClient(mongo_id=mongo_id)
return conn.getConn()