def __init__(self, client=None):
if client is None:
self.client = PyMongoClient()
else:
self.client = client
self.dbname = "jh"
self.modename = "UserIP"
def __init__(self, mongo_id=1):
self.client = PyMongoClient(mongo_id=mongo_id)
self.conn = self.client.getConn()
self.modename = "uvfile"
# self.store_attachmode = UserIP()
self.attachmode_storers = []
try:
# self.attachmode_storers = [UserIP(), UserProfileUpdateWriter()]
self.attachmode_storers = [UserIP()]
except:
import traceback
print(traceback.print_exc())
def usetimeDistribute(num, appkey="BIQU_ANDROID", delta=120):
# def usetimeDistribute(num, appkey="biqu", delta=120):
curday = datetime.datetime.today().strftime("%Y%m%d")
dayStr = time.strftime("%Y-%m-%d",
time.localtime(time.time() - 86400 * num))
client = PyMongoClient()
result = {}
# for item in client.find(appkey, "uvfile", {"tm": dayStr, "jhd_userkey": userkey}):
m, n = 0, 0
for item in client.find(appkey, "uvfile", {"tm": dayStr}):
opas = ["action", "page", "in", "end"]
uid = item["jhd_userkey"]
end_sum = item["item_add"].get("end", 0)
opatms = list(
set(
reduce(
lambda a, b: a + b,
map(
lambda opa: item["item_count"].get(opa, {}).get(
"opatm", []), opas))))
opatms.sort()
opsdtsmps = map(
lambda opatm: int(
time.mktime(
time.strptime("".join([curday, opatm]), "%Y%m%d%H:%M:%S"))
), opatms)
tmp = [
0,
]
for opastamp, pos in zip(opsdtsmps, range(len(opsdtsmps) - 1)):
a = opsdtsmps[pos]
b = opsdtsmps[pos + 1]
tmp.append(b - a)
if end_sum >= 600:
print(uid, end_sum, sum([i for i in tmp if i <= delta]), tmp)
print(uid, end_sum, sum([i for i in tmp if i <= delta]), opatms)
m += 1
print(i, end_sum)
else:
print(uid, end_sum, sum([i for i in tmp if i <= delta]), tmp)
print(uid, end_sum, sum([i for i in tmp if i <= delta]), opatms)
n += 1
total_opatm = sum([i for i in tmp if i <= delta])
if total_opatm != 0:
result.setdefault(uid, total_opatm)
print(m, n)
def getData(self, appkey, *args, **kwargs):
modename = "UserMapMeta"
self.get_mongoid(appkey)
self.client = PyMongoClient(self.mongo_id)
cur = self.client.getConn()[appkey][modename].find({})
result = {}
# format: { "_id" : "ac7", "fields" : [ { "type" : "string", "name" : "type" } ], "field_elems" : { "type" : [ "分类视图" ] } }
for item in cur:
eventid = item["_id"]
fields = item["fields"]
field_elems = item.get("field_elems", {})
# 保存 type/elems
for field in fields:
try:
mapkey = field["name"]
mapkey_type = field["type"]
elems = field_elems.get(mapkey, [])
elems_tmp = []
for elem in elems:
if isinstance(elem, str) or isinstance(elem, unicode):
# 排除URL类型属性
if "http" in elem:
continue
if len(elem) >= 60:
continue
elems_tmp.append(elem)
if len(elems_tmp) >= 100:
break
result.setdefault(eventid,
{}).setdefault(mapkey, {
"type": mapkey_type,
"elems": elems_tmp,
})
except:
continue
# 不包含map的情况
if bool(fields) == False:
result.setdefault(eventid, {})
return result
def __init__(self):
self.client = PyMongoClient()
def __init__(self, mongo_id=1):
self.client = PyMongoClient(mongo_id=mongo_id)
self.conn = self.client.getConn()
self.dbname = "jh"
self.modename = "UserIP"
def __init__(self):
self.client = PyMongoClient()
self.modename = "UserActiveUpdate"
def __init__(self, mongo_id=1):
self.client = PyMongoClient(mongo_id=mongo_id)
self.modename = "UserMapMeta"
def __init__(self):
self.client = PyMongoClient()
self.groupwriter = UserEventGroupWriter()
self.modename = "UserEvent"
def __init__(self):
self.client = PyMongoClient()
self.modename = "UserEventGroup"
def write(self, dataDict, appkey, modename, modetools, *args, **kwargs):
num = kwargs["num"]
client = PyMongoClient(self.mongo_id)
uvfile_path = get_uvfile_path(num, appkey)
cur_day = time.strftime("%Y-%m-%d",
time.localtime(time.time() - 86400 * num))
uids = dataDict.keys()
# userProfile = client.find(appkey, "UserProfile", {"_id": {"$in": uids}})
uvfile = client.findElemIn(
appkey, "uvfile", "jhd_userkey", uids,
OrderedDict([("tm", cur_day)]), {
"_id": False,
"jhd_userkey": True,
"jhd_loc": True,
"firstLoginTime": True,
"lastOpaTime": True,
"jhd_pb": True
})
for item in uvfile:
try:
uid = item["jhd_userkey"]
comepub = item.get("jhd_pb", ["#"])
firstLoginTime = item.get("firstLoginTime", "#")
lastLoginTime = item.get("lastOpaTime", "#")
data = dataDict[uid]
data["firstLoginTime"] = firstLoginTime
data["lastLoginTime"] = lastLoginTime
data["comepub"] = comepub[0] if comepub else "#"
locs = item.get("jhd_loc", None)
if locs:
data["locs"] = []
for item in locs:
if isinstance(item, dict):
prov, city = item.get("prov",
"#"), item.get("city", "#")
else:
prov, city = item.split("_")
if (prov, city) not in data["locs"]:
data["locs"].append((prov, city))
except:
import traceback
print traceback.print_exc()
for key in dataDict:
data = dataDict[key]
uid = key
pushid = data["pushid"]
plat = data["plat"]
ua = data["ua"]
net = "#".join(list(data["net"]))
curpub = "#".join(list(data["curpubs"]))
comepub = data["comepub"]
firstLoginTime = data["firstLoginTime"]
lastLoginTime = data["lastLoginTime"]
ver = "#".join(list(data["vers"]))
loc = "#".join(map(lambda item: "_".join(item), data["locs"]))
in_num = data["in"][0]
dur = "#".join(map(str, data["end"])) if data["end"] else "#"
actions = {}
[
actions.setdefault(key, data["action"][key][0])
for key in data["action"]
]
actionDict = json.dumps(actions)
pages = {}
[
pages.setdefault(key, data["page"][key][0])
for key in data["page"]
]
pageDict = json.dumps(pages)
isactive = data["isactive"]
line = []
line.append(uid) # 1
line.append(isactive)
line.append(comepub)
line.append(curpub)
line.append(plat) # 5
line.append(ver)
line.append(ua)
line.append(net)
line.append(firstLoginTime)
line.append(lastLoginTime) # 10
line.append(loc)
line.append(in_num)
line.append(dur)
line.append(actionDict)
line.append(pageDict) # 15
line.append(pushid) # 16
JHWrite(uvfile_path, "\t".join(map(str, line)))
JHWrite.finished(iszip=True)
def __init__(self):
self.client = PyMongoClient()
self.conn = self.client.getConn()
self.modename = "UserProfile"
def __init__(self):
self.client = PyMongoClient()
self.conn = self.client.getConn()
{
"_id": _id,
"field_elems.%s.100" % (key, ): {
"$exists": False
}
}, update_query, True))
except:
import traceback
print(traceback.print_exc(), op)
try:
if op:
op = self.client.bulkWrite(appkey, modename, op)
except:
print(
"Warn: bulkStore 'UserProfile' Rise a error; Switch to Single Mode"
)
for op_item in op:
try:
self.client.bulkWrite(appkey, modename, [op_item])
except:
import traceback
print(traceback.print_exc())
# self.client.getConn()[appkey][modename].remove({"field_elems": {"$exists": True}})
finallyMask(appkey, modename, self.client)
print("UserMapMetaWriter cost seconds %.10f" % ((time.time() - a), ))
if __name__ == "__main__":
client = PyMongoClient()
uewriter = dropIndex("caiyu_ios_free", "UserEvent", client)
# uewriter.remove("feeling", )
from apscheduler.triggers.cron import CronTrigger
from apscheduler.events import (
EVENT_SCHEDULER_STARTED, EVENT_SCHEDULER_SHUTDOWN, EVENT_SCHEDULER_PAUSED,
EVENT_SCHEDULER_RESUMED, EVENT_EXECUTOR_ADDED, EVENT_EXECUTOR_REMOVED,
EVENT_JOBSTORE_ADDED, EVENT_JOBSTORE_REMOVED, EVENT_ALL_JOBS_REMOVED,
EVENT_JOB_ADDED, EVENT_JOB_REMOVED, EVENT_JOB_MODIFIED, EVENT_JOB_EXECUTED,
EVENT_JOB_ERROR, EVENT_JOB_MISSED, EVENT_JOB_SUBMITTED,
EVENT_JOB_MAX_INSTANCES)
from apscheduler.executors.pool import ThreadPoolExecutor, ProcessPoolExecutor
from JobHandler import JobHandler
from DBClient.PyMongoClient import PyMongoClient
global _mongoclient
_mongoclient = PyMongoClient().getConn()
class SchedulerManager(object):
global _mongoclient
def __init__(self):
self._jobs = {}
self._jobhandlers = {} # format, key: jobid, value: jobhandler
self.create_scheduler()
self.start()
def create_scheduler(self):
self.jobstores = {
'mongo':
def __init__(self):
self.client = PyMongoClient()
self.IPContainer = StackSet()
def get_mongo_conn(appkey):
m_client = MysqlClient()
mongo_id = m_client.get_mongoid(appkey)[0]
m_client.closeMysql()
conn = PyMongoClient(mongo_id=mongo_id)
return conn.getConn()