def __init__(self, client=None): if client is None: self.client = PyMongoClient() else: self.client = client self.dbname = "jh" self.modename = "UserIP"
def __init__(self, mongo_id=1): self.client = PyMongoClient(mongo_id=mongo_id) self.conn = self.client.getConn() self.modename = "uvfile" # self.store_attachmode = UserIP() self.attachmode_storers = [] try: # self.attachmode_storers = [UserIP(), UserProfileUpdateWriter()] self.attachmode_storers = [UserIP()] except: import traceback print(traceback.print_exc())
def usetimeDistribute(num, appkey="BIQU_ANDROID", delta=120): # def usetimeDistribute(num, appkey="biqu", delta=120): curday = datetime.datetime.today().strftime("%Y%m%d") dayStr = time.strftime("%Y-%m-%d", time.localtime(time.time() - 86400 * num)) client = PyMongoClient() result = {} # for item in client.find(appkey, "uvfile", {"tm": dayStr, "jhd_userkey": userkey}): m, n = 0, 0 for item in client.find(appkey, "uvfile", {"tm": dayStr}): opas = ["action", "page", "in", "end"] uid = item["jhd_userkey"] end_sum = item["item_add"].get("end", 0) opatms = list( set( reduce( lambda a, b: a + b, map( lambda opa: item["item_count"].get(opa, {}).get( "opatm", []), opas)))) opatms.sort() opsdtsmps = map( lambda opatm: int( time.mktime( time.strptime("".join([curday, opatm]), "%Y%m%d%H:%M:%S")) ), opatms) tmp = [ 0, ] for opastamp, pos in zip(opsdtsmps, range(len(opsdtsmps) - 1)): a = opsdtsmps[pos] b = opsdtsmps[pos + 1] tmp.append(b - a) if end_sum >= 600: print(uid, end_sum, sum([i for i in tmp if i <= delta]), tmp) print(uid, end_sum, sum([i for i in tmp if i <= delta]), opatms) m += 1 print(i, end_sum) else: print(uid, end_sum, sum([i for i in tmp if i <= delta]), tmp) print(uid, end_sum, sum([i for i in tmp if i <= delta]), opatms) n += 1 total_opatm = sum([i for i in tmp if i <= delta]) if total_opatm != 0: result.setdefault(uid, total_opatm) print(m, n)
def getData(self, appkey, *args, **kwargs): modename = "UserMapMeta" self.get_mongoid(appkey) self.client = PyMongoClient(self.mongo_id) cur = self.client.getConn()[appkey][modename].find({}) result = {} # format: { "_id" : "ac7", "fields" : [ { "type" : "string", "name" : "type" } ], "field_elems" : { "type" : [ "分类视图" ] } } for item in cur: eventid = item["_id"] fields = item["fields"] field_elems = item.get("field_elems", {}) # 保存 type/elems for field in fields: try: mapkey = field["name"] mapkey_type = field["type"] elems = field_elems.get(mapkey, []) elems_tmp = [] for elem in elems: if isinstance(elem, str) or isinstance(elem, unicode): # 排除URL类型属性 if "http" in elem: continue if len(elem) >= 60: continue elems_tmp.append(elem) if len(elems_tmp) >= 100: break result.setdefault(eventid, {}).setdefault(mapkey, { "type": mapkey_type, "elems": elems_tmp, }) except: continue # 不包含map的情况 if bool(fields) == False: result.setdefault(eventid, {}) return result
def __init__(self): self.client = PyMongoClient()
def __init__(self, mongo_id=1): self.client = PyMongoClient(mongo_id=mongo_id) self.conn = self.client.getConn() self.dbname = "jh" self.modename = "UserIP"
def __init__(self): self.client = PyMongoClient() self.modename = "UserActiveUpdate"
def __init__(self, mongo_id=1): self.client = PyMongoClient(mongo_id=mongo_id) self.modename = "UserMapMeta"
def __init__(self): self.client = PyMongoClient() self.groupwriter = UserEventGroupWriter() self.modename = "UserEvent"
def __init__(self): self.client = PyMongoClient() self.modename = "UserEventGroup"
def write(self, dataDict, appkey, modename, modetools, *args, **kwargs): num = kwargs["num"] client = PyMongoClient(self.mongo_id) uvfile_path = get_uvfile_path(num, appkey) cur_day = time.strftime("%Y-%m-%d", time.localtime(time.time() - 86400 * num)) uids = dataDict.keys() # userProfile = client.find(appkey, "UserProfile", {"_id": {"$in": uids}}) uvfile = client.findElemIn( appkey, "uvfile", "jhd_userkey", uids, OrderedDict([("tm", cur_day)]), { "_id": False, "jhd_userkey": True, "jhd_loc": True, "firstLoginTime": True, "lastOpaTime": True, "jhd_pb": True }) for item in uvfile: try: uid = item["jhd_userkey"] comepub = item.get("jhd_pb", ["#"]) firstLoginTime = item.get("firstLoginTime", "#") lastLoginTime = item.get("lastOpaTime", "#") data = dataDict[uid] data["firstLoginTime"] = firstLoginTime data["lastLoginTime"] = lastLoginTime data["comepub"] = comepub[0] if comepub else "#" locs = item.get("jhd_loc", None) if locs: data["locs"] = [] for item in locs: if isinstance(item, dict): prov, city = item.get("prov", "#"), item.get("city", "#") else: prov, city = item.split("_") if (prov, city) not in data["locs"]: data["locs"].append((prov, city)) except: import traceback print traceback.print_exc() for key in dataDict: data = dataDict[key] uid = key pushid = data["pushid"] plat = data["plat"] ua = data["ua"] net = "#".join(list(data["net"])) curpub = "#".join(list(data["curpubs"])) comepub = data["comepub"] firstLoginTime = data["firstLoginTime"] lastLoginTime = data["lastLoginTime"] ver = "#".join(list(data["vers"])) loc = "#".join(map(lambda item: "_".join(item), data["locs"])) in_num = data["in"][0] dur = "#".join(map(str, data["end"])) if data["end"] else "#" actions = {} [ actions.setdefault(key, data["action"][key][0]) for key in data["action"] ] actionDict = json.dumps(actions) pages = {} [ pages.setdefault(key, data["page"][key][0]) for key in data["page"] ] pageDict = json.dumps(pages) isactive = data["isactive"] line = [] line.append(uid) # 1 line.append(isactive) line.append(comepub) line.append(curpub) line.append(plat) # 5 line.append(ver) line.append(ua) line.append(net) line.append(firstLoginTime) line.append(lastLoginTime) # 10 line.append(loc) line.append(in_num) line.append(dur) line.append(actionDict) line.append(pageDict) # 15 line.append(pushid) # 16 JHWrite(uvfile_path, "\t".join(map(str, line))) JHWrite.finished(iszip=True)
def __init__(self): self.client = PyMongoClient() self.conn = self.client.getConn() self.modename = "UserProfile"
def __init__(self): self.client = PyMongoClient() self.conn = self.client.getConn()
{ "_id": _id, "field_elems.%s.100" % (key, ): { "$exists": False } }, update_query, True)) except: import traceback print(traceback.print_exc(), op) try: if op: op = self.client.bulkWrite(appkey, modename, op) except: print( "Warn: bulkStore 'UserProfile' Rise a error; Switch to Single Mode" ) for op_item in op: try: self.client.bulkWrite(appkey, modename, [op_item]) except: import traceback print(traceback.print_exc()) # self.client.getConn()[appkey][modename].remove({"field_elems": {"$exists": True}}) finallyMask(appkey, modename, self.client) print("UserMapMetaWriter cost seconds %.10f" % ((time.time() - a), )) if __name__ == "__main__": client = PyMongoClient() uewriter = dropIndex("caiyu_ios_free", "UserEvent", client) # uewriter.remove("feeling", )
from apscheduler.triggers.cron import CronTrigger from apscheduler.events import ( EVENT_SCHEDULER_STARTED, EVENT_SCHEDULER_SHUTDOWN, EVENT_SCHEDULER_PAUSED, EVENT_SCHEDULER_RESUMED, EVENT_EXECUTOR_ADDED, EVENT_EXECUTOR_REMOVED, EVENT_JOBSTORE_ADDED, EVENT_JOBSTORE_REMOVED, EVENT_ALL_JOBS_REMOVED, EVENT_JOB_ADDED, EVENT_JOB_REMOVED, EVENT_JOB_MODIFIED, EVENT_JOB_EXECUTED, EVENT_JOB_ERROR, EVENT_JOB_MISSED, EVENT_JOB_SUBMITTED, EVENT_JOB_MAX_INSTANCES) from apscheduler.executors.pool import ThreadPoolExecutor, ProcessPoolExecutor from JobHandler import JobHandler from DBClient.PyMongoClient import PyMongoClient global _mongoclient _mongoclient = PyMongoClient().getConn() class SchedulerManager(object): global _mongoclient def __init__(self): self._jobs = {} self._jobhandlers = {} # format, key: jobid, value: jobhandler self.create_scheduler() self.start() def create_scheduler(self): self.jobstores = { 'mongo':
def __init__(self): self.client = PyMongoClient() self.IPContainer = StackSet()
def get_mongo_conn(appkey): m_client = MysqlClient() mongo_id = m_client.get_mongoid(appkey)[0] m_client.closeMysql() conn = PyMongoClient(mongo_id=mongo_id) return conn.getConn()