Ejemplo n.º 1
0
 def __init__(self, client=None):
     if client is None:
         self.client = PyMongoClient()
     else:
         self.client = client
     self.dbname = "jh"
     self.modename = "UserIP"
Ejemplo n.º 2
0
 def __init__(self, mongo_id=1):
     self.client = PyMongoClient(mongo_id=mongo_id)
     self.conn = self.client.getConn()
     self.modename = "uvfile"
     # self.store_attachmode = UserIP()
     self.attachmode_storers = []
     try:
         # self.attachmode_storers = [UserIP(), UserProfileUpdateWriter()]
         self.attachmode_storers = [UserIP()]
     except:
         import traceback
         print(traceback.print_exc())
Ejemplo n.º 3
0
def usetimeDistribute(num, appkey="BIQU_ANDROID", delta=120):
    # def usetimeDistribute(num, appkey="biqu", delta=120):
    curday = datetime.datetime.today().strftime("%Y%m%d")
    dayStr = time.strftime("%Y-%m-%d",
                           time.localtime(time.time() - 86400 * num))
    client = PyMongoClient()
    result = {}
    # for item in client.find(appkey, "uvfile", {"tm": dayStr, "jhd_userkey": userkey}):
    m, n = 0, 0
    for item in client.find(appkey, "uvfile", {"tm": dayStr}):
        opas = ["action", "page", "in", "end"]
        uid = item["jhd_userkey"]
        end_sum = item["item_add"].get("end", 0)
        opatms = list(
            set(
                reduce(
                    lambda a, b: a + b,
                    map(
                        lambda opa: item["item_count"].get(opa, {}).get(
                            "opatm", []), opas))))
        opatms.sort()
        opsdtsmps = map(
            lambda opatm: int(
                time.mktime(
                    time.strptime("".join([curday, opatm]), "%Y%m%d%H:%M:%S"))
            ), opatms)
        tmp = [
            0,
        ]
        for opastamp, pos in zip(opsdtsmps, range(len(opsdtsmps) - 1)):
            a = opsdtsmps[pos]
            b = opsdtsmps[pos + 1]
            tmp.append(b - a)
        if end_sum >= 600:
            print(uid, end_sum, sum([i for i in tmp if i <= delta]), tmp)
            print(uid, end_sum, sum([i for i in tmp if i <= delta]), opatms)
            m += 1
            print(i, end_sum)
        else:
            print(uid, end_sum, sum([i for i in tmp if i <= delta]), tmp)
            print(uid, end_sum, sum([i for i in tmp if i <= delta]), opatms)
            n += 1
        total_opatm = sum([i for i in tmp if i <= delta])
        if total_opatm != 0:
            result.setdefault(uid, total_opatm)
    print(m, n)
Ejemplo n.º 4
0
 def getData(self, appkey, *args, **kwargs):
     modename = "UserMapMeta"
     self.get_mongoid(appkey)
     self.client = PyMongoClient(self.mongo_id)
     cur = self.client.getConn()[appkey][modename].find({})
     result = {}
     # format: { "_id" : "ac7", "fields" : [ { "type" : "string", "name" : "type" } ], "field_elems" : { "type" : [ "分类视图" ] } }
     for item in cur:
         eventid = item["_id"]
         fields = item["fields"]
         field_elems = item.get("field_elems", {})
         # 保存 type/elems
         for field in fields:
             try:
                 mapkey = field["name"]
                 mapkey_type = field["type"]
                 elems = field_elems.get(mapkey, [])
                 elems_tmp = []
                 for elem in elems:
                     if isinstance(elem, str) or isinstance(elem, unicode):
                         # 排除URL类型属性
                         if "http" in elem:
                             continue
                         if len(elem) >= 60:
                             continue
                     elems_tmp.append(elem)
                     if len(elems_tmp) >= 100:
                         break
                 result.setdefault(eventid,
                                   {}).setdefault(mapkey, {
                                       "type": mapkey_type,
                                       "elems": elems_tmp,
                                   })
             except:
                 continue
         # 不包含map的情况
         if bool(fields) == False:
             result.setdefault(eventid, {})
     return result
Ejemplo n.º 5
0
 def __init__(self):
     self.client = PyMongoClient()
Ejemplo n.º 6
0
 def __init__(self, mongo_id=1):
     self.client = PyMongoClient(mongo_id=mongo_id)
     self.conn = self.client.getConn()
     self.dbname = "jh"
     self.modename = "UserIP"
Ejemplo n.º 7
0
 def __init__(self):
     self.client = PyMongoClient()
     self.modename = "UserActiveUpdate"
Ejemplo n.º 8
0
 def __init__(self, mongo_id=1):
     self.client = PyMongoClient(mongo_id=mongo_id)
     self.modename = "UserMapMeta"
Ejemplo n.º 9
0
 def __init__(self):
     self.client = PyMongoClient()
     self.groupwriter = UserEventGroupWriter()
     self.modename = "UserEvent"
Ejemplo n.º 10
0
 def __init__(self):
     self.client = PyMongoClient()
     self.modename = "UserEventGroup"
Ejemplo n.º 11
0
 def write(self, dataDict, appkey, modename, modetools, *args, **kwargs):
     num = kwargs["num"]
     client = PyMongoClient(self.mongo_id)
     uvfile_path = get_uvfile_path(num, appkey)
     cur_day = time.strftime("%Y-%m-%d",
                             time.localtime(time.time() - 86400 * num))
     uids = dataDict.keys()
     # userProfile = client.find(appkey, "UserProfile", {"_id": {"$in": uids}})
     uvfile = client.findElemIn(
         appkey, "uvfile", "jhd_userkey", uids,
         OrderedDict([("tm", cur_day)]), {
             "_id": False,
             "jhd_userkey": True,
             "jhd_loc": True,
             "firstLoginTime": True,
             "lastOpaTime": True,
             "jhd_pb": True
         })
     for item in uvfile:
         try:
             uid = item["jhd_userkey"]
             comepub = item.get("jhd_pb", ["#"])
             firstLoginTime = item.get("firstLoginTime", "#")
             lastLoginTime = item.get("lastOpaTime", "#")
             data = dataDict[uid]
             data["firstLoginTime"] = firstLoginTime
             data["lastLoginTime"] = lastLoginTime
             data["comepub"] = comepub[0] if comepub else "#"
             locs = item.get("jhd_loc", None)
             if locs:
                 data["locs"] = []
                 for item in locs:
                     if isinstance(item, dict):
                         prov, city = item.get("prov",
                                               "#"), item.get("city", "#")
                     else:
                         prov, city = item.split("_")
                     if (prov, city) not in data["locs"]:
                         data["locs"].append((prov, city))
         except:
             import traceback
             print traceback.print_exc()
     for key in dataDict:
         data = dataDict[key]
         uid = key
         pushid = data["pushid"]
         plat = data["plat"]
         ua = data["ua"]
         net = "#".join(list(data["net"]))
         curpub = "#".join(list(data["curpubs"]))
         comepub = data["comepub"]
         firstLoginTime = data["firstLoginTime"]
         lastLoginTime = data["lastLoginTime"]
         ver = "#".join(list(data["vers"]))
         loc = "#".join(map(lambda item: "_".join(item), data["locs"]))
         in_num = data["in"][0]
         dur = "#".join(map(str, data["end"])) if data["end"] else "#"
         actions = {}
         [
             actions.setdefault(key, data["action"][key][0])
             for key in data["action"]
         ]
         actionDict = json.dumps(actions)
         pages = {}
         [
             pages.setdefault(key, data["page"][key][0])
             for key in data["page"]
         ]
         pageDict = json.dumps(pages)
         isactive = data["isactive"]
         line = []
         line.append(uid)  # 1
         line.append(isactive)
         line.append(comepub)
         line.append(curpub)
         line.append(plat)  # 5
         line.append(ver)
         line.append(ua)
         line.append(net)
         line.append(firstLoginTime)
         line.append(lastLoginTime)  # 10
         line.append(loc)
         line.append(in_num)
         line.append(dur)
         line.append(actionDict)
         line.append(pageDict)  # 15
         line.append(pushid)  # 16
         JHWrite(uvfile_path, "\t".join(map(str, line)))
     JHWrite.finished(iszip=True)
Ejemplo n.º 12
0
 def __init__(self):
     self.client = PyMongoClient()
     self.conn = self.client.getConn()
     self.modename = "UserProfile"
Ejemplo n.º 13
0
 def __init__(self):
     self.client = PyMongoClient()
     self.conn = self.client.getConn()
Ejemplo n.º 14
0
                            {
                                "_id": _id,
                                "field_elems.%s.100" % (key, ): {
                                    "$exists": False
                                }
                            }, update_query, True))
            except:
                import traceback
                print(traceback.print_exc(), op)
        try:
            if op:
                op = self.client.bulkWrite(appkey, modename, op)
        except:
            print(
                "Warn: bulkStore 'UserProfile' Rise a error; Switch to Single Mode"
            )
            for op_item in op:
                try:
                    self.client.bulkWrite(appkey, modename, [op_item])
                except:
                    import traceback
                    print(traceback.print_exc())
        # self.client.getConn()[appkey][modename].remove({"field_elems": {"$exists": True}})
        finallyMask(appkey, modename, self.client)
        print("UserMapMetaWriter cost seconds %.10f" % ((time.time() - a), ))


if __name__ == "__main__":
    client = PyMongoClient()
    uewriter = dropIndex("caiyu_ios_free", "UserEvent", client)
    # uewriter.remove("feeling", )
Ejemplo n.º 15
0
from apscheduler.triggers.cron import CronTrigger
from apscheduler.events import (
    EVENT_SCHEDULER_STARTED, EVENT_SCHEDULER_SHUTDOWN, EVENT_SCHEDULER_PAUSED,
    EVENT_SCHEDULER_RESUMED, EVENT_EXECUTOR_ADDED, EVENT_EXECUTOR_REMOVED,
    EVENT_JOBSTORE_ADDED, EVENT_JOBSTORE_REMOVED, EVENT_ALL_JOBS_REMOVED,
    EVENT_JOB_ADDED, EVENT_JOB_REMOVED, EVENT_JOB_MODIFIED, EVENT_JOB_EXECUTED,
    EVENT_JOB_ERROR, EVENT_JOB_MISSED, EVENT_JOB_SUBMITTED,
    EVENT_JOB_MAX_INSTANCES)

from apscheduler.executors.pool import ThreadPoolExecutor, ProcessPoolExecutor

from JobHandler import JobHandler
from DBClient.PyMongoClient import PyMongoClient
global _mongoclient
_mongoclient = PyMongoClient().getConn()


class SchedulerManager(object):
    global _mongoclient

    def __init__(self):

        self._jobs = {}
        self._jobhandlers = {}  # format, key: jobid,  value: jobhandler
        self.create_scheduler()
        self.start()

    def create_scheduler(self):
        self.jobstores = {
            'mongo':
Ejemplo n.º 16
0
 def __init__(self):
     self.client = PyMongoClient()
     self.IPContainer = StackSet()
Ejemplo n.º 17
0
def get_mongo_conn(appkey):
    m_client = MysqlClient()
    mongo_id = m_client.get_mongoid(appkey)[0]
    m_client.closeMysql()
    conn = PyMongoClient(mongo_id=mongo_id)
    return conn.getConn()