Ejemplo n.º 1
0
def rollbackErrorSchedule(error_filter={}):
    """重新运行所有Error schedule
    """

    server = getRedisConn2()
    for (k, v) in MAPPING.items():
        if k in exclude_key:
            continue
        for vi in v:
            if not server.exists(vi):
                break

            # no filter

            ids = server.hkeys(vi)
            server.delete(vi)
            server.sadd(k, *ids)
            print vi, len(ids), k

            # filter
            # vals = server.hgetall(vi)
            # ids = set()
            # for k,v in vals.items():
            #     for fit in error_filter[k]:
            #         if fit in v:
            #             ids.add(k)
            # server.delete(vi)
            # server.sadd(k, *ids)
            # print vi, len(ids), k

    del server
Ejemplo n.º 2
0
def rollbackErrorSchedule(error_filter={}):
    """重新运行所有Error schedule
    """

    server = getRedisConn2()
    for (k, v) in MAPPING.items():
        if k in exclude_key:
            continue
        for vi in v:
            if not server.exists(vi):
                break

            # no filter

            ids = server.hkeys(vi)
            server.delete(vi)
            server.sadd(k, *ids)
            print vi, len(ids), k

            # filter
            # vals = server.hgetall(vi)
            # ids = set()
            # for k,v in vals.items():
            #     for fit in error_filter[k]:
            #         if fit in v:
            #             ids.add(k)
            # server.delete(vi)
            # server.sadd(k, *ids)
            # print vi, len(ids), k

    del server
Ejemplo n.º 3
0
def rollbackShopAllitemSchedule():
    server = getRedisConn2()
    uids = []
    for schedule in ['Schedule_shopAllitemInfo_process', 'Schedule_shopAllitemInfoError']:
        uids.extend(server.hkeys(schedule))
        server.delete(schedule)
    server.sadd('Schedule_shopAllitemInfo', *uids)
    del server
Ejemplo n.º 4
0
def rollbackKeywordSearchResultSchedule():
    server = getRedisConn2()
    kids = []
    for schedule in ['Schedule_keywordSearchResultError', 'Schedule_keywordSearchResult_process']:
        kids.extend(server.hkeys(schedule))
        server.delete(schedule)
    server.sadd('Schedule_keywordSearchResult', *kids)
    del server
Ejemplo n.º 5
0
def main():
    interval = (atoi(sys.argv[1]) if len(sys.argv) > 1 else 60)
    slaveId = uuid.getnode()
    server = getRedisConn2()
    server.set('SlaveSleep%s' % slaveId, interval)
    server.expire('SlaveSleep%s' % slaveId, 60)
    del server
    print 'paused'
    raw_input('')
Ejemplo n.º 6
0
def rollbackErrorItemTradeSchedule():
    server = getRedisConn2()
    vals = server.hgetall('Schedule_itemTradeError')
    iids = []
    for (iid, val) in vals.items():
        if val.find('None') != -1 or val.find('lost') != -1:
            iids.append(iid)
    server.sadd('Schedule_itemTrade', *iids)
    server.delete('Schedule_itemTradeError')
    del server
Ejemplo n.º 7
0
def clearAllErrorSchedule():
    """清除所有错误Schedule
    """

    server = getRedisConn2()
    for (k, v) in MAPPING.items():
        for vi in v:
            print vi, server.hlen(vi)
            server.delete(vi)
    del server
Ejemplo n.º 8
0
def runSchedule():
    reConn = getRedisConn2()
    uids = reConn.hkeys('Schedule_shopAllitemInfoError')
    for i, uid in enumerate(uids):
        url = 'http://store.taobao.com/shop/view_shop.htm?user_number_id=%s' % uid
        try:
            metadata = getShopItemsOverview(url)
            saveTopAllitemInfo(metadata)
        except Exception, e:
            print uid
Ejemplo n.º 9
0
def rollbackErrorItemTradeSchedule():
    server = getRedisConn2()
    vals = server.hgetall('Schedule_itemTradeError')
    iids = []
    for (iid, val) in vals.items():
        if val.find('None') != -1 or val.find('lost') != -1:
            iids.append(iid)
    server.sadd('Schedule_itemTrade', *iids)
    server.delete('Schedule_itemTradeError')
    del server
Ejemplo n.º 10
0
def clearAllErrorSchedule():
    """清除所有错误Schedule
    """

    server = getRedisConn2()
    for (k, v) in MAPPING.items():
        for vi in v:
            print vi, server.hlen(vi)
            server.delete(vi)
    del server
Ejemplo n.º 11
0
def runSchedule():
    reConn = getRedisConn2()
    uids = reConn.hkeys('Schedule_shopAllitemInfoError')
    for i, uid in enumerate(uids):
        url = 'http://store.taobao.com/shop/view_shop.htm?user_number_id=%s' % uid
        try:
            metadata = getShopItemsOverview(url)
            saveTopAllitemInfo(metadata)
        except Exception, e:
            print uid
Ejemplo n.º 12
0
def rollbackShopAllitemSchedule():
    server = getRedisConn2()
    uids = []
    for schedule in [
            'Schedule_shopAllitemInfo_process', 'Schedule_shopAllitemInfoError'
    ]:
        uids.extend(server.hkeys(schedule))
        server.delete(schedule)
    server.sadd('Schedule_shopAllitemInfo', *uids)
    del server
Ejemplo n.º 13
0
def rollbackNullSiteIdItemSchedule():
    print 'init...'
    server = getRedisConn2()
    iids = []
    db_conn = ibbdlib.get_db_conn()
    for row in db_conn.iter("SELECT item_id FROM ibbd2.top_cat_items WHERE site_id=''"):
        iids.append(row.item_id)
    db_conn.close()
    print 'Schedule_itemFullInfo', len(iids)
    server.sadd('Schedule_itemFullInfo', *iids)
    del server
Ejemplo n.º 14
0
def rollbackKeywordSearchResultSchedule():
    server = getRedisConn2()
    kids = []
    for schedule in [
            'Schedule_keywordSearchResultError',
            'Schedule_keywordSearchResult_process'
    ]:
        kids.extend(server.hkeys(schedule))
        server.delete(schedule)
    server.sadd('Schedule_keywordSearchResult', *kids)
    del server
Ejemplo n.º 15
0
def main():
    connpool = ConnectionPool(max_connections=20, **{
        'host': 'localhost',
        'user': '******',
        'passwd': '',
        'db': 'ibbd2',
        })
    reConn = getRedisConn2()
    test_schedule_cat(reConn, connpool)
    del reConn
    connpool.disconnect()
Ejemplo n.º 16
0
def rollbackNullSiteIdItemSchedule():
    print 'init...'
    server = getRedisConn2()
    iids = []
    db_conn = ibbdlib.get_db_conn()
    for row in db_conn.iter(
            "SELECT item_id FROM ibbd2.top_cat_items WHERE site_id=''"):
        iids.append(row.item_id)
    db_conn.close()
    print 'Schedule_itemFullInfo', len(iids)
    server.sadd('Schedule_itemFullInfo', *iids)
    del server
Ejemplo n.º 17
0
def saveRedisSchedulePlan():
    rdSvr = getRedisConn2()
    db = getDBConn('ibbd2')
    dbCursor = db.cursor()
    for schedule in SCHEDULES:
        ids = rdSvr.smembers(schedule)
        dbCursor.executemany('INSERT INTO topspider.schedule_plan_log VALUES(%s,CURDATE(),%s)', [[schedule, iid]
                             for iid in ids])
    db.commit()
    dbCursor.close()
    db.close()
    del rdSvr
Ejemplo n.º 18
0
def saveRedisSchedulePlan():
    rdSvr = getRedisConn2()
    db = getDBConn('ibbd2')
    dbCursor = db.cursor()
    for schedule in SCHEDULES:
        ids = rdSvr.smembers(schedule)
        dbCursor.executemany(
            'INSERT INTO topspider.schedule_plan_log VALUES(%s,CURDATE(),%s)',
            [[schedule, iid] for iid in ids])
    db.commit()
    dbCursor.close()
    db.close()
    del rdSvr
Ejemplo n.º 19
0
def saveRedisScheduleError(runDate):
    rdSvr = getRedisConn2()
    db = getDBConn('ibbd2')
    dbCursor = db.cursor()

    def parseLog(val):
        j = {}
        try:
            j = json.loads(val)
            return [j['msg'], j['SlaveID']]
        except:
            try:
                j = json.loads(re.sub(r'(\d+)L', r'\1', val.replace("'", '"')))
            except:
                j = {'msg': None, 'SlaveID': None}
        return [j['msg'], j['SlaveID']]

    def resort(arr):

        # resort [schedule,k,schedule_error,msg,slaveid] to [schedule,slaveid,k,schedule_error,msg]

        arr.insert(2, arr[-1])
        return arr[:-1]

    for (schedule, schedule_error) in SCHEDULE_ERROR_MAP.items():
        print schedule, schedule_error, rdSvr.hlen(schedule_error)
        if rdSvr.hlen(schedule_error) > 0:
            dbCursor.executemany(
                'INSERT INTO topspider.schedule_error_log VALUES(%s,%s,%s,%s,%s,%s,NOW())',
                [
                    resort([schedule, runDate, k, schedule_error] +
                           parseLog(v))
                    for (k, v) in rdSvr.hgetall(schedule_error).items()
                ])

        # rdSvr.delete(schedule_error)

    db.commit()
    dbCursor.close()
    db.close()
    del rdSvr
Ejemplo n.º 20
0
def saveRedisScheduleError(runDate):
    rdSvr = getRedisConn2()
    db = getDBConn('ibbd2')
    dbCursor = db.cursor()

    def parseLog(val):
        j = {}
        try:
            j = json.loads(val)
            return [j['msg'], j['SlaveID']]
        except:
            try:
                j = json.loads(re.sub(r'(\d+)L', r'\1', val.replace("'", '"')))
            except:
                j = {'msg': None, 'SlaveID': None}
        return [j['msg'], j['SlaveID']]

    def resort(arr):

        # resort [schedule,k,schedule_error,msg,slaveid] to [schedule,slaveid,k,schedule_error,msg]

        arr.insert(2, arr[-1])
        return arr[:-1]

    for (schedule, schedule_error) in SCHEDULE_ERROR_MAP.items():
        print schedule, schedule_error, rdSvr.hlen(schedule_error)
        if rdSvr.hlen(schedule_error) > 0:
            dbCursor.executemany('INSERT INTO topspider.schedule_error_log VALUES(%s,%s,%s,%s,%s,%s,NOW())',
                                 [resort([schedule, runDate, k, schedule_error] + parseLog(v)) for (k, v) in
                                 rdSvr.hgetall(schedule_error).items()])

        # rdSvr.delete(schedule_error)

    db.commit()
    dbCursor.close()
    db.close()
    del rdSvr