예제 #1
0
def get_count(jd_or_cv, channel):

    if jd_or_cv == "jd":
        store = PageStore(channel)
    elif jd_or_cv == "cv":
        store = CVPageStore(channel)
    elif jd_or_cv == "co":
        store = CoPageStore(channel)

    else:
        raise Exception("unknown jd_or_cv type: %s" % jd_or_cv)

    return store.count_all()
예제 #2
0
    def __init__(self, channel):
        self.channel = channel
        BaseTask.__init__(self, "{}_expire_detect".format(channel))
        self.page_store = PageStore(channel)
        self.process_items = []
        self.page_expire_detect = PageExpireDetect()

        self.test_mode = False
예제 #3
0
    def __init__(self, owner, queue_size, thread_cnt):
        BaseTask.__init__(self, owner, queue_size, thread_cnt)
        self.owner = owner
        self.process_item = {}
        self.expired_items = []
        self._raw_store = self._get_raw_store(owner)
        self._measure_store = self._get_measure_store(owner)
        self._page_store = PageStore(owner)
        self.page_store_db = 'admin'
        self.page_store_coll = "page_store_{}".format(owner)

        self.thrift_client = TClient('../../conf/thrift_conf.ini')
        self.jd_measure_client = self.thrift_client.jd_measure_server_client

        self.test_mode = False

        self.rs_file = FileSave(BaseTask.PathConfig.result_file)
        self.failfile = FileSave(BaseTask.PathConfig.etl_failids_file)
예제 #4
0
class ETLTask(BaseTask):
    def __init__(self, owner, queue_size, thread_cnt):
        BaseTask.__init__(self, owner, queue_size, thread_cnt)
        self.owner = owner
        self.process_item = {}
        self.expired_items = []
        self._raw_store = self._get_raw_store(owner)
        self._measure_store = self._get_measure_store(owner)
        self._page_store = PageStore(owner)
        self.page_store_db = 'admin'
        self.page_store_coll = "page_store_{}".format(owner)

        self.thrift_client = TClient('../../conf/thrift_conf.ini')
        self.jd_measure_client = self.thrift_client.jd_measure_server_client

        self.test_mode = False

        self.rs_file = FileSave(BaseTask.PathConfig.result_file)
        self.failfile = FileSave(BaseTask.PathConfig.etl_failids_file)

    def _get_raw_store(self, owner):
        if "jd_lagou" == owner:
            return LgEtlStore('raw')
        if "jd_51job" == owner:
            return Job51EtlStore('raw')
        if "jd_zhilian" == owner:
            return ZLEtlStore('raw')

        raise Exception(" unknown owner ")

    def _get_measure_store(self, owner):
        if "jd_lagou" == owner:
            return LgEtlStore('measure')
        if "jd_51job" == owner:
            return Job51EtlStore('measure')
        if "jd_zhilian" == owner:
            return ZLEtlStore('measure')

        raise Exception(" unknown owner ")

    def fill_data_with_flag(self, indexUrl, realUrl, contentSign, updateTime, filePath, flag):

        if indexUrl in self.process_item:
            if self.process_item[indexUrl]['updateTime'] < updateTime:
                self.process_item[indexUrl]['updateTime'] = updateTime
                self.process_item[indexUrl]['contentSign'] = contentSign
                self.process_item[indexUrl]['realUrl'] = realUrl
                self.process_item[indexUrl]['flag'] = flag
                self.process_item[indexUrl]['filePath'] = filePath

        else:
            self.process_item[indexUrl] = {
                'updateTime': updateTime,
                'contentSign': contentSign,
                'realUrl': realUrl,
                'filePath': filePath,
                'flag': flag,
            }

    def check(self, item):

        status = item.get('status', 0)
        isUpdated = item.get('isUpdated', 0)
        updateTime = item.get('updateTime')
        indexUrl = item.get('indexUrl')
        contentSign = item.get('contentSign')
        file_path = item.get('pageContentPath')
        jdUrl = item.get('realUrl')
        expired = item.get('isExpired', 0)

        if status == 0:
            self.fill_data_with_flag(indexUrl, jdUrl, contentSign, updateTime, file_path, 0)
        if status == 1 and isUpdated == 1:
            self.fill_data_with_flag(indexUrl, jdUrl, contentSign, updateTime, file_path, 1)
        if expired == 1:
            self.fill_data_with_flag(indexUrl, jdUrl, contentSign, updateTime,  file_path, 2)

    def _load_data(self):
        page_client = self._raw_store.cmgClient.page_store_mongo_client
        for item in page_client[self.page_store_db][self.page_store_coll].find():
            self.check(item)
            if len(self.process_item) and len(self.process_item) % 10000 == 0:
                print "load {} items".format(len(self.process_item))

        print "finish load {} items".format(len(self.process_item))
        print "finish load {} expired items".format(len(self.expired_items))
        # print "======================start dump ids to files================="
        # fd = FileSave('../data/results.txt')
        # for indexUrl in self.process_item:
        #     fd.append_end_with(indexUrl)
        #
        # print "=======================dump finish============================="

    def dispatcher(self, q):

        for item in self.process_item:
            q.put(item)

        for item in self.expired_items:
            q.put(item)

        q.put(None)
        self.wait_q()

    def getPageContent(self, filename):
        parts = filename.split("::")
        if len(parts) == 3:
            binReader = BinReader(parts[1])
            _, content = binReader.readone_at(int(parts[2]))
            if len(content) == 0:
                raise Exception("file name:{} , content error".format(filename))
            return content

        if len(parts) == 1:
            with open(filename) as f:
                content = f.read()
                if len(content) == 0:
                    raise Exception("file name:{} , content error".format(filename))
                return content

    def update_jd(self, item):
        jd_store_key = {'jdId': item}
        page_store_key = {'indexUrl': item, 'contentSign': self.process_item[item].get('contentSign')}
        updateTime = self.process_item[item].get("updateTime")
        strTime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(updateTime/1000))

        jkey = json.dumps(jd_store_key)
        jdoc = json.dumps({"$set": {"pubDate": strTime}})
        jdoc_m = json.dumps({"$set": {"pubDateStamp": updateTime}})

        if self.test_mode:
            print " Update ===> jdkey: {} \n jdoc: {} \n".format(jkey, jdoc)
            return

        if not self._raw_store.save_one(jkey, jdoc, False):
            raise Exception("update jd_raw pubTime Exception")
        if not self._measure_store.save_one(jkey, jdoc_m, False):
            raise Exception("update jd_measure pubTime Exception")
        if not self._page_store.save_one(json.dumps(page_store_key), json.dumps({"$set": {"isUpdated": 0}}), False):
            raise Exception("set page store isUpdated status Exception")

    def check_is_student_job(self, jd_raw):
        if jd_raw.jobWorkAge in [u"在读学生", u"应届毕业生", u"无经验", u"无要求"]\
                or jd_raw.jobCate in [u"应届毕业生", u"储备干部", u"培训生", u"兼职", u"临时", u"实习生"]\
                or jd_raw.jobType in [u"兼职", u"实习"]:
            print "student job ===> ", jd_raw.jdId

            expired = 0
            if self.process_item[jd_raw.jdId]['flag'] == 2:
                expired = 1

            self.rs_file.append_end_with('\t'.join((jd_raw.jdId, str(expired))))
            return True
        return False

    def parse_measure_jd(self, item):
        try:
            jd_store_key = json.dumps({'jdId': item})
            page_store_key = json.dumps({'indexUrl': item, 'contentSign': self.process_item[item].get('contentSign')})
            fileName = self.process_item[item].get("filePath")
            pageContent = self.getPageContent(fileName)

            jd_raw = self.parse_by_owner(item, pageContent, self.owner)
            jd_measure = self.measure(jd_raw)

            if self.test_mode:
                print "Raw ===> {}".format(jd_raw.to_json())
                print "Measure ===> {}".format(jd_measure.to_json())
                return

            if not self._raw_store.save_one(jd_store_key, jd_raw.to_json(), True):
                raise Exception("set raw exception")

            if not self._measure_store.save_one(jd_store_key, jd_measure.to_json(), True):
                raise Exception("set measure exception")

            if not self._page_store.save_one(page_store_key, json.dumps({"$set": {"status": 1}}), False):
                raise Exception("set page store status exception")
            self.check_is_student_job(jd_raw)
        except Exception as e:
            self.failfile.append_end_with(item)
            raise e

    def parse_by_owner(self, jdId, pageContent, owner):
        if "jd_lagou" == owner:
            jdRaw = JdLagouHtmlFind(pageContent).find_fields()
        elif "jd_51job" == owner:
            jdRaw = Jd51JobHtmlFind(pageContent).find_fields()
        elif "jd_zhilian" == owner:
            jdRaw = JdZhilianHtmlFind(pageContent).find_fields()
        else:
            raise Exception("unknown owner")

        jdRaw.jdId = jdId
        jdRaw.jdFrom = self.owner
        jdRaw.jdUrl = self.process_item[jdId].get('realUrl')

        if not jdRaw.pubDate:
            stamp = self.process_item[jdId].get("updateTime")
            str_uptime = time.strftime("%Y-%m-%d %H:%m:%S", time.localtime(stamp/1000))
            jdRaw.pubDate = str_uptime

        return jdRaw

    def measure(self, jd_raw):
        raw_for_measure = construct_jd_raw(jd_raw)
        jd_measure = self.jd_measure_client.measureJd(raw_for_measure)
        jdMeasureObj = convertToJdMeasure(self.owner, jd_measure)

        self.set_md5_SimHash(jdMeasureObj, jd_raw)

        return jdMeasureObj

    def set_md5_SimHash(self, jd_measure, jd_raw):
        jd_measure.jdMd5 = self.get_jd_md5(jd_measure, jd_raw)
        jd_measure.jdSimHash = gen_sim_hash(jd_raw.jobDescription)

    def get_jd_md5(self, jd_measure, jd_raw):

        temp = dict({})
        temp["jdPosition"] = jd_raw.jobPosition
        temp["incName"] = jd_measure.incSegmentId
        temp["jdWorkLoc"] = jd_measure.jobWorkLocId

        return get_jd_measure_hash(temp)

    def event_handler(self, evt, msg, **kwargs):
        if "START" == evt:
            util.send_email(["<*****@*****.**>"], "{}_etl 任务".format(self.owner), msg)
            return

        if "DONE" == evt:
            util.send_email(["<*****@*****.**>"], "{}_etl 任务".format(self.owner), msg)
            return
예제 #5
0
class JdExpireDetect(BaseTask):
    def __init__(self, channel):
        self.channel = channel
        BaseTask.__init__(self, "{}_expire_detect".format(channel))
        self.page_store = PageStore(channel)
        self.process_items = []
        self.page_expire_detect = PageExpireDetect()

        self.test_mode = False

    def pre_check(self, item):
        now = int(time.time() * 1000)
        # 只检测3天之内,没标记为过期的
        if "updateTime" in item and item["updateTime"] < now - 3 * 24 * 3600:
            if "isExpired" not in item or item['isExpired'] == 0:
                self.process_items.append({"indexUrl": item["indexUrl"], "realUrl": item["realUrl"]})

    def _load_data(self):
        for item in self.page_store.get_all():
            self.pre_check(item)
            if len(self.process_items) and len(self.process_items) % 10000 == 0:
                print "load {} items".format(len(self.process_items))

        print "totally load {} items".format(len(self.process_items))

    def dispatcher(self, q):
        for item in self.process_items:
            q.put(item)

        q.put(None)
        self.wait_q()

    def check_expire_by_channel(self, job, channel):
        if "jd_lagou" == channel:
            return self.page_expire_detect.lagou_page_detect(job["realUrl"])
        if "jd_51job" == channel:
            return self.page_expire_detect.jd51job_page_detect(job["realUrl"])
        if "jd_zhilian" == channel:
            return self.page_expire_detect.zhilian_page_detect(job["realUrl"])
        if "jd_wealink" == channel:
            return self.page_expire_detect.wealink_page_detect(job["realUrl"])
        if 'jd_liepin' == channel:
            return self.page_expire_detect.liepin_page_detect(job['realUrl'])

    def run_job(self, job):
        if not isinstance(job, dict):
            return
        try:
            if not self.check_expire_by_channel(job, self.channel):
                if not self.test_mode:
                    self.page_store.set_expire({"indexUrl": job["indexUrl"]})
                print "set expired , indexUrl: {}".format(job["indexUrl"])
        except Exception as e:
            print e
            print "failed , indexUrl: {}".format(job["indexUrl"])

    def event_handler(self, evt, msg, **kwargs):
        if "START" == evt:
            util.send_email(["<*****@*****.**>"], "{} 过期检测".format(self.channel), msg)
            return

        if "DONE" == evt:
            util.send_email(["<*****@*****.**>"], "{} 过期检测".format(self.channel), msg)
            return