def __init__(self, channel, thread_cnt=5):
BaseTask.__init__(self, "fixbug", thread_cnt=thread_cnt)
self.chanel = channel
self.bugfix = BugFixDispatcher(channel, self._queue)
self.dispatcher = lambda q: self.bugfix.dispatcher()
self.etl_runner = ETLRunner(channel)
def __init__(self, channel):
self.channel = channel
BaseTask.__init__(self, "{}_expire_detect".format(channel))
self.page_store = PageStore(channel)
self.process_items = []
self.page_expire_detect = PageExpireDetect()
self.test_mode = False
def cancel(self):
"""If the task is cancelled, delete the file."""
BaseTask.cancel(self)
if self.file:
self.file = None
if self.filename and os.path.exists(self.filename):
os.remove(self.filename)
def __init__(self, channel, thread_cnt, need_cnt):
BaseTask.__init__(self, "fetch_csv_sample", thread_cnt=thread_cnt)
self.channel = channel
self.rand = RandomDispatcher(channel, self._queue, need_cnt)
self.dispatcher = lambda q: self.rand.dispatcher()
self.dir_path = self.get_save_dir_path()
CsvWriteBase.__init__(self, self.dir_path, self.channel)
self.cv_measure_store = CVRawStore(channel, 'measure')
def __init__(self, channel, field, field_pattern):
BaseTask.__init__(self, "fix_cv_field")
self.channel = channel
self.field = field
self.field_pattern = field_pattern
self.bugfix = BugFixDispatcher(channel, self._queue)
self.dispatcher = lambda q: self.bugfix.dispatcher()
self.etl_runner = ETLRunnerFromRaw(channel)
def __init__(self, channel, thread_cnt=5):
BaseTask.__init__(self, "gets_raw", thread_cnt=thread_cnt)
self.rawdispatcher = BugFixDispatcher(channel, self._queue)
self.dispatcher = lambda q: self.rawdispatcher.dispatcher()
self.channel = channel
self.dir_path = self.get_save_file_dir()
CsvWriteBase.__init__(self, self.dir_path, self.channel)
def __init__(self, thread_cnt):
BaseTask.__init__(self, 'fix_loc',thread_cnt=thread_cnt)
self.measure_mongo_client = pymongo.MongoClient('mongodb://localhost:28017/cv_51job_measure')
self.raw_mongo_client = pymongo.MongoClient('mongodb://localhost:28017/cv_51job_raw')
self.raw_db = 'cv_51job_raw'
self.raw_coll = 'cv_raw'
self.measure_db = 'cv_51job_measure'
self.measure_coll = 'cv_measure'
self.update_cnt = 0
self.r_lock = threading.RLock()
self.locIdMap={}
def __init__(self, owner, queue_size, thread_cnt):
BaseTask.__init__(self, owner, queue_size, thread_cnt)
self.owner = owner
self.process_item = {}
self.expired_items = []
self._raw_store = self._get_raw_store(owner)
self._measure_store = self._get_measure_store(owner)
self._page_store = PageStore(owner)
self.page_store_db = 'admin'
self.page_store_coll = "page_store_{}".format(owner)
self.thrift_client = TClient('../../conf/thrift_conf.ini')
self.jd_measure_client = self.thrift_client.jd_measure_server_client
self.test_mode = False
self.rs_file = FileSave(BaseTask.PathConfig.result_file)
self.failfile = FileSave(BaseTask.PathConfig.etl_failids_file)
def __init__(self, thread_cnt):
BaseTask.__init__(self, '2c_import', thread_cnt=thread_cnt)
self.process_items = []
self.fail_fn = BaseTask.PathConfig.toc_failids_file
self.rs_fn = BaseTask.PathConfig.result_file
self.lgstore = LgEtlStore()
self.job51store = Job51EtlStore()
self.zlstore = ZLEtlStore()
self.tocstore = ToCMeasureStore()
self.thrift_client = TClient('../../conf/thrift_conf.ini')
self.inc_stats2_client = pymongo.MongoClient(self.lgstore.cmgClient.inc_stats2_mongo_url)
self.zhineng_salary5_charts_client = pymongo.MongoClient(self.lgstore.cmgClient.zhineng_salary5_charts_mongo_url)
self.edu_info_client = self.thrift_client.edu_info_client
self.inc_info_clients = self.thrift_client.inc_idinfo_client
self.toc_failids_fd = FileSave(BaseTask.PathConfig.toc_failids_file)
def __init__(self, background=False):
"""Initialize the task.
Parameters:
background (default False): should the task run in bthe background?
"""
BaseTask.__init__(self)
self.worlds = []
self.title = t("task.import_worlds.title")
self.message = t("task.import_worlds.message")
self.confirmation = t("task.import_worlds.confirmation")
if background:
self.dialog = None
else:
self.dialog = TaskDialog(self, self.title.format(progress=0))
self.dialog.message = self.message
self.dialog.confirmation = self.confirmation
def __init__(self, world, filename, configuration, to_copy=None):
"""Initialize the task.
Parameters:
world (World): the world to be exported.
filename (str): the file name to be written as a ZIP archive.
configuration (str): the content of the 'config.set' file to be created.
to_copy (list): optional list of file names to copy in the archive.
"""
BaseTask.__init__(self)
self.world = world
self.filename = filename
self.configuration = configuration
self.to_copy = to_copy or []
self.title = t("task.export_world.title")
self.message = t("task.export_world.message")
self.confirmation = t("task.export_world.confirmation")
self.dialog = TaskDialog(self, self.title.format(progress=0))
self.dialog.message = self.message
self.dialog.confirmation = self.confirmation
def __init__(self, channel, prefix=None):
BaseTask.__init__(self, channel, thread_cnt=2)
self.channel = channel
self.cv_parser = CvParser()
self._cvs_data = {}
#对比 临时存放文件
self.diff_rs_dir = '%s_rs_diff' % self.channel if not prefix else '%s/%s_job_diff' % (prefix, self.channel)
#机器解析
self.mechine_rs_dir = '%s_mechine_parsed' % self.channel if not prefix else '%s/%s_mechine_parsed' % (prefix, self.channel)
#人工解析
self.person_rs_dir = '%s_person_parsed' % self.channel if not prefix else '%s/%s_person_parsed' % (prefix, self.channel)
#结果
self.result_file = '%s_result.txt' % self.channel if not prefix else '%s/%s_result.txt' % (prefix, self.channel)
# util.check_and_clear(os.path.dirname(self.result_file))
self.excel_save = ExcelFileSave(self.result_file)
#样本
self.sample_dir = '%s_sample' % self.channel if not prefix else '%s/%s_sample' % (prefix, self.channel)
def __init__(self, filename, url, background=False):
"""Initialize the task.
Parameters:
filename: the name of file to be created (or None).
url: the url of the file to be downloaded.
background (default False): should the task run in the background?
If the filename is None, then download in memory. The file
attribute will contain a StringIO pointing to this object.
"""
BaseTask.__init__(self)
self.filename = filename
self.file = StringIO() if filename is None else None
self.url = url
if background:
self.dialog = None
else:
self.dialog = TaskDialog(self, t("task.download.title",
url=url, progress=0))
self.dialog.confirmation = t("task.download.confirmation")
def __init__(self, background=False, title="Loading",
message="Downloading the world list..",
confirmation="Do you want to cancel loading?"):
"""Initialize the task.
Parameters:
background (default False): should the task run in bthe background?
title: title of the dialog to display.
message: message of the dialog to display.
confirmation: message when pressing on cancel in the dialog.
"""
BaseTask.__init__(self)
self.worlds = []
self.title = title
self.message = message
self.confirmation = confirmation
if background:
self.dialog = None
else:
self.dialog = TaskDialog(self, title.format(progress=0))
self.dialog.message = message
self.dialog.confirmation = confirmation
def __init__(self):
BaseTask.__init__(self, "fix_bug")
self.process_item = []
self.client = pymongo.MongoClient(mongo_url)
self.test_mode = False
def __init__(self):
BaseTask.__init__(self, "fields_fix")