def worker_proc_main(self, from_main_queue, to_main_queue): try: signal.signal(signal.SIGTERM, signal.SIG_IGN) signal.signal(signal.SIGINT, signal.SIG_IGN) set_proc_name(self.process.name) try: log.init_log(self.conf, level=self.conf['logger']['level'], console_out=self.conf['logger']['console'], name=self.process.name) except Exception as e: packet = from_main_queue.get(block=True) to_main_queue.put(ResponsePacket(packet, succ=False), block=True) raise e # with daemon.DaemonContext: log.debug(str(self.process.name) + " : started") self.running = True while self.running: pack = from_main_queue.get(block=True) self.controller.process_packet(pack, from_main_queue, to_main_queue) log.debug(str(self.process.name) + " : gracefully stopped") finally: log.debug(str(self.process.name) + " : exited")
def load_conf(self, filename): with open(filename, 'rb') as config: conf = pytoml.load(config) log.init_log(conf, console_out=conf['logger']['console']) conf['log'] = log conf['log_etl'] = logging return conf
def get_conf(file_path='./crawler_merge.toml'): try: opt, args = getopt.getopt(sys.argv[1:], 'f:', ['help']) for name, value in opt: if name == "-f": file_path = value else: assert False, "unhandled option" with open(file_path, 'rb') as config: conf = pytoml.load(config) log.init_log(conf=conf, level=conf['logger']['level'], console_out=conf['logger']['console'], name=conf['server']['name'] + "-" + "main") conf['log'] = log return conf except: print traceback.format_exc() sys.exit(1)
return value return u'' if __name__ == '__main__': import pytoml import sys import time from common.log import log sys.path.append('../../') with open('../../entity.toml', 'rb') as config: conf = pytoml.load(config) log.init_log(conf, console_out=conf['logger']['console']) conf['log'] = log topic_id = 33 from i_entity_extractor.entity_extractor_route import EntityExtractorRoute from i_entity_extractor.common_parser_lib.mongo import MongDb import json route = EntityExtractorRoute(conf) topic_info = route.all_topics.get(topic_id, None) begin_time = time.time() obj = PppProjectExtractor(topic_info, log) extract_data = { "demonstration_project_batch": "", "money_unit": "元", "_record_id": "c81fd13bd09d7248857d2a5f6d8b968d",
if __name__ == '__main__': try: file_path = './scheduler.toml' opt, args = getopt.getopt(sys.argv[1:], 'f:', ['help']) for name, value in opt: if name == "-f": file_path = value elif name in ("-h", "--help"): usage() sys.exit() else: assert False, "unhandled option" with open(file_path, 'rb') as config: conf = pytoml.load(config) log.init_log(conf, level=conf['logger']['level'], console_out=conf['logger']['console'], name=conf['server']['name'] + str(conf['server']['port'])) conf['log'] = log main(conf) exit(0) except Exception, e: log.error("调度器启动失败...") log.exception(e) print traceback.format_exc() sys.exit(1)
if __name__ == '__main__': from conf import get_config try: file_path = './entity.toml' topic_ids = [] from_db = False opt, args = getopt.getopt(sys.argv[1:], 'f:t:d', ['help']) for name, value in opt: if name == "-f": file_path = value elif name == "-t": topic_ids = [int(i) for i in value.split(',')] elif name == '-d': from_db = True elif name in ("-h", "--help"): usaget() sys.exit() else: assert False, "unhandled option" with open(file_path, 'rb') as config: conf = pytoml.load(config) conf['topic_ids'] = topic_ids conf['from_db'] = from_db log.init_log(conf, console_out=conf['logger']['console'], name="ee_main_proc") conf['log'] = log main(conf) except: print traceback.format_exc()
remove_pid_file(sys.path[0]) os.kill(os.getpid(), signal.SIGKILL) def usaget(): pass if __name__ == '__main__': try: file_path = './downloader.toml' opt, args = getopt.getopt(sys.argv[1:], 'f:', ['help']) for name, value in opt: if name == "-f": file_path = value elif name in ("-h", "--help"): usaget() sys.exit() else: assert False, "unhandled option" with open(file_path, 'rb') as config: config = pytoml.load(config) log.init_log(conf=config, level=config['logger']['level'], console_out=config['logger']['console'], name=config['server']['name'] + "-" + "main") config['log'] = log main(config) except getopt.GetoptError: sys.exit()