def worker_proc_main(self, from_main_queue, to_main_queue):
try:
signal.signal(signal.SIGTERM, signal.SIG_IGN)
signal.signal(signal.SIGINT, signal.SIG_IGN)
set_proc_name(self.process.name)
try:
log.init_log(self.conf,
level=self.conf['logger']['level'],
console_out=self.conf['logger']['console'],
name=self.process.name)
except Exception as e:
packet = from_main_queue.get(block=True)
to_main_queue.put(ResponsePacket(packet, succ=False),
block=True)
raise e
# with daemon.DaemonContext:
log.debug(str(self.process.name) + " : started")
self.running = True
while self.running:
pack = from_main_queue.get(block=True)
self.controller.process_packet(pack, from_main_queue,
to_main_queue)
log.debug(str(self.process.name) + " : gracefully stopped")
finally:
log.debug(str(self.process.name) + " : exited")
def load_conf(self, filename):
with open(filename, 'rb') as config:
conf = pytoml.load(config)
log.init_log(conf, console_out=conf['logger']['console'])
conf['log'] = log
conf['log_etl'] = logging
return conf
def get_conf(file_path='./crawler_merge.toml'):
try:
opt, args = getopt.getopt(sys.argv[1:], 'f:', ['help'])
for name, value in opt:
if name == "-f":
file_path = value
else:
assert False, "unhandled option"
with open(file_path, 'rb') as config:
conf = pytoml.load(config)
log.init_log(conf=conf,
level=conf['logger']['level'],
console_out=conf['logger']['console'],
name=conf['server']['name'] + "-" + "main")
conf['log'] = log
return conf
except:
print traceback.format_exc()
sys.exit(1)
return value
return u''
if __name__ == '__main__':
import pytoml
import sys
import time
from common.log import log
sys.path.append('../../')
with open('../../entity.toml', 'rb') as config:
conf = pytoml.load(config)
log.init_log(conf, console_out=conf['logger']['console'])
conf['log'] = log
topic_id = 33
from i_entity_extractor.entity_extractor_route import EntityExtractorRoute
from i_entity_extractor.common_parser_lib.mongo import MongDb
import json
route = EntityExtractorRoute(conf)
topic_info = route.all_topics.get(topic_id, None)
begin_time = time.time()
obj = PppProjectExtractor(topic_info, log)
extract_data = {
"demonstration_project_batch": "",
"money_unit": "元",
"_record_id": "c81fd13bd09d7248857d2a5f6d8b968d",
if __name__ == '__main__':
try:
file_path = './scheduler.toml'
opt, args = getopt.getopt(sys.argv[1:], 'f:', ['help'])
for name, value in opt:
if name == "-f":
file_path = value
elif name in ("-h", "--help"):
usage()
sys.exit()
else:
assert False, "unhandled option"
with open(file_path, 'rb') as config:
conf = pytoml.load(config)
log.init_log(conf, level=conf['logger']['level'], console_out=conf['logger']['console'],
name=conf['server']['name'] + str(conf['server']['port']))
conf['log'] = log
main(conf)
exit(0)
except Exception, e:
log.error("调度器启动失败...")
log.exception(e)
print traceback.format_exc()
sys.exit(1)
if __name__ == '__main__':
from conf import get_config
try:
file_path = './entity.toml'
topic_ids = []
from_db = False
opt, args = getopt.getopt(sys.argv[1:], 'f:t:d', ['help'])
for name, value in opt:
if name == "-f":
file_path = value
elif name == "-t":
topic_ids = [int(i) for i in value.split(',')]
elif name == '-d':
from_db = True
elif name in ("-h", "--help"):
usaget()
sys.exit()
else:
assert False, "unhandled option"
with open(file_path, 'rb') as config:
conf = pytoml.load(config)
conf['topic_ids'] = topic_ids
conf['from_db'] = from_db
log.init_log(conf,
console_out=conf['logger']['console'],
name="ee_main_proc")
conf['log'] = log
main(conf)
except:
print traceback.format_exc()
remove_pid_file(sys.path[0])
os.kill(os.getpid(), signal.SIGKILL)
def usaget():
pass
if __name__ == '__main__':
try:
file_path = './downloader.toml'
opt, args = getopt.getopt(sys.argv[1:], 'f:', ['help'])
for name, value in opt:
if name == "-f":
file_path = value
elif name in ("-h", "--help"):
usaget()
sys.exit()
else:
assert False, "unhandled option"
with open(file_path, 'rb') as config:
config = pytoml.load(config)
log.init_log(conf=config,
level=config['logger']['level'],
console_out=config['logger']['console'],
name=config['server']['name'] + "-" + "main")
config['log'] = log
main(config)
except getopt.GetoptError:
sys.exit()
