def main():
logging.basicConfig(level=logging.INFO)
r = StrictRedis()
while True:
logging.info("Waiting for a screenshot...")
# block if no screenshots
r.brpop('pending_screenshot_blocked_list')[1]
# get a pending screenshot
url = r.spop('pending_screenshots')
if url:
pipeline = r.pipeline()
pipeline.get('pending_screenshot_{}'.format(url))
pipeline.delete('pending_screenshot_{}'.format(url))
filename, _ = pipeline.execute()
logging.info("Taking screenshot of {} to {}".format(url, filename))
screenshot_path = os.path.join('/Users/dhumbert/Code/Projects/Misc/clementia/public/user/screenshots', filename)
subprocess.Popen(['/usr/local/bin/phantomjs', 'screenshot.js', url, screenshot_path], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()
class RedisQueue(object):
_cli = None
_connection_parameters = dict()
_queue = None
logger = getLogger()
def __init__(self, **kwargs):
self._queue = kwargs.pop('queue', None)
self._connection_parameters.update(kwargs)
def connect(self, **kwargs):
if self._cli and not kwargs:
return self
self._queue = kwargs.pop('queue') if kwargs.get('queue') else self._queue
self._connection_parameters.update(kwargs)
if not self._connection_parameters:
raise QueueConnectionError
self._cli = StrictRedis(**self._connection_parameters)
return self
def put(self, message, **kwargs):
self.connect() if not self._cli else None
queue = kwargs.pop('queue') if kwargs.get('queue') else self._queue
publisher = kwargs.pop('publisher') if kwargs.get('publisher') else 'default'
pipe = self._cli.pipeline()
return pipe.lpush(queue, dumps(message)).publish(publisher, queue).execute()
def get(self, **kwargs):
self.connect() if not self._cli else None
queue = kwargs.pop('queue') if kwargs.get('queue') else self._queue
timeout = kwargs.pop('timeout') if isinstance(kwargs.get('timeout'), int) else 1
try:
return loads(self._cli.brpop(queue, timeout)[1])
except TypeError:
return None
except ValueError:
return None
def length(self, queue=None):
self.connect() if not self._cli else None
queue = queue if queue else self._queue
return self._cli.llen('{0}'.format(queue))
@property
def queue(self):
return self._queue
@queue.setter
def queue(self, value):
self._queue = value
@queue.deleter
def queue(self):
self._queue = None
class ServiceBase(object):
def __init__(self,
service_name,
output_dir,
redis_host='127.0.0.1',
redis_port=6379,
redis_db=0,
redis_pwd='',
logger=None):
assert service_name in ('classify_extract', 'dl_extract',
'crf_extract', 'auto_rule_extract',
'sents_search', 'fields_analysis',
'fuzzy_extract', 'table_extract',
'diff_extract')
self.redis = StrictRedis(redis_host, redis_port, redis_db, redis_pwd)
self.service_name = service_name
self.queue_b = QUEUE_B_FORMATTER.format(
self.service_name) #'queue:b:{table_extract}'
self.logger = logger
self.output_dir = output_dir
def start(self):
if self.logger:
self.logger.info('service {} starts'.format(self.service_name))
while True:
print('我要执行message_bmessage_b')
message_b = self._read_message() #根据服务名获取redis里的信息(一直在等在不然不往下执行)
print('message_b', message_b)
message_c = self._process(message_b)
self._write_message(message_c)
def _process(self, message_b):
config = message_b.config
service_name = self.service_name
status = 'OK'
msg = 'OK'
# create version dir
version_path = '{}/{}'.format(self.output_dir, config.version)
if not os.path.isdir(version_path):
os.system('mkdir {}'.format(version_path))
# delete old status file if exists
error_file = '{}/{}/{}.{}'.format(self.output_dir, config.version,
config.field_id, 'ERROR')
ok_file = '{}/{}/{}.{}'.format(self.output_dir, config.version,
config.field_id, 'OK')
if os.path.isfile(error_file):
os.system('rm {}'.format(error_file))
if os.path.isfile(ok_file):
os.system('rm {}'.format(ok_file))
# run
try:
self._run(config)
except Exception as e:
if self.logger:
self.logger.exception(e)
status = 'ERROR'
msg = str(e)
# write status to file
# codecs.open('{}/{}/{}.{}'.format(self.output_dir, config.version, config.field_id, status), 'w').write(msg)
message_c = MessageC(config.version, service_name, config.field_id,
status, msg)
return message_c
def _run(self, config):
pass # TODO
def _read_message(self):
# read message B from queue B
return MessageB.load(self.redis.brpop(
self.queue_b)[1]) #'queue:b:table_extract'
def _write_message(self, message_c):
# write message C to queue C
if self.service_name == 'fields_analysis':
self.redis.lpush(QUEUE_FIELDS_ANALYSIS, str(message_c))
else:
self.redis.lpush(QUEUE_C, str(message_c))
args = parser.parse_args()
argsdict = vars(args)
instance = argsdict['instance']
FILESIZE=1024*1024*1024 #1MB
handler=('LI ARCHIVER-['+instance+']')
formatter = ('\n'+handler+':%(asctime)s-[%(filename)s:%(lineno)s]-%(levelname)s - %(message)s')
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format=formatter)
logger = logging.getLogger('liarchiver'+instance)
rclient = StrictRedis()
liarchivebackup = 'liarchivebackup_'+instance
db = dbops.MongoORM()
while True:
if (rclient.llen(liarchivebackup)):
item = rclient.brpop (liarchivebackup)
logger.info("Getting Mails from {}".format(liarchivebackup))
itemlist = pickle.loads(item[1])
if (len(itemlist) == 2):
db.lidump(itemlist, Binary(str(itemlist[1]).encode(), 128))
else:
item = rclient.brpoplpush('liarchive', liarchivebackup)
itemlist = pickle.loads(item)
if (len(itemlist) == 2):
db.lidump(itemlist, Binary(str(itemlist[1]).encode(), 128))
logger.info ('len of {} is : {}'.format(liarchivebackup, rclient.llen(liarchivebackup)))
rclient.lrem(liarchivebackup, 0, item)
logger.info ('len of {} is : {}'.format(liarchivebackup, rclient.llen(liarchivebackup)))
pickledEv = pickle.dumps(ev)
supportMailHandler(ev, pickledEv)
exit()
supportChannelBackUp = 'supportChannelBackUp_' + instance
logger.info(
"supportChannelBackUp ListName : {} ".format(supportChannelBackUp))
while True:
#Read config changes only while processing the mesage
readdress_configs = ReConfig()
valids.re_readconfig()
backupmail = False
if (rclient.llen(supportChannelBackUp)):
evt = rclient.brpop(supportChannelBackUp)
backupmail = True
ev = pickle.loads(evt[1])
#pickledEv = pickle.dumps(ev)
pickledEv = evt[1]
logger.info("Getting events from {}".format(supportChannelBackUp))
else:
pickledEv = rclient.brpoplpush('supportChannel',
supportChannelBackUp)
ev = pickle.loads(pickledEv)
logger.info("Getting events from {}".format('supportChannel'))
#mail handler
origmsg, msg = supportMailHandler(ev, pickledEv)
del origmsg
del msg
logging.basicConfig(stream=sys.stdout,
level=logging.DEBUG,
format=formatter)
logger = logging.getLogger('mailHandler' + instance)
rclient = StrictRedis()
if 'debug' in argsdict and argsdict['debug'] is not None:
logger.info("Loegger testing\n")
exit()
sendmailbackup = 'sendmailbackup_' + instance
while True:
backmail = False
if (rclient.llen(sendmailbackup)):
tupitem = rclient.brpop(sendmailbackup)
logger.info("Getting Mails from {}".format(sendmailbackup))
backmail = True
item = (tupitem[1]).decode()
else:
item = rclient.brpoplpush('sendmail', sendmailbackup)
logger.info("Getting Mails from {}".format('sendmail'))
item = item.decode()
#Get the smtp msg from redis
logger.info("Item : {}".format(item))
msgtuplepickle = rclient.get(item)
if msgtuplepickle:
msgtuple = pickle.loads(msgtuplepickle)
#Get the inbound json obj from redis
logger.info('item is {} '.format(item))
f.close()
pickledEv = pickle.dumps(ev)
emailModifyHandler(ev, pickledEv)
exit()
mailModifyhandlerBackUp = 'mailModifyhandler_' + instance
logger.info("mailModifyhandlerBackUp ListName : {} ".format(mailModifyhandlerBackUp))
while True:
del readdress_configs
readdress_configs = ReConfig()
valids.re_readconfig()
backupmail = False
if (rclient.llen(mailModifyhandlerBackUp)):
evt = rclient.brpop (mailModifyhandlerBackUp)
backupmail = True
ev = pickle.loads(evt[1])
pickledEv = pickle.dumps(ev)
logger.info("Getting events from {}".format(mailModifyhandlerBackUp))
else:
pickledEv = rclient.brpoplpush('mailModifyhandler', mailModifyhandlerBackUp)
ev = pickle.loads(pickledEv)
logger.info("Getting events from {}".format('mailModifyhandler'))
#mail handler
emailModifyHandler(ev, pickledEv)
if(not backupmail):
logger.info('len of {} is : {}'.format(
mailModifyhandlerBackUp, rclient.llen(mailModifyhandlerBackUp)))
