def main(): logging.basicConfig(level=logging.INFO) r = StrictRedis() while True: logging.info("Waiting for a screenshot...") # block if no screenshots r.brpop('pending_screenshot_blocked_list')[1] # get a pending screenshot url = r.spop('pending_screenshots') if url: pipeline = r.pipeline() pipeline.get('pending_screenshot_{}'.format(url)) pipeline.delete('pending_screenshot_{}'.format(url)) filename, _ = pipeline.execute() logging.info("Taking screenshot of {} to {}".format(url, filename)) screenshot_path = os.path.join('/Users/dhumbert/Code/Projects/Misc/clementia/public/user/screenshots', filename) subprocess.Popen(['/usr/local/bin/phantomjs', 'screenshot.js', url, screenshot_path], stderr=subprocess.STDOUT, stdout=subprocess.PIPE).communicate()
class RedisQueue(object): _cli = None _connection_parameters = dict() _queue = None logger = getLogger() def __init__(self, **kwargs): self._queue = kwargs.pop('queue', None) self._connection_parameters.update(kwargs) def connect(self, **kwargs): if self._cli and not kwargs: return self self._queue = kwargs.pop('queue') if kwargs.get('queue') else self._queue self._connection_parameters.update(kwargs) if not self._connection_parameters: raise QueueConnectionError self._cli = StrictRedis(**self._connection_parameters) return self def put(self, message, **kwargs): self.connect() if not self._cli else None queue = kwargs.pop('queue') if kwargs.get('queue') else self._queue publisher = kwargs.pop('publisher') if kwargs.get('publisher') else 'default' pipe = self._cli.pipeline() return pipe.lpush(queue, dumps(message)).publish(publisher, queue).execute() def get(self, **kwargs): self.connect() if not self._cli else None queue = kwargs.pop('queue') if kwargs.get('queue') else self._queue timeout = kwargs.pop('timeout') if isinstance(kwargs.get('timeout'), int) else 1 try: return loads(self._cli.brpop(queue, timeout)[1]) except TypeError: return None except ValueError: return None def length(self, queue=None): self.connect() if not self._cli else None queue = queue if queue else self._queue return self._cli.llen('{0}'.format(queue)) @property def queue(self): return self._queue @queue.setter def queue(self, value): self._queue = value @queue.deleter def queue(self): self._queue = None
class ServiceBase(object): def __init__(self, service_name, output_dir, redis_host='127.0.0.1', redis_port=6379, redis_db=0, redis_pwd='', logger=None): assert service_name in ('classify_extract', 'dl_extract', 'crf_extract', 'auto_rule_extract', 'sents_search', 'fields_analysis', 'fuzzy_extract', 'table_extract', 'diff_extract') self.redis = StrictRedis(redis_host, redis_port, redis_db, redis_pwd) self.service_name = service_name self.queue_b = QUEUE_B_FORMATTER.format( self.service_name) #'queue:b:{table_extract}' self.logger = logger self.output_dir = output_dir def start(self): if self.logger: self.logger.info('service {} starts'.format(self.service_name)) while True: print('我要执行message_bmessage_b') message_b = self._read_message() #根据服务名获取redis里的信息(一直在等在不然不往下执行) print('message_b', message_b) message_c = self._process(message_b) self._write_message(message_c) def _process(self, message_b): config = message_b.config service_name = self.service_name status = 'OK' msg = 'OK' # create version dir version_path = '{}/{}'.format(self.output_dir, config.version) if not os.path.isdir(version_path): os.system('mkdir {}'.format(version_path)) # delete old status file if exists error_file = '{}/{}/{}.{}'.format(self.output_dir, config.version, config.field_id, 'ERROR') ok_file = '{}/{}/{}.{}'.format(self.output_dir, config.version, config.field_id, 'OK') if os.path.isfile(error_file): os.system('rm {}'.format(error_file)) if os.path.isfile(ok_file): os.system('rm {}'.format(ok_file)) # run try: self._run(config) except Exception as e: if self.logger: self.logger.exception(e) status = 'ERROR' msg = str(e) # write status to file # codecs.open('{}/{}/{}.{}'.format(self.output_dir, config.version, config.field_id, status), 'w').write(msg) message_c = MessageC(config.version, service_name, config.field_id, status, msg) return message_c def _run(self, config): pass # TODO def _read_message(self): # read message B from queue B return MessageB.load(self.redis.brpop( self.queue_b)[1]) #'queue:b:table_extract' def _write_message(self, message_c): # write message C to queue C if self.service_name == 'fields_analysis': self.redis.lpush(QUEUE_FIELDS_ANALYSIS, str(message_c)) else: self.redis.lpush(QUEUE_C, str(message_c))
args = parser.parse_args() argsdict = vars(args) instance = argsdict['instance'] FILESIZE=1024*1024*1024 #1MB handler=('LI ARCHIVER-['+instance+']') formatter = ('\n'+handler+':%(asctime)s-[%(filename)s:%(lineno)s]-%(levelname)s - %(message)s') logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format=formatter) logger = logging.getLogger('liarchiver'+instance) rclient = StrictRedis() liarchivebackup = 'liarchivebackup_'+instance db = dbops.MongoORM() while True: if (rclient.llen(liarchivebackup)): item = rclient.brpop (liarchivebackup) logger.info("Getting Mails from {}".format(liarchivebackup)) itemlist = pickle.loads(item[1]) if (len(itemlist) == 2): db.lidump(itemlist, Binary(str(itemlist[1]).encode(), 128)) else: item = rclient.brpoplpush('liarchive', liarchivebackup) itemlist = pickle.loads(item) if (len(itemlist) == 2): db.lidump(itemlist, Binary(str(itemlist[1]).encode(), 128)) logger.info ('len of {} is : {}'.format(liarchivebackup, rclient.llen(liarchivebackup))) rclient.lrem(liarchivebackup, 0, item) logger.info ('len of {} is : {}'.format(liarchivebackup, rclient.llen(liarchivebackup)))
pickledEv = pickle.dumps(ev) supportMailHandler(ev, pickledEv) exit() supportChannelBackUp = 'supportChannelBackUp_' + instance logger.info( "supportChannelBackUp ListName : {} ".format(supportChannelBackUp)) while True: #Read config changes only while processing the mesage readdress_configs = ReConfig() valids.re_readconfig() backupmail = False if (rclient.llen(supportChannelBackUp)): evt = rclient.brpop(supportChannelBackUp) backupmail = True ev = pickle.loads(evt[1]) #pickledEv = pickle.dumps(ev) pickledEv = evt[1] logger.info("Getting events from {}".format(supportChannelBackUp)) else: pickledEv = rclient.brpoplpush('supportChannel', supportChannelBackUp) ev = pickle.loads(pickledEv) logger.info("Getting events from {}".format('supportChannel')) #mail handler origmsg, msg = supportMailHandler(ev, pickledEv) del origmsg del msg
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format=formatter) logger = logging.getLogger('mailHandler' + instance) rclient = StrictRedis() if 'debug' in argsdict and argsdict['debug'] is not None: logger.info("Loegger testing\n") exit() sendmailbackup = 'sendmailbackup_' + instance while True: backmail = False if (rclient.llen(sendmailbackup)): tupitem = rclient.brpop(sendmailbackup) logger.info("Getting Mails from {}".format(sendmailbackup)) backmail = True item = (tupitem[1]).decode() else: item = rclient.brpoplpush('sendmail', sendmailbackup) logger.info("Getting Mails from {}".format('sendmail')) item = item.decode() #Get the smtp msg from redis logger.info("Item : {}".format(item)) msgtuplepickle = rclient.get(item) if msgtuplepickle: msgtuple = pickle.loads(msgtuplepickle) #Get the inbound json obj from redis logger.info('item is {} '.format(item))
f.close() pickledEv = pickle.dumps(ev) emailModifyHandler(ev, pickledEv) exit() mailModifyhandlerBackUp = 'mailModifyhandler_' + instance logger.info("mailModifyhandlerBackUp ListName : {} ".format(mailModifyhandlerBackUp)) while True: del readdress_configs readdress_configs = ReConfig() valids.re_readconfig() backupmail = False if (rclient.llen(mailModifyhandlerBackUp)): evt = rclient.brpop (mailModifyhandlerBackUp) backupmail = True ev = pickle.loads(evt[1]) pickledEv = pickle.dumps(ev) logger.info("Getting events from {}".format(mailModifyhandlerBackUp)) else: pickledEv = rclient.brpoplpush('mailModifyhandler', mailModifyhandlerBackUp) ev = pickle.loads(pickledEv) logger.info("Getting events from {}".format('mailModifyhandler')) #mail handler emailModifyHandler(ev, pickledEv) if(not backupmail): logger.info('len of {} is : {}'.format( mailModifyhandlerBackUp, rclient.llen(mailModifyhandlerBackUp)))