class WorkerPool(object): def __init__(self, queue_name, routing_keys=None, backend='mongodb', conn_url='localhost:27017', dbname='skunkqueue', logfile=sys.stdout, pidfile=None): """ routing_keys are a required parameter to specify an n-length list of routing keys, which will each be assigned to one worker """ self.stop = False self.queue_name = queue_name self.persister = get_backend(backend)(conn_url=conn_url, dbname=dbname) self.log = Logger('pool-'+queue_name, logfile=logfile) self.host = socket.gethostbyname(socket.gethostname()) self.workers = {} if pidfile: self.log.info("writing to pidfile %s" % pidfile) with open(pidfile) as f: f.write(str(self.pid)) f.close() # TODO this needs to be in shared memory wnums = {} self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.s.bind((self.host, 0)) self.s.listen(5) self.port = self.s.getsockname()[1] for key in routing_keys: if key not in wnums: wnums[key] = 0 wnums[key] += 1 worker = Worker(queue_name, key, self.persister, wnums[key], self.port, logfile) thread = KillableThread(target=worker.begin_execution) thread.start() self.workers[worker.worker_id] = (worker, thread) def __enter__(self, *args, **kwargs): self.log.info("starting") def gentle(signum, frame): self.log.info("Received gentle shutdown signal %d" % signum) self.shutdown() sys.exit(0) def rough(signum, frame): self.log.warn("Received non-gentle kill signal %d" % signum) self.die() sys.exit(0) signal.signal(signal.SIGINT, rough ) signal.signal(signal.SIGHUP, gentle) signal.signal(signal.SIGTERM, gentle) signal.signal(signal.SIGALRM, gentle) signal.signal(signal.SIGQUIT, gentle) return self def __exit__(self, *args, **kwargs): self.shutdown() def shutdown(self): for _id in self.workers.keys(): worker = self.workers[_id] worker[0].stop_worker() del self.workers[_id] self.s.close() self.stop = True def die(self): for _id in self.workers.keys(): worker = self.workers[_id] try: worker[1].raise_exc(TriggeredInterrupt) self.log.warn("raised an exception in %s" % str(_id)) del self.workers[_id] except ValueError: # it was dead already self.log.debug("ignored killing thread %s" % str(_id)) continue self.s.close() self.stop = True def run_cmd(self, terminate=None, kill=None, suicide=None): # requests a termination # terminate looks like: <worker_id> if terminate: self.workers[terminate][0].stop_worker() del self.workers[terminate] # performs a hard kill # kill looks like: <worker_id> if kill: try: self.workers[kill][1].raise_exc(TriggeredInterrupt) del self.workers[kill] except ValueError: # it was already dead self.log.warn("tried to kill a thread when it was dead already") # shuts down the entire pool # suicide looks like: 1 if suicide == 1: self.stop = True def listen(self): while not self.stop: conn, addr = self.s.accept() datalen = conn.recv(2) data = "" while len(data) < int(datalen): data += conn.recv(1024) data = json.loads(data) self.run_cmd(**data) # no need to run the pool if it has no workers if len(self.workers) == 0: self.shutdown()
class Worker(object): def __init__(self, queue_name, route, persister, wnum, port, logfile=sys.stdout): self.queue_name = queue_name self.route = route self.persister = persister self.stop = False self.pid = os.getpid() self.worker_id = '-'.join(['worker', str(wnum), queue_name, route, str(self.pid)]) self.log = Logger(self.worker_id,logfile=logfile, loglevel=logging.DEBUG) self.log.info("starting") self.host = socket.gethostbyname(socket.gethostname()) self.port = port self.register() def begin_execution(self, *args): while not self.stop: try: job = self.persister.get_job_from_queue(self.queue_name, self.worker_id, self.route) if job: self.do_job(job) sleep(0.1) except TriggeredInterrupt: # we were cut off by an interrupt trigger self.log.warn("received interrupt request; stopping current job") self.log.warn("no result will be committed and this job will not be restarted") self.stop_worker() def register(self): self.persister.add_worker(self.worker_id, self.host, self.port) def unregister(self): self.persister.delete_worker(self.worker_id) def do_job(self, job): # depickle body = pickle.loads(job['body']) directory = body['dir'] # FIXME a horrible hack where we add ourselves to the pythonpath sys.path.append(directory) mod = __import__(body['mod']) self.log.debug("successfully imported module "+str(mod)) if job['fn_type'] == 'method': parent = dill.loads(body['parent']) fn = getattr(parent, body['fn']) else: fn = getattr(mod, body['fn']) args = body['args'] kwargs = body['kwargs'] # call it self.persister.set_working(self.worker_id) try: ret = fn(*args, **kwargs) self.persister.save_result(job['job_id'], ret, 'complete') self.log.info(ret) except Exception as e: self.persister.save_result(job['job_id'], None, 'error') self.log.error(str(e)) exc_t, exc_v, exc_tr = sys.exc_info() self.log.error(str( '\n'.join(traceback.format_exception(exc_t, exc_v, exc_tr)))) self.log.debug("python path is %s" % str(sys.path)) finally: a = sys.path.pop() self.persister.unset_working(self.worker_id) def stop_worker(self): self.log.info("shutting down") self.unregister() self.stop = True
def main(): Logger.debug("Hello, this line is for test|int:%u|double:%u|string:%s", 100, 3.14, "abc")
class WorkerPool(object): def __init__(self, queue_name, routing_keys=None, backend='mongodb', conn_url='localhost:27017', dbname='fwbots', logfile=sys.stdout, pidfile=None): """ routing_keys are a required parameter to specify an n-length list of routing keys, which will each be assigned to one worker FWBOTS: we are using routing_keys as the account names to load """ self.stop = False self.name = queue_name self.log = Logger('pool-' + queue_name, logfile=logfile) self.persister = get_backend(backend)(conn_url, dbname) self.port = -1 # too lazy to actually remove this self.workers = {} self.manual = {} self.auto = {} self.insta = {} if pidfile: self.log.info("writing to pidfile %s" % pidfile) with open(pidfile) as f: f.write(str(self.pid)) f.close() # TODO this needs to be in shared memory wnums = {} self.ircbot = IrcListener(self, "#fwbots", self.name, "irc.freenode.net") for key in routing_keys: errnum = 0 try: acc = get_twitter_account(key) if acc.ty == 'auto': self.log.info('found auto for %s' % key) self.auto[acc.name] = acc # if there are multiple manual accs defined, pick only the last one elif acc.ty == 'manual': self.log.info('found manual for %s' % key) self.manual[acc.name] = acc except NameError: errnum += 1 try: acc = get_instagram_account(key) self.insta[acc.name] = acc except NameError: errnum += 1 if errnum > 1: self.log.warn("Could not find any account called %s" % key) if key not in wnums: wnums[key] = 0 wnums[key] += 1 worker = Worker(queue_name, key, self.persister, wnums[key], self.port, logfile) thread = KillableThread(target=worker.start_worker) thread.start() self.workers[worker.worker_id] = (worker, thread) # lastly, register yourself self.persister.add_pool(self) def __enter__(self, *args, **kwargs): self.log.info("starting") def gentle(signum, frame): self.log.info("Received gentle shutdown signal %d" % signum) self.shutdown() sys.exit(0) def rough(signum, frame): self.log.warn("Received non-gentle kill signal %d" % signum) self.die() sys.exit(0) signal.signal(signal.SIGINT, rough) signal.signal(signal.SIGHUP, gentle) signal.signal(signal.SIGTERM, gentle) signal.signal(signal.SIGALRM, gentle) signal.signal(signal.SIGQUIT, gentle) return self def __exit__(self, *args, **kwargs): self.shutdown() def shutdown(self): for _id in self.workers.keys(): worker = self.workers[_id] worker[0].stop_worker() del self.workers[_id] self.stop = True self.persister.delete_pool(self.name) def die(self): for _id in self.workers.keys(): worker = self.workers[_id] try: worker[1].raise_exc(TriggeredInterrupt) self.log.warn("raised an exception in %s" % str(_id)) del self.workers[_id] except ValueError: # it was dead already self.log.debug("ignored killing thread %s" % str(_id)) continue self.stop = True def work(self, f): w = self.persister.get_avail_workers()[0] w.todo = f def run_cmd(self, terminate=None, kill=None, suicide=None, tweet=None, rtp=None): # requests a termination # terminate looks like: <worker_id> if terminate: self.workers[terminate][0].stop_worker() del self.workers[terminate] # performs a hard kill # kill looks like: <worker_id> if kill: try: self.workers[kill][1].raise_exc(TriggeredInterrupt) del self.workers[kill] except ValueError: # it was already dead self.log.warn( "tried to kill a thread when it was dead already") # shuts down the entire pool # suicide looks like: 1 if suicide == 1: self.stop = True # sends a tweet from self.manual # tweet looks like: <msg> if tweet: self.log.info("tweeting with manual account %s: '%s'" % (self.manual.name, tweet)) self.work(lambda: self.manual.make_tweet(tweet)) # if rtp is set, have dummies retweet it with probability rtp if rtp: def f(): for a in self.auto: if rtp < random.random(): a.retweet_last(self.manual.name) self.work(f) def listen(self): self.ircbot.start()
class WorkerPool(object): def __init__(self, queue_name, routing_keys=None, backend='mongodb', conn_url='localhost:27017', dbname='fwbots', logfile=sys.stdout, pidfile=None): """ routing_keys are a required parameter to specify an n-length list of routing keys, which will each be assigned to one worker FWBOTS: we are using routing_keys as the account names to load """ self.stop = False self.name = queue_name self.log = Logger('pool-'+queue_name, logfile=logfile) self.persister = get_backend(backend)(conn_url,dbname) self.port = -1 # too lazy to actually remove this self.workers = {} self.manual = {} self.auto = {} self.insta = {} if pidfile: self.log.info("writing to pidfile %s" % pidfile) with open(pidfile) as f: f.write(str(self.pid)) f.close() # TODO this needs to be in shared memory wnums = {} self.ircbot = IrcListener(self,"#fwbots",self.name,"irc.freenode.net") for key in routing_keys: errnum = 0 try: acc = get_twitter_account(key) if acc.ty == 'auto': self.log.info('found auto for %s' % key) self.auto[acc.name] = acc # if there are multiple manual accs defined, pick only the last one elif acc.ty == 'manual': self.log.info('found manual for %s' % key) self.manual[acc.name] = acc except NameError: errnum += 1 try: acc = get_instagram_account(key) self.insta[acc.name] = acc except NameError: errnum += 1 if errnum > 1: self.log.warn("Could not find any account called %s"%key) if key not in wnums: wnums[key] = 0 wnums[key] += 1 worker = Worker(queue_name, key, self.persister, wnums[key], self.port, logfile) thread = KillableThread(target=worker.start_worker) thread.start() self.workers[worker.worker_id] = (worker, thread) # lastly, register yourself self.persister.add_pool(self) def __enter__(self, *args, **kwargs): self.log.info("starting") def gentle(signum, frame): self.log.info("Received gentle shutdown signal %d" % signum) self.shutdown() sys.exit(0) def rough(signum, frame): self.log.warn("Received non-gentle kill signal %d" % signum) self.die() sys.exit(0) signal.signal(signal.SIGINT, rough ) signal.signal(signal.SIGHUP, gentle) signal.signal(signal.SIGTERM, gentle) signal.signal(signal.SIGALRM, gentle) signal.signal(signal.SIGQUIT, gentle) return self def __exit__(self, *args, **kwargs): self.shutdown() def shutdown(self): for _id in self.workers.keys(): worker = self.workers[_id] worker[0].stop_worker() del self.workers[_id] self.stop = True self.persister.delete_pool(self.name) def die(self): for _id in self.workers.keys(): worker = self.workers[_id] try: worker[1].raise_exc(TriggeredInterrupt) self.log.warn("raised an exception in %s" % str(_id)) del self.workers[_id] except ValueError: # it was dead already self.log.debug("ignored killing thread %s" % str(_id)) continue self.stop = True def work(self, f): w = self.persister.get_avail_workers()[0] w.todo = f def run_cmd(self, terminate=None, kill=None, suicide=None, tweet=None,rtp=None): # requests a termination # terminate looks like: <worker_id> if terminate: self.workers[terminate][0].stop_worker() del self.workers[terminate] # performs a hard kill # kill looks like: <worker_id> if kill: try: self.workers[kill][1].raise_exc(TriggeredInterrupt) del self.workers[kill] except ValueError: # it was already dead self.log.warn("tried to kill a thread when it was dead already") # shuts down the entire pool # suicide looks like: 1 if suicide == 1: self.stop = True # sends a tweet from self.manual # tweet looks like: <msg> if tweet: self.log.info("tweeting with manual account %s: '%s'"%(self.manual.name,tweet)) self.work(lambda: self.manual.make_tweet(tweet)) # if rtp is set, have dummies retweet it with probability rtp if rtp: def f(): for a in self.auto: if rtp < random.random(): a.retweet_last(self.manual.name) self.work(f) def listen(self): self.ircbot.start()