class BeanstalkStack(threading.Thread): def setConnectParams(self, host, port, sendQueue,yowsUpStack): self.host = host self.port = port self.sendQueue = sendQueue self.yowsUpStack = yowsUpStack def run(self): self.beanstalk = Connection(host=self.host, port=int(self.port)) self.beanstalk.watch('whatsapp-send') while 1: time.sleep(1) job = self.beanstalk.reserve(0.05) if not job is None: try: messageBody = str(job.body) message = json.loads(messageBody) if message["type"] == "simple": print(messageBody) self.sendMessageToWhatsapp(message["address"], message["body"]) elif message["type"] == "image": self.sendImage(message["address"], message["image"]) pass else: raise Exception("Unrecognized Message: %s" % message) print("Sucessfully sended Message") job.delete() except Exception as e: job.bury() traceback.print_exc() else: time.sleep(1) try: messageToSend = self.sendQueue.get(True, 0.05) self.sendMessage2BeanStalkd(messageToSend) except queue.Empty as e: pass def sendMessage2BeanStalkd(self, message): self.beanstalk.use("whatsapp-receive") if type(message) is not str: message = json.dumps(message) self.beanstalk.put(message) pass def sendMessageToWhatsapp(self, number, msg): #self.output(msg) #self.output(number) self.yowsUpStack.broadcastEvent(YowLayerEvent(name=QueueLayer.EVENT_SEND_MESSAGE, msg=msg, number=number)) def sendImage(self, number, imgData): data = base64.b64decode(imgData) tf = tempfile.NamedTemporaryFile(prefix="yowsup-queue-tmp",suffix='.jpg',delete=False) tf.write(data) tf.close() self.yowsUpStack.broadcastEvent(YowLayerEvent(name=QueueLayer.EVENT_SEND_IMAGE, path=tf.name, number=number))
class BeanstalkStack(threading.Thread): def setConnectParams(self, host, port, sendQueue, yowsUpStack): self.host = host self.port = port self.sendQueue = sendQueue self.yowsUpStack = yowsUpStack def run(self): self.beanstalk = Connection(host=self.host, port=int(self.port)) self.beanstalk.watch('whatsapp-send') while 1: time.sleep(1) job = self.beanstalk.reserve(0.05) if not job is None: try: messageBody = str(job.body) message = json.loads(messageBody) if message["type"] == "simple": print(messageBody) self.sendMessageToWhatsapp(message["address"], message["body"]) elif message["type"] == "image": self.sendImage(message["address"], message["image"]) elif message["type"] == "video": self.sendVideo(message["address"], message["video"]) elif message["type"] == "audio": self.sendAudio(message["address"], message["video"]) else: raise Exception("Unrecognized Message: %s" % message) print("Sucessfully sended Message") job.delete() except Exception as e: job.bury() traceback.print_exc() else: time.sleep(1) try: messageToSend = self.sendQueue.get(True, 0.05) self.sendMessage2BeanStalkd(messageToSend) except queue.Empty as e: pass def sendMessage2BeanStalkd(self, message): self.beanstalk.use("whatsapp-receive") if type(message) is not str: message = json.dumps(message) self.beanstalk.put(message) pass def sendMessageToWhatsapp(self, number, msg): self.yowsUpStack.broadcastEvent(YowLayerEvent(name=QueueLayer.EVENT_SEND_MESSAGE, msg=msg, number=number)) def sendImage(self, number, imgPath): self.yowsUpStack.broadcastEvent(YowLayerEvent(name=QueueLayer.EVENT_SEND_IMAGE, path=imgPath, number=number)) def sendVideo(self, number, vidPath): self.yowsUpStack.broadcastEvent(YowLayerEvent(name=QueueLayer.EVENT_SEND_VIDEO, path=vidPath, number=number)) def sendAudio(self, number, audPath): self.yowsUpStack.broadcastEvent(YowLayerEvent(name=QueueLayer.EVENT_SEND_AUDIO, path=audPath, number=number))
def test_gmpush(): bsc = Connection('localhost', 11300) logging.debug('Connected bsc') gitcmdline = base64.b64encode("git push origin master".encode()).decode() gmcmdline = '--cmd gmpush --pwd /home/gzleo/opensource/wxagent --rawcmdline %s --hehere 123' % gitcmdline bsc.put(gmcmdline) return
class BeanstalkStack(threading.Thread): def setConnectParams(self, host, port, sendQueue, yowsUpStack): self.host = host self.port = port self.sendQueue = sendQueue self.yowsUpStack = yowsUpStack def run(self): self.beanstalk = Connection(host=self.host, port=int(self.port)) self.beanstalk.watch('whatsapp-send') while 1: job = self.beanstalk.reserve(0.05) if not job is None: try: messageBody = str(job.body) message = json.loads(messageBody) if message["type"] == "simple": print(messageBody) self.sendMessageToWhatsapp(message["address"], message["body"]) elif message["type"] == "image": print(messageBody) self.sendImage(message["address"], message["body"]) else: raise Exception("Unrecognized Message: %s" % message) print("Sucessfully sent Message") job.delete() except Exception as e: job.bury() traceback.print_exc() try: messageToSend = self.sendQueue.get(True, 0.05) self.sendMessage2BeanStalkd(messageToSend) except queue.Empty as e: pass def sendMessage2BeanStalkd(self, message): self.beanstalk.use("whatsapp-receive") if type(message) is not str: message = json.dumps(message) self.beanstalk.put(message) pass def sendMessageToWhatsapp(self, number, msg): #self.output(msg) #self.output(number) self.yowsUpStack.broadcastEvent( YowLayerEvent(name=QueueLayer.EVENT_SEND_MESSAGE, msg=msg, number=number)) def sendImage(self, number, path): self.yowsUpStack.broadcastEvent( YowLayerEvent(name=QueueLayer.EVENT_SEND_IMAGE, path=path, number=number))
def start(self): self.user.server.ip = self.ip self.user.server.port = random_port() self.db.commit() # only args, not the base command reply = yield self.do(action='spawn', args=self.get_args(), env=self.env) connection = Connection("localhost") connection.use("tarpit_queue_for_server_{}".format(server_id)) connection.put(json.dumps(dict(username=self.user.name))) self.pid = reply['pid']
def test123(): c = Connection() c.put('Hey!') c.put('Hey123!') job = c.reserve() print(job.body) pprint(job.stats()) job.touch() job.delete() job = c.reserve() print(job.body) pprint(job.stats()) job.touch() job.delete() pprint(c.stats_tube('default')) pprint(c.stats()) return
def push_city_jobs(city, sample_order): """get image download jobs for a city from job api then push them to image download queue.""" dst_tube = 'backlog_' + city.replace(' ', '_').lower() beanstalk = Connection(host='localhost', port=11300) print("tubes:", beanstalk.tubes()) print("switching to", beanstalk.use(dst_tube)) print("now using", beanstalk.using()) job_api = API() ok, jobs = job_api.jobs(city, sample_order) if not ok: return 0 for job in jobs: job_json = json.dumps(job) beanstalk.put(job_json) print("pushed {}_{}_{}_{}".format(job['city'], job['osm_way_id'], job['sequence'], job['cam_dir'])) beanstalk.close() return len(jobs)
class BeanstalkdBroker(BaseBroker): def __init__(self, queue_name: str): self.queue_name = queue_name self.connection = Connection(host=settings.beanstalkd_host, port=settings.beanstalkd_port) self.connection.watch(name=queue_name) self.connection.use(name=queue_name) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.connection.close() def add_to_queue(self, task_uuid: uuid.UUID): self.connection.put(body=str(task_uuid)) def reserve(self, timeout: int = None) -> pystalkd.Job: job = self.connection.reserve(timeout=timeout) return job def delete(self, job: pystalkd.Job): job.delete()
#!/usr/bin/env python3 from pystalkd.Beanstalkd import Connection import notification import pickle import base64 setup = { 'connection': ('localhost', 11300), 'tubes': { 'use': 'python', 'ignore': ['default'], } } c = Connection(*setup['connection']) c.use(setup['tubes']['use']) print("Using tube {}".format(c.using())) notifications = [] notifications.append(notification.Notification("*****@*****.**", "Hello")) for notification in notifications: thing = pickle.dumps(notification) # beanstalk jobs only allows str objects body = base64.b64encode(thing) c.put(body.decode('ascii'))
import datetime import json import requests import os beanstalks = Connection(host="127.0.0.1", port=int(11300)) beanstalkr = Connection(host="127.0.0.1", port=int(11300)) beanstalks.use('whatsapp-send') beanstalkr.watch('whatsapp-receive') job = None message_received_url = os.environ.get('MESSAGE_RECEIVED_URL') while (job==None): job = beanstalkr.reserve(timeout=1) if (job!=None): body = json.loads(job.body) job.delete() date = str(datetime.datetime.utcnow()) address = str(body["address"]).split('@')[0] text = body["body"] to_send = {"from": address, "text": "Mensaje: "+text+". Recibido a las "+str(body["timestamp"])+" desde el número: " + address, "date": date} headers = {"Accept": "Application/json","Content-Type": "application/json"} r = requests.post(message_received_url, data = json.dumps(to_send,ensure_ascii=False).encode('utf8'),headers=headers) to_send["to"] = str(body["address"]) message = json.dumps(to_send) beanstalks.put(message) job = None
while (job == None): job = beanstalkr.reserve(timeout=1) if (job != None): body = json.loads(job.body) job.delete() date = str(datetime.datetime.utcnow()) address = str(body["address"]).split('@')[0] text = body["body"] to_send = { "from": address, "text": "Mensaje: " + text + ". Recibido a las " + str(body["timestamp"]) + " desde el número: " + address, "date": date } headers = { "Accept": "Application/json", "Content-Type": "application/json" } r = requests.post(message_received_url, data=json.dumps(to_send, ensure_ascii=False).encode('utf8'), headers=headers) to_send["to"] = str(body["address"]) message = json.dumps(to_send) beanstalks.put(message) job = None
class BeanstalkStack(threading.Thread): def setConnectParams(self, host, port, sendQueue, yowsUpStack): self.host = host self.port = port self.sendQueue = sendQueue self.yowsUpStack = yowsUpStack def run(self): self.beanstalk = Connection(host=self.host, port=int(self.port)) self.beanstalk.watch('whatsapp-send') while 1: time.sleep(1) job = self.beanstalk.reserve(0.05) if not job is None: try: messageBody = str(job.body) message = json.loads(messageBody) if message["type"] == "simple": print(messageBody) self.sendMessageToWhatsapp(message["address"], message["body"]) elif message["type"] == "image": self.sendImage(message["address"], message["image"]) pass else: raise Exception("Unrecognized Message: %s" % message) print("Sucessfully sended Message") job.delete() except Exception as e: job.bury() traceback.print_exc() else: time.sleep(1) try: messageToSend = self.sendQueue.get(True, 0.05) self.sendMessage2BeanStalkd(messageToSend) except queue.Empty as e: pass def sendMessage2BeanStalkd(self, message): self.beanstalk.use("whatsapp-receive") if type(message) is not str: message = json.dumps(message) self.beanstalk.put(message) pass def sendMessageToWhatsapp(self, number, msg): #self.output(msg) #self.output(number) self.yowsUpStack.broadcastEvent( YowLayerEvent(name=QueueLayer.EVENT_SEND_MESSAGE, msg=msg, number=number)) def sendImage(self, number, imgData): data = base64.b64decode(imgData) tf = tempfile.NamedTemporaryFile(prefix="yowsup-queue-tmp", suffix='.jpg', delete=False) tf.write(data) tf.close() self.yowsUpStack.broadcastEvent( YowLayerEvent(name=QueueLayer.EVENT_SEND_IMAGE, path=tf.name, number=number))
class Shovel(): def __init__(self, beanstalk_host, beanstalk_port): self.logger = logging.getLogger(__name__) self.beanstalk = Connection(host=beanstalk_host, port=beanstalk_port) self.logger.info("host: {} {}".format(beanstalk_host, beanstalk_port)) def watch_single_tube(self, tube): """watch a single tube.""" # todo: is this necessary? self.beanstalk.watch(tube) watching = [x for x in self.beanstalk.watching() if x != tube] for x in watching: self.beanstalk.ignore(x) self.logger.info("now watching {}".format(self.beanstalk.watching())) def move_jobs(self, src_tube, dst_tube, n=0): """move n jobs from one tube to another.""" self.watch_single_tube(src_tube) self.beanstalk.watch(src_tube) self.beanstalk.use(dst_tube) # BATCH DRAIN INTO THIS (note that this bit is not persistent!) lifo = [] while (n > 0): job = self.beanstalk.reserve(timeout=60) if job is None: print("timed out. nothing to do?!") return lifo.append(job) n -= 1 stack_len = len(lifo) # dump stack into destination work queue. while (len(lifo) > 0): job = lifo.pop() self.beanstalk.put(job.body) job.delete() self.logger.info("drained {} jobs".format(stack_len)) def drain(self, total_shovel, target_queue, queue_prefix="backlog"): self.logger.info( "total_shovel: [{}] target_queue: [{}] queue_prefix: [{}]".format( total_shovel, target_queue, queue_prefix)) backlog = [ self.beanstalk.stats_tube(x) for x in self.beanstalk.tubes() if x.startswith(queue_prefix + "_") ] # shuffle cities. # we do this because there is a chance that some of the jobs in the last # city to be processed may be left on backlog if number of jobs shoveled # so far exceeds maximum processing limit. this happens due to # accumulation of rounding error. shuffle(backlog) total_jobs = sum(city['current-jobs-ready'] for city in backlog) total_shovel = min(total_jobs, total_shovel) self.logger.info("jobs remaining: [{}] jobs to shovel: [{}]".format( total_jobs, total_shovel)) done = 0 for city in backlog: name, jobs = city['name'], city['current-jobs-ready'] weight = jobs / total_jobs shovel = ceil(weight * total_shovel) done += shovel if done > total_shovel: excess = done - total_shovel shovel = max(0, shovel - excess) self.logger.info( "tube: {} jobs: {} weight: {:0.1f}%, shovel: {}".format( name, jobs, 100 * weight, shovel)) self.move_jobs(name, target_queue, shovel) self.beanstalk.close()
def test_add_job(): bsc = Connection('localhost', 11300) logging.debug('Connected bsc') bsc.put('--cmd abc --hehere 123') return
from pystalkd.Beanstalkd import Connection beanstalk = Connection(host='localhost', port=11300) print("tubes:", beanstalk.tubes()) print("switching to", beanstalk.use('manchester')) print("now using", beanstalk.using()) print("sending job...") beanstalk.put("meh")