def beanstalk(host='localhost', port=11300, watch=None, use=None):
beanstalk = Connection(host=host, port=port)
if use:
beanstalk.use(use)
if watch:
beanstalk.watch(watch)
return beanstalk
def run(self):
self.beanstalk = Connection(host=self.host, port=int(self.port))
self.beanstalk.watch('whatsapp-send')
while 1:
time.sleep(1)
job = self.beanstalk.reserve(0.05)
if not job is None:
try:
messageBody = str(job.body)
message = json.loads(messageBody)
if message["type"] == "simple":
print(messageBody)
self.sendMessageToWhatsapp(message["address"],
message["body"])
elif message["type"] == "image":
self.sendImage(message["address"], message["image"])
elif message["type"] == "video":
self.sendVideo(message["address"], message["video"])
elif message["type"] == "audio":
self.sendAudio(message["address"], message["video"])
else:
raise Exception("Unrecognized Message: %s" % message)
print("Sucessfully sended Message")
job.delete()
except Exception as e:
job.bury()
traceback.print_exc()
else:
time.sleep(1)
try:
messageToSend = self.sendQueue.get(True, 0.05)
self.sendMessage2BeanStalkd(messageToSend)
except queue.Empty as e:
pass
def beanstalk(self):
if self.connection is None:
self.connection = Connection(host=self.host, port=self.port)
self.connection.use('ching-shop-print-jobs')
self.connection.watch('ching-shop-print-jobs')
return self.connection
def connect(host, port, dst_tube):
"""connect to beanstalk, use dst_tube tube."""
beanstalk = Connection(host, port))
print("tubes:", beanstalk.tubes())
print("switching to", beanstalk.use(dst_tube))
print("now using", beanstalk.using())
return beanstalk
def test_gmpush():
bsc = Connection('localhost', 11300)
logging.debug('Connected bsc')
gitcmdline = base64.b64encode("git push origin master".encode()).decode()
gmcmdline = '--cmd gmpush --pwd /home/gzleo/opensource/wxagent --rawcmdline %s --hehere 123' % gitcmdline
bsc.put(gmcmdline)
return
def test_gmpush():
bsc = Connection('localhost', 11300)
logging.debug('Connected bsc')
gitcmdline = base64.b64encode("git push origin master".encode()).decode()
gmcmdline = '--cmd gmpush --pwd /home/gzleo/opensource/wxagent --rawcmdline %s --hehere 123' % gitcmdline
bsc.put(gmcmdline)
return
def main_loop():
bsc = Connection('localhost', 11300)
logging.debug('Connected bsc')
while True:
job = bsc.reserve() # default tube only
logging.debug('got 1 new job')
run_job(job)
job.delete()
return
def start(self):
self.user.server.ip = self.ip
self.user.server.port = random_port()
self.db.commit()
# only args, not the base command
reply = yield self.do(action='spawn', args=self.get_args(), env=self.env)
connection = Connection("localhost")
connection.use("tarpit_queue_for_server_{}".format(server_id))
connection.put(json.dumps(dict(username=self.user.name)))
self.pid = reply['pid']
def main_loop():
bsc = Connection('localhost', 11300)
logging.debug('Connected bsc')
while True:
job = bsc.reserve() # default tube only
logging.debug('got 1 new job')
run_job(job)
job.delete()
return
def run(self):
self.beanstalk = Connection(host=self.host, port=int(self.port))
self.beanstalk.watch('whatsapp-send')
while 1:
time.sleep(1)
job = self.beanstalk.reserve(0.05)
if not job is None:
try:
messageBody = str(job.body)
message = json.loads(messageBody)
if message["type"] == "simple":
print(messageBody)
self.sendMessageToWhatsapp(message["address"], message["body"])
elif message["type"] == "image":
self.sendImage(message["address"], message["image"])
pass
else:
raise Exception("Unrecognized Message: %s" % message)
print("Sucessfully sended Message")
job.delete()
except Exception as e:
job.bury()
traceback.print_exc()
else:
time.sleep(1)
try:
messageToSend = self.sendQueue.get(True, 0.05)
self.sendMessage2BeanStalkd(messageToSend)
except queue.Empty as e:
pass
def __init__(self, worker_name, beanstalk_host, beanstalk_port, src_store, plugin):
super().__init__(beanstalk_host, beanstalk_port)
self.logger = logging.getLogger(__name__)
self.worker_name = worker_name
self.beanstalk = Connection(host=beanstalk_host, port=beanstalk_port)
self.src_store = src_store
self.sample_api = API()
self.plugin = PluginFactory.instance(plugin)
class BeanstalkStack(threading.Thread):
def setConnectParams(self, host, port, sendQueue,yowsUpStack):
self.host = host
self.port = port
self.sendQueue = sendQueue
self.yowsUpStack = yowsUpStack
def run(self):
self.beanstalk = Connection(host=self.host, port=int(self.port))
self.beanstalk.watch('whatsapp-send')
while 1:
time.sleep(1)
job = self.beanstalk.reserve(0.05)
if not job is None:
try:
messageBody = str(job.body)
message = json.loads(messageBody)
if message["type"] == "simple":
print(messageBody)
self.sendMessageToWhatsapp(message["address"], message["body"])
elif message["type"] == "image":
self.sendImage(message["address"], message["image"])
pass
else:
raise Exception("Unrecognized Message: %s" % message)
print("Sucessfully sended Message")
job.delete()
except Exception as e:
job.bury()
traceback.print_exc()
else:
time.sleep(1)
try:
messageToSend = self.sendQueue.get(True, 0.05)
self.sendMessage2BeanStalkd(messageToSend)
except queue.Empty as e:
pass
def sendMessage2BeanStalkd(self, message):
self.beanstalk.use("whatsapp-receive")
if type(message) is not str:
message = json.dumps(message)
self.beanstalk.put(message)
pass
def sendMessageToWhatsapp(self, number, msg):
#self.output(msg)
#self.output(number)
self.yowsUpStack.broadcastEvent(YowLayerEvent(name=QueueLayer.EVENT_SEND_MESSAGE, msg=msg, number=number))
def sendImage(self, number, imgData):
data = base64.b64decode(imgData)
tf = tempfile.NamedTemporaryFile(prefix="yowsup-queue-tmp",suffix='.jpg',delete=False)
tf.write(data)
tf.close()
self.yowsUpStack.broadcastEvent(YowLayerEvent(name=QueueLayer.EVENT_SEND_IMAGE, path=tf.name, number=number))
class BeanstalkStack(threading.Thread):
def setConnectParams(self, host, port, sendQueue, yowsUpStack):
self.host = host
self.port = port
self.sendQueue = sendQueue
self.yowsUpStack = yowsUpStack
def run(self):
self.beanstalk = Connection(host=self.host, port=int(self.port))
self.beanstalk.watch('whatsapp-send')
while 1:
time.sleep(1)
job = self.beanstalk.reserve(0.05)
if not job is None:
try:
messageBody = str(job.body)
message = json.loads(messageBody)
if message["type"] == "simple":
print(messageBody)
self.sendMessageToWhatsapp(message["address"], message["body"])
elif message["type"] == "image":
self.sendImage(message["address"], message["image"])
elif message["type"] == "video":
self.sendVideo(message["address"], message["video"])
elif message["type"] == "audio":
self.sendAudio(message["address"], message["video"])
else:
raise Exception("Unrecognized Message: %s" % message)
print("Sucessfully sended Message")
job.delete()
except Exception as e:
job.bury()
traceback.print_exc()
else:
time.sleep(1)
try:
messageToSend = self.sendQueue.get(True, 0.05)
self.sendMessage2BeanStalkd(messageToSend)
except queue.Empty as e:
pass
def sendMessage2BeanStalkd(self, message):
self.beanstalk.use("whatsapp-receive")
if type(message) is not str:
message = json.dumps(message)
self.beanstalk.put(message)
pass
def sendMessageToWhatsapp(self, number, msg):
self.yowsUpStack.broadcastEvent(YowLayerEvent(name=QueueLayer.EVENT_SEND_MESSAGE, msg=msg, number=number))
def sendImage(self, number, imgPath):
self.yowsUpStack.broadcastEvent(YowLayerEvent(name=QueueLayer.EVENT_SEND_IMAGE, path=imgPath, number=number))
def sendVideo(self, number, vidPath):
self.yowsUpStack.broadcastEvent(YowLayerEvent(name=QueueLayer.EVENT_SEND_VIDEO, path=vidPath, number=number))
def sendAudio(self, number, audPath):
self.yowsUpStack.broadcastEvent(YowLayerEvent(name=QueueLayer.EVENT_SEND_AUDIO, path=audPath, number=number))
class BeanstalkQueue:
connection = None
def __init__(self, host='localhost', port=11333):
self.host = host
self.port = port
@property
def beanstalk(self):
if self.connection is None:
self.connection = Connection(host=self.host, port=self.port)
self.connection.use('ching-shop-print-jobs')
self.connection.watch('ching-shop-print-jobs')
return self.connection
def pop(self):
return self.beanstalk.reserve(timeout=0)
class BeanstalkStack(threading.Thread):
def setConnectParams(self, host, port, sendQueue, yowsUpStack):
self.host = host
self.port = port
self.sendQueue = sendQueue
self.yowsUpStack = yowsUpStack
def run(self):
self.beanstalk = Connection(host=self.host, port=int(self.port))
self.beanstalk.watch('whatsapp-send')
while 1:
job = self.beanstalk.reserve(0.05)
if not job is None:
try:
messageBody = str(job.body)
message = json.loads(messageBody)
if message["type"] == "simple":
print(messageBody)
self.sendMessageToWhatsapp(message["address"],
message["body"])
elif message["type"] == "image":
print(messageBody)
self.sendImage(message["address"], message["body"])
else:
raise Exception("Unrecognized Message: %s" % message)
print("Sucessfully sent Message")
job.delete()
except Exception as e:
job.bury()
traceback.print_exc()
try:
messageToSend = self.sendQueue.get(True, 0.05)
self.sendMessage2BeanStalkd(messageToSend)
except queue.Empty as e:
pass
def sendMessage2BeanStalkd(self, message):
self.beanstalk.use("whatsapp-receive")
if type(message) is not str:
message = json.dumps(message)
self.beanstalk.put(message)
pass
def sendMessageToWhatsapp(self, number, msg):
#self.output(msg)
#self.output(number)
self.yowsUpStack.broadcastEvent(
YowLayerEvent(name=QueueLayer.EVENT_SEND_MESSAGE,
msg=msg,
number=number))
def sendImage(self, number, path):
self.yowsUpStack.broadcastEvent(
YowLayerEvent(name=QueueLayer.EVENT_SEND_IMAGE,
path=path,
number=number))
def test123():
c = Connection()
c.put('Hey!')
c.put('Hey123!')
job = c.reserve()
print(job.body)
pprint(job.stats())
job.touch()
job.delete()
job = c.reserve()
print(job.body)
pprint(job.stats())
job.touch()
job.delete()
pprint(c.stats_tube('default'))
pprint(c.stats())
return
def beanstalk(host='localhost', port=11300, watch=None, use=None):
beanstalk = Connection(host=host, port=port)
if use:
beanstalk.use(use)
if watch:
beanstalk.watch(watch)
return beanstalk
def test123():
c = Connection()
c.put('Hey!')
c.put('Hey123!')
job = c.reserve()
print(job.body)
pprint(job.stats())
job.touch()
job.delete()
job = c.reserve()
print(job.body)
pprint(job.stats())
job.touch()
job.delete()
pprint(c.stats_tube('default'))
pprint(c.stats())
return
def push_city_jobs(city, sample_order):
"""get image download jobs for a city from job api then push them to image
download queue."""
dst_tube = 'backlog_' + city.replace(' ', '_').lower()
beanstalk = Connection(host='localhost', port=11300)
print("tubes:", beanstalk.tubes())
print("switching to", beanstalk.use(dst_tube))
print("now using", beanstalk.using())
job_api = API()
ok, jobs = job_api.jobs(city, sample_order)
if not ok:
return 0
for job in jobs:
job_json = json.dumps(job)
beanstalk.put(job_json)
print("pushed {}_{}_{}_{}".format(job['city'], job['osm_way_id'],
job['sequence'], job['cam_dir']))
beanstalk.close()
return len(jobs)
class BeanstalkdBroker(BaseBroker):
def __init__(self, queue_name: str):
self.queue_name = queue_name
self.connection = Connection(host=settings.beanstalkd_host,
port=settings.beanstalkd_port)
self.connection.watch(name=queue_name)
self.connection.use(name=queue_name)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.connection.close()
def add_to_queue(self, task_uuid: uuid.UUID):
self.connection.put(body=str(task_uuid))
def reserve(self, timeout: int = None) -> pystalkd.Job:
job = self.connection.reserve(timeout=timeout)
return job
def delete(self, job: pystalkd.Job):
job.delete()
class Consumer():
def __init__(self, beanstalk_host, beanstalk_port):
self.beanstalk = Connection(host=beanstalk_host, port=beanstalk_port)
def consume(self, max_jobs, src_queue):
"""consume from the incoming job queue."""
# incoming jobs
self.beanstalk.watch(src_queue)
self.beanstalk.ignore('default')
print("now watching", self.beanstalk.watching())
queue_timeout = 10 if max_jobs >= 0 else None
done = 0
while(max_jobs <= 0 or done < max_jobs):
job = self.beanstalk.reserve(timeout=queue_timeout)
if job is None:
break
try:
res = self.process_job(job.body)
if res is True:
job.delete()
else:
job.bury()
except Exception as e:
print("error from process_job()", e)
job.bury()
finally:
done += 1
self.beanstalk.close()
def process_job(self, json_job):
"""default: do nothing."""
return False
def __init__(self, beanstalk_host, beanstalk_port):
self.logger = logging.getLogger(__name__)
self.beanstalk = Connection(host=beanstalk_host, port=beanstalk_port)
self.logger.info("host: {} {}".format(beanstalk_host, beanstalk_port))
def test_add_job():
bsc = Connection('localhost', 11300)
logging.debug('Connected bsc')
bsc.put('--cmd abc --hehere 123')
return
from pystalkd.Beanstalkd import Connection
import datetime
import json
import requests
import os
beanstalks = Connection(host="127.0.0.1", port=int(11300))
beanstalkr = Connection(host="127.0.0.1", port=int(11300))
beanstalks.use('whatsapp-send')
beanstalkr.watch('whatsapp-receive')
job = None
message_received_url = os.environ.get('MESSAGE_RECEIVED_URL')
while (job==None):
job = beanstalkr.reserve(timeout=1)
if (job!=None):
body = json.loads(job.body)
job.delete()
date = str(datetime.datetime.utcnow())
address = str(body["address"]).split('@')[0]
text = body["body"]
to_send = {"from": address,
"text": "Mensaje: "+text+". Recibido a las "+str(body["timestamp"])+" desde el número: " + address,
"date": date}
headers = {"Accept": "Application/json","Content-Type": "application/json"}
r = requests.post(message_received_url,
data = json.dumps(to_send,ensure_ascii=False).encode('utf8'),headers=headers)
to_send["to"] = str(body["address"])
message = json.dumps(to_send)
beanstalks.put(message)
job = None
def __init__(self, queue_name: str):
self.queue_name = queue_name
self.connection = Connection(host=settings.beanstalkd_host,
port=settings.beanstalkd_port)
self.connection.watch(name=queue_name)
self.connection.use(name=queue_name)
def get_connection(self):
return Connection(host='localhost', port=11300)
from pystalkd.Beanstalkd import Connection
import datetime
import json
import requests
import os
beanstalks = Connection(host="127.0.0.1", port=int(11300))
beanstalkr = Connection(host="127.0.0.1", port=int(11300))
beanstalks.use('whatsapp-send')
beanstalkr.watch('whatsapp-receive')
job = None
message_received_url = os.environ.get('MESSAGE_RECEIVED_URL')
while (job == None):
job = beanstalkr.reserve(timeout=1)
if (job != None):
body = json.loads(job.body)
job.delete()
date = str(datetime.datetime.utcnow())
address = str(body["address"]).split('@')[0]
text = body["body"]
to_send = {
"from":
address,
"text":
"Mensaje: " + text + ". Recibido a las " + str(body["timestamp"]) +
" desde el número: " + address,
"date":
date
}
headers = {
from pystalkd.Beanstalkd import Connection
import notification
import pickle
import base64
from time import sleep
setup = {
'connection': ('localhost', 11300),
'tubes': {
'watch': ['python'],
'ignore': ['default'],
},
'timeout': 1,
}
c = Connection(*setup['connection'])
for tube in setup['tubes']['watch']:
c.watch(tube)
print("Watching tubes {}".format(c.watching()))
for tube in setup['tubes']['ignore']:
c.ignore(tube)
print("Ignoring tubes {}".format(c.watching()))
#for action, tubes in setup['tubes'].items():
# for tube in tubes:
# getattr(c, action)(tube)
class BeanstalkStack(threading.Thread):
def setConnectParams(self, host, port, sendQueue, yowsUpStack):
self.host = host
self.port = port
self.sendQueue = sendQueue
self.yowsUpStack = yowsUpStack
def run(self):
self.beanstalk = Connection(host=self.host, port=int(self.port))
self.beanstalk.watch('whatsapp-send')
while 1:
time.sleep(1)
job = self.beanstalk.reserve(0.05)
if not job is None:
try:
messageBody = str(job.body)
message = json.loads(messageBody)
if message["type"] == "simple":
print(messageBody)
self.sendMessageToWhatsapp(message["address"],
message["body"])
elif message["type"] == "image":
self.sendImage(message["address"], message["image"])
pass
else:
raise Exception("Unrecognized Message: %s" % message)
print("Sucessfully sended Message")
job.delete()
except Exception as e:
job.bury()
traceback.print_exc()
else:
time.sleep(1)
try:
messageToSend = self.sendQueue.get(True, 0.05)
self.sendMessage2BeanStalkd(messageToSend)
except queue.Empty as e:
pass
def sendMessage2BeanStalkd(self, message):
self.beanstalk.use("whatsapp-receive")
if type(message) is not str:
message = json.dumps(message)
self.beanstalk.put(message)
pass
def sendMessageToWhatsapp(self, number, msg):
#self.output(msg)
#self.output(number)
self.yowsUpStack.broadcastEvent(
YowLayerEvent(name=QueueLayer.EVENT_SEND_MESSAGE,
msg=msg,
number=number))
def sendImage(self, number, imgData):
data = base64.b64decode(imgData)
tf = tempfile.NamedTemporaryFile(prefix="yowsup-queue-tmp",
suffix='.jpg',
delete=False)
tf.write(data)
tf.close()
self.yowsUpStack.broadcastEvent(
YowLayerEvent(name=QueueLayer.EVENT_SEND_IMAGE,
path=tf.name,
number=number))
from pystalkd.Beanstalkd import Connection
beanstalk = Connection(host='localhost', port=11300)
print("tubes:", beanstalk.tubes())
print("switching to", beanstalk.watch('manchester'))
print("now watching", beanstalk.watching())
while True:
print("blocking...")
job = beanstalk.reserve()
print("got job:", job.body)
job.delete()
from pystalkd.Beanstalkd import Connection
import notification
import pickle
import base64
from time import sleep
setup = {
'connection': ('localhost', 11300),
'tubes': {
'watch': ['python'],
'ignore': ['default'],
},
'timeout': 1,
}
c = Connection(*setup['connection'])
for tube in setup['tubes']['watch']:
c.watch(tube)
print("Watching tubes {}".format(c.watching()))
for tube in setup['tubes']['ignore']:
c.ignore(tube)
print("Ignoring tubes {}".format(c.watching()))
#for action, tubes in setup['tubes'].items():
# for tube in tubes:
# getattr(c, action)(tube)
#!/usr/bin/env python3
from pystalkd.Beanstalkd import Connection
import notification
import pickle
import base64
setup = {
'connection': ('localhost', 11300),
'tubes': {
'use': 'python',
'ignore': ['default'],
}
}
c = Connection(*setup['connection'])
c.use(setup['tubes']['use'])
print("Using tube {}".format(c.using()))
notifications = []
notifications.append(notification.Notification("*****@*****.**", "Hello"))
for notification in notifications:
thing = pickle.dumps(notification)
# beanstalk jobs only allows str objects
body = base64.b64encode(thing)
c.put(body.decode('ascii'))
#!/usr/bin/env python3
from pystalkd.Beanstalkd import Connection
import notification
import pickle
import base64
setup = {
'connection': ('localhost', 11300),
'tubes': {
'use': 'python',
'ignore': ['default'],
}
}
c = Connection(*setup['connection'])
c.use(setup['tubes']['use'])
print("Using tube {}".format(c.using()))
notifications = []
notifications.append(notification.Notification("*****@*****.**", "Hello"))
for notification in notifications:
thing = pickle.dumps(notification)
# beanstalk jobs only allows str objects
body = base64.b64encode(thing)
c.put(body.decode('ascii'))
def test_add_job():
bsc = Connection('localhost', 11300)
logging.debug('Connected bsc')
bsc.put('--cmd abc --hehere 123')
return
class Shovel():
def __init__(self, beanstalk_host, beanstalk_port):
self.logger = logging.getLogger(__name__)
self.beanstalk = Connection(host=beanstalk_host, port=beanstalk_port)
self.logger.info("host: {} {}".format(beanstalk_host, beanstalk_port))
def watch_single_tube(self, tube):
"""watch a single tube."""
# todo: is this necessary?
self.beanstalk.watch(tube)
watching = [x for x in self.beanstalk.watching() if x != tube]
for x in watching:
self.beanstalk.ignore(x)
self.logger.info("now watching {}".format(self.beanstalk.watching()))
def move_jobs(self, src_tube, dst_tube, n=0):
"""move n jobs from one tube to another."""
self.watch_single_tube(src_tube)
self.beanstalk.watch(src_tube)
self.beanstalk.use(dst_tube)
# BATCH DRAIN INTO THIS (note that this bit is not persistent!)
lifo = []
while (n > 0):
job = self.beanstalk.reserve(timeout=60)
if job is None:
print("timed out. nothing to do?!")
return
lifo.append(job)
n -= 1
stack_len = len(lifo)
# dump stack into destination work queue.
while (len(lifo) > 0):
job = lifo.pop()
self.beanstalk.put(job.body)
job.delete()
self.logger.info("drained {} jobs".format(stack_len))
def drain(self, total_shovel, target_queue, queue_prefix="backlog"):
self.logger.info(
"total_shovel: [{}] target_queue: [{}] queue_prefix: [{}]".format(
total_shovel, target_queue, queue_prefix))
backlog = [
self.beanstalk.stats_tube(x) for x in self.beanstalk.tubes()
if x.startswith(queue_prefix + "_")
]
# shuffle cities.
# we do this because there is a chance that some of the jobs in the last
# city to be processed may be left on backlog if number of jobs shoveled
# so far exceeds maximum processing limit. this happens due to
# accumulation of rounding error.
shuffle(backlog)
total_jobs = sum(city['current-jobs-ready'] for city in backlog)
total_shovel = min(total_jobs, total_shovel)
self.logger.info("jobs remaining: [{}] jobs to shovel: [{}]".format(
total_jobs, total_shovel))
done = 0
for city in backlog:
name, jobs = city['name'], city['current-jobs-ready']
weight = jobs / total_jobs
shovel = ceil(weight * total_shovel)
done += shovel
if done > total_shovel:
excess = done - total_shovel
shovel = max(0, shovel - excess)
self.logger.info(
"tube: {} jobs: {} weight: {:0.1f}%, shovel: {}".format(
name, jobs, 100 * weight, shovel))
self.move_jobs(name, target_queue, shovel)
self.beanstalk.close()
def _connect(self, host='localhost', port=11300, tube='politwoops'):
beanstalk = Connection(host=host, port=port)
beanstalk.use(tube)
beanstalk.watch(tube)
return beanstalk
from pystalkd.Beanstalkd import Connection
beanstalk = Connection(host='localhost', port=11300)
print("tubes:", beanstalk.tubes())
print("switching to", beanstalk.use('manchester'))
print("now using", beanstalk.using())
print("sending job...")
beanstalk.put("meh")
def __init__(self, beanstalk_host, beanstalk_port):
self.beanstalk = Connection(host=beanstalk_host, port=beanstalk_port)
