def run(self):
coros = []
queue_url = self.conf.get('queue_url', '127.0.0.1:11300')
concurrency = int_value(self.conf.get('concurrency'), 10)
server_gt = greenthread.getcurrent()
for i in range(concurrency):
beanstalk = Beanstalk.from_url(queue_url)
gt = eventlet.spawn(self.handle, beanstalk)
gt.link(_eventlet_stop, server_gt, beanstalk)
coros.append(gt)
beanstalk, gt = None, None
while self.alive:
self.notify()
try:
eventlet.sleep(1.0)
except AssertionError:
self.alive = False
break
self.notify()
try:
with Timeout(self.graceful_timeout) as t:
[c.kill(StopServe()) for c in coros]
[c.wait() for c in coros]
except Timeout as te:
if te != t:
raise
[c.kill() for c in coros]
def run(self):
coros = []
queue_url = self.conf.get('queue_url', 'beanstalk://127.0.0.1:11300')
concurrency = int_value(self.conf.get('concurrency'), 10)
server_gt = greenthread.getcurrent()
for url in queue_url.split(';'):
for i in range(concurrency):
beanstalk = Beanstalk.from_url(url)
gt = eventlet.spawn(self.handle, beanstalk)
gt.link(_eventlet_stop, server_gt, beanstalk)
coros.append(gt)
beanstalk, gt = None, None
while self.alive:
self.notify()
try:
eventlet.sleep(1.0)
except AssertionError:
self.alive = False
break
self.notify()
try:
with Timeout(self.graceful_timeout) as t:
[c.kill(StopServe()) for c in coros]
[c.wait() for c in coros]
except Timeout as te:
if te != t:
raise
[c.kill() for c in coros]
def init(self):
queue_url = self.conf.get('queue_url')
if not queue_url:
raise ValueError("Missing 'queue_url' in the configuration")
self.beanstalk = Beanstalk.from_url(queue_url)
self.tube = self.conf.get('tube', 'notif')
self.beanstalk.use(self.tube)
def _fetch_chunks_from_beanstalk(self):
self.beanstalk = Beanstalk.from_url(self.beanstalkd_addr)
self.beanstalk.use(self.beanstalkd_tube)
self.beanstalk.watch(self.beanstalkd_tube)
conn_error = False
while 1:
for chunk in self._handle_beanstalk_event(conn_error):
yield chunk
def locate_tube(services, tube):
"""
Get a list of beanstalkd services hosting the specified tube.
:param services: known beanstalkd services.
:type services: iterable of dictionaries
:param tube: the tube to locate.
:returns: a list of beanstalkd services hosting the the specified tube.
:rtype: `list` of `dict`
"""
available = list()
for bsd in services:
tubes = Beanstalk.from_url('beanstalk://' + bsd['addr']).tubes()
if tube in tubes:
available.append(bsd)
return available
def zmq_loop():
queue_location = self.conf.get('queue_url',
'tcp://127.0.0.1:11300')
beanstalk = Beanstalk.from_url(queue_location)
context = zmq.Context.instance()
socket = context.socket(zmq.ROUTER)
socket.set(zmq.LINGER, 1000)
bind_addr = self.conf.get('bind_addr',
'ipc:///tmp/run/event-agent.sock')
socket.bind(bind_addr)
context = context
while True:
msg = socket.recv_multipart()
if validate_msg(msg):
data = msg[3]
beanstalk.put(data)
ack = msg[0:3]
socket.send_multipart(ack)
def zmq_loop():
queue_location = self.conf.get('queue_url',
'tcp://127.0.0.1:11300')
beanstalk = Beanstalk.from_url(queue_location)
context = zmq.Context.instance()
socket = context.socket(zmq.ROUTER)
socket.set(zmq.LINGER, 1000)
bind_addr = self.conf.get('bind_addr',
'ipc:///tmp/run/event-agent.sock')
socket.bind(bind_addr)
context = context
while True:
msg = socket.recv_multipart()
if validate_msg(msg):
data = msg[3]
beanstalk.put(data)
ack = msg[0:3]
socket.send_multipart(ack)
def setUp(self):
super(TestContentRebuildFilter, self).setUp()
self.namespace = self.conf['namespace']
self.gridconf = {"namespace": self.namespace}
self.container = "TestContentRebuildFilter%f" % time.time()
self.ref = self.container
self.container_client = ContainerClient(self.conf)
self.container_client.container_create(self.account, self.container)
syst = self.container_client.container_get_properties(
self.account, self.container)['system']
self.container_id = syst['sys.name'].split('.', 1)[0]
self.object_storage_api = ObjectStorageApi(namespace=self.namespace)
queue_addr = choice(self.conf['services']['beanstalkd'])['addr']
self.queue_url = queue_addr
self.conf['queue_url'] = 'beanstalk://' + self.queue_url
self.conf['tube'] = DEFAULT_REBUILDER_TUBE
self.notify_filter = NotifyFilter(app=_App, conf=self.conf)
bt = Beanstalk.from_url(self.conf['queue_url'])
bt.drain_tube(DEFAULT_REBUILDER_TUBE)
bt.close()
def setUp(self):
super(TestContentRebuildFilter, self).setUp()
self.namespace = self.conf['namespace']
self.gridconf = {"namespace": self.namespace}
self.container = "TestContentRebuildFilter%f" % time.time()
self.ref = self.container
self.container_client = ContainerClient(self.conf)
self.container_client.container_create(self.account, self.container)
syst = self.container_client.container_get_properties(
self.account, self.container)['system']
self.container_id = syst['sys.name'].split('.', 1)[0]
self.object_storage_api = ObjectStorageApi(namespace=self.namespace)
self.stgpol = "SINGLE"
self.conf['tube'] = 'rebuild'
self.conf['queue_url'] = 'beanstalk://127.0.0.1:11300'
self.notify_filter = NotifyFilter(app=_App, conf=self.conf)
queue_url = self.conf.get('queue_url', 'tcp://127.0.0.1:11300')
self.tube = self.conf.get('tube', 'rebuild')
self.beanstalk = Beanstalk.from_url(queue_url)
self.beanstalk.use(self.tube)
def setUp(self):
super(TestContentRebuildFilter, self).setUp()
self.gridconf = {"namespace": self.ns}
self.container = "TestContentRebuildFilter%f" % time.time()
self.ref = self.container
self.container_client = self.storage.container
self.container_client.container_create(self.account, self.container)
syst = self.container_client.container_get_properties(
self.account, self.container)['system']
self.container_id = syst['sys.name'].split('.', 1)[0]
self.object_storage_api = self.storage
queue_addr = choice(self.conf['services']['beanstalkd'])['addr']
self.queue_url = queue_addr
self.conf['queue_url'] = 'beanstalk://' + self.queue_url
self.conf['tube'] = BlobRebuilder.DEFAULT_BEANSTALKD_WORKER_TUBE
self.notify_filter = NotifyFilter(app=_App, conf=self.conf)
bt = Beanstalk.from_url(self.conf['queue_url'])
bt.drain_tube(BlobRebuilder.DEFAULT_BEANSTALKD_WORKER_TUBE)
bt.close()
self.wait_for_score(('rawx', ))
def reply(self, job_id, task_ids, task_results, task_errors,
beanstalkd_reply_info):
beanstalkd_reply_addr = beanstalkd_reply_info['addr']
beanstalkd_reply_tube = beanstalkd_reply_info['tube']
beanstalkd_reply_info = (beanstalkd_reply_addr, beanstalkd_reply_tube)
beanstalkd_reply = self.beanstalkd_replies.get(beanstalkd_reply_info)
if not beanstalkd_reply:
beanstalkd_reply = Beanstalk.from_url(beanstalkd_reply_addr)
beanstalkd_reply.use(beanstalkd_reply_tube)
beanstalkd_reply.watch(beanstalkd_reply_tube)
self.beanstalkd_replies[beanstalkd_reply_info] = beanstalkd_reply
reply_payload = json.dumps({
'job_id': job_id,
'task_ids': task_ids,
'task_results': task_results,
'task_errors': task_errors,
})
beanstalkd_reply.put(reply_payload)
def main(_):
maybe_download_and_extract()
create_graph()
b = Beanstalk.from_url("beanstalk://127.0.0.1:6014")
b.watch("oio-process")
while True:
try:
job_id, data = b.reserve()
except ResponseError:
continue
meta = json.loads(data)
url = meta["url"]
print(url)
if url["path"].split('.')[len(url["path"].split('.')) - 1] == 'png':
b.delete(job_id)
continue
s = object_storage.ObjectStorageAPI(url["ns"], "http://127.0.0.1:6006")
meta, stream = s.object_fetch(url["account"], url["user"], url["path"])
image = (np.frombuffer("".join(stream), np.uint8))
#image = np.array(image)[:, :, 0:3]
result = run_inference_on_image(image)
print(json.dumps(result))
s.object_update(url["account"], url["user"], url["path"], result)
# /!\ Change the ip /!\
es = Elasticsearch(['http://192.168.99.1:9200'])
# Retrieve the metadata from the object
meta, stream = s.object_fetch(url["account"], url["user"], url["path"])
# Create the index in ElasticSearch if it does not exist
if not es.indices.exists(url["account"].lower()):
es.indices.create(index=url["account"].lower())
# Push the metadatas to Elasticsearch
res = es.index(index=url["account"].lower(),
doc_type=url["user"].lower(),
body=meta)
es.indices.refresh(index=url["account"].lower())
b.delete(job_id)
def run(self):
queue_url = self.conf.get('queue_url', 'tcp://127.0.0.1:11300')
self.beanstalk = Beanstalk.from_url(queue_url)
gt = eventlet.spawn(self.handle)
while self.alive:
self.notify()
try:
eventlet.sleep(1.0)
except AssertionError:
self.alive = False
break
self.notify()
try:
with Timeout(self.graceful_timeout) as t:
gt.kill(StopServe())
gt.wait()
except Timeout as te:
if te != t:
raise
gt.kill()
def run(self):
queue_url = self.conf.get('queue_url', 'tcp://127.0.0.1:11300')
self.beanstalk = Beanstalk.from_url(queue_url)
gt = eventlet.spawn(
self.handle)
while self.alive:
self.notify()
try:
eventlet.sleep(1.0)
except AssertionError:
self.alive = False
break
self.notify()
try:
with Timeout(self.graceful_timeout) as t:
gt.kill(StopServe())
gt.wait()
except Timeout as te:
if te != t:
raise
gt.kill()
def __init__(self, conf, tool):
super(_DistributedDispatcher, self).__init__(conf, tool)
self.sending = False
# All available beanstalkd
conscience_client = ConscienceClient(self.conf)
all_beanstalkd = conscience_client.all_services('beanstalkd')
all_available_beanstalkd = dict()
for beanstalkd in all_beanstalkd:
if beanstalkd['score'] <= 0:
continue
all_available_beanstalkd[beanstalkd['addr']] = beanstalkd
if not all_available_beanstalkd:
raise OioException('No beanstalkd available')
# Beanstalkd workers
workers_tube = self.conf.get('distributed_beanstalkd_worker_tube') \
or self.tool.DEFAULT_DISTRIBUTED_BEANSTALKD_WORKER_TUBE
self.beanstalkd_workers = dict()
for _, beanstalkd in all_available_beanstalkd.items():
beanstalkd_worker_addr = beanstalkd['addr']
# If the tube exists,
# there should be a service that listens to this tube
tubes = Beanstalk.from_url('beanstalk://' +
beanstalkd_worker_addr).tubes()
if workers_tube not in tubes:
continue
beanstalkd_worker = BeanstalkdSender(beanstalkd_worker_addr,
workers_tube, self.logger)
self.beanstalkd_workers[beanstalkd_worker_addr] = beanstalkd_worker
self.logger.info(
'Beanstalkd %s using tube %s is selected as a worker',
beanstalkd_worker.addr, beanstalkd_worker.tube)
if not self.beanstalkd_workers:
raise OioException('No beanstalkd worker available')
# Beanstalkd reply
beanstalkd_reply = dict()
try:
local_services = conscience_client.local_services()
for local_service in local_services:
if local_service['type'] != 'beanstalkd':
continue
beanstalkd = all_available_beanstalkd.get(
local_service['addr'])
if beanstalkd is None:
continue
if beanstalkd_reply \
and beanstalkd_reply['score'] >= beanstalkd['score']:
continue
beanstalkd_reply = beanstalkd
except Exception as exc: # pylint: disable=broad-except
self.logger.warning(
'ERROR when searching for beanstalkd locally: %s', exc)
if not beanstalkd_reply:
self.logger.warn('No beanstalkd available locally')
try:
beanstalkd = conscience_client.next_instance('beanstalkd')
beanstalkd_reply = all_available_beanstalkd[beanstalkd['addr']]
except Exception as exc: # pylint: disable=broad-except
self.logger.warning('ERROR when searching for beanstalkd: %s',
exc)
beanstalkd_reply_addr = beanstalkd_reply['addr']
# If the tube exists, another service must have already used this tube
tube_reply = workers_tube + '.reply.' + str(time.time())
tubes = Beanstalk.from_url('beanstalk://' +
beanstalkd_reply_addr).tubes()
if tube_reply in tubes:
raise OioException('Beanstalkd %s using tube %s is already used')
self.beanstalkd_reply = BeanstalkdListener(beanstalkd_reply_addr,
tube_reply, self.logger)
self.logger.info(
'Beanstalkd %s using tube %s is selected for the replies',
self.beanstalkd_reply.addr, self.beanstalkd_reply.tube)
def _connect(self, **kwargs):
self.close()
self.beanstalkd = Beanstalk.from_url('beanstalk://' + self.addr)
self.beanstalkd.use(self.tube)
self.beanstalkd.watch(self.tube)
self.connected = True
def beanstalkd(self):
if self._beanstalkd is None:
host = self.stat_conf['host']
port = int(self.stat_conf['port'])
self._beanstalkd = Beanstalk(host, port)
return self._beanstalkd
def init(self):
queue_url = self.conf.get('queue_url', 'tcp://127.0.0.1:11300')
self.beanstalk = Beanstalk.from_url(queue_url)
self.tube = self.conf.get('tube', 'notif')
self.beanstalk.use(self.tube)
def beanstalk(self):
if not self._beanstalk:
self._beanstalk = Beanstalk.from_url(self.queue_url)
return self._beanstalk
def beanstalk(self):
if not self._beanstalk:
from oio.event.beanstalk import Beanstalk
self._beanstalk = Beanstalk.from_url(self.conf['queue_url'])
return self._beanstalk
def _check_beanstalkd_worker(self, beanstalkd_info):
"""
Check beanstalkd worker by looking at the score,
existing tubes and tube statistics.
"""
beanstalkd_addr = 'beanstalk://' + beanstalkd_info['addr']
beanstalkd_score = beanstalkd_info['score']
if beanstalkd_score == 0:
self.logger.debug(
'Ignore beanstalkd %s: score=0', beanstalkd_addr)
return None
beanstalkd = self.beanstalkd_workers.get(beanstalkd_addr)
if not beanstalkd:
beanstalkd = Beanstalk.from_url(beanstalkd_addr)
beanstalkd.addr = beanstalkd_addr
beanstalkd_tubes = beanstalkd.tubes()
if self.beanstalkd_workers_tube not in beanstalkd_tubes:
self.logger.debug(
'Ignore beanstalkd %s: '
'No worker has ever listened to the tube %s',
beanstalkd_addr, self.beanstalkd_workers_tube)
return None
current_stats = beanstalkd.stats_tube(
self.beanstalkd_workers_tube)
beanstalkd_jobs_ready = current_stats['current-jobs-ready']
if beanstalkd_jobs_ready > 0:
beanstalkd_jobs_reserved = current_stats['current-jobs-reserved']
if beanstalkd_jobs_reserved <= 0:
self.logger.warn(
'Ignore beanstalkd %s: The worker doesn\'t process task '
'(current-jobs-ready=%d, current-jobs-reserved=%d)',
beanstalkd_addr, beanstalkd_jobs_ready,
beanstalkd_jobs_reserved)
return None
if beanstalkd_jobs_ready >= self.max_jobs_per_beanstalkd:
self.logger.warn(
'Ignore beanstalkd %s: The queue is full '
'(current-jobs-ready=%d, current-jobs-reserved=%d)',
beanstalkd_addr, beanstalkd_jobs_ready,
beanstalkd_jobs_reserved)
return None
if hasattr(beanstalkd, 'is_broken') and beanstalkd.is_broken:
self.logger.info(
'Beanstalkd %s was broken, and now it\'s coming back',
beanstalkd_addr)
beanstalkd.is_broken = False
# Favor the workers with a good score
# 50% -> beanstalkd score
worker_score = beanstalkd_score * 50. / 100.
# 50% -> beanstalkd tube size
worker_score += 50 - (beanstalkd_jobs_ready * 50.
/ self.max_jobs_per_beanstalkd)
beanstalkd.occurrence = int(math.ceil(worker_score / 10.))
self.logger.debug(
'Give the green light to beanstalkd %s (worker_score=%d)',
beanstalkd_addr, worker_score)
return beanstalkd
def _connect_to_beanstalk(self):
self.logger.debug('Connecting to %s', self.beanstalkd_addr)
self.beanstalk = Beanstalk.from_url(self.beanstalkd_addr)
self.logger.debug('Using tube %s', self.beanstalkd_tube)
self.beanstalk.use(self.beanstalkd_tube)
self.beanstalk.watch(self.beanstalkd_tube)
def _connect_to_beanstalk(self):
self.beanstalk = Beanstalk.from_url(self.beanstalkd_addr)
self.beanstalk.use(self.beanstalkd_tube)
self.beanstalk.watch(self.beanstalkd_tube)
def beanstalk(self):
if not self._beanstalk:
self._beanstalk = Beanstalk.from_url(self.queue_url)
return self._beanstalk
def _bt_make_connections(self, bt_list):
for bt_entry in bt_list:
self.bt_connections.append(
Beanstalk.from_url('beanstalk://{0}'.format(bt_entry['addr'])))
def __init__(self, conf, tool):
super(_DistributedDispatcher, self).__init__(conf, tool)
self.sending = None
self.max_items_per_second = int_value(
self.conf.get('items_per_second'),
self.tool.DEFAULT_ITEM_PER_SECOND)
# All available beanstalkd
conscience_client = ConscienceClient(self.conf)
all_beanstalkd = conscience_client.all_services('beanstalkd')
all_available_beanstalkd = dict()
for beanstalkd in all_beanstalkd:
if beanstalkd['score'] <= 0:
continue
all_available_beanstalkd[beanstalkd['addr']] = beanstalkd
if not all_available_beanstalkd:
raise OioException('No beanstalkd available')
# Beanstalkd workers
workers_tube = self.conf.get('distributed_beanstalkd_worker_tube') \
or self.tool.DEFAULT_DISTRIBUTED_BEANSTALKD_WORKER_TUBE
self.beanstalkd_workers = dict()
for beanstalkd in locate_tube(all_available_beanstalkd.values(),
workers_tube):
beanstalkd_worker = BeanstalkdSender(beanstalkd['addr'],
workers_tube, self.logger)
self.beanstalkd_workers[beanstalkd['addr']] = beanstalkd_worker
self.logger.info(
'Beanstalkd %s using tube %s is selected as a worker',
beanstalkd_worker.addr, beanstalkd_worker.tube)
if not self.beanstalkd_workers:
raise OioException('No beanstalkd worker available')
nb_workers = len(self.beanstalkd_workers)
if self.max_items_per_second > 0:
# Max 2 seconds in advance
queue_size_per_worker = self.max_items_per_second * 2 / nb_workers
else:
queue_size_per_worker = 64
for _, beanstalkd_worker in self.beanstalkd_workers.items():
beanstalkd_worker.low_limit = queue_size_per_worker / 2
beanstalkd_worker.high_limit = queue_size_per_worker
# Beanstalkd reply
beanstalkd_reply = dict()
try:
local_services = conscience_client.local_services()
for local_service in local_services:
if local_service['type'] != 'beanstalkd':
continue
beanstalkd = all_available_beanstalkd.get(
local_service['addr'])
if beanstalkd is None:
continue
if beanstalkd_reply \
and beanstalkd_reply['score'] >= beanstalkd['score']:
continue
beanstalkd_reply = beanstalkd
except Exception as exc: # pylint: disable=broad-except
self.logger.warning(
'ERROR when searching for beanstalkd locally: %s', exc)
if not beanstalkd_reply:
self.logger.warn('No beanstalkd available locally')
try:
beanstalkd = conscience_client.next_instance('beanstalkd')
beanstalkd_reply = all_available_beanstalkd[beanstalkd['addr']]
except Exception as exc: # pylint: disable=broad-except
self.logger.warning('ERROR when searching for beanstalkd: %s',
exc)
beanstalkd_reply_addr = beanstalkd_reply['addr']
# If the tube exists, another service must have already used this tube
tube_reply = workers_tube + '.reply.' + str(time.time())
tubes = Beanstalk.from_url('beanstalk://' +
beanstalkd_reply_addr).tubes()
if tube_reply in tubes:
raise OioException('Beanstalkd %s using tube %s is already used')
self.beanstalkd_reply = BeanstalkdListener(beanstalkd_reply_addr,
tube_reply, self.logger)
self.logger.info(
'Beanstalkd %s using tube %s is selected for the replies',
self.beanstalkd_reply.addr, self.beanstalkd_reply.tube)
def init(self):
queue_url = self.conf.get('queue_url', 'tcp://127.0.0.1:11300')
self.beanstalk = Beanstalk.from_url(queue_url)
self.tube = self.conf.get('tube', 'notif')
self.beanstalk.use(self.tube)
def beanstalk(self):
if not self._beanstalk:
self._beanstalk = Beanstalk.from_url(
random.choice(self.queue_url))
return self._beanstalk
def beanstalkd(self):
if not self._beanstalk:
self._beanstalk = Beanstalk.from_url(self.conf['queue_url'])
return self._beanstalk
def beanstalkd0(self):
if not self._beanstalkd0:
self._beanstalkd0 = Beanstalk.from_url(self.conf['main_queue_url'])
return self._beanstalkd0
