def delete_queue(*queues):
"""Delete queues from RabbitMQ"""
k = Kuyruk()
with k.connection() as conn:
for name in queues:
logger.debug("deleting queue %s", name)
with conn.channel() as ch:
ch.queue_delete(queue=name)
def delete_queue(*queues):
"""Delete queues from RabbitMQ"""
k = Kuyruk()
with k.connection() as conn:
for name in queues:
logger.debug("deleting queue %s", name)
with conn.channel() as ch:
ch.queue_delete(queue=name)
def delete_queue(*queues):
"""Delete queues from RabbitMQ"""
for name in queues:
try:
ch = Kuyruk().channel()
RabbitQueue(name, ch).delete()
ch.close()
except ChannelClosed:
pass
def test_task_queue_name(self):
"""Task queue name is correct"""
k = Kuyruk()
def f():
return
t = k.task(f)
self.assertEqual(t._queue_for_host(None), 'kuyruk')
self.assertEqual(t._queue_for_host('foo'), 'kuyruk.foo')
def test_task_queue_name(self):
"""Task queue name is correct"""
k = Kuyruk()
def f():
return
t = k.task()(f)
self.assertEqual(t._queue_for_host(None), 'kuyruk')
self.assertEqual(t._queue_for_host('foo'), 'kuyruk.foo')
def len_queue(queue):
with Kuyruk().channel() as ch:
_, count, _ = ch.queue_declare(queue=queue,
durable=True,
passive=True,
auto_delete=False)
return count
def test_invalid_json(self):
"""Message is dropped when JSON is not valid"""
with run_worker() as worker:
worker.expect('Consumer started')
with Kuyruk().channel() as ch:
msg = amqp.Message(body='foo')
ch.basic_publish(msg, exchange='', routing_key='kuyruk')
worker.expect('Cannot decode message')
self.assertEqual(len_queue('kuyruk'), 0)
def test_queue_names(self):
"""Hostname is appended to local queues"""
given = ['foo', 'bar.localhost']
k = Kuyruk()
w = Worker(k, Args(queues=given))
hostname = socket.gethostname()
expected = ['foo', 'bar.%s' % hostname]
self.assertListEqual(w.queues, expected)
def test_invalid_task_path(self):
"""Message is dropped when task cannot be imported"""
with run_worker() as worker:
worker.expect('Consumer started')
with Kuyruk().channel() as ch:
desc = {'module': 'kuyruk', 'function': 'foo'}
body = json.dumps(desc)
msg = amqp.Message(body=body)
ch.basic_publish(msg, exchange='', routing_key='kuyruk')
worker.expect('Cannot import task')
self.assertEqual(len_queue('kuyruk'), 0)
def _test_function_name(self, args, cwd, name):
with Kuyruk(config=config).channel() as ch:
print(cwd, args, name)
ch.queue_delete("kuyruk")
# Every call sends a task to the queue
run_python(args, cwd=cwd)
# Can we load the task by name?
got = get_name()
assert got == name, got
def test_function_name(self):
cases = [
('onefile.py', 'loader', 'onefile.print_message'),
('main.py', 'loader/appdirectory', 'tasks.print_message'),
('-m apppackage.main', 'loader', 'apppackage.tasks.print_message'),
('-m apppackage.scripts.send_message', 'loader',
'apppackage.tasks.print_message'),
]
with Kuyruk().channel() as ch:
for args, cwd, name in cases:
print(cwd, args, name)
ch.queue_delete("kuyruk")
# Every call sends a task to the queue
run_python(args, cwd=cwd)
# Can we load the task by name?
got = get_name()
assert got == name, got
"ENV variable KUYRUK is not set, assuming a worker setup")
return function(fn)
elif mytype.upper() == 'SEND':
logger.debug(
"ENV variable KUYRUK='%s', assuming you will be sending commands",
mytype)
return fn
else:
logger.debug("ENV variable KUYRUK='%s', assuming a worker setup",
mytype)
return function(fn)
return decorate
kuyruk = Kuyruk(config)
# taskify is our own conditional decorator
@taskify(kuyruk.task(socket.gethostname()))
def sendfile(filename, contents):
"""
copy a file
"""
f = open(filename, "w")
result = f.write(contents)
f.close()
logger.info("> SENDILE: '%s' received", filename)
return result
import os
import sys
import signal
import subprocess
from time import sleep
from kuyruk import Kuyruk
kuyruk = Kuyruk()
@kuyruk.task(queue='forever')
def run_forever():
# Execute this script
path = os.path.abspath(__file__)
p = subprocess.Popen([sys.executable, path])
p.wait()
print "Done."
if __name__ == '__main__':
# A script that cannot be interrupted
def handle_signal(signum, frame):
print 'SIGNAL', signum
# Ignore all signals
signal.signal(signal.SIGINT, handle_signal)
signal.signal(signal.SIGTERM, handle_signal)
def new_instance():
config = Config()
config.from_pyfile('/tmp/kuyruk_config.py')
return Kuyruk(config=config)
def test_default_queue(self):
"""Consume from "kuyruk" if no queue is given"""
k = Kuyruk()
w = Worker(k, Args())
self.assertListEqual(w.queues, ['kuyruk'])
_Splitter = getattr(
__import__(custom_array['Splitter']['path'] + '.' +
custom_array['Splitter']['module'],
fromlist=[custom_array['Splitter']['module']]),
custom_array['Splitter']['module'])
if 'OCForInvoices_splitter' not in custom_array:
from bin.src.process import OCForInvoices_splitter
else:
OCForInvoices_splitter = getattr(
__import__(custom_array['OCForInvoices_splitter']['path'],
fromlist=[custom_array['OCForInvoices_splitter']['module']
]),
custom_array['OCForInvoices_splitter']['module'])
OCforInvoices = Kuyruk()
OCforInvoices.config.MANAGER_HOST = "127.0.0.1"
OCforInvoices.config.MANAGER_PORT = 16502
OCforInvoices.config.MANAGER_HTTP_PORT = 16503
m = Manager(OCforInvoices)
def check_file(Files, path, Config, Log):
if not Files.check_file_integrity(path, Config):
Log.error('The integrity of file could\'nt be verified : ' + str(path))
return False
def timer(startTime, endTime):
"""
from __future__ import print_function
import sys
import string
import random
from time import sleep
from kuyruk import Kuyruk, Task
from kuyruk import signals
from kuyruk.config import Config
# Override defaults for testing
Config.WORKER_LOGGING_LEVEL = "debug"
Config.WORKER_MAX_LOAD = 999
kuyruk = Kuyruk()
# These functions below needs to be at module level in order that
# Kuyruk worker to determine their fully qualified name.
@kuyruk.task
def echo(message):
print(message)
must_be_called()
@kuyruk.task(queue='another_queue')
def echo_another(message):
print(message)
if 'OCForInvoices' not in custom_array:
from bin.src.process import OCForInvoices as OCForInvoices_process
else:
OCForInvoices_process = getattr(
__import__(custom_array['OCForInvoices']['path'],
fromlist=[custom_array['OCForInvoices']['module']]),
custom_array['OCForInvoices']['module'])
# if 'invoice_classification' not in custom_array:
# from bin.src.invoice_classification import invoice_classification
# else:
# invoice_classification = getattr(__import__(custom_array['invoice_classification']['path'], fromlist=[custom_array['invoice_classification']['module']]),
# custom_array['invoice_classification']['module'])
OCforInvoices_worker = Kuyruk()
OCforInvoices_worker.config.MANAGER_HOST = "127.0.0.1"
OCforInvoices_worker.config.MANAGER_PORT = 16501
OCforInvoices_worker.config.MANAGER_HTTP_PORT = 16500
m = Manager(OCforInvoices_worker)
def check_file(files, path, config, log):
if not files.check_file_integrity(path, config):
log.error('The integrity of file could\'nt be verified : ' + str(path))
return False
def timer(start_time, end_time):
from __future__ import print_function
from kuyruk import Kuyruk, Config
config = Config()
config.from_pyfile('/tmp/kuyruk_config.py')
kuyruk = Kuyruk(config=config)
@kuyruk.task()
def print_message(m):
print(m)
cfg_parser.read([section, inifile])
conf.RABBIT_HOST = cfg_parser[section]['RABBIT_HOST']
conf.RABBIT_PORT = int(cfg_parser[section]['RABBIT_PORT'])
conf.RABBIT_USER = cfg_parser[section]['RABBIT_USER']
conf.RABBIT_PASSWORD = cfg_parser[section]['RABBIT_PASSWORD']
conf.RABBIT_VIRTUAL_HOST = cfg_parser[section]['RABBIT_VIRTUAL_HOST']
conf.WORKER_MAX_RUN_TIME = int(cfg_parser[section]['WORKER_MAX_RUN_TIME'])
conf.WORKER_LOGGING_LEVEL = cfg_parser[section]['WORKER_LOGGING_LEVEL']
conf.RABBIT_SSL = cfg_parser[section].getboolean('RABBIT_SSL')
return conf
# don't let these lines fool you, they do a lot of work
try:
kuyruk = Kuyruk(readconfig("/etc/kuyruk.ini", "ansible_tasks"))
except:
Logger.err("cannot initialize kuyruk")
exit(1)
#taskify 'playbook' and bind it to queue ha-ansible
@kuyruk.task("ha-ansible")
def playbook(hostname, tag=""):
"""
The actual task to send over rabbitmq and execute when received.
We are using a multithreaded IO reader because the output is quite long
and we would rather wathc it go by rather than wait for the end
of the task.
"""
def _outputLoop(fd, dummy=0):
def get_name():
with Kuyruk().channel() as ch:
message = ch.basic_get("kuyruk")
desc = json.loads(message.body)
return '.'.join([desc['module'], desc['function']])
# useful to use the worker and avoid ModuleNotFoundError
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
from kuyruk import Kuyruk
from kuyruk_manager import Manager
import src.classes.Log as logClass
import src.classes.Locale as localeClass
import src.classes.Images as imagesClass
import src.classes.Config as configClass
import src.classes.PyTesseract as ocrClass
from src.process.OCForMaarch import process, get_process_name
from src.classes.Mail import move_batch_to_error, send_email_error_pj
from src.classes.SMTP import SMTP
import src.classes.Separator as separatorClass
import src.classes.WebServices as webserviceClass
OCforMaarch = Kuyruk()
OCforMaarch.config.MANAGER_HOST = "127.0.0.1"
OCforMaarch.config.MANAGER_PORT = 16501
OCforMaarch.config.MANAGER_HTTP_PORT = 16500
m = Manager(OCforMaarch)
def str2bool(value):
"""
Function to convert string to boolean
:return: Boolean
"""
return value.lower() in "true"
