def delete_queue(*queues): """Delete queues from RabbitMQ""" k = Kuyruk() with k.connection() as conn: for name in queues: logger.debug("deleting queue %s", name) with conn.channel() as ch: ch.queue_delete(queue=name)
def delete_queue(*queues): """Delete queues from RabbitMQ""" for name in queues: try: ch = Kuyruk().channel() RabbitQueue(name, ch).delete() ch.close() except ChannelClosed: pass
def test_task_queue_name(self): """Task queue name is correct""" k = Kuyruk() def f(): return t = k.task(f) self.assertEqual(t._queue_for_host(None), 'kuyruk') self.assertEqual(t._queue_for_host('foo'), 'kuyruk.foo')
def test_task_queue_name(self): """Task queue name is correct""" k = Kuyruk() def f(): return t = k.task()(f) self.assertEqual(t._queue_for_host(None), 'kuyruk') self.assertEqual(t._queue_for_host('foo'), 'kuyruk.foo')
def len_queue(queue): with Kuyruk().channel() as ch: _, count, _ = ch.queue_declare(queue=queue, durable=True, passive=True, auto_delete=False) return count
def test_invalid_json(self): """Message is dropped when JSON is not valid""" with run_worker() as worker: worker.expect('Consumer started') with Kuyruk().channel() as ch: msg = amqp.Message(body='foo') ch.basic_publish(msg, exchange='', routing_key='kuyruk') worker.expect('Cannot decode message') self.assertEqual(len_queue('kuyruk'), 0)
def test_queue_names(self): """Hostname is appended to local queues""" given = ['foo', 'bar.localhost'] k = Kuyruk() w = Worker(k, Args(queues=given)) hostname = socket.gethostname() expected = ['foo', 'bar.%s' % hostname] self.assertListEqual(w.queues, expected)
def test_invalid_task_path(self): """Message is dropped when task cannot be imported""" with run_worker() as worker: worker.expect('Consumer started') with Kuyruk().channel() as ch: desc = {'module': 'kuyruk', 'function': 'foo'} body = json.dumps(desc) msg = amqp.Message(body=body) ch.basic_publish(msg, exchange='', routing_key='kuyruk') worker.expect('Cannot import task') self.assertEqual(len_queue('kuyruk'), 0)
def _test_function_name(self, args, cwd, name): with Kuyruk(config=config).channel() as ch: print(cwd, args, name) ch.queue_delete("kuyruk") # Every call sends a task to the queue run_python(args, cwd=cwd) # Can we load the task by name? got = get_name() assert got == name, got
def test_function_name(self): cases = [ ('onefile.py', 'loader', 'onefile.print_message'), ('main.py', 'loader/appdirectory', 'tasks.print_message'), ('-m apppackage.main', 'loader', 'apppackage.tasks.print_message'), ('-m apppackage.scripts.send_message', 'loader', 'apppackage.tasks.print_message'), ] with Kuyruk().channel() as ch: for args, cwd, name in cases: print(cwd, args, name) ch.queue_delete("kuyruk") # Every call sends a task to the queue run_python(args, cwd=cwd) # Can we load the task by name? got = get_name() assert got == name, got
"ENV variable KUYRUK is not set, assuming a worker setup") return function(fn) elif mytype.upper() == 'SEND': logger.debug( "ENV variable KUYRUK='%s', assuming you will be sending commands", mytype) return fn else: logger.debug("ENV variable KUYRUK='%s', assuming a worker setup", mytype) return function(fn) return decorate kuyruk = Kuyruk(config) # taskify is our own conditional decorator @taskify(kuyruk.task(socket.gethostname())) def sendfile(filename, contents): """ copy a file """ f = open(filename, "w") result = f.write(contents) f.close() logger.info("> SENDILE: '%s' received", filename) return result
import os import sys import signal import subprocess from time import sleep from kuyruk import Kuyruk kuyruk = Kuyruk() @kuyruk.task(queue='forever') def run_forever(): # Execute this script path = os.path.abspath(__file__) p = subprocess.Popen([sys.executable, path]) p.wait() print "Done." if __name__ == '__main__': # A script that cannot be interrupted def handle_signal(signum, frame): print 'SIGNAL', signum # Ignore all signals signal.signal(signal.SIGINT, handle_signal) signal.signal(signal.SIGTERM, handle_signal)
def new_instance(): config = Config() config.from_pyfile('/tmp/kuyruk_config.py') return Kuyruk(config=config)
def test_default_queue(self): """Consume from "kuyruk" if no queue is given""" k = Kuyruk() w = Worker(k, Args()) self.assertListEqual(w.queues, ['kuyruk'])
_Splitter = getattr( __import__(custom_array['Splitter']['path'] + '.' + custom_array['Splitter']['module'], fromlist=[custom_array['Splitter']['module']]), custom_array['Splitter']['module']) if 'OCForInvoices_splitter' not in custom_array: from bin.src.process import OCForInvoices_splitter else: OCForInvoices_splitter = getattr( __import__(custom_array['OCForInvoices_splitter']['path'], fromlist=[custom_array['OCForInvoices_splitter']['module'] ]), custom_array['OCForInvoices_splitter']['module']) OCforInvoices = Kuyruk() OCforInvoices.config.MANAGER_HOST = "127.0.0.1" OCforInvoices.config.MANAGER_PORT = 16502 OCforInvoices.config.MANAGER_HTTP_PORT = 16503 m = Manager(OCforInvoices) def check_file(Files, path, Config, Log): if not Files.check_file_integrity(path, Config): Log.error('The integrity of file could\'nt be verified : ' + str(path)) return False def timer(startTime, endTime):
""" from __future__ import print_function import sys import string import random from time import sleep from kuyruk import Kuyruk, Task from kuyruk import signals from kuyruk.config import Config # Override defaults for testing Config.WORKER_LOGGING_LEVEL = "debug" Config.WORKER_MAX_LOAD = 999 kuyruk = Kuyruk() # These functions below needs to be at module level in order that # Kuyruk worker to determine their fully qualified name. @kuyruk.task def echo(message): print(message) must_be_called() @kuyruk.task(queue='another_queue') def echo_another(message): print(message)
if 'OCForInvoices' not in custom_array: from bin.src.process import OCForInvoices as OCForInvoices_process else: OCForInvoices_process = getattr( __import__(custom_array['OCForInvoices']['path'], fromlist=[custom_array['OCForInvoices']['module']]), custom_array['OCForInvoices']['module']) # if 'invoice_classification' not in custom_array: # from bin.src.invoice_classification import invoice_classification # else: # invoice_classification = getattr(__import__(custom_array['invoice_classification']['path'], fromlist=[custom_array['invoice_classification']['module']]), # custom_array['invoice_classification']['module']) OCforInvoices_worker = Kuyruk() OCforInvoices_worker.config.MANAGER_HOST = "127.0.0.1" OCforInvoices_worker.config.MANAGER_PORT = 16501 OCforInvoices_worker.config.MANAGER_HTTP_PORT = 16500 m = Manager(OCforInvoices_worker) def check_file(files, path, config, log): if not files.check_file_integrity(path, config): log.error('The integrity of file could\'nt be verified : ' + str(path)) return False def timer(start_time, end_time):
from __future__ import print_function from kuyruk import Kuyruk, Config config = Config() config.from_pyfile('/tmp/kuyruk_config.py') kuyruk = Kuyruk(config=config) @kuyruk.task() def print_message(m): print(m)
cfg_parser.read([section, inifile]) conf.RABBIT_HOST = cfg_parser[section]['RABBIT_HOST'] conf.RABBIT_PORT = int(cfg_parser[section]['RABBIT_PORT']) conf.RABBIT_USER = cfg_parser[section]['RABBIT_USER'] conf.RABBIT_PASSWORD = cfg_parser[section]['RABBIT_PASSWORD'] conf.RABBIT_VIRTUAL_HOST = cfg_parser[section]['RABBIT_VIRTUAL_HOST'] conf.WORKER_MAX_RUN_TIME = int(cfg_parser[section]['WORKER_MAX_RUN_TIME']) conf.WORKER_LOGGING_LEVEL = cfg_parser[section]['WORKER_LOGGING_LEVEL'] conf.RABBIT_SSL = cfg_parser[section].getboolean('RABBIT_SSL') return conf # don't let these lines fool you, they do a lot of work try: kuyruk = Kuyruk(readconfig("/etc/kuyruk.ini", "ansible_tasks")) except: Logger.err("cannot initialize kuyruk") exit(1) #taskify 'playbook' and bind it to queue ha-ansible @kuyruk.task("ha-ansible") def playbook(hostname, tag=""): """ The actual task to send over rabbitmq and execute when received. We are using a multithreaded IO reader because the output is quite long and we would rather wathc it go by rather than wait for the end of the task. """ def _outputLoop(fd, dummy=0):
def get_name(): with Kuyruk().channel() as ch: message = ch.basic_get("kuyruk") desc = json.loads(message.body) return '.'.join([desc['module'], desc['function']])
# useful to use the worker and avoid ModuleNotFoundError sys.path.append(os.path.join(os.path.dirname(__file__), '..')) from kuyruk import Kuyruk from kuyruk_manager import Manager import src.classes.Log as logClass import src.classes.Locale as localeClass import src.classes.Images as imagesClass import src.classes.Config as configClass import src.classes.PyTesseract as ocrClass from src.process.OCForMaarch import process, get_process_name from src.classes.Mail import move_batch_to_error, send_email_error_pj from src.classes.SMTP import SMTP import src.classes.Separator as separatorClass import src.classes.WebServices as webserviceClass OCforMaarch = Kuyruk() OCforMaarch.config.MANAGER_HOST = "127.0.0.1" OCforMaarch.config.MANAGER_PORT = 16501 OCforMaarch.config.MANAGER_HTTP_PORT = 16500 m = Manager(OCforMaarch) def str2bool(value): """ Function to convert string to boolean :return: Boolean """ return value.lower() in "true"