def __init__(self, name, in_queue, out_queue, function):
Thread.__init__(self, name, self.event_loop)
self.in_queue = in_queue
self.out_queue = out_queue
self.function = function
self.num_runs = 0
self.start()
def __init__(
self,
exchange, # name of the Pulse exchange
topic, # message name pattern to subscribe to ('#' is wildcard)
target=None, # WILL BE CALLED WITH PULSE PAYLOADS AND ack() IF COMPLETE$ED WITHOUT EXCEPTION
target_queue=None, # (aka self.queue) WILL BE FILLED WITH PULSE PAYLOADS
host='pulse.mozilla.org', # url to connect,
port=5671, # tcp port
user=None,
password=None,
vhost="/",
start=0, # USED AS STARTING POINT FOR ASSIGNING THE _meta.count ATTRIBUTE
ssl=True,
applabel=None,
heartbeat=False, # True to also get the Pulse heartbeat message
durable=False, # True to keep queue after shutdown
serializer='json',
broker_timezone='GMT',
settings=None
):
self.target_queue = target_queue
self.pulse_target = target
if (target_queue == None and target == None) or (target_queue != None and target != None):
Log.error("Expecting a queue (for fast digesters) or a target (for slow digesters)")
Thread.__init__(self, name="Pulse consumer for " + settings.exchange, target=self._worker)
self.settings = settings
settings.callback = self._got_result
settings.user = coalesce(settings.user, settings.username)
settings.applabel = coalesce(settings.applable, settings.queue, settings.queue_name)
settings.topic = topic
self.pulse = ModifiedGenericConsumer(settings, connect=True, **settings)
self.count = coalesce(start, 0)
self.start()
def __init__(
self,
exchange, # name of the Pulse exchange
topic, # message name pattern to subscribe to ('#' is wildcard)
target=None, # WILL BE CALLED WITH PULSE PAYLOADS AND ack() IF COMPLETE$ED WITHOUT EXCEPTION
target_queue=None, # (aka self.queue) WILL BE FILLED WITH PULSE PAYLOADS
host='pulse.mozilla.org', # url to connect,
port=5671, # tcp port
user=None,
password=None,
vhost="/",
start=0, # USED AS STARTING POINT FOR ASSIGNING THE _meta.count ATTRIBUTE
ssl=True,
applabel=None,
heartbeat=False, # True to also get the Pulse heartbeat message
durable=False, # True to keep queue after shutdown
serializer='json',
broker_timezone='GMT',
settings=None):
global count
count = coalesce(start, 0)
self.target_queue = target_queue
self.pulse_target = target
if (target_queue == None and target == None) or (target_queue != None
and target != None):
Log.error(
"Expecting a queue (for fast digesters) or a target (for slow digesters)"
)
Thread.__init__(self,
name="Pulse consumer for " + settings.exchange,
target=self._worker)
self.settings = settings
settings.callback = self._got_result
settings.user = coalesce(settings.user, settings.username)
settings.applabel = coalesce(settings.applable, settings.queue,
settings.queue_name)
settings.topic = topic
self.pulse = ModifiedGenericConsumer(settings,
connect=True,
**settings)
self.start()
class ETL(Thread):
@use_settings
def __init__(self,
name,
work_queue,
workers,
resources,
please_stop,
wait_forever=False,
settings=None):
# FIND THE WORKERS METHODS
settings.workers = []
for w in workers:
w = deepcopy(w)
for existing_worker in settings.workers:
try:
fuzzytestcase.assertAlmostEqual(existing_worker.source,
w.source)
fuzzytestcase.assertAlmostEqual(
existing_worker.transformer, w.transformer)
# SAME SOURCE AND TRANSFORMER, MERGE THE destinations
except Exception, e:
continue
destination = get_container(w.destination)
existing_worker._destination = Split(
existing_worker._destination, destination)
break
else:
t_name = w.transformer
w._transformer = dot.get_attr(sys.modules, t_name)
if not w._transformer:
Log.error(
"Can not find {{path}} to transformer (are you sure you are pointing to a function?)",
path=t_name)
w._source = get_container(w.source)
w._destination = get_container(w.destination)
settings.workers.append(w)
w._notify = []
for notify in listwrap(w.notify):
w._notify.append(aws.Queue(notify))
self.resources = resources
self.settings = settings
if isinstance(work_queue, Mapping):
self.work_queue = aws.Queue(work_queue)
else:
self.work_queue = work_queue
Thread.__init__(self, name, self.loop, please_stop=please_stop)
self.start()
