def __init__(self, name, in_queue, out_queue, function): Thread.__init__(self, name, self.event_loop) self.in_queue = in_queue self.out_queue = out_queue self.function = function self.num_runs = 0 self.start()
def __init__( self, exchange, # name of the Pulse exchange topic, # message name pattern to subscribe to ('#' is wildcard) target=None, # WILL BE CALLED WITH PULSE PAYLOADS AND ack() IF COMPLETE$ED WITHOUT EXCEPTION target_queue=None, # (aka self.queue) WILL BE FILLED WITH PULSE PAYLOADS host='pulse.mozilla.org', # url to connect, port=5671, # tcp port user=None, password=None, vhost="/", start=0, # USED AS STARTING POINT FOR ASSIGNING THE _meta.count ATTRIBUTE ssl=True, applabel=None, heartbeat=False, # True to also get the Pulse heartbeat message durable=False, # True to keep queue after shutdown serializer='json', broker_timezone='GMT', settings=None ): self.target_queue = target_queue self.pulse_target = target if (target_queue == None and target == None) or (target_queue != None and target != None): Log.error("Expecting a queue (for fast digesters) or a target (for slow digesters)") Thread.__init__(self, name="Pulse consumer for " + settings.exchange, target=self._worker) self.settings = settings settings.callback = self._got_result settings.user = coalesce(settings.user, settings.username) settings.applabel = coalesce(settings.applable, settings.queue, settings.queue_name) settings.topic = topic self.pulse = ModifiedGenericConsumer(settings, connect=True, **settings) self.count = coalesce(start, 0) self.start()
def __init__( self, exchange, # name of the Pulse exchange topic, # message name pattern to subscribe to ('#' is wildcard) target=None, # WILL BE CALLED WITH PULSE PAYLOADS AND ack() IF COMPLETE$ED WITHOUT EXCEPTION target_queue=None, # (aka self.queue) WILL BE FILLED WITH PULSE PAYLOADS host='pulse.mozilla.org', # url to connect, port=5671, # tcp port user=None, password=None, vhost="/", start=0, # USED AS STARTING POINT FOR ASSIGNING THE _meta.count ATTRIBUTE ssl=True, applabel=None, heartbeat=False, # True to also get the Pulse heartbeat message durable=False, # True to keep queue after shutdown serializer='json', broker_timezone='GMT', settings=None): global count count = coalesce(start, 0) self.target_queue = target_queue self.pulse_target = target if (target_queue == None and target == None) or (target_queue != None and target != None): Log.error( "Expecting a queue (for fast digesters) or a target (for slow digesters)" ) Thread.__init__(self, name="Pulse consumer for " + settings.exchange, target=self._worker) self.settings = settings settings.callback = self._got_result settings.user = coalesce(settings.user, settings.username) settings.applabel = coalesce(settings.applable, settings.queue, settings.queue_name) settings.topic = topic self.pulse = ModifiedGenericConsumer(settings, connect=True, **settings) self.start()
class ETL(Thread): @use_settings def __init__(self, name, work_queue, workers, resources, please_stop, wait_forever=False, settings=None): # FIND THE WORKERS METHODS settings.workers = [] for w in workers: w = deepcopy(w) for existing_worker in settings.workers: try: fuzzytestcase.assertAlmostEqual(existing_worker.source, w.source) fuzzytestcase.assertAlmostEqual( existing_worker.transformer, w.transformer) # SAME SOURCE AND TRANSFORMER, MERGE THE destinations except Exception, e: continue destination = get_container(w.destination) existing_worker._destination = Split( existing_worker._destination, destination) break else: t_name = w.transformer w._transformer = dot.get_attr(sys.modules, t_name) if not w._transformer: Log.error( "Can not find {{path}} to transformer (are you sure you are pointing to a function?)", path=t_name) w._source = get_container(w.source) w._destination = get_container(w.destination) settings.workers.append(w) w._notify = [] for notify in listwrap(w.notify): w._notify.append(aws.Queue(notify)) self.resources = resources self.settings = settings if isinstance(work_queue, Mapping): self.work_queue = aws.Queue(work_queue) else: self.work_queue = work_queue Thread.__init__(self, name, self.loop, please_stop=please_stop) self.start()