def get_dataset_bounds(self, dataset_id=''): """@brief Get the bounding coordinates of the dataset using a couch map/reduce query @param dataset_id @result bounds is a dictionary containing spatial and temporal bounds of the dataset in standard units @param dataset_id str @retval bounds Unknown """ dataset = self.read_dataset(dataset_id=dataset_id) key = dataset.primary_view_key # stream_id ar = gevent.event.AsyncResult() def ar_timeout(db): opts = {'start_key': [key, 0], 'end_key': [key, 2]} try: results = db.query_view("datasets/bounds", opts=opts)[0]['value'] except IndexError: # Means there are no results results = {} ar.set(results) db = self.container.datastore_manager.get_datastore( dataset.datastore_name) g = Greenlet(ar_timeout, db) g.start() bounds = ar.get(timeout=5) return bounds
def launch_benchmark(transform_number=1, primer=1, message_length=4): import gevent from gevent.greenlet import Greenlet from pyon.util.containers import DotDict from pyon.net.transport import NameTrio from pyon.net.endpoint import Publisher import uuid num = transform_number msg_len = message_length transforms = list() pids = 1 TransformBenchTesting.message_length = message_length cc = Container.instance pub = Publisher(to_name=NameTrio(get_sys_name(), str(uuid.uuid4())[0:6])) for i in xrange(num): tbt = cc.proc_manager._create_service_instance( str(pids), 'tbt', 'prototype.transforms.linear', 'TransformInPlace', DotDict({ 'process': { 'name': 'tbt%d' % pids, 'transform_id': pids } })) tbt.init() tbt.start() gevent.sleep(0.2) for i in xrange(primer): pub.publish(list(xrange(msg_len))) g = Greenlet(tbt.perf) g.start() transforms.append(tbt) pids += 1
def execute_replay(self): ''' @brief Spawns a greenlet to take care of the query and work ''' if not hasattr(self, 'output'): raise Inconsistent( 'The replay process requires an output stream publisher named output. Invalid configuration!' ) datastore_name = self.datastore_name key_id = self.key_id view_name = self.view_name opts = { 'start_key': [key_id, 0], 'end_key': [key_id, 2], 'include_docs': True } g = Greenlet(self._query, datastore_name=datastore_name, view_name=view_name, opts=opts, callback=lambda results: self._publish_query(results)) g.start()
def handle_multi_device_job(self, job_handler, result_handler): job_worker_pool = Pool(self.max_job_task) job_percent_per_task = \ self.job_log_utils.calculate_job_percentage( len(self.device_json), buffer_task_percent=False, total_percent=self.job_percent)[0] for device_id in self.device_json: device_data = self.device_json.get(device_id) device_fqname = ':'.join(map(str, device_data.get('device_fqname'))) job_template_fq_name = ':'.join(map(str, self.job_template.fq_name)) pr_fabric_job_template_fq_name = device_fqname + ":" + \ self.fabric_fq_name + ":" + \ job_template_fq_name self.job_log_utils.send_prouter_job_uve( self.job_template.fq_name, pr_fabric_job_template_fq_name, self.job_execution_id, job_status="IN_PROGRESS") job_worker_pool.start( Greenlet(job_handler.handle_job, result_handler, job_percent_per_task, device_id)) job_worker_pool.join()
def handle_multi_device_job(self, job_handler, result_handler): job_worker_pool = Pool(self.max_job_task) job_percent_per_task = \ self.job_log_utils.calculate_job_percentage( len(self.device_json), buffer_task_percent=False, total_percent=self.job_percent)[0] for device_id in self.device_json: if device_id in result_handler.failed_device_jobs: self._logger.debug("Not executing the next operation" "in the workflow for device: %s" % device_id) continue device_data = self.device_json.get(device_id) device_fqname = ':'.join(map(str, device_data.get('device_fqname'))) device_name = device_data.get('device_fqname', [""])[-1] job_template_fq_name = ':'.join(map(str, self.job_template.fq_name)) pr_fabric_job_template_fq_name = device_fqname + ":" + \ self.fabric_fq_name + ":" + \ job_template_fq_name self.job_log_utils.send_prouter_job_uve( self.job_template.fq_name, pr_fabric_job_template_fq_name, self.job_execution_id, job_status="IN_PROGRESS") job_worker_pool.start( Greenlet(job_handler.handle_job, result_handler, job_percent_per_task, device_id, device_name)) job_worker_pool.join()
def on_start(self): super(VizTransformProcForMatplotlibGraphs, self).on_start() #assert len(self.streams)==1 self.initDataFlag = True self.graph_data = { } # Stores a dictionary of variables : [List of values] # Need some clients self.rr_cli = ResourceRegistryServiceProcessClient( process=self, node=self.container.node) self.pubsub_cli = PubsubManagementServiceClient( node=self.container.node) # extract the various parameters passed to the transform process self.out_stream_id = self.CFG.get('process').get( 'publish_streams').get('visualization_service_submit_stream_id') # Create a publisher on the output stream #stream_route = self.pubsub_cli.register_producer(stream_id=self.out_stream_id) out_stream_pub_registrar = StreamPublisherRegistrar( process=self.container, node=self.container.node) self.out_stream_pub = out_stream_pub_registrar.create_publisher( stream_id=self.out_stream_id) self.data_product_id = self.CFG.get('data_product_id') self.stream_def_id = self.CFG.get("stream_def_id") self.stream_def = self.rr_cli.read(self.stream_def_id) # Start the thread responsible for keeping track of time and generating graphs # Mutex for ensuring proper concurrent communications between threads self.lock = RLock() self.rendering_proc = Greenlet(self.rendering_thread) self.rendering_proc.start()
def run(self, blog): ''' Initiate the thread to query, organize and publish the data ''' production = Greenlet(self._grab, blog=blog, callback=lambda: self._on_done()) production.start() self.greenlet_queue.append(production)
def generate_worker(self, target: Callable, *args, **kwargs) -> Greenlet: @functools.wraps(target) @CoroutineStrategy.save_return_value def _target_function(*_args, **_kwargs): result_value = target(*_args, **_kwargs) return result_value # return gevent.spawn(_target_function, *args, **kwargs) return Greenlet(_target_function, *args, **kwargs)
def start(self): log.debug("StreamProducer start") # Threads become efficent Greenlets with gevent streams = self.CFG.get('process', {}).get('publish_streams', None) if streams: self.output_streams = list(k for k in streams) else: self.output_streams = None self.producer_proc = Greenlet(self._trigger_func) self.producer_proc.start()
def on_start(self): super(ExampleDataProducer, self).on_start() stream_id = self.CFG.process.out_stream_id g = Greenlet(self._trigger_func, stream_id) log.debug('Starting publisher thread for simple ctd data.') g.start() log.info('Publisher Greenlet started in "%s"' % self.__class__.__name__) self.greenlet_queue = [] self.greenlet_queue.append(g) self.finished = gevent.event.Event()
def launch_benchmark(transform_number=1, primer=1, message_length=4): import gevent from gevent.greenlet import Greenlet from pyon.util.containers import DotDict from pyon.net.transport import NameTrio from pyon.net.endpoint import Publisher import numpy from pyon.ion.granule.record_dictionary import RecordDictionaryTool from pyon.ion.granule.taxonomy import TaxyTool from pyon.ion.granule.granule import build_granule tt = TaxyTool() tt.add_taxonomy_set('a') import uuid num = transform_number msg_len = message_length transforms = list() pids = 1 TransformBenchTesting.message_length = message_length cc = Container.instance pub = Publisher(to_name=NameTrio(get_sys_name(), str(uuid.uuid4())[0:6])) for i in xrange(num): tbt = cc.proc_manager._create_service_instance( str(pids), 'tbt', 'prototype.transforms.linear', 'TransformInPlaceNewGranule', DotDict({ 'process': { 'name': 'tbt%d' % pids, 'transform_id': pids } })) tbt.init() tbt.start() gevent.sleep(0.2) for i in xrange(primer): rd = RecordDictionaryTool(tt, message_length) rd['a'] = numpy.arange(message_length) gran = build_granule(data_producer_id='dp_id', taxonomy=tt, record_dictionary=rd) pub.publish(gran) g = Greenlet(tbt.perf) g.start() transforms.append(tbt) pids += 1
def test_dispatcher(self): options = { 'enabled': True, 'server': '127.0.0.1', 'active_range': '00:00 - 23:59', 'sleep_interval': '1', 'activation_probability': '1', 'username': '******', 'password': '******', 'port': 8080} dispatcher = BaitDispatcher(Mock(), options) dispatcher_greenlet = Greenlet(dispatcher.start) dispatcher_greenlet.start() gevent.sleep(2) dispatcher_greenlet.kill()
def test_dispatcher(self): options = { 'enabled': True, 'server': '127.0.0.1', 'active_range': '00:00 - 23:59', 'sleep_interval': '1', 'activation_probability': '1', 'username': '******', 'password': '******', 'port': 8080 } dispatcher = BaitDispatcher({}, None, options) dispatcher.bait_type = Mock() dispatcher_greenlet = Greenlet(dispatcher.start) dispatcher_greenlet.start() time.sleep(1) dispatcher_greenlet.kill() dispatcher.bait_type.start.assert_called()
def on_start(self): ''' Creates a publisher for each stream_id passed in as publish_streams Creates an attribute with the name matching the stream name which corresponds to the publisher ex: say we have publish_streams:{'output': my_output_stream_id } then the instance has an attribute output which corresponds to the publisher for the stream in my_output_stream_id ''' # Get the stream(s) stream_id = self.CFG.get('process',{}).get('stream_id','') self.greenlet_queue = [] self._usgs_def = USGS_stream_definition() # Stream creation is done in SA, but to make the example go for demonstration create one here if it is not provided... if not stream_id: pubsub_cli = PubsubManagementServiceClient(node=self.container.node) stream_id = pubsub_cli.create_stream( name='Example USGS Data', stream_definition=self._usgs_def, original=True, encoding='ION R2') self.stream_publisher_registrar = StreamPublisherRegistrar(process=self,node=self.container.node) # Needed to get the originator's stream_id self.stream_id = stream_id self.publisher = self.stream_publisher_registrar.create_publisher(stream_id=stream_id) self.last_time = 0 g = Greenlet(self._trigger_func, stream_id) log.warn('Starting publisher thread for simple usgs data.') g.start() self.greenlet_queue.append(g)
def on_start(self): log.warn('Entering On Start!!!') # Get the stream(s) stream_id = self.CFG.get_safe('process.stream_id', {}) self.greenlet_queue = [] # Stream creation is done in SA, but to make the example go for demonstration create one here if it is not provided... if not stream_id: pubsub_cli = PubsubManagementServiceClient( node=self.container.node) stream_def_id = pubsub_cli.create_stream_definition( name='Producer stream %s' % str(uuid4()), container=self.outgoing_stream_def) stream_id = pubsub_cli.create_stream( name='Example CTD Data', stream_definition_id=stream_def_id, original=True, encoding='ION R2') self.stream_publisher_registrar = StreamPublisherRegistrar( process=self, node=self.container.node) # Needed to get the originator's stream_id self.stream_id = stream_id self.publisher = self.stream_publisher_registrar.create_publisher( stream_id=stream_id) self.last_time = 0 g = Greenlet(self._trigger_func, stream_id) log.debug('Starting publisher thread for simple ctd data.') g.start() log.warn('Publisher Greenlet started in "%s"' % self.__class__.__name__) self.greenlet_queue.append(g)
def wrapper(run, *args, **kwargs): return Greenlet(app_context(app)(run), *args, **kwargs)
def handle_multi_device_job(self, job_handler, result_handler): job_worker_pool = Pool(POOL_SIZE) for device_id in self.job_params['device_list']: job_worker_pool.start(Greenlet(job_handler.handle_device_job, device_id, result_handler)) job_worker_pool.join()
def on_start(self): log.debug("StreamProducer start") self.producer_proc = Greenlet(self._trigger_func) self.producer_proc.start()
def _go_greenlet(self, greenlet_count, put_count, get_count, bench_item_count, watchdog_interval_ms=60000, max_item=128000, max_bytes=32 * 1024 * 1024, max_single_item_bytes=1 * 1024 * 1024, purge_min_bytes=8 * 1024 * 1024, purge_min_count=1000): """ Doc :param greenlet_count: greenlet_count :param put_count: put_count :param get_count: get_count :param bench_item_count: bench_item_count :param watchdog_interval_ms: watchdog_interval_ms :param max_item: max_item :param max_bytes: max_bytes :param max_single_item_bytes: max_single_item_bytes :param purge_min_bytes: purge_min_bytes :param purge_min_count: purge_min_count """ g_event = None g_array = None try: # Settings g_count = greenlet_count g_ms = 10000 # Continue callback loop self.callback_return = True # Go self.mem_cache = MemoryCache( watchdog_interval_ms=watchdog_interval_ms, max_item=max_item, max_bytes=max_bytes, max_single_item_bytes=max_single_item_bytes, purge_min_bytes=purge_min_bytes, purge_min_count=purge_min_count) # Item count self.bench_item_count = bench_item_count self.bench_put_weight = put_count self.bench_get_weight = get_count self.bench_ttl_min_ms = 1000 self.bench_ttl_max_ms = int(g_ms / 2) # Go self.run_event = Event() self.exception_raised = 0 self.open_count = 0 self.thread_running = AtomicIntSafe() self.thread_running_ok = AtomicIntSafe() # Item per greenlet item_per_greenlet = self.bench_item_count / g_count # Signal self.gorun_event = Event() # Alloc greenlet g_array = list() g_event = list() for _ in range(0, g_count): greenlet = Greenlet() g_array.append(greenlet) g_event.append(Event()) # Run them cur_idx = 0 for idx in range(0, len(g_array)): greenlet = g_array[idx] event = g_event[idx] greenlet.spawn(self._run_cache_bench, event, cur_idx, cur_idx + item_per_greenlet) cur_idx += item_per_greenlet SolBase.sleep(0) # Signal self.gorun_event.set() # Wait a bit dt = SolBase.mscurrent() while SolBase.msdiff(dt) < g_ms: SolBase.sleep(500) # Stat ms = SolBase.msdiff(dt) sec = float(ms / 1000.0) total_put = Meters.aig("mcs.cache_put") per_sec_put = round(float(total_put) / sec, 2) total_get = Meters.aig("mcs.cache_get_hit") + Meters.aig( "mcs.cache_get_miss") per_sec_get = round(float(total_get) / sec, 2) logger.info( "Running..., count=%s, run=%s, ok=%s, put/sec=%s get/sec=%s, cache=%s", self.open_count, self.thread_running.get(), self.thread_running_ok.get(), per_sec_put, per_sec_get, self.mem_cache) self.assertEqual(self.exception_raised, 0) # Over, signal logger.info("Signaling, count=%s", self.open_count) self.run_event.set() # Wait for g in g_event: g.wait(30.0) self.assertTrue(g.isSet()) g_event = None g_array = None # Log Meters.write_to_logger() finally: self.run_event.set() if g_event: for g in g_event: g.set() if g_array: for g in g_array: g.kill() if self.mem_cache: max_count = 0 total_size = 0 i = 0 for (k, v) in self.mem_cache._hash_key.items(): i += 1 total_size += len(k) + len(v[1]) if i < max_count: logger.info("%s => %s", k, v) self.assertEqual(total_size, self.mem_cache._current_data_bytes.get()) self.mem_cache.stop_cache() self.mem_cache = None
def _go_greenlet(self, greenlet_count, put_count, get_count, bench_item_count): """ Doc :param greenlet_count: greenlet_count :param put_count: put_count :param get_count: get_count :param bench_item_count: bench_item_count """ g_event = None g_array = None try: # Settings g_count = greenlet_count g_ms = 10000 # Continue callback loop self.callback_return = True # Go self.redis_cache = RedisCache() # Item count self.bench_item_count = bench_item_count self.bench_put_weight = put_count self.bench_get_weight = get_count self.bench_ttl_min_ms = 1000 self.bench_ttl_max_ms = int(g_ms / 2) # Go self.run_event = Event() self.exception_raised = 0 self.open_count = 0 self.thread_running = AtomicIntSafe() self.thread_running_ok = AtomicIntSafe() # Item per greenlet item_per_greenlet = self.bench_item_count / g_count # Signal self.gorun_event = Event() # Alloc greenlet g_array = list() g_event = list() for _ in range(0, g_count): greenlet = Greenlet() g_array.append(greenlet) g_event.append(Event()) # Run them cur_idx = 0 for idx in range(0, len(g_array)): greenlet = g_array[idx] event = g_event[idx] greenlet.spawn(self._run_cache_bench, event, cur_idx, cur_idx + item_per_greenlet) cur_idx += item_per_greenlet SolBase.sleep(0) # Signal self.gorun_event.set() # Wait a bit dt = SolBase.mscurrent() while SolBase.msdiff(dt) < g_ms: SolBase.sleep(500) # Stat ms = SolBase.msdiff(dt) sec = float(ms / 1000.0) total_put = Meters.aig("rcs.cache_put") per_sec_put = round(float(total_put) / sec, 2) total_get = Meters.aig("rcs.cache_get_hit") + Meters.aig( "rcs.cache_get_miss") per_sec_get = round(float(total_get) / sec, 2) logger.info( "Running..., count=%s, run=%s, ok=%s, put/sec=%s get/sec=%s, cache=%s", self.open_count, self.thread_running.get(), self.thread_running_ok.get(), per_sec_put, per_sec_get, self.redis_cache) self.assertEqual(self.exception_raised, 0) # Over, signal logger.info("Signaling, count=%s", self.open_count) self.run_event.set() # Wait for g in g_event: g.wait(30.0) self.assertTrue(g.isSet()) g_event = None g_array = None # Log Meters.write_to_logger() finally: self.run_event.set() if g_event: for g in g_event: g.set() if g_array: for g in g_array: g.kill() if self.redis_cache: self.redis_cache.stop_cache() self.redis_cache = None
def handle_multi_device_job(self, job_handler, result_handler): job_worker_pool = Pool(self.max_job_task) for device_id in self.job_params['device_list']: job_worker_pool.start( Greenlet(job_handler.handle_job, result_handler, device_id)) job_worker_pool.join()
# gevent depends on a gevent library, greenlet is a simple library for wrapping functions and arguments togther. # Lets see them in action. # In[ ]: # Creating a simple function and wrapping it in greenlet def myfunction(arg1, arg2, **kwargs): print(arg1, arg2, kwargs) return 100 # In[ ]: g = Greenlet(myfunction, 'One', 'Two', now='Buckle my shoe') # create a Greenlet instance using c'tor g.start( ) # and then call start(), which immediately calls it, returns None always g.join() print('Finished') print('Greenlet.value', g.value) # Stores the function return value # In[ ]: # the other way is to use the spawn() method, which creates an instance and calls start() too. # So it is a shorthand and widely used jobs = [ gevent.spawn(myfunction, '1', '2', now='Buckle my shoe') for i in range(0, 5) ] gevent.joinall(
def spawn(self, function, *args, **kwargs): greenlet = Greenlet(function, *args, **kwargs) self.start(greenlet) return greenlet