async def test_multiple_execute(self):
m = await asyncio.gather(send('arbiter', 'run', wait, 1.2),
send('arbiter', 'ping'),
send('arbiter', 'echo', 'ciao!'),
send('arbiter', 'run', wait, 2.1),
send('arbiter', 'echo', 'ciao again!'))
self.assertTrue(m[0] >= 1.1)
self.assertEqual(m[1], 'pong')
self.assertEqual(m[2], 'ciao!')
self.assertTrue(m[3] >= 2.0)
self.assertEqual(m[4], 'ciao again!')
async def test_multiple_execute(self):
m = await asyncio.gather(
send('arbiter', 'run', wait, 1.2),
send('arbiter', 'ping'),
send('arbiter', 'echo', 'ciao!'),
send('arbiter', 'run', wait, 2.1),
send('arbiter', 'echo', 'ciao again!')
)
self.assertTrue(m[0] >= 1.1)
self.assertEqual(m[1], 'pong')
self.assertEqual(m[2], 'ciao!')
self.assertTrue(m[3] >= 2.0)
self.assertEqual(m[4], 'ciao again!')
async def download_homepages(request, message):
print("DOWNLOAD", message)
current_actor = get_actor()
request.actor.logger.info("Download homepages: " + str(current_actor) +
" Urls: " + str(message))
urls = message
https = []
for url in urls:
with urllib.request.urlopen(url) as conn:
http = conn.read()
https.append([http])
spawned_actors = []
for i, http in enumerate(https):
spawned_actors.append(
spawn(name=current_actor.name + "_extractor_" + str(i)))
for actor in spawned_actors:
await actor
sent_actions = []
for actor, http in zip(spawned_actors, https):
sent_actions.append(send(actor, 'extract_data', http))
for action in sent_actions:
result = await action
for actor in spawned_actors:
await send(actor, 'stop')
return get_actor().name
async def _work(self, a=None):
indexes_pair_dict = self.prepare_indexes()
print(indexes_pair_dict)
#czyszczenie tekstowego pliku
f = open(CONFERENCES_HOMEPAGES_FILENAME, 'w')
f.close()
spawned_actors = []
for actor_name in FIRST_LEVEL_ACTORS:
spawned_actors.append(spawn(name=actor_name))
for actor in spawned_actors:
await actor
sent_actions = []
for actor, actor_name in zip(spawned_actors, FIRST_LEVEL_ACTORS):
sent_actions.append(
send(actor, 'parse_homepages_from_wiki_lvl1',
indexes_pair_dict[actor_name]))
print("Awaiting")
for action in sent_actions:
await action
print("CLOSING")
for actor in spawned_actors:
await send(actor, 'stop')
self._loop.call_later(500, self, a)
async def on_serialized_message(self, serialized_message: str) -> None:
"""Send an actor command of the serialized message."""
if callable(self.__command_targets):
command_targets = self.__command_targets()
else:
command_targets = self.__command_targets
await asyncio.gather(*[
ps.send(command_target, 'send_serialized_message',
serialized_message) for command_target in command_targets
])
async def _gather(self):
self._print('Gathering results...')
cycle_result = []
for act in self._actors:
# DO NOT AWAIT
# run all cancel commands async
send(act.aid, 'run', actor_cancel_process)
for act in self._actors:
while True:
report_events = await send(act, 'run', actor_gather_process)
if report_events is not None:
break
await asyncio.sleep(0.5)
cycle_result.extend(report_events)
self._print('Stopping actors...')
for act in self._actors:
await send(act, 'stop')
return cycle_result
async def parse_homepages_from_wiki_lvl1(request, message):
print("PARSE_FROM_WIKI", message)
current_actor = get_actor()
request.actor.logger.info("Actor: " + str(current_actor) + " Indexes: " + str(message))
wikicfp_file = open(WIKICFP_FILENAME, "r")
wikicfp_file_lines = wikicfp_file.readlines()[message[0]: message[1]]
wikicfp_file.close()
conference_links = []
for i in range(len(wikicfp_file_lines)):
stripped_line = wikicfp_file_lines[i].strip()
print("Stripped line:" , stripped_line)
conference_link = find_conf_link_one(stripped_line)
print("conferecne link", conference_link)
if conference_link == None:
continue
conference_links.append(conference_link)
f = open(CONFERENCES_HOMEPAGES_FILENAME, 'a')
for link in conference_links:
f.write(link + '\n')
f.close()
print(conference_links)
conference_links_chunks = split_to_chunks(conference_links, 4)
spawned_actors = []
for i, links_chunk in enumerate(conference_links_chunks):
spawned_actors.append(spawn(name=current_actor.name+"_"+str(i)))
for actor in spawned_actors:
await actor
sent_actions = []
for actor, chunk in zip(spawned_actors, conference_links_chunks):
sent_actions.append(send(actor, 'download_homepages', chunk))
for action in sent_actions:
result = await action
print(result)
for actor in spawned_actors:
await send(actor, 'stop')
return
async def run_network_instances(args):
num_instances = NUMBER_NETWORK_INSTANCES
activ = 'relu'
if 'instances' in args:
num_instances = args['instances']
if 'activation' in args:
activ = args['activation']
await spawn_network_instances(num_instances)
dpinfo = await pulsar.send('data_provider', 'data_provider_info')
input_format = 'hot_vector'
output_format = 'categorical'
input_length = dpinfo['model_params']['input_length'][input_format]
output_length = dpinfo['model_params']['output_length'][output_format]
if NETWORK_STATE['model'] is None:
NETWORK_STATE['input_length'] = input_length
NETWORK_STATE['output_length'] = output_length
NETWORK_STATE['activation'] = activ
model = create_model(input_length, output_length, activation=activ)
logging.info(
'NETWORK SUPER-MODEL INIT (size = {}, X:{}->Y:{})'.format(model_sizeMB(serialize_model(model)),
input_length, output_length))
NETWORK_STATE['model'] = serialize_model(model)
instance_args = dict(args, **{'model': NETWORK_STATE['model'], 'in_len': input_length,
'out_len': output_length, 'activation': activ,
'input_format': input_format, 'output_format': output_format})
#logging.warning(instance_args)
for (aid, inst) in NETWORK_STATE['instances'].items():
asyncio.ensure_future(
pulsar.send(aid, 'run', run_network_instance, args=instance_args)
)
def stop_actors(self, *args):
all = args or self.all_spawned
return as_gather(*[send(a, 'stop') for a in all])
def tearDownClass(cls):
coros = [
send('arbiter', 'kill_actor', a.name)
for a in (cls.tq_app, cls.rpc) if a is not None
]
return asyncio.gather(*coros)
async def run_bayes_instances(args):
await spawn_bayes_instances(NUMBER_BAYES_INSTANCES)
for (aid, inst) in BAYES_STATE['instances'].items():
asyncio.ensure_future(
pulsar.send(aid, 'run', run_bayes_instance, args=args))
def stop_actors(self, *args):
all = args or self.all_spawned
return as_gather(*[send(a, 'stop') for a in all])
def __call__(self, *args, **kwargs):
return send(self.remote.proxy, 'remote_call', self.remote.__class__,
self.name, args, kwargs)
def rpc_kill_actor(self, request, aid):
'''Kill the actor with id equal to *aid*.'''
return send('arbiter', 'kill_actor', aid)
def __call__(self, *args, **kwargs):
return send(self.remote.proxy, 'remote_call', self.remote.__class__,
self.name, args, kwargs)
def tearDownClass(cls):
if cls.server_cfg:
return send('arbiter', 'kill_actor', cls.server_cfg.name)
def tearDownClass(cls):
if cls.app_cfg is not None:
return send('arbiter', 'kill_actor', cls.app_cfg.name)
def tearDownClass(cls):
if cls.server_cfg:
return send('arbiter', 'kill_actor', cls.server_cfg.name)
