def run_parallel(args):
worker_factory, simulation_cases = factory.resolve_configuration(args)
logger.debug("Shuffling the job queue")
random.shuffle(simulation_cases)
logger.debug("Creating the pool")
processes_no = int(args["-j"])
rxtools.configure_default_executor(processes_no)
wall_time = []
start_time = datetime.now()
results = []
logger.debug("Simulation cases: %s", simulation_cases)
logger.debug("Work will be divided into %d processes", processes_no)
sys = ActorSystem("multiprocTCPBase", logDefs=log_helper.EVOGIL_LOG_CONFIG)
with log_time(system_time, logger, "Pool evaluated in {time_res}s", out=wall_time):
def process_result(subres):
results.append(subres)
log_simulation_stats(start_time, subres[-1], len(simulation_cases))
rx.from_iterable(range(len(simulation_cases))).pipe(
ops.map(lambda i: worker_factory(simulation_cases[i], i)),
# ops.map(lambda w: rxtools.from_process(w.run)),
ops.map(lambda w : w.run()),
# ops.merge(max_concurrent=1)
ops.do_action(on_next=process_result)
).run()
log_summary(args, results, simulation_cases, wall_time)
rxtools.shutdown_default_executor()
sys.shutdown()
def get_tickets(customer_id):
"""Get the tickets of a specific customer."""
try:
customer_id = request.headers.get('Customer-ID')
if not customer_id:
return jsonify({'error': "You do not have permissions."}), 403
asys = ActorSystem()
actor = asys.createActor(actorClass=CustomersActor)
customer_id = request.headers.get('Customer-ID')
order_date = request.args.get('order_date', default=None, type=float)
event_date = request.args.get('event_date', default=None, type=float)
payload = {
'customer_id':
int(customer_id),
'order_date':
datetime.fromtimestamp(order_date).date() if order_date else None,
'event_date':
datetime.fromtimestamp(event_date).date() if event_date else None
}
message = ActorMessage(action=CustomersActorAction.CUSTOMERS_TICKETS,
payload=payload,
customer_id=customer_id)
tickets_dict = []
response = asys.ask(actor, message)
if response.error:
return jsonify({'error': str(response.error.message)
}), response.error.http_code
for ticket in response.payload.get('tickets'):
tickets_dict.append(Ticket.to_dict(ticket))
return jsonify(tickets_dict)
except Exception as ex:
return jsonify({'error': str(ex)}), 500
def shutdown_system():
"""
fixture that shutdown all multiproQeueuBase actor system after the test end
"""
yield None
syst = ActorSystem(systemBase='multiprocQueueBase', logDefs=LOG_DEF)
syst.shutdown()
def system():
"""
fixture that start an Actor system with log enabled before launching the test and shutdown it after the test end
"""
syst = ActorSystem(systemBase='multiprocQueueBase', logDefs=LOG_DEF)
yield syst
syst.shutdown()
def asys(request):
caps = {'Foo Allowed': True,
'Cows Allowed': True,
'Dogs Allowed': True,
'dog': 'food'}
if request.param.startswith('multiprocTCP') or \
request.param.startswith('multiprocUDP'):
caps['Admin Port'] = get_free_admin_port()
caps['Convention Address.IPv4'] = '', caps['Admin Port']
if request.param.endswith('-AdminRouting'):
caps['Admin Routing'] = True
if request.param.endswith('-AdminRoutingTXOnly'):
caps['Admin Routing'] = True
caps['Outbound Only'] = True
asys = ActorSystem(systemBase=request.param.partition('-')[0],
capabilities=caps,
logDefs=(simpleActorTestLogging()
if request.param.startswith('multiproc')
else False),
transientUnique=True)
asys.base_name = request.param
asys.port_num = caps.get('Admin Port', None)
asys.txonly = request.param.endswith('-AdminRoutingTXOnly')
request.addfinalizer(lambda asys=asys: asys.shutdown())
return asys
def test_simple_message(self):
# Create System 2 and have it join the convention, this will force the registrar to create an actor on system2
self.sys2_capabilities = {
'Convention Address.IPv4': ('10.128.108.62', 2219),
'Admin Port': 2215,
'uuid': uuid.uuid4().hex
}
# the uuid is included here to uniquely identify system2 from other actor systems so the registrar can create
# an actor on this system
self.sys2 = ActorSystem(capabilities=self.sys2_capabilities)
print(self.sys2_capabilities)
try:
rn = self.sys2.createActor(
'Actors.RegistrarActor',
{'uuid': self.sys2_capabilities['uuid']},
globalName='rnode')
self.assertEqual(
rn, self.registrar.rnodes[self.sys2_capabilities['uuid']])
self.sys2.tell(rn, 'Hello String Me')
self.sys2.tell(rn, ActorExitRequest(True))
finally:
print("Shutting Down System 2")
self.sys2.shutdown()
def spawn(sys_base, app_id):
asys = ActorSystem(sys_base)
clerk = asys.createActor(Clerk, globalName="clerk-%d" % app_id)
asys.ask(
clerk,
Clerk.View(
asys.createActor(DirectoryServer, globalName="directory-server"),
KEYSPACE,
))
for i in range(0, 5):
oids = set()
while len(oids) != 10:
key_ = random.randint(1, KEYSPACE)
if key_ in oids:
continue
oids.add(key_)
success = False
while not success:
time.sleep(0.5)
trx = asys.ask(clerk, Clerk.Read(list(oids)))
if trx is False:
continue
mods = random.sample(oids, 5)
for mod in mods:
trx.write_set[mod] = app_id
logging.debug("Clients initialized")
success = asys.ask(clerk, Clerk.Commit(trx))
print(success)
time.sleep(2)
asys.tell(clerk, ActorExitRequest())
def setSystemBase(self, newBase='simpleSystemBase', systemCapabilities=None, logDefs='BestForBase'):
newBaseStr = str(newBase)
if not hasattr(self, 'currentBase') or self.currentBase != newBaseStr:
ldefs = logDefs if logDefs != 'BestForBase' else (simpleActorTestLogging() if newBase.startswith('multiproc') else False)
# In case the ActorSystem was *already* setup, break the singleton aspect and re-init
ActorSystem(logDefs = ldefs).shutdown()
ActorSystem(newBase, systemCapabilities, logDefs = ldefs)
self.currentBase = newBaseStr
def __init__(self, newBase='simpleSystemBase',
systemCapabilities=None,
logDefs='BestForBase'):
self._asys = ActorSystem(systemBase=newBase,
capabilities=systemCapabilities,
logDefs=logDefs if logDefs != 'BestForBase' else (
simpleActorTestLogging() if newBase.startswith('multiproc')
else False),
transientUnique=True)
def actor_counter():
global ACTOR_COUNTER
if not ACTOR_COUNTER:
counter = ActorSystem().createActor(TicketCounter)
response = ActorSystem().ask(counter, "What's my count?")
ACTOR_COUNTER = counter
return response
else:
response = ActorSystem().ask(ACTOR_COUNTER, "What's my count?")
return response
def start_thor(sys_base):
asys = ActorSystem(sys_base, logDefs=logcfg)
# ds = asys.createActor(DirectoryServer, globalName="directory-server")
db_servers = {}
for i in range(0, 3):
name = "server-%d" % i
db_servers[name] = asys.createActor(Server, globalName=name)
for server in db_servers.values():
asys.ask(server, Server.View({}, list(db_servers.values())))
def main(systembase):
actor_name = sys.argv[0].split('.')[0]
behavior = sys.argv[1]
capabilities = get_capability(actor_name, behavior)
asys = ActorSystem(systembase, capabilities=capabilities, logDefs=logcfg)
logging.info("Service Node system started.")
while True:
try:
time.sleep(2.0)
except KeyboardInterrupt:
asys.shutdown()
def setSystemBase(self,
newBase='simpleSystemBase',
systemCapabilities=None,
logDefs=None):
newBaseStr = str(newBase)
if not hasattr(self, 'currentBase') or self.currentBase != newBaseStr:
# In case the ActorSystem was *already* setup, break the singleton aspect and re-init
ActorSystem().shutdown()
if logDefs is None:
logDefs = ActorSystemTestCase.getDefaultTestLogging()
ActorSystem(newBase, systemCapabilities, logDefs)
self.currentBase = newBaseStr
def __init__(self):
icon = pystray.Icon(
"KSCollector",
menu=pystray.Menu(
pystray.MenuItem(
"Enabled",
on_activate,
default=True,
checked=lambda item: self.enabled,
),
pystray.MenuItem("Quit", on_quit),
),
)
icon.icon = self.__generate_icon()
img = self.__generate_icon("yellow")
img.save("N.ico")
self.icon = icon
self.sequence = None
self.actors = []
self.stop_event = Event()
self.user = g_user
self.enabled = True
self.dnaref = ActorSystem().createActor(
DisplayNotificationActorNew, globalName="DisplayNotification")
self.datastore = ActorSystem().createActor(DataStoreActor,
globalName="DataStore")
self.downKeys = {}
self.filters = []
# Ugly hack that needs to be fixed eventually
global KSAPP
KSAPP = self
if os.path.exists('filters.ini'):
#Load the config file to get the filtered apps
cfg = configparser.ConfigParser()
cfg.read("filters.ini")
if 'Filters' in cfg:
if 'Apps' in cfg['Filters']:
apps = cfg['Filters']['Apps']
apps = list(map(lambda x: x.strip(), apps.split(',')))
for a in apps:
if a == '':
continue
self.filters.append(a)
else:
cfg = configparser.ConfigParser()
cfg['Filters'] = {'Apps': ''}
with open('filters.ini', 'w') as cfile:
cfg.write(cfile)
def goodbye(process=None):
from thespian.actors import ActorSystem
print("Shutting down mTree Actor land now...")
#ActorSystemStartup.shutdown()
# capabilities = dict([('Admin Port', 19000)])
# actors = ActorSystem('multiprocTCPBase', capabilities)
actors = ActorSystem('multiprocTCPBase')
time.sleep(3)
actors.shutdown()
time.sleep(1)
process.kill()
process.terminate()
print("mTree finished shutting down")
def start(sys_base):
asys = ActorSystem(sys_base, logDefs=logcfg)
ds = asys.createActor(DirectoryServer, globalName="directory-server")
db_servers = {}
for i in range(0, REPLICAS):
name = "db-server-%d" % i
db_servers[name] = asys.createActor(Server, globalName=name)
clusters = []
server_list = list(db_servers.keys())
for i in range(0, CLUSTER_COUNT):
cluster = Cluster(
"cluster-{}".format(i),
server_list[i * CLUSTER_SIZE:i * CLUSTER_SIZE + CLUSTER_SIZE])
clusters.append(cluster)
cluster_map = {}
for key in range(1, KEY_SPACE + 1):
cluster_map[key] = clusters[key % CLUSTER_COUNT]
asys.ask(ds, DirectoryServer.WhoServes(cluster_map))
grouped_cluster_map = _group_by_cluster(cluster_map)
for cluster, oid_set in grouped_cluster_map.items():
for server_name in cluster.members:
asys.ask(db_servers[server_name],
Server.View(oid_set, cluster, "directory-server"))
def on_activate(self, icon):
self.enabled = not self.enabled
if self.enabled:
self.icon.icon = self.__generate_icon("green")
ActorSystem().tell(self.dnaref, {
"title": "KS Collector",
"text": "Collecting Enabled"
})
else:
self.icon.icon = self.__generate_icon("blue")
ActorSystem().tell(self.dnaref, {
"title": "KS Collector",
"text": "Collecting Paused"
})
def step(self):
epoch_end_message = ActorSystem().ask(
self.node_supervisor, HgsMessage(HgsOperation.NEW_METAEPOCH))
self.cost = epoch_end_message.data
ActorSystem().ask(self.node_supervisor,
HgsMessage(HgsOperation.TRIM_NOT_PROGRESSING))
ActorSystem().ask(self.node_supervisor,
HgsMessage(HgsOperation.TRIM_REDUNDANT))
ActorSystem().ask(self.node_supervisor,
HgsMessage(HgsOperation.RELEASE_SPROUTS))
ActorSystem().ask(self.node_supervisor,
HgsMessage(HgsOperation.REVIVE))
logger.info(f"step finished, current cost = {self.cost}")
def __key_press_handler(self, event):
# If we are not collecting then simply skip this event
if not self.enabled:
return
# This really should be done in the base keyboard library instead of here... but I didn't want to modify the base
# library so instead I simply set the key time event here... if it turns out there is too much jitter in the results
# we can change the base library so that the perf_counter call is closer to the actual OS keystroke event. However given that this runs as a
# separate actor that only collects the data it shouldn't be too bad.
event.time = time.perf_counter()
if event.event_type == "down":
if event.scan_code in self.downKeys:
print("Held Key {}".format(event.scan_code))
return
else:
self.downKeys[event.scan_code] = True
if event.event_type == "up":
if event.scan_code in self.downKeys:
del self.downKeys[event.scan_code]
for actor in self.actors:
ActorSystem().tell(actor['aref'], {
"kbe": event,
"app": getActiveApp()
})
def get_sales():
"""Get the number of tickets sold per event."""
try:
customer_id = request.headers.get('Customer-ID')
if customer_id:
return jsonify({'error': "You do not have permissions."}), 403
asys = ActorSystem()
actor = asys.createActor(actorClass=EventsActor)
message = ActorMessage(action=EventsActorAction.EVENTS_SALES)
response = asys.ask(actor, message)
if response.error:
return jsonify({'error': str(response.error.message)
}), response.error.http_code
return jsonify(response.payload.get('sales_dict'))
except Exception as ex:
return jsonify({'error': str(ex)}), 500
def tearDown(self):
# Unset environment.
try:
del (os.environ['DB_URI'])
except KeyError:
pass
# Shutdown base actor system.
ActorSystem().shutdown()
def __init__(self, verbose_mode: bool):
"""
:param verbose_mode: Specify the log level : True -> DEBUG False -> INFO
"""
if verbose_mode:
LOG_DEF['handlers']['h1']['level'] = logging.DEBUG
LOG_DEF['handlers']['h2']['level'] = logging.DEBUG
else:
LOG_DEF['handlers']['h1']['level'] = logging.INFO
LOG_DEF['handlers']['h2']['level'] = logging.INFO
self.system = ActorSystem(systemBase='multiprocQueueBase',
logDefs=LOG_DEF)
self.pushers = {}
self.pullers = {}
self.dispatchers = {}
self.actors = {}
class TestSystem(object):
"Functions as a context manager for a transient system base"
def __init__(self, newBase='simpleSystemBase',
systemCapabilities=None,
logDefs='BestForBase'):
self._asys = ActorSystem(systemBase=newBase,
capabilities=systemCapabilities,
logDefs=logDefs if logDefs != 'BestForBase' else (
simpleActorTestLogging() if newBase.startswith('multiproc')
else False),
transientUnique=True)
def __enter__(self):
return self._asys
def __exit__(self, exc_type, exc_val, exc_tb):
self._asys.shutdown()
self._asys = None
def get(event_id):
"""Get event by ID."""
try:
customer_id = request.headers.get('Customer-ID')
asys = ActorSystem()
actor = asys.createActor(actorClass=EventsActor)
payload = {'event_id': int(event_id)}
message = ActorMessage(action=EventsActorAction.EVENTS_GET,
payload=payload,
customer_id=customer_id)
response = asys.ask(actor, message)
if response.error:
return jsonify({'error': str(response.error.message)
}), response.error.http_code
return jsonify(Event.to_dict(response.payload.get('event')))
except Exception as ex:
return jsonify({'error': str(ex)}), 500
def setUp(self):
self.sys1 = None
self.sys2 = None
self.sys2_capabilities = None
self.sys1_capabilities = None
print("Starting Main System")
self.sys1_capabilities = {
'Convention Address.IPv4': ('10.128.108.62', 2219),
'Admin Port': 2211
}
self.sys1 = ActorSystem('multiprocTCPBase',
capabilities=self.sys1_capabilities)
# Create the registrar on System 1 and send an Init Packet
self.registrar = self.sys1.createActor('ConventionLead.Registrar')
self.sys1.tell(self.registrar, messages.InitPacket())
class TestSystem(object):
"Functions as a context manager for a transient system base"
def __init__(self, newBase='simpleSystemBase',
systemCapabilities=None,
logDefs='BestForBase'):
self._asys = ActorSystem(systemBase=newBase,
capabilities=systemCapabilities,
logDefs=logDefs if logDefs != 'BestForBase' else (
simpleActorTestLogging() if newBase.startswith('multiproc')
else False),
transientUnique=True)
def __enter__(self):
return self._asys
def __exit__(self, exc_type, exc_val, exc_tb):
self._asys.shutdown()
self._asys = None
def __init__(self, newBase='simpleSystemBase',
systemCapabilities=None,
logDefs='BestForBase'):
self._asys = ActorSystem(systemBase=newBase,
capabilities=systemCapabilities,
logDefs=logDefs if logDefs != 'BestForBase' else (
simpleActorTestLogging() if newBase.startswith('multiproc')
else False),
transientUnique=True)
def add_actor(self, actor_name):
class_str = actor_name
aref = None
try:
aref = ActorSystem().createActor(eval(class_str))
except NameError:
try:
class_str = "Actors.{}".format(class_str)
aref = ActorSystem().createActor(eval(class_str))
except NameError:
print("Error: Actor Class {} not found".format(actor_name))
return
ActorSystem().tell(aref, {"load": "{}_data".format(self.user)})
for f in self.filters:
ActorSystem().tell(aref, {"filter_app": f})
adata = {'actor': class_str, 'aref': aref}
self.actors.append(adata)
def index():
"""Get event list."""
try:
customer_id = request.headers.get('Customer-ID')
asys = ActorSystem()
actor = asys.createActor(actorClass=EventsActor)
message = ActorMessage(action=EventsActorAction.EVENTS_LIST,
customer_id=customer_id)
response = asys.ask(actor, message)
if response.error:
return jsonify({'error': str(response.error.message)
}), response.error.http_code
events = response.payload.get('events')
events_dict = []
for event in events:
events_dict.append(Event.to_dict(event))
return jsonify(events_dict)
except Exception as ex:
return jsonify({'error': str(ex)}), 500
def main():
asys = ActorSystem()
try:
while True:
print("Type what you want to search:")
query = input()
master = asys.createActor(MasterActor, globalName="master")
res = asys.ask(master, query, timeout=timedelta(seconds=5))
print(res)
asys.tell(master, ActorExitRequest())
finally:
asys.shutdown()
def add():
"""Add a new event."""
try:
customer_id = request.headers.get('Customer-ID')
if customer_id:
return jsonify({'error': "You do not have permissions."}), 403
asys = ActorSystem()
actor = asys.createActor(actorClass=EventsActor)
event = Event.from_json(request.get_json())
payload = {'event': event}
message = ActorMessage(action=EventsActorAction.EVENTS_ADD,
payload=payload)
response = asys.ask(actor, message)
if response.error:
return jsonify({'error': str(response.error.message)
}), response.error.http_code
return '', 204
except Exception as ex:
return jsonify({'error': str(ex)}), 500
def test_ask(self):
print("Testing ask")
asys = ActorSystem('multiprocQueueBase')
source = asys.createActor(DoTestSource)
targ = asys.createActor(DoTestTarget)
msg = SourceAsk("Hello From Source", target=targ, sender=source)
logging.info("Sending {}".format(str(msg)))
print(asys.ask(targ, msg))
asys.shutdown()
def similar_asys(asys, in_convention=True, start_wait=True, capabilities=None):
caps = capabilities or {}
if asys.base_name.startswith('multiprocTCP') or \
asys.base_name.startswith('multiprocUDP'):
caps['Admin Port'] = get_free_admin_port()
if in_convention:
caps['Convention Address.IPv4'] = '', asys.port_num
if asys.base_name.endswith('-AdminRouting'):
caps['Admin Routing'] = True
asys2 = ActorSystem(systemBase=asys.base_name.partition('-')[0],
capabilities=caps,
logDefs=(simpleActorTestLogging()
if asys.base_name.startswith('multiproc')
else False),
transientUnique=True)
asys2.base_name = asys.base_name
asys2.port_num = caps.get('Admin Port', None)
if in_convention and start_wait:
time.sleep(0.25) # Wait for Actor Systems to start and connect together
return asys2
events = pygame.event.get()
for event in events:
if event.type == MOUSEBUTTONUP:
self.send(
self.clickNotifier, ClickData(event.pos, self.itemsToDraw, ActorExitRequest(), self.myAddress)
)
elif event.type == KEYDOWN:
ant_type = {pygame.K_b: BlackAntSpriteActor, pygame.K_r: RedAntSpriteActor}.get(event.key, None)
if ant_type:
sprite = self.createActor(ant_type)
self.send(sprite, StartSprite(self.myAddress, []))
elif event.type == QUIT:
print("QUIT")
pygame.quit()
self.send(self.origSender, "STOP")
self.send(self.myAddress, ActorExitRequest())
def _create_text_obj(self, centerLoc=(SURFACE_X_SIZE / 2, SURFACE_Y_SIZE / 2), text=""):
font_obj = pygame.font.Font("freesansbold.ttf", 32)
self.textSurfaceObj = font_obj.render(text, True, GREEN, BLUE)
self.textRectObj = self.textSurfaceObj.get_rect()
self.textRectObj.center = (centerLoc[0], centerLoc[1])
asys = ActorSystem("multiprocQueueBase")
drawerActor = asys.createActor(DrawerActor)
asys.ask(drawerActor, "start")
print("COMPLETE")
asys.shutdown()
from thespian.actors import ActorSystem
import sys
portnum = int(sys.argv[1])
addrmv = sys.argv[2]
assert addrmv in '+-'
cap = sys.argv[3]
asys = ActorSystem('multiprocTCPBase', {'Admin Port': portnum})
asys.updateCapability(cap, True if addrmv == '+' else None)
from thespian.actors import ActorSystem, Actor, ValidateSource, ValidatedSource
import sys
portnum = int(sys.argv[1])
capability_names = (sys.argv + [''])[2].split(',')
capabilities = dict([('Admin Port', portnum),
('Convention Address.IPv4', ('', 1900)),
] +
list(zip(capability_names, [True] * len(capability_names))))
class SimpleSourceAuthority(Actor):
def receiveMessage(self, msg, sender):
if msg is True:
self.registerSourceAuthority()
if isinstance(msg, ValidateSource):
self.send(sender,
ValidatedSource(msg.sourceHash,
msg.sourceData,
# Thespian pre 3.2.0 has no sourceInfo
getattr(msg, 'sourceInfo', None)))
asys = ActorSystem('multiprocTCPBase', capabilities)
sa = asys.createActor(SimpleSourceAuthority)
asys.tell(sa, True) # cause source authority to register itself as such
from thespian.actors import ActorSystem, Actor, ValidateSource, ValidatedSource
import sys
portnum = int(sys.argv[1])
srchash = sys.argv[2]
asys = ActorSystem('multiprocTCPBase', {'Admin Port': portnum})
asys.unloadActorSource(srchash)
