def __init__(self):
self.daemon = None
self.workers = []
SerializerBase.register_dict_to_class("worker_node.WorkerNode", WorkerNode.node_dict_to_class)
SerializerBase.register_dict_to_class("job.Job", Job.job_dict_to_class)
self.nfs_exporter = CertCheckingProxy('PYRO:NfsExporter@localhost:9091')
self.first_run = True
def setup(self):
signal.signal(signal.SIGINT, self.termination_handler)
with open(OD_FOLDER + SSL_CERTS_DIR + 'whoismaster') as f:
self.MASTER_ADDRESS = f.read().strip()
self.MOUNTPOINT_DEFAULT = str(Path.home()) + '/olive-share/'
self.job_dispatcher = CertCheckingProxy('PYRO:JobDispatcher@' +
self.MASTER_ADDRESS + ':9090')
self.nfs_mounter = CertCheckingProxy('PYRO:NfsMounter@' + 'localhost' +
':9092')
SerializerBase.register_dict_to_class(
"job.ExportRange", ExportRange.export_range_dict_to_class)
SerializerBase.register_dict_to_class("job.Job", Job.job_dict_to_class)
def __init__(self):
super().__init__()
self.split_job = None
self.parts_lock = threading.Lock()
self.worker_fails = dict()
self.ongoing_ranges = set()
self.failed_ranges = set()
self.completed_ranges = set()
self.cleanup_queue = set()
self.last_assigned_frame = 0
self.job_parts = 0
SerializerBase.register_dict_to_class("job.ExportRange", ExportRange.export_range_dict_to_class)
SerializerBase.register_class_to_dict(ExportRange, ExportRange.export_range_class_to_dict)
def __serialized_dict__(self):
"""serialized form as a dictionary"""
from Pyro4.util import SerializerBase # XXX circular
return {
"__class__": "Pyro4.futures._ExceptionWrapper",
"exception": SerializerBase.class_to_dict(self.exception)
}
def __serialized_dict__(self):
"""serialized form as a dictionary"""
from Pyro4.util import SerializerBase # XXX circular
return {
"__class__": "Pyro4.futures._ExceptionWrapper",
"exception": SerializerBase.class_to_dict(self.exception)
}
def __init__(self, address):
with open(SSL_CERTS_DIR + 'whoismaster') as f:
self.MASTER_ADDRESS = f.read().strip()
self.MOUNTPOINT_DEFAULT = str(Path.home()) + '/olive-share'
self.address = address
self.cpu_score = 0
self.net_score = 0
self._job_start_time = None
self._job = None
self.sample_weight = None
self.sample_time = None
self.worker_options = dict()
self.job_dispatcher = CertCheckingProxy('PYRO:JobDispatcher@' +
self.MASTER_ADDRESS + ':9090')
self.nfs_mounter = CertCheckingProxy('PYRO:NfsMounter@' + 'localhost' +
':9092')
SerializerBase.register_dict_to_class(
"job.ExportRange", ExportRange.export_range_dict_to_class)
SerializerBase.register_dict_to_class("job.Job", Job.job_dict_to_class)
def register_deserialization_handlers():
for supported_class in supported_classes:
SerializerBase.register_dict_to_class(supported_class, deserialize)
for supported_exc in supported_exceptions:
SerializerBase.register_dict_to_class(supported_exc, deserialize_exception)
import subprocess
SerializerBase.register_class_to_dict(subprocess.CalledProcessError, serialize_called_process_error)
def register_deserialization_handlers():
for supported_class in supported_classes:
SerializerBase.register_dict_to_class(supported_class, deserialize)
for supported_exc in supported_exceptions:
SerializerBase.register_dict_to_class(supported_exc,
deserialize_exception)
import subprocess
SerializerBase.register_class_to_dict(subprocess.CalledProcessError,
serialize_called_process_error)
bias: float = 0.0) -> Generator[sample.Sample, None, None]:
gen = self.synth.semicircle_gen(frequency, amplitude, phase, bias)
while True:
chunk = next(gen)
yield sample.Sample.from_array(chunk, self.synth.samplerate, 1)
def pointy(self,
frequency: int,
duration: float,
amplitude: float = 0.9999,
phase: float = 0.0,
bias: float = 0.0) -> sample.Sample:
return self.synth.pointy(frequency, duration, amplitude, phase, bias)
def pointy_gen(self,
frequency: int,
amplitude: float = 0.9999,
phase: float = 0.0,
bias: float = 0.0) -> Generator[sample.Sample, None, None]:
gen = self.synth.pointy_gen(frequency, amplitude, phase, bias)
while True:
chunk = next(gen)
yield sample.Sample.from_array(chunk, self.synth.samplerate, 1)
if __name__ == "__main__":
Pyro4.config.SERIALIZER = "marshal"
Pyro4.config.SERIALIZERS_ACCEPTED = {"marshal"}
SerializerBase.register_class_to_dict(sample.Sample, sample_serializer)
Pyro4.Daemon.serveSimple({WaveSynthServer: "synth.wavesynth"})
from common import config
from common.module import ShellModule
from common.infra_modules.shell_manager_module import MODULE_NAME
from Pyro4.util import SerializerBase
logger = config.get_log(MODULE_NAME)
def statement_dict_to_class(classname, d):
# Method that deserialize cmd2.Statement class to tuple of tuple
return tuple(d['raw-attribute'].split())
# Register method in pyro4 for correct deserialization
SerializerBase.register_dict_to_class("statement", statement_dict_to_class)
class ShellManagerModule(ShellModule):
def initialize(self):
"""
This method create and initialize all variables and resources needed
:return: None
"""
from typing import Any, Dict
from synthplayer import sample
from synthplayer.playback import Output
from Pyro4.util import SerializerBase
import Pyro4
def sample_deserializer(classname: str, data: Dict[str, Any]) -> sample.Sample:
return sample.Sample.from_raw_frames(data["frames"], data["samplewidth"],
data["samplerate"], data["nchannels"],
data["name"])
Pyro4.config.SERIALIZER = "marshal"
SerializerBase.register_dict_to_class("synthplayer.sample.Sample",
sample_deserializer)
synth = Pyro4.Proxy("PYRONAME:synth.wavesynth")
synth.setup(44100)
with Output(44100, nchannels=1, samplewidth=2, mixing="sequential") as output:
silence = sample.Sample.from_raw_frames(b"",
samplewidth=2,
samplerate=44100,
numchannels=1)
silence.add_silence(0.1)
output.play_sample(synth.sine(220, .5))
output.play_sample(silence)
output.play_sample(synth.sine(330, .5))
output.play_sample(silence)
output.play_sample(synth.sine(440, .5))
output.play_sample(silence)
for (id, value) in ts.replicas.items():
if id not in self.replicas:
self.replicas[id] = value
elif value > self.replicas[id]:
self.replicas[id] = ts.replicas[id]
def to_dict(self) -> Dict:
"""
Used for serpent serialisation
:return: A dict representing this Timestamp
"""
return {"__class__": "Timestamp", "replicas": self.replicas}
@staticmethod
def from_dict(classname: str, dict: Dict) -> 'Timestamp':
"""
Used for serpent deserialisation
:return: A Log
"""
return Timestamp(dict["replicas"])
SerializerBase.register_class_to_dict(Timestamp, Timestamp.to_dict)
SerializerBase.register_dict_to_class('Timestamp', Timestamp.from_dict)
import base64
import os
import socket
import sys
import Pyro4
from Pyro4.util import SerializerBase
from dispatcher_queue import DispatcherQueue
from job import Job
# For "job.Job" we register a deserialisation hook to be able to get these back from Pyro
SerializerBase.register_dict_to_class("job.Job", Job.from_dict)
# For "dispatcher_queue.DispatcherQueue" we register a deserialisation hook to be able to get these back from Pyro
SerializerBase.register_dict_to_class("dispatcher_queue.DispatcherQueue", DispatcherQueue.from_dict)
SERVER_NAME = ""
SUBDIR = ""
def handle_upload_init(file_name):
"""
Handles an upload request with just the file name.
"""
print("Dispatcher wanted to upload a file: {}".format(file_name))
return {"outcome": "ready to receive"}
def handle_upload_data(file_name, file_contents):
"""
Handles a full upload request.
"__class__": "waheeee-custom-thingy",
"number-attribute": obj.number
}
def thingy_dict_to_class(classname, d):
print("{deserializer hook, converting to class: %s}" % d)
return mycustomclasses.Thingy(d["number-attribute"])
def otherthingy_dict_to_class(classname, d):
print("{deserializer hook, converting to class: %s}" % d)
return mycustomclasses.OtherThingy(d["number"])
# for 'Thingy' we register both serialization and deserialization hooks
SerializerBase.register_dict_to_class("waheeee-custom-thingy", thingy_dict_to_class)
SerializerBase.register_class_to_dict(mycustomclasses.Thingy, thingy_class_to_dict)
# for 'OtherThingy' we only register a deserialization hook (and for serialization depend on serpent's default behavior)
SerializerBase.register_dict_to_class("mycustomclasses.OtherThingy", otherthingy_dict_to_class)
# regular Pyro server stuff
@Pyro4.expose
class Server(object):
def method(self, arg):
print("\nmethod called, arg=", arg)
response = mycustomclasses.Thingy(999)
return response
def deserialize_exception(fq_classname, data):
class_object = get_class_object(fq_classname)
return SerializerBase.make_exception(class_object, data)
from __future__ import print_function
import os
import socket
import sys
from math import sqrt
import Pyro4
from Pyro4.util import SerializerBase
from workitem import Workitem
# For 'workitem.Workitem' we register a deserialization hook to be able to get these back from Pyro
SerializerBase.register_dict_to_class("workitem.Workitem", Workitem.from_dict)
if sys.version_info < (3, 0):
range = xrange # make sure to use the memory efficient range generator
WORKERNAME = "Worker_%d@%s" % (os.getpid(), socket.gethostname())
def factorize(n):
"""simple algorithm to find the prime factorials of the given number n"""
def isPrime(n):
return not any(x for x in range(2, int(sqrt(n)) + 1) if n % x == 0)
primes = []
candidates = range(2, n + 1)
candidate = 2
while not primes and candidate in candidates:
if n % candidate == 0 and isPrime(candidate):
primes = primes + [candidate] + factorize(n // candidate)
f.ps = d["ps"]
f.ps_distro = d["ps_distro"]
f.protocol = d["protocol"]
f.starting_date = d["starting_date"]
f.trans_duration = d["trans_duration"]
f.mesure = d["mesure"]
m = mesure_config()
m.metrics = d["mesure"]["metrics"]
m.sampling_interval = d["mesure"]["sampling_interval"]
m.finish_date = d["mesure"]["finish_date"]
m.start_date = d["mesure"]["start_date"]
f.mesure = m
return f
SerializerBase.register_class_to_dict(flow_config, flow_config_class_to_dict)
SerializerBase.register_dict_to_class("flow_config", flow_config_dict_to_class)
def mesure_config_class_to_dict(obj):
return dict(__class__="mesure_config", metrics=obj.metrics, start_date=obj.start_date, finish_date=obj.finish_date,
sampling_interval=obj.sampling_interval)
def mesure_config_dict_to_class(classname, d):
m = mesure_config()
m.metrics = d["metrics"]
m.sampling_interval = d["sampling_interval"]
m.finish_date = d["finish_date"]
m.start_date = d["start_date"]
return m
# TaskProcessingError is designed to be returned on the result
# "slot" of Task, and when I've attempted to register
# (de)serialization hooks for both Task and TaskProcessingError,
# the end result is that the TaskProcessingError
# de-serialization hook never gets called (leaving it in its
# serialized dict form on the deserialized Task object). This
# defeats the purpose of creating a TaskProcessingError type.
# Perhaps there is a way around this, but for now I am giving
# priority to having an error type that can be transmitted
# for all serializers in both Pyro and Pyro4 (over making
# Task a class again).
#
if using_pyro4:
import Pyro4
from Pyro4.util import SerializerBase
# register hooks for TaskProcessingError
def TaskProcessingError_to_dict(obj):
return {
"__class__": "pyutilib.pyro.task.TaskProcessingError",
"message": obj.args[0]
}
def dict_to_TaskProcessingError(classname, d):
return TaskProcessingError(d['message'])
SerializerBase.register_class_to_dict(TaskProcessingError,
TaskProcessingError_to_dict)
SerializerBase.register_dict_to_class(
"pyutilib.pyro.task.TaskProcessingError", dict_to_TaskProcessingError)
def to_dict(obj):
return {
"__class__": "Pyro4.utils.messagebus.message",
"msgid": str(obj.msgid),
"created": obj.created.isoformat(),
"data": obj.data
}
@classmethod
def from_dict(cls, classname, d):
return cls(uuid.UUID(d["msgid"]),
datetime.datetime.strptime(d["created"], "%Y-%m-%dT%H:%M:%S.%f"),
d["data"])
# make sure Pyro knows how to serialize the custom Message class
SerializerBase.register_class_to_dict(Message, Message.to_dict)
SerializerBase.register_dict_to_class("Pyro4.utils.messagebus.message", Message.from_dict)
@Pyro4.expose
class Subscriber(object):
"""
This class is meant to be subclassed, and the consume_message method reimplemented.
As messages arrive from the publisher, the consume_message method gets called.
"""
def __init__(self, auto_consume=True, max_queue_size=5000, host='localhost', port=9090, **kwargs):
"""
kwargs:
- auto_consume (bool): Automatically start subscribing (True)
- max_queue_size (int): Max number of elements that can be placed in queue.Queue (5000)
- host (str): The nameserver hostname ('localhost')
if not (record.id in self):
self.records.append(record)
def to_dict(self) -> Dict:
"""
Used for serpent serialisation
:return: A dict representing this Log
"""
return {
"__class__": "Log",
"records": [record.to_dict() for record in self.records]
}
@staticmethod
def from_dict(classname: str, dict: Dict) -> 'Log':
"""
Used for serpent deserialisation
:return: A Log
"""
return Log(
[Record.from_dict("Record", record) for record in dict["records"]])
SerializerBase.register_class_to_dict(Record, Record.to_dict)
SerializerBase.register_dict_to_class("Record", Record.from_dict)
SerializerBase.register_class_to_dict(Log, Log.to_dict)
SerializerBase.register_dict_to_class("Log", Log.from_dict)
f.ps = d["ps"]
f.ps_distro = d["ps_distro"]
f.protocol = d["protocol"]
f.starting_date = d["starting_date"]
f.trans_duration = d["trans_duration"]
f.mesure = d["mesure"]
m = mesure_config()
m.metrics = d["mesure"]["metrics"]
m.sampling_interval = d["mesure"]["sampling_interval"]
m.finish_date = d["mesure"]["finish_date"]
m.start_date = d["mesure"]["start_date"]
f.mesure = m
return f
SerializerBase.register_class_to_dict(flow_config, flow_config_class_to_dict)
SerializerBase.register_dict_to_class("flow_config", flow_config_dict_to_class)
def mesure_config_class_to_dict(obj):
return dict(__class__="mesure_config",
metrics=obj.metrics,
start_date=obj.start_date,
finish_date=obj.finish_date,
sampling_interval=obj.sampling_interval)
def mesure_config_dict_to_class(classname, d):
m = mesure_config()
m.metrics = d["metrics"]
m.sampling_interval = d["sampling_interval"]
uri = nameserver.lookup(conninfo.virtual_host)
return pyro.Proxy(uri)
except NamingError:
reraise(NamingError, NamingError(E_LOOKUP.format(conninfo)),
sys.exc_info()[2])
def driver_version(self):
return pyro.__version__
@cached_property
def shared_queues(self):
return self._open()
if pyro is not None:
SerializerBase.register_dict_to_class("queue.Empty",
lambda cls, data: Empty())
@pyro.expose
@pyro.behavior(instance_mode="single")
class KombuBroker(object):
"""Kombu Broker used by the Pyro transport.
You have to run this as a separate (Pyro) service.
"""
def __init__(self):
self.queues = {}
def get_queue_names(self):
return list(self.queues)
def unregister_deserialization_handlers():
for supported_class in supported_classes:
SerializerBase.unregister_dict_to_class(supported_class, deserialize)
for supported_exc in supported_exceptions:
SerializerBase.unregister_dict_to_class(supported_exc,
deserialize_exception)
# Note: We can use this same mechanism to allow Task to be
# re-implemented as a custom class. HOWEVER,
# TaskProcessingError is designed to be returned on the result
# "slot" of Task, and when I've attempted to register
# (de)serialization hooks for both Task and TaskProcessingError,
# the end result is that the TaskProcessingError
# de-serialization hook never gets called (leaving it in its
# serialized dict form on the deserialized Task object). This
# defeats the purpose of creating a TaskProcessingError type.
# Perhaps there is a way around this, but for now I am giving
# priority to having an error type that can be transmitted
# for all serializers in both Pyro and Pyro4 (over making
# Task a class again).
#
if using_pyro4:
import Pyro4
from Pyro4.util import SerializerBase
# register hooks for TaskProcessingError
def TaskProcessingError_to_dict(obj):
return {"__class__": "pyutilib.pyro.task.TaskProcessingError",
"message": obj.args[0]}
def dict_to_TaskProcessingError(classname, d):
return TaskProcessingError(d['message'])
SerializerBase.register_class_to_dict(
TaskProcessingError,
TaskProcessingError_to_dict)
SerializerBase.register_dict_to_class(
"pyutilib.pyro.task.TaskProcessingError",
dict_to_TaskProcessingError)
@staticmethod
def to_dict(obj):
return {
"__class__": "Pyro4.utils.messagebus.message",
"msgid": str(obj.msgid),
"created": obj.created.isoformat(),
"data": obj.data,
}
@classmethod
def from_dict(cls, classname, d):
return cls(uuid.UUID(d["msgid"]), datetime.datetime.strptime(d["created"], "%Y-%m-%dT%H:%M:%S.%f"), d["data"])
# make sure Pyro knows how to serialize the custom Message class
SerializerBase.register_class_to_dict(Message, Message.to_dict)
SerializerBase.register_dict_to_class("Pyro4.utils.messagebus.message", Message.from_dict)
@Pyro4.expose
class Subscriber(object):
def __init__(self, auto_consume=True, max_queue_size=5000):
self.bus = Pyro4.Proxy("PYRONAME:" + PYRO_MSGBUS_NAME)
self.received_messages = queue.Queue(maxsize=max_queue_size)
if auto_consume:
self.__bus_consumer_thread = threading.Thread(target=self.__bus_consume_message)
self.__bus_consumer_thread.daemon = True
self.__bus_consumer_thread.start()
def incoming_message(self, topic, message):
self.received_messages.put((topic, message))
def SerializeMenuToDict(menu):
out = {}
out["__class__"] = "Menu"
out["_active"] = menu._active
out["_info"] = menu._info
for key in menu._entries.keys():
out["_entries.{0}".format(key)] = menu._entries[key]
if key in menu._mask:
out["_mask.{0}".format(key)] = True
else:
out["_mask.{0}".format(key)] = False
return out
## need to define functions to serialize so Menu can be passed with Pyro
def DeserializeMenuDict(classname, mdict):
menu = Menu()
menu._active = mdict["_active"]
menu._info = mdict["_info"]
for key in mdict.keys():
if key[:8] == '_entries':
xkey = key[9:]
menu._entries[xkey] = mdict[key]
if mdict["_mask.{0}".format(xkey)]:
menu._mask.add(xkey)
return menu
SerializerBase.register_class_to_dict(Menu, SerializeMenuToDict)
SerializerBase.register_dict_to_class("Menu", DeserializeMenuDict)
from __future__ import print_function
import random
import socket
from Worker import Worker
import Pyro4
try:
import queue # py3
except ImportError:
import Queue as queue # py2
from Pyro4.util import SerializerBase
from Job import Job
SerializerBase.register_dict_to_class("Job.Job", Job.from_dict)
class Scheduler(object):
METHOD_RAND = "RANDOM"
METHOD_TWO = "CHOOSE_TWO"
METHOD_BATCH = "BATCH"
METHOD_LATE = "BATCH+LATE_BINDING"
def __init__(self, scheduling_method=METHOD_RAND, nameserver_hostname="newyork", scheduler_number=1):
self.no_of_workers_per_scheduler = 10
self.scheduling_method = scheduling_method
print("Scheduling method: ", self.scheduling_method)
# Number of the scheduler
self.scheduler_number = scheduler_number
# List of jobs that have been scheduled/reserved
Z3ValFactory_T = Callable[[str], z3.ExprRef]
Z3TypeEnv = Dict[str, Z3ValFactory_T]
def fi_deserialize(classname: str, d: Dict[Any, Any]) -> OpaqueVal:
return OpaqueVal.from_dict(d)
def fi_serialize(obj: OpaqueVal) -> Dict[Any, Any]:
d = OpaqueVal.to_dict(obj)
assert "__class__" not in d
d["__class__"] = "OpaqueVal"
return d
SerializerBase.register_dict_to_class("OpaqueVal", fi_deserialize)
SerializerBase.register_class_to_dict(OpaqueVal, fi_serialize)
def type_to_z3sort(ast_typ: AstType) -> z3.SortRef:
if isinstance(ast_typ, AstIntType):
return IntSort()
elif isinstance(ast_typ, AstBoolType):
return BoolSort()
else:
# TODO: Multi-dimensional maps NYI
assert isinstance(ast_typ, AstMapType) and len(ast_typ.domainT) == 1
return z3.ArraySort(type_to_z3sort(ast_typ.domainT[0]),
type_to_z3sort(ast_typ.rangeT))
def to_dict(self) -> dict:
"""
Serpent serialization requires to_dict
Represent class InvalidUserError as a dictionary
:return: A dictionary representation of this InvalidUserError
"""
return {
"__class__": "InvalidUserError",
"message": self.message
}
@staticmethod
def to_class(class_name: str, dictionary: dict) -> 'InvalidUserError':
"""
This converts a dictionary representing an InvalidUserError
to an object of type InvalidUserError
This exception is thrown when there is a problem with the provided user_id
:param class_name: parameter is required for serialization
:param dictionary: dictionary representing an InvalidUserError
:return:
"""
del class_name
return InvalidUserError(dictionary["message"])
# Register serialization and deserialization hooks for InvalidUserError and InvalidMovieError
SerializerBase.register_class_to_dict(InvalidUserError, InvalidUserError.to_dict)
SerializerBase.register_dict_to_class("InvalidUserError", InvalidUserError.to_class)
SerializerBase.register_class_to_dict(InvalidMovieError, InvalidMovieError.to_dict)
SerializerBase.register_dict_to_class("InvalidMovieError", InvalidMovieError.to_class)
def unregister_deserialization_handlers():
for supported_class in supported_classes:
SerializerBase.unregister_dict_to_class(supported_class, deserialize)
for supported_exc in supported_exceptions:
SerializerBase.unregister_dict_to_class(supported_exc, deserialize_exception)
# from __future__ import print_function
import os
import socket
import sys
from math import sqrt
import Pyro4
from Pyro4.util import SerializerBase
from workitem import Workitem
import psutil
import time
import pickle
import json
# For 'workitem.Workitem' we register a deserialization hook to be able to get these back from Pyro
SerializerBase.register_dict_to_class("workitem.Workitem", Workitem.from_dict)
if sys.version_info < (3, 0):
range = xrange # make sure to use the memory efficient range generator
#Loads the worker's info
# WORKERINFO = pickle.load(open("config_worker.pickle", "rb"))
def main():
#connects to the dispatcher
with open("config/slave_config.json","r") as f:
configfile = f.read()
config = json.loads(configfile)
def deserialize_exception(fq_classname, data):
class_object = get_class_object(fq_classname)
return SerializerBase.make_exception(class_object, data)
"number-attribute": obj.number
}
def thingy_dict_to_class(classname, d):
print("{deserializer hook, converting to class: %s}" % d)
return mycustomclasses.Thingy(d["number-attribute"])
def otherthingy_dict_to_class(classname, d):
print("{deserializer hook, converting to class: %s}" % d)
return mycustomclasses.OtherThingy(d["number"])
# for 'Thingy' we register both serialization and deserialization hooks
SerializerBase.register_dict_to_class("waheeee-custom-thingy",
thingy_dict_to_class)
SerializerBase.register_class_to_dict(mycustomclasses.Thingy,
thingy_class_to_dict)
# for 'OtherThingy' we only register a deserialization hook (and for serialization depend on serpent's default behavior)
SerializerBase.register_dict_to_class("mycustomclasses.OtherThingy",
otherthingy_dict_to_class)
# regular Pyro server stuff
@Pyro4.expose
class Server(object):
def method(self, arg):
print("\nmethod called, arg=", arg)
response = mycustomclasses.Thingy(999)
def register_classes():
"""
Register the serializers
"""
SerializerBase.register_class_to_dict(Path, path_class_to_dict)
SerializerBase.register_dict_to_class('Path', path_dict_to_class)