예제 #1
0
 def __init__(self):
     self.daemon = None
     self.workers = []
     SerializerBase.register_dict_to_class("worker_node.WorkerNode", WorkerNode.node_dict_to_class)
     SerializerBase.register_dict_to_class("job.Job", Job.job_dict_to_class)
     self.nfs_exporter = CertCheckingProxy('PYRO:NfsExporter@localhost:9091')
     self.first_run = True
예제 #2
0
def register_deserialization_handlers():
    for supported_class in supported_classes:
        SerializerBase.register_dict_to_class(supported_class, deserialize)
    for supported_exc in supported_exceptions:
        SerializerBase.register_dict_to_class(supported_exc, deserialize_exception)
    import subprocess
    SerializerBase.register_class_to_dict(subprocess.CalledProcessError, serialize_called_process_error)
예제 #3
0
def register_deserialization_handlers():
    for supported_class in supported_classes:
        SerializerBase.register_dict_to_class(supported_class, deserialize)
    for supported_exc in supported_exceptions:
        SerializerBase.register_dict_to_class(supported_exc,
                                              deserialize_exception)
    import subprocess
    SerializerBase.register_class_to_dict(subprocess.CalledProcessError,
                                          serialize_called_process_error)
예제 #4
0
 def setup(self):
     signal.signal(signal.SIGINT, self.termination_handler)
     with open(OD_FOLDER + SSL_CERTS_DIR + 'whoismaster') as f:
         self.MASTER_ADDRESS = f.read().strip()
     self.MOUNTPOINT_DEFAULT = str(Path.home()) + '/olive-share/'
     self.job_dispatcher = CertCheckingProxy('PYRO:JobDispatcher@' +
                                             self.MASTER_ADDRESS + ':9090')
     self.nfs_mounter = CertCheckingProxy('PYRO:NfsMounter@' + 'localhost' +
                                          ':9092')
     SerializerBase.register_dict_to_class(
         "job.ExportRange", ExportRange.export_range_dict_to_class)
     SerializerBase.register_dict_to_class("job.Job", Job.job_dict_to_class)
 def __init__(self):
     super().__init__()
     self.split_job = None
     self.parts_lock = threading.Lock()
     self.worker_fails = dict()
     self.ongoing_ranges = set()
     self.failed_ranges = set()
     self.completed_ranges = set()
     self.cleanup_queue = set()
     self.last_assigned_frame = 0
     self.job_parts = 0
     SerializerBase.register_dict_to_class("job.ExportRange", ExportRange.export_range_dict_to_class)
     SerializerBase.register_class_to_dict(ExportRange, ExportRange.export_range_class_to_dict)
예제 #6
0
 def __init__(self, address):
     with open(SSL_CERTS_DIR + 'whoismaster') as f:
         self.MASTER_ADDRESS = f.read().strip()
     self.MOUNTPOINT_DEFAULT = str(Path.home()) + '/olive-share'
     self.address = address
     self.cpu_score = 0
     self.net_score = 0
     self._job_start_time = None
     self._job = None
     self.sample_weight = None
     self.sample_time = None
     self.worker_options = dict()
     self.job_dispatcher = CertCheckingProxy('PYRO:JobDispatcher@' +
                                             self.MASTER_ADDRESS + ':9090')
     self.nfs_mounter = CertCheckingProxy('PYRO:NfsMounter@' + 'localhost' +
                                          ':9092')
     SerializerBase.register_dict_to_class(
         "job.ExportRange", ExportRange.export_range_dict_to_class)
     SerializerBase.register_dict_to_class("job.Job", Job.job_dict_to_class)
예제 #7
0
from __future__ import print_function

import random
import socket
from Worker import Worker
import Pyro4
try:
    import queue  # py3
except ImportError:
    import Queue as queue  # py2

from Pyro4.util import SerializerBase
from Job import Job
SerializerBase.register_dict_to_class("Job.Job", Job.from_dict)


class Scheduler(object):
    METHOD_RAND = "RANDOM"
    METHOD_TWO = "CHOOSE_TWO"
    METHOD_BATCH = "BATCH"
    METHOD_LATE = "BATCH+LATE_BINDING"

    def __init__(self, scheduling_method=METHOD_RAND, nameserver_hostname="newyork", scheduler_number=1):
        self.no_of_workers_per_scheduler = 10
        self.scheduling_method = scheduling_method
        print("Scheduling method: ", self.scheduling_method)

        # Number of the scheduler
        self.scheduler_number = scheduler_number

        # List of jobs that have been scheduled/reserved
예제 #8
0
파일: server.py 프로젝트: Peque/Pyro4
        "__class__": "waheeee-custom-thingy",
        "number-attribute": obj.number
    }


def thingy_dict_to_class(classname, d):
    print("{deserializer hook, converting to class: %s}" % d)
    return mycustomclasses.Thingy(d["number-attribute"])


def otherthingy_dict_to_class(classname, d):
    print("{deserializer hook, converting to class: %s}" % d)
    return mycustomclasses.OtherThingy(d["number"])

# for 'Thingy' we register both serialization and deserialization hooks
SerializerBase.register_dict_to_class("waheeee-custom-thingy", thingy_dict_to_class)
SerializerBase.register_class_to_dict(mycustomclasses.Thingy, thingy_class_to_dict)

# for 'OtherThingy' we only register a deserialization hook (and for serialization depend on serpent's default behavior)
SerializerBase.register_dict_to_class("mycustomclasses.OtherThingy", otherthingy_dict_to_class)


# regular Pyro server stuff

@Pyro4.expose
class Server(object):
    def method(self, arg):
        print("\nmethod called, arg=", arg)
        response = mycustomclasses.Thingy(999)
        return response
예제 #9
0
    def to_dict(obj):
        return {
            "__class__": "Pyro4.utils.messagebus.message",
            "msgid": str(obj.msgid),
            "created": obj.created.isoformat(),
            "data": obj.data,
        }

    @classmethod
    def from_dict(cls, classname, d):
        return cls(uuid.UUID(d["msgid"]), datetime.datetime.strptime(d["created"], "%Y-%m-%dT%H:%M:%S.%f"), d["data"])


# make sure Pyro knows how to serialize the custom Message class
SerializerBase.register_class_to_dict(Message, Message.to_dict)
SerializerBase.register_dict_to_class("Pyro4.utils.messagebus.message", Message.from_dict)


@Pyro4.expose
class Subscriber(object):
    def __init__(self, auto_consume=True, max_queue_size=5000):
        self.bus = Pyro4.Proxy("PYRONAME:" + PYRO_MSGBUS_NAME)
        self.received_messages = queue.Queue(maxsize=max_queue_size)
        if auto_consume:
            self.__bus_consumer_thread = threading.Thread(target=self.__bus_consume_message)
            self.__bus_consumer_thread.daemon = True
            self.__bus_consumer_thread.start()

    def incoming_message(self, topic, message):
        self.received_messages.put((topic, message))
예제 #10
0
파일: menu.py 프로젝트: asmeyer2012/tarock
def SerializeMenuToDict(menu):
    out = {}
    out["__class__"] = "Menu"
    out["_active"] = menu._active
    out["_info"] = menu._info
    for key in menu._entries.keys():
        out["_entries.{0}".format(key)] = menu._entries[key]
        if key in menu._mask:
            out["_mask.{0}".format(key)] = True
        else:
            out["_mask.{0}".format(key)] = False
    return out


## need to define functions to serialize so Menu can be passed with Pyro
def DeserializeMenuDict(classname, mdict):
    menu = Menu()
    menu._active = mdict["_active"]
    menu._info = mdict["_info"]
    for key in mdict.keys():
        if key[:8] == '_entries':
            xkey = key[9:]
            menu._entries[xkey] = mdict[key]
            if mdict["_mask.{0}".format(xkey)]:
                menu._mask.add(xkey)
    return menu


SerializerBase.register_class_to_dict(Menu, SerializeMenuToDict)
SerializerBase.register_dict_to_class("Menu", DeserializeMenuDict)
예제 #11
0
from common import config
from common.module import ShellModule
from common.infra_modules.shell_manager_module import MODULE_NAME
from Pyro4.util import SerializerBase

logger = config.get_log(MODULE_NAME)


def statement_dict_to_class(classname, d):
    # Method that deserialize cmd2.Statement class to tuple of tuple
    return tuple(d['raw-attribute'].split())


# Register method in pyro4 for correct deserialization
SerializerBase.register_dict_to_class("statement", statement_dict_to_class)


class ShellManagerModule(ShellModule):
    def initialize(self):
        """
        This method create and initialize all variables and resources needed
        :return: None
        """
예제 #12
0
파일: z3_embed.py 프로젝트: d1m0/pyboogie
Z3ValFactory_T = Callable[[str], z3.ExprRef]
Z3TypeEnv = Dict[str, Z3ValFactory_T]


def fi_deserialize(classname: str, d: Dict[Any, Any]) -> OpaqueVal:
    return OpaqueVal.from_dict(d)


def fi_serialize(obj: OpaqueVal) -> Dict[Any, Any]:
    d = OpaqueVal.to_dict(obj)
    assert "__class__" not in d
    d["__class__"] = "OpaqueVal"
    return d


SerializerBase.register_dict_to_class("OpaqueVal", fi_deserialize)
SerializerBase.register_class_to_dict(OpaqueVal, fi_serialize)


def type_to_z3sort(ast_typ: AstType) -> z3.SortRef:
    if isinstance(ast_typ, AstIntType):
        return IntSort()
    elif isinstance(ast_typ, AstBoolType):
        return BoolSort()
    else:
        # TODO: Multi-dimensional maps NYI
        assert isinstance(ast_typ, AstMapType) and len(ast_typ.domainT) == 1
        return z3.ArraySort(type_to_z3sort(ast_typ.domainT[0]),
                            type_to_z3sort(ast_typ.rangeT))

예제 #13
0
            uri = nameserver.lookup(conninfo.virtual_host)
            return pyro.Proxy(uri)
        except NamingError:
            reraise(NamingError, NamingError(E_LOOKUP.format(conninfo)),
                    sys.exc_info()[2])

    def driver_version(self):
        return pyro.__version__

    @cached_property
    def shared_queues(self):
        return self._open()


if pyro is not None:
    SerializerBase.register_dict_to_class("queue.Empty",
                                          lambda cls, data: Empty())

    @pyro.expose
    @pyro.behavior(instance_mode="single")
    class KombuBroker(object):
        """Kombu Broker used by the Pyro transport.

        You have to run this as a separate (Pyro) service.
        """

        def __init__(self):
            self.queues = {}

        def get_queue_names(self):
            return list(self.queues)
예제 #14
0
파일: serialization.py 프로젝트: HMazen/pcd
    f.ps_distro = d["ps_distro"]
    f.protocol = d["protocol"]
    f.starting_date = d["starting_date"]
    f.trans_duration = d["trans_duration"]
    f.mesure = d["mesure"]
    m = mesure_config()
    m.metrics = d["mesure"]["metrics"]
    m.sampling_interval = d["mesure"]["sampling_interval"]
    m.finish_date = d["mesure"]["finish_date"]
    m.start_date = d["mesure"]["start_date"]
    f.mesure = m
    return f


SerializerBase.register_class_to_dict(flow_config, flow_config_class_to_dict)
SerializerBase.register_dict_to_class("flow_config", flow_config_dict_to_class)


def mesure_config_class_to_dict(obj):
    return dict(__class__="mesure_config", metrics=obj.metrics, start_date=obj.start_date, finish_date=obj.finish_date,
                sampling_interval=obj.sampling_interval)


def mesure_config_dict_to_class(classname, d):
    m = mesure_config()
    m.metrics = d["metrics"]
    m.sampling_interval = d["sampling_interval"]
    m.finish_date = d["finish_date"]
    m.start_date = d["start_date"]
    return m
예제 #15
0
파일: task.py 프로젝트: nerdoc/pyutilib
# Note: We can use this same mechanism to allow Task to be
#       re-implemented as a custom class.  HOWEVER,
#       TaskProcessingError is designed to be returned on the result
#       "slot" of Task, and when I've attempted to register
#       (de)serialization hooks for both Task and TaskProcessingError,
#       the end result is that the TaskProcessingError
#       de-serialization hook never gets called (leaving it in its
#       serialized dict form on the deserialized Task object). This
#       defeats the purpose of creating a TaskProcessingError type.
#       Perhaps there is a way around this, but for now I am giving
#       priority to having an error type that can be transmitted
#       for all serializers in both Pyro and Pyro4 (over making
#       Task a class again).
#
if using_pyro4:
    import Pyro4
    from Pyro4.util import SerializerBase

    # register hooks for TaskProcessingError
    def TaskProcessingError_to_dict(obj):
        return {"__class__": "pyutilib.pyro.task.TaskProcessingError",
                "message": obj.args[0]}
    def dict_to_TaskProcessingError(classname, d):
        return TaskProcessingError(d['message'])
    SerializerBase.register_class_to_dict(
        TaskProcessingError,
        TaskProcessingError_to_dict)
    SerializerBase.register_dict_to_class(
        "pyutilib.pyro.task.TaskProcessingError",
        dict_to_TaskProcessingError)
예제 #16
0
#       TaskProcessingError is designed to be returned on the result
#       "slot" of Task, and when I've attempted to register
#       (de)serialization hooks for both Task and TaskProcessingError,
#       the end result is that the TaskProcessingError
#       de-serialization hook never gets called (leaving it in its
#       serialized dict form on the deserialized Task object). This
#       defeats the purpose of creating a TaskProcessingError type.
#       Perhaps there is a way around this, but for now I am giving
#       priority to having an error type that can be transmitted
#       for all serializers in both Pyro and Pyro4 (over making
#       Task a class again).
#
if using_pyro4:
    import Pyro4
    from Pyro4.util import SerializerBase

    # register hooks for TaskProcessingError
    def TaskProcessingError_to_dict(obj):
        return {
            "__class__": "pyutilib.pyro.task.TaskProcessingError",
            "message": obj.args[0]
        }

    def dict_to_TaskProcessingError(classname, d):
        return TaskProcessingError(d['message'])

    SerializerBase.register_class_to_dict(TaskProcessingError,
                                          TaskProcessingError_to_dict)
    SerializerBase.register_dict_to_class(
        "pyutilib.pyro.task.TaskProcessingError", dict_to_TaskProcessingError)
예제 #17
0
        return {
            "__class__": "Pyro4.utils.messagebus.message",
            "msgid": str(obj.msgid),
            "created": obj.created.isoformat(),
            "data": obj.data
        }

    @classmethod
    def from_dict(cls, classname, d):
        return cls(uuid.UUID(d["msgid"]),
                   datetime.datetime.strptime(d["created"], "%Y-%m-%dT%H:%M:%S.%f"),
                   d["data"])

# make sure Pyro knows how to serialize the custom Message class
SerializerBase.register_class_to_dict(Message, Message.to_dict)
SerializerBase.register_dict_to_class("Pyro4.utils.messagebus.message", Message.from_dict)


@Pyro4.expose
class Subscriber(object):
    """
    This class is meant to be subclassed, and the consume_message method reimplemented.
    As messages arrive from the publisher, the consume_message method gets called.
    """
    def __init__(self, auto_consume=True, max_queue_size=5000, host='localhost', port=9090, **kwargs):
        """
        kwargs:
            - auto_consume (bool): Automatically start subscribing (True)
            - max_queue_size (int): Max number of elements that can be placed in queue.Queue (5000)
            - host (str): The nameserver hostname ('localhost')
            - port (int): The nameserver port (9090)
예제 #18
0
                if not (record.id in self):
                    self.records.append(record)

    def to_dict(self) -> Dict:
        """
        Used for serpent serialisation
        :return: A dict representing this Log
        """

        return {
            "__class__": "Log",
            "records": [record.to_dict() for record in self.records]
        }

    @staticmethod
    def from_dict(classname: str, dict: Dict) -> 'Log':
        """
        Used for serpent deserialisation
        :return: A Log
        """

        return Log(
            [Record.from_dict("Record", record) for record in dict["records"]])


SerializerBase.register_class_to_dict(Record, Record.to_dict)
SerializerBase.register_dict_to_class("Record", Record.from_dict)

SerializerBase.register_class_to_dict(Log, Log.to_dict)
SerializerBase.register_dict_to_class("Log", Log.from_dict)
예제 #19
0
import base64
import os
import socket
import sys
import Pyro4
from Pyro4.util import SerializerBase

from dispatcher_queue import DispatcherQueue
from job import Job

# For "job.Job" we register a deserialisation hook to be able to get these back from Pyro
SerializerBase.register_dict_to_class("job.Job", Job.from_dict)

# For "dispatcher_queue.DispatcherQueue" we register a deserialisation hook to be able to get these back from Pyro
SerializerBase.register_dict_to_class("dispatcher_queue.DispatcherQueue", DispatcherQueue.from_dict)

SERVER_NAME = ""
SUBDIR = ""


def handle_upload_init(file_name):
    """
    Handles an upload request with just the file name.
    """
    print("Dispatcher wanted to upload a file: {}".format(file_name))
    return {"outcome": "ready to receive"}


def handle_upload_data(file_name, file_contents):
    """
    Handles a full upload request.
예제 #20
0
from __future__ import print_function
import os
import socket
import sys
from math import sqrt
import Pyro4
from Pyro4.util import SerializerBase
from workitem import Workitem


# For 'workitem.Workitem' we register a deserialization hook to be able to get these back from Pyro
SerializerBase.register_dict_to_class("workitem.Workitem", Workitem.from_dict)

if sys.version_info < (3, 0):
    range = xrange   # make sure to use the memory efficient range generator

WORKERNAME = "Worker_%d@%s" % (os.getpid(), socket.gethostname())


def factorize(n):
    """simple algorithm to find the prime factorials of the given number n"""

    def isPrime(n):
        return not any(x for x in range(2, int(sqrt(n)) + 1) if n % x == 0)

    primes = []
    candidates = range(2, n + 1)
    candidate = 2
    while not primes and candidate in candidates:
        if n % candidate == 0 and isPrime(candidate):
            primes = primes + [candidate] + factorize(n // candidate)
예제 #21
0
        for (id, value) in ts.replicas.items():

            if id not in self.replicas:

                self.replicas[id] = value

            elif value > self.replicas[id]:

                self.replicas[id] = ts.replicas[id]

    def to_dict(self) -> Dict:
        """
        Used for serpent serialisation
        :return: A dict representing this Timestamp
        """

        return {"__class__": "Timestamp", "replicas": self.replicas}

    @staticmethod
    def from_dict(classname: str, dict: Dict) -> 'Timestamp':
        """
        Used for serpent deserialisation
        :return: A Log
        """

        return Timestamp(dict["replicas"])


SerializerBase.register_class_to_dict(Timestamp, Timestamp.to_dict)
SerializerBase.register_dict_to_class('Timestamp', Timestamp.from_dict)
예제 #22
0
    def to_dict(self) -> dict:
        """
            Serpent serialization requires to_dict
            Represent class InvalidUserError as a dictionary
            :return: A dictionary representation of this InvalidUserError
        """
        return {
            "__class__": "InvalidUserError",
            "message": self.message
        }

    @staticmethod
    def to_class(class_name: str, dictionary: dict) -> 'InvalidUserError':
        """
            This converts a dictionary representing an InvalidUserError
            to an object of type InvalidUserError
            This exception is thrown when there is a problem with the provided user_id
            :param class_name: parameter is required for serialization
            :param dictionary: dictionary representing an InvalidUserError
            :return:
        """
        del class_name
        return InvalidUserError(dictionary["message"])


# Register serialization and deserialization hooks for InvalidUserError and InvalidMovieError
SerializerBase.register_class_to_dict(InvalidUserError, InvalidUserError.to_dict)
SerializerBase.register_dict_to_class("InvalidUserError", InvalidUserError.to_class)
SerializerBase.register_class_to_dict(InvalidMovieError, InvalidMovieError.to_dict)
SerializerBase.register_dict_to_class("InvalidMovieError", InvalidMovieError.to_class)
from typing import Any, Dict
from synthplayer import sample
from synthplayer.playback import Output
from Pyro4.util import SerializerBase
import Pyro4


def sample_deserializer(classname: str, data: Dict[str, Any]) -> sample.Sample:
    return sample.Sample.from_raw_frames(data["frames"], data["samplewidth"],
                                         data["samplerate"], data["nchannels"],
                                         data["name"])


Pyro4.config.SERIALIZER = "marshal"
SerializerBase.register_dict_to_class("synthplayer.sample.Sample",
                                      sample_deserializer)
synth = Pyro4.Proxy("PYRONAME:synth.wavesynth")
synth.setup(44100)

with Output(44100, nchannels=1, samplewidth=2, mixing="sequential") as output:
    silence = sample.Sample.from_raw_frames(b"",
                                            samplewidth=2,
                                            samplerate=44100,
                                            numchannels=1)
    silence.add_silence(0.1)
    output.play_sample(synth.sine(220, .5))
    output.play_sample(silence)
    output.play_sample(synth.sine(330, .5))
    output.play_sample(silence)
    output.play_sample(synth.sine(440, .5))
    output.play_sample(silence)
예제 #24
0
# from __future__ import print_function
import os
import socket
import sys
from math import sqrt
import Pyro4
from Pyro4.util import SerializerBase
from workitem import Workitem
import psutil
import time
import pickle
import json


# For 'workitem.Workitem' we register a deserialization hook to be able to get these back from Pyro
SerializerBase.register_dict_to_class("workitem.Workitem", Workitem.from_dict)

if sys.version_info < (3, 0):
    range = xrange   # make sure to use the memory efficient range generator

#Loads the worker's info
# WORKERINFO = pickle.load(open("config_worker.pickle", "rb"))

def main():
	#connects to the dispatcher

	with open("config/slave_config.json","r") as f:
		configfile = f.read()

	config = json.loads(configfile)
예제 #25
0
    f.ps_distro = d["ps_distro"]
    f.protocol = d["protocol"]
    f.starting_date = d["starting_date"]
    f.trans_duration = d["trans_duration"]
    f.mesure = d["mesure"]
    m = mesure_config()
    m.metrics = d["mesure"]["metrics"]
    m.sampling_interval = d["mesure"]["sampling_interval"]
    m.finish_date = d["mesure"]["finish_date"]
    m.start_date = d["mesure"]["start_date"]
    f.mesure = m
    return f


SerializerBase.register_class_to_dict(flow_config, flow_config_class_to_dict)
SerializerBase.register_dict_to_class("flow_config", flow_config_dict_to_class)


def mesure_config_class_to_dict(obj):
    return dict(__class__="mesure_config",
                metrics=obj.metrics,
                start_date=obj.start_date,
                finish_date=obj.finish_date,
                sampling_interval=obj.sampling_interval)


def mesure_config_dict_to_class(classname, d):
    m = mesure_config()
    m.metrics = d["metrics"]
    m.sampling_interval = d["sampling_interval"]
    m.finish_date = d["finish_date"]
예제 #26
0
        "number-attribute": obj.number
    }


def thingy_dict_to_class(classname, d):
    print("{deserializer hook, converting to class: %s}" % d)
    return mycustomclasses.Thingy(d["number-attribute"])


def otherthingy_dict_to_class(classname, d):
    print("{deserializer hook, converting to class: %s}" % d)
    return mycustomclasses.OtherThingy(d["number"])


# for 'Thingy' we register both serialization and deserialization hooks
SerializerBase.register_dict_to_class("waheeee-custom-thingy",
                                      thingy_dict_to_class)
SerializerBase.register_class_to_dict(mycustomclasses.Thingy,
                                      thingy_class_to_dict)

# for 'OtherThingy' we only register a deserialization hook (and for serialization depend on serpent's default behavior)
SerializerBase.register_dict_to_class("mycustomclasses.OtherThingy",
                                      otherthingy_dict_to_class)

# regular Pyro server stuff


@Pyro4.expose
class Server(object):
    def method(self, arg):
        print("\nmethod called, arg=", arg)
        response = mycustomclasses.Thingy(999)
예제 #27
0
def register_classes():
    """
    Register the serializers
    """
    SerializerBase.register_class_to_dict(Path, path_class_to_dict)
    SerializerBase.register_dict_to_class('Path', path_dict_to_class)