def main():
def noop_error_handler(env, error):
pass
env = marcel.env.Environment(None, old_namespace=None)
version = env.getvar('MARCEL_VERSION')
TRACE.write(f'Marcel version {version}')
# Use sys.stdin.buffer because we want binary data, not the text version
input = dill.Unpickler(sys.stdin.buffer)
pipeline = input.load()
pipeline.set_error_handler(noop_error_handler)
TRACE.write(f'pipeline: {pipeline}')
pipeline_runner = PipelineRunner(env, pipeline)
pipeline_runner.start()
try:
signal_id = input.load()
TRACE.write(f'Received signal {signal_id}')
kill_descendents(signal_id)
except EOFError:
TRACE.write('Received EOF')
while pipeline_runner.is_alive():
TRACE.write(f'PipelineRunner alive: {pipeline_runner.is_alive()}')
pipeline_runner.join(0.1)
TRACE.write(f'PipelineRunner alive: {pipeline_runner.is_alive()}')
kill_descendents(signal.SIGTERM)
finally:
TRACE.write('Exiting')
TRACE.close()
def receive(self, _):
# Start the remote process
command = ' '.join(['sudo'] + self.args + ['farcel.py'])
self.process = subprocess.Popen(command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
universal_newlines=False)
# Pickle the pipeline so that it can be sent to the remote process
buffer = io.BytesIO()
pickler = dill.Pickler(buffer)
pickler.dump(self.pipeline)
buffer.seek(0)
stdout, stderr = self.process.communicate(input=buffer.getvalue())
# Wait for completion (already guaranteed by communicate returning?)
self.process.wait()
# Handle results
stderr_lines = stderr.decode('utf-8').split('\n')
if len(stderr_lines[-1]) == 0:
del stderr_lines[-1]
sys.stdout.flush()
for line in stderr_lines:
print(line, file=sys.stderr)
sys.stderr.flush()
input = dill.Unpickler(io.BytesIO(stdout))
try:
while True:
self.send(input.load())
except EOFError:
self.send_complete()
def __getitem__(self, key):
try:
value = self.cache[key]
except KeyError:
f = BytesIO(self.dict[key.encode(self.keyencoding)])
value = pickle.Unpickler(f).load()
if self.writeback:
self.cache[key] = value
return value
def load_set(self, Path):
with open(Path, 'rb') as fichier:
mon_depickler = pickle.Unpickler(fichier)
save = mon_depickler.load()
#print(save[0])
self.master = [save[0]]
self.tracks = save[1].copy()
self.init_osc()
print("loaded", Path)
def __init__(self, filename: Union[str, PathLike]):
self.f = _open_compressed(filename, "rb")
self.unpickler = dill.Unpickler(self.f)
version = self.unpickler.load()
if version > DillFormat.VERSION:
raise ValueError(f"File {filename} is too recent for this version of {self.__class__}.")
iterator = self.unpickler.load()
if not iterator:
raise ValueError(
f"Tried to open {filename} as an iterator, but it does not store an iterator."
)
def read(self, fname='.'):
"""Load model, data and space from disk.
:param str fname: path to a directory.
"""
path = os.path.join(fname, self.dir['surrogate'])
with open(path, 'rb') as f:
unpickler = pickle.Unpickler(f)
self.predictor = unpickler.load()
self.logger.debug('Model read from {}'.format(path))
path = os.path.join(fname, self.dir['space'])
self.space.read(path)
path = os.path.join(fname, self.dir['data'])
with open(path, 'rb') as f:
unpickler = pickle.Unpickler(f)
self.data = unpickler.load()[:len(self.space)]
self.logger.debug('Data read from {}'.format(path))
self.logger.info('Model, data and space loaded.')
def copy(x):
try:
buffer = io.BytesIO()
pickler = dill.Pickler(buffer)
pickler.dump(x)
buffer.seek(0)
unpickler = dill.Unpickler(buffer)
return unpickler.load()
except Exception as e:
sys.stdout.flush()
print(f'Cloning error: ({type(e)}) {e}',
file=sys.__stderr__,
flush=True)
def read(self, dir: Union[str, PathLike]) -> T:
filename = pathlib.Path(dir) / ("data.dill" + _SUFFIXES[self.open])
with self.open(filename, "rb") as f:
unpickler = dill.Unpickler(file=f)
version = unpickler.load()
if version > self.VERSION:
raise ValueError(
f"File {filename} is too recent for this version of {self.__class__}."
)
iterator = unpickler.load()
if iterator:
return DillFormatIterator(filename) # type: ignore
else:
return unpickler.load()
def test_extend():
obj = lambda: my_fn(34)
assert obj() == 578
obj_io = StringIO()
pickler = pickle.Pickler(obj_io)
pickler.dump(obj)
obj_str = obj_io.getvalue()
obj2_io = StringIO(obj_str)
unpickler = pickle.Unpickler(obj2_io)
obj2 = unpickler.load()
assert obj2() == 578
def load_dill(file_name):
if os.path.isfile(file_name): # file exists
# print(file_name, 'exists')
try:
# print("loading pickle: ", file_name)
program_path = os.path.dirname(os.path.realpath("__file__")) + '/'
file = open(program_path + file_name, "rb")
unpickler = dill.Unpickler(file)
t = unpickler.load()
return t
except Exception as error:
print('Error opening file:', error)
return False
else:
print(file_name, 'does not exist')
return False
def importPickle(filename):
"""
To import an object in a binary file
Parameters
----------
filename: string
the path and name of the file
Returns
-------
obj :
the imported object
"""
datafile = open(filename, "rb")
mon_depickler = pickle.Unpickler(datafile)
obj = mon_depickler.load()
datafile.close()
return obj
def fix_broken_pickles(file, new_name, return_data=False):
pickle.Unpickler = pickle._Unpickler
import dill
obj = open(file, 'rb')
unpickler = dill.Unpickler(obj)
try:
unpickler.load()
except EOFError:
pass
data = unpickler.memo[0]
with open(new_name, 'wb') as handle:
pickle.dump(data, handle)
if return_data:
return data
def main():
def noop_error_handler(env, error):
pass
try:
namespace = marcel.nestednamespace.NestedNamespace(read_config())
# Use sys.stdin.buffer because we want binary data, not the text version
input = dill.Unpickler(sys.stdin.buffer)
env = input.load()
namespace.update(env.namespace)
env.namespace = namespace
env.main_pid = os.getpid()
pipeline = input.load()
version = env.getvar('MARCEL_VERSION')
TRACE.write(f'Marcel version {version}')
TRACE.write(f'pipeline: {pipeline}')
atexit.register(shutdown)
pipeline.set_env(env)
pipeline.set_error_handler(noop_error_handler)
pipeline_runner = PipelineRunner(env, pipeline)
pipeline_runner.start()
except Exception as e:
TRACE.write(f'Caught {type(e)}: {e}')
marcel.util.print_stack(TRACE.file)
try:
signal_id = input.load()
TRACE.write(f'Received signal {signal_id}')
kill_descendents(signal_id)
except EOFError:
TRACE.write('Received EOF')
while pipeline_runner.is_alive():
TRACE.write(f'PipelineRunner alive: {pipeline_runner.is_alive()}')
pipeline_runner.join(0.1)
TRACE.write(f'PipelineRunner alive: {pipeline_runner.is_alive()}')
kill_descendents(signal.SIGTERM)
finally:
TRACE.write('Exiting')
TRACE.close()
def receive(self, _):
# Start the remote process
command = ' '.join(
['ssh', '-l', self.host.user, self.host.addr, 'farcel.py'])
self.process = subprocess.Popen(command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
universal_newlines=False)
buffer = io.BytesIO()
pickler = dill.Pickler(buffer)
pickler.dump(self.env().remotify())
pickler.dump(self.pipeline)
buffer.seek(0)
try:
stdout, stderr = self.process.communicate(input=buffer.getvalue())
stderr_lines = stderr.decode('utf-8').split('\n')
if len(stderr_lines[-1]) == 0:
del stderr_lines[-1]
sys.stdout.flush()
for line in stderr_lines:
print(line, file=sys.stderr)
sys.stderr.flush()
input = dill.Unpickler(io.BytesIO(stdout))
try:
while True:
x = input.load()
if isinstance(x, marcel.object.error.Error):
self.send_error(x)
else:
self.send(x)
except EOFError as e:
self.send_complete()
except BaseException as e:
marcel.util.print_stack()
print(e)
def last(self):
(key, value) = self.dict.last()
f = BytesIO(value)
return (key.decode(self.keyencoding), pickle.Unpickler(f).load())
def set_location(self, key):
(key, value) = self.dict.set_location(key)
f = BytesIO(value)
return (key.decode(self.keyencoding), pickle.Unpickler(f).load())
import dill
import io
# dill
pick = dill.dumps({'a': 'b', 'c': 'd'})
print(dill.loads(pick))
file_obj = io.BytesIO()
dill.dump([1, 2, '3'], file_obj)
file_obj.seek(0)
print(dill.load(file_obj))
file_obj.seek(0)
print(dill.Unpickler(file_obj).load())
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 2008-2014 California Institute of Technology.
# License: 3-clause BSD. The full license text is available at:
# - http://trac.mystic.cacr.caltech.edu/project/pathos/browser/dill/LICENSE
import dill as pickle
try:
from StringIO import StringIO
except ImportError:
from io import BytesIO as StringIO
def my_fn(x):
return x * 17
obj = lambda: my_fn(34)
assert obj() == 578
obj_io = StringIO()
pickler = pickle.Pickler(obj_io)
pickler.dump(obj)
obj_str = obj_io.getvalue()
obj2_io = StringIO(obj_str)
unpickler = pickle.Unpickler(obj2_io)
obj2 = unpickler.load()
assert obj2() == 578
def loads(filename_):
fh = file(filename_, 'rb')
unpickler = pickle.Unpickler(fh)
unpickler.dispatch[pickle.GLOBAL] = mapped_load_global
return unpickler.load()
def load(fp):
return dill.Unpickler(fp).load()
