def loads(self, s):
up = Unpickler(BytesIO(s))
up.persistent_load = self._get_object
try:
return up.load()
except KeyError, e:
raise UnpicklingError("Could not find Node class for %s" % e)
def deserialize(self, message, task_id=None):
"""Deserialize an object
:param message: A serialized object (string).
:param deferred: When true load deferreds. When false
raise an error if the message contains deferreds.
"""
fail = []
if task_id is None:
persistent_load = []
else:
args = self._queue.get_arguments(task_id)
args = {k: loads(v) for k, v in args.items()}
def persistent_load(arg_id):
value = args[arg_id]
if isinstance(value, TaskFailure):
fail.append(value)
return value
data = StringIO(message)
pickle = Unpickler(data)
pickle.persistent_load = persistent_load
obj = pickle.load()
if task_id is None and persistent_load:
raise UnpicklingError(
'message contained references to '
'external objects: %s', persistent_load)
if fail and not obj.on_error_pass:
# TODO detect errors earlier, fail earlier, cancel enqueued tasks
self.set_result(obj, fail[0])
obj = None
return obj
def restricted_find_global(mod_name, fn_name):
__import__(mod_name)
mod = sys.modules[mod_name]
fn = getattr(mod, fn_name)
if not (fn in JOB_REGISTRY or fn in _UNPICKLE_WHITELIST):
raise UnpicklingError('{}.{} is not allowed in jobs'.format(
mod_name, fn_name))
return fn
def findGlobal(self, module, klass):
"""Find class name."""
if (module, klass) not in self.allowedGlobals():
raise UnpicklingError("For security reasons, you can\'t unpickle"
" objects from module %s with type %s." %
(module, klass))
g = {}
exec 'from %s import %s as theClass' % (module, klass) in g
return g['theClass']
def _next_case(self):
""" Generator which returns Cases one at a time. """
# figure out which selectors are for cases and which are for variables
sql = ["SELECT * FROM cases"]
if self.selectors is not None:
for sel in self.selectors:
rhs, rel, lhs = _query_split(sel)
if rhs in _casetable_attrs:
if len(sql) == 1:
sql.append("WHERE %s%s%s" % (rhs, rel, lhs))
else:
sql.append("AND %s%s%s" % (rhs, rel, lhs))
casecur = self._connection.cursor()
casecur.execute(' '.join(sql))
sql = [
'SELECT var_id,name,case_id,sense,value from casevars WHERE case_id=%s'
]
if self.selectors is not None:
for sel in self.selectors:
rhs, rel, lhs = _query_split(sel)
if rhs in _vartable_attrs:
sql.append("AND %s%s%s" % (rhs, rel, lhs))
combined = ' '.join(sql)
varcur = self._connection.cursor()
for cid, text_id, parent, msg, model_id, timeEnter in casecur:
varcur.execute(combined % cid)
inputs = []
outputs = []
for var_id, vname, case_id, sense, value in varcur:
if isinstance(value, (float, int, str)):
pass
elif value is None: # Result when recorded value was NaN.
value = float('NaN')
else:
try:
value = loads(str(value))
except UnpicklingError as err:
print 'value', type(value), repr(value)
raise UnpicklingError("can't unpickle value '%s' for"
" case '%s' from database: %s" %
(vname, text_id, str(err)))
if sense == 'i':
inputs.append((vname, value))
elif sense == 'o':
outputs.append((vname, value))
if len(inputs) > 0 or len(outputs) > 0:
exc = Exception(msg) if msg else None
yield Case(inputs=inputs,
outputs=outputs,
exc=exc,
case_uuid=text_id,
parent_uuid=parent)
def _next_case(self):
""" Generator which returns Cases one at a time. """
# figure out which selectors are for cases and which are for variables
sql = ["SELECT * FROM cases"]
if self.selectors is not None:
for sel in self.selectors:
rhs, rel, lhs = _query_split(sel)
if rhs in _casetable_attrs:
if len(sql) == 1:
sql.append("WHERE %s%s%s" % (rhs, rel, lhs))
else:
sql.append("AND %s%s%s" % (rhs, rel, lhs))
casecur = self._connection.cursor()
casecur.execute(' '.join(sql))
sql = [
'SELECT var_id,name,case_id,sense,value from casevars WHERE case_id=%s'
]
if self.selectors is not None:
for sel in self.selectors:
rhs, rel, lhs = _query_split(sel)
if rhs in _vartable_attrs:
sql.append("AND %s%s%s" % (rhs, rel, lhs))
combined = ' '.join(sql)
varcur = self._connection.cursor()
for cid, text_id, parent, label, msg, retries, model_id, timeEnter in casecur:
varcur.execute(combined % cid)
inputs = []
outputs = []
for var_id, vname, case_id, sense, value in varcur:
if not isinstance(value, (float, int, str)):
try:
value = loads(str(value))
except UnpicklingError as err:
raise UnpicklingError(
"can't unpickle value '%s' for case '%s' from database: %s"
% (vname, cname, str(err)))
if sense == 'i':
inputs.append((vname, value))
else:
outputs.append((vname, value))
if len(inputs) > 0 or len(outputs) > 0:
yield Case(inputs=inputs,
outputs=outputs,
retries=retries,
msg=msg,
label=label,
case_uuid=text_id,
parent_uuid=parent)
def broadcast(buffer, root, vanilla=False, bypass=False):
"""Wrapper for MPI bcast.
Broadcast buffer from the process with rank root to all other processes.
Automatically determine appropriate protocol
and call corresponding send function.
The variable buffer can be any (picklable) type, but
numpy variables and text strings will most efficient.
Setting vanilla = 1 forces vanilla mode for any type.
If bypass is True, all admin and error checks
get bypassed to reduce the latency.
"""
if bypass:
broadcast_array(buffer, root)
return
import types
# Input check
errmsg = 'Root id (%s) must be an integer.' % root
assert type(root) == types.IntType, errmsg
# Create metadata about object to be sent
protocol = create_control_info(buffer, vanilla)[0]
# Broadcast
if protocol == 'array':
broadcast_array(buffer, root)
elif protocol == 'string':
broadcast_string(buffer, root)
elif protocol == 'vanilla':
from cPickle import loads, dumps, UnpicklingError
s = dumps(buffer, protocol=2)
s = s + ' ' * int(0.1 * len(s)) #safety
broadcast_string(s, root)
try:
buffer = loads(s)
except UnpicklingError, err:
raise UnpicklingError(str(err) + " - '%s'" % s)
def get_data(db_file_name):
conn = sqlite3.connect(db_file_name)
conn.text_factory = sqlite3.OptimizedUnicode
cur = conn.cursor()
sql_count = "SELECT count(*) from cases"
print "total cases:", [x for x in cur.execute(sql_count)][0][0]
sql_count = 'SELECT case_id from casevars where name=="Objective_0"'
ids = [x[0] for x in cur.execute(sql_count)]
id_index_map = dict([(id, i) for i, id in enumerate(ids)])
n_cases = len(ids)
print "top level cases:", n_cases
data = {}
sql = "SELECT var_id,name,case_id,sense,value FROM cases INNER JOIN casevars ON casevars.case_id=cases.id WHERE case_id in (%s)" % ','.join(
map(str, ids))
cur = conn.cursor()
cur.execute(sql)
objective_cases = []
for var_id, vname, case_id, sense, value in cur:
if not isinstance(value, (float, int, str)):
try:
value = loads(str(value))
except UnpicklingError as err:
raise UnpicklingError("can't unpickle value '%s' for"
" case '%s' from database: %s" %
(vname, case_id, str(err)))
index = id_index_map[case_id]
if vname in data:
data[vname][index] = value
else:
data[vname] = [
None,
] * n_cases
data[vname][index] = value
return n_cases, data
def ireceive(source, buffer=None, vanilla=False, tag=default_tag,
return_status=False, bypass=False):
"""receive - blocking MPI receive
Receive data from source.
Optional parameters:
buffer: Use specified buffer for received data (faster). Default None.
vanilla: Specify to enforce vanilla protocol for any type.
Default False
tag: Only received messages tagged as specified. Default default_tag
return_status: Return Status object along with result. Default False.
If no buffer is specified, receive will try to receive a
preceding message containing protocol, type, size and shape and
then create a suitable buffer.
If buffer is specified the corresponding send must specify
use_buffer = True.
The variable buffer can be any (picklable) type, but
numpy variables and text strings will most efficient.
Appropriate protocol will be automatically determined
and corresponding receive function called.
If bypass is True, all admin and error checks
get bypassed to reduce the latency. Should only
be used for receiving numpy arrays and should
be matched with a bypass in the corresponding send command.
Also buffer must be specified.
"""
if bypass:
# errmsg = 'bypass mode must be used with specified buffer'
# assert buffer is not None, msg
stat = ireceive_array(buffer, source, tag)
else:
import types
# Input check
errmsg = 'Source id (%s) must be an integer.' %source
assert type(source) == types.IntType, errmsg
errmsg = 'Tag %d is reserved by pypar - please use another.'\
% control_tag
assert tag != control_tag, errmsg
# Either receive or create metadata about object to receive
if buffer is None:
control_info, source = receive_control_info(source,
return_source=True)
protocol, typecode, size, shape = control_info
else:
protocol, typecode, size, shape = create_control_info(buffer,
vanilla)
# Receive payload data
if protocol == 'array':
if buffer is None:
buffer = zeros(size,typecode)
buffer = reshape(buffer, shape)
stat = receive_array(buffer, source, tag)
elif protocol == 'string':
if buffer is None:
buffer = ' '*size
stat = receive_string(buffer, source, tag)
elif protocol == 'vanilla':
from cPickle import dumps, loads, UnpicklingError
if buffer is None:
s = ' '*size
else:
s = dumps(buffer, protocol=2)
s = s + ' '*int(0.1*len(s)) #safety
stat = receive_string(s, source, tag)
try:
buffer = loads(s) #Replace buffer with received result
except UnpicklingError, err:
raise UnpicklingError(str(err) + " - '%s'" % s)
else:
def case_db_to_dict(dbname, varnames, case_sql='', var_sql='', include_errors=False):
"""
Retrieve the values of specified variables from a sqlite DB containing
Case data.
Returns a dict containing a list of values for each entry, keyed on
variable name.
Only data from cases containing ALL of the specified variables will
be returned so that all data values with the same index will correspond
to the same case.
dbname: str
The name of the sqlite DB file.
varnames: list[str]
Iterator of names of variables to be retrieved.
case_sql: str (optional)
SQL syntax that will be placed in the WHERE clause for Case retrieval.
var_sql: str (optional)
SQL syntax that will be placed in the WHERE clause for variable retrieval.
include_errors: bool (optional) [False]
If True, include data from cases that reported an error.
"""
connection = sqlite3.connect(dbname)
vardict = dict([(name, []) for name in varnames])
sql = ["SELECT id FROM cases"]
qlist = []
if case_sql:
qlist.append(case_sql)
if not include_errors:
qlist.append("msg = ''")
if qlist:
sql.append("WHERE %s" % ' AND '.join(qlist))
casecur = connection.cursor()
casecur.execute(' '.join(sql))
sql = ["SELECT name, value from casevars WHERE case_id=%s"]
vars_added = False
for i, name in enumerate(vardict.keys()):
if i == 0:
sql.append("AND (")
else:
sql.append("OR")
sql.append("name='%s'" % name)
vars_added = True
if vars_added:
sql.append(")")
if var_sql:
sql.append(" AND %s" % var_sql)
combined = ' '.join(sql)
varcur = connection.cursor()
for case_id in casecur:
casedict = {}
varcur.execute(combined % case_id)
for vname, value in varcur:
if not isinstance(value, (float, int, str)):
try:
value = loads(str(value))
except UnpicklingError as err:
raise UnpicklingError("can't unpickle value '%s' from"
" database: %s" % (vname, str(err)))
casedict[vname] = value
if len(casedict) != len(vardict):
continue # case doesn't contain a complete set of specified vars,
# so skip it to avoid data mismatches
for name, value in casedict.items():
vardict[name].append(value)
return vardict
def persistent_load(task_id):
raise UnpicklingError('message contained references to '
'external objects: %s' % task_id)
def persistent_load(persid):
if persid.startswith(KEY):
value = persid[len(KEY):]
return value
else:
raise UnpicklingError('Invalid persistent id')