def loads(self, s): up = Unpickler(BytesIO(s)) up.persistent_load = self._get_object try: return up.load() except KeyError, e: raise UnpicklingError("Could not find Node class for %s" % e)
def deserialize(self, message, task_id=None): """Deserialize an object :param message: A serialized object (string). :param deferred: When true load deferreds. When false raise an error if the message contains deferreds. """ fail = [] if task_id is None: persistent_load = [] else: args = self._queue.get_arguments(task_id) args = {k: loads(v) for k, v in args.items()} def persistent_load(arg_id): value = args[arg_id] if isinstance(value, TaskFailure): fail.append(value) return value data = StringIO(message) pickle = Unpickler(data) pickle.persistent_load = persistent_load obj = pickle.load() if task_id is None and persistent_load: raise UnpicklingError( 'message contained references to ' 'external objects: %s', persistent_load) if fail and not obj.on_error_pass: # TODO detect errors earlier, fail earlier, cancel enqueued tasks self.set_result(obj, fail[0]) obj = None return obj
def restricted_find_global(mod_name, fn_name): __import__(mod_name) mod = sys.modules[mod_name] fn = getattr(mod, fn_name) if not (fn in JOB_REGISTRY or fn in _UNPICKLE_WHITELIST): raise UnpicklingError('{}.{} is not allowed in jobs'.format( mod_name, fn_name)) return fn
def findGlobal(self, module, klass): """Find class name.""" if (module, klass) not in self.allowedGlobals(): raise UnpicklingError("For security reasons, you can\'t unpickle" " objects from module %s with type %s." % (module, klass)) g = {} exec 'from %s import %s as theClass' % (module, klass) in g return g['theClass']
def _next_case(self): """ Generator which returns Cases one at a time. """ # figure out which selectors are for cases and which are for variables sql = ["SELECT * FROM cases"] if self.selectors is not None: for sel in self.selectors: rhs, rel, lhs = _query_split(sel) if rhs in _casetable_attrs: if len(sql) == 1: sql.append("WHERE %s%s%s" % (rhs, rel, lhs)) else: sql.append("AND %s%s%s" % (rhs, rel, lhs)) casecur = self._connection.cursor() casecur.execute(' '.join(sql)) sql = [ 'SELECT var_id,name,case_id,sense,value from casevars WHERE case_id=%s' ] if self.selectors is not None: for sel in self.selectors: rhs, rel, lhs = _query_split(sel) if rhs in _vartable_attrs: sql.append("AND %s%s%s" % (rhs, rel, lhs)) combined = ' '.join(sql) varcur = self._connection.cursor() for cid, text_id, parent, msg, model_id, timeEnter in casecur: varcur.execute(combined % cid) inputs = [] outputs = [] for var_id, vname, case_id, sense, value in varcur: if isinstance(value, (float, int, str)): pass elif value is None: # Result when recorded value was NaN. value = float('NaN') else: try: value = loads(str(value)) except UnpicklingError as err: print 'value', type(value), repr(value) raise UnpicklingError("can't unpickle value '%s' for" " case '%s' from database: %s" % (vname, text_id, str(err))) if sense == 'i': inputs.append((vname, value)) elif sense == 'o': outputs.append((vname, value)) if len(inputs) > 0 or len(outputs) > 0: exc = Exception(msg) if msg else None yield Case(inputs=inputs, outputs=outputs, exc=exc, case_uuid=text_id, parent_uuid=parent)
def _next_case(self): """ Generator which returns Cases one at a time. """ # figure out which selectors are for cases and which are for variables sql = ["SELECT * FROM cases"] if self.selectors is not None: for sel in self.selectors: rhs, rel, lhs = _query_split(sel) if rhs in _casetable_attrs: if len(sql) == 1: sql.append("WHERE %s%s%s" % (rhs, rel, lhs)) else: sql.append("AND %s%s%s" % (rhs, rel, lhs)) casecur = self._connection.cursor() casecur.execute(' '.join(sql)) sql = [ 'SELECT var_id,name,case_id,sense,value from casevars WHERE case_id=%s' ] if self.selectors is not None: for sel in self.selectors: rhs, rel, lhs = _query_split(sel) if rhs in _vartable_attrs: sql.append("AND %s%s%s" % (rhs, rel, lhs)) combined = ' '.join(sql) varcur = self._connection.cursor() for cid, text_id, parent, label, msg, retries, model_id, timeEnter in casecur: varcur.execute(combined % cid) inputs = [] outputs = [] for var_id, vname, case_id, sense, value in varcur: if not isinstance(value, (float, int, str)): try: value = loads(str(value)) except UnpicklingError as err: raise UnpicklingError( "can't unpickle value '%s' for case '%s' from database: %s" % (vname, cname, str(err))) if sense == 'i': inputs.append((vname, value)) else: outputs.append((vname, value)) if len(inputs) > 0 or len(outputs) > 0: yield Case(inputs=inputs, outputs=outputs, retries=retries, msg=msg, label=label, case_uuid=text_id, parent_uuid=parent)
def broadcast(buffer, root, vanilla=False, bypass=False): """Wrapper for MPI bcast. Broadcast buffer from the process with rank root to all other processes. Automatically determine appropriate protocol and call corresponding send function. The variable buffer can be any (picklable) type, but numpy variables and text strings will most efficient. Setting vanilla = 1 forces vanilla mode for any type. If bypass is True, all admin and error checks get bypassed to reduce the latency. """ if bypass: broadcast_array(buffer, root) return import types # Input check errmsg = 'Root id (%s) must be an integer.' % root assert type(root) == types.IntType, errmsg # Create metadata about object to be sent protocol = create_control_info(buffer, vanilla)[0] # Broadcast if protocol == 'array': broadcast_array(buffer, root) elif protocol == 'string': broadcast_string(buffer, root) elif protocol == 'vanilla': from cPickle import loads, dumps, UnpicklingError s = dumps(buffer, protocol=2) s = s + ' ' * int(0.1 * len(s)) #safety broadcast_string(s, root) try: buffer = loads(s) except UnpicklingError, err: raise UnpicklingError(str(err) + " - '%s'" % s)
def get_data(db_file_name): conn = sqlite3.connect(db_file_name) conn.text_factory = sqlite3.OptimizedUnicode cur = conn.cursor() sql_count = "SELECT count(*) from cases" print "total cases:", [x for x in cur.execute(sql_count)][0][0] sql_count = 'SELECT case_id from casevars where name=="Objective_0"' ids = [x[0] for x in cur.execute(sql_count)] id_index_map = dict([(id, i) for i, id in enumerate(ids)]) n_cases = len(ids) print "top level cases:", n_cases data = {} sql = "SELECT var_id,name,case_id,sense,value FROM cases INNER JOIN casevars ON casevars.case_id=cases.id WHERE case_id in (%s)" % ','.join( map(str, ids)) cur = conn.cursor() cur.execute(sql) objective_cases = [] for var_id, vname, case_id, sense, value in cur: if not isinstance(value, (float, int, str)): try: value = loads(str(value)) except UnpicklingError as err: raise UnpicklingError("can't unpickle value '%s' for" " case '%s' from database: %s" % (vname, case_id, str(err))) index = id_index_map[case_id] if vname in data: data[vname][index] = value else: data[vname] = [ None, ] * n_cases data[vname][index] = value return n_cases, data
def ireceive(source, buffer=None, vanilla=False, tag=default_tag, return_status=False, bypass=False): """receive - blocking MPI receive Receive data from source. Optional parameters: buffer: Use specified buffer for received data (faster). Default None. vanilla: Specify to enforce vanilla protocol for any type. Default False tag: Only received messages tagged as specified. Default default_tag return_status: Return Status object along with result. Default False. If no buffer is specified, receive will try to receive a preceding message containing protocol, type, size and shape and then create a suitable buffer. If buffer is specified the corresponding send must specify use_buffer = True. The variable buffer can be any (picklable) type, but numpy variables and text strings will most efficient. Appropriate protocol will be automatically determined and corresponding receive function called. If bypass is True, all admin and error checks get bypassed to reduce the latency. Should only be used for receiving numpy arrays and should be matched with a bypass in the corresponding send command. Also buffer must be specified. """ if bypass: # errmsg = 'bypass mode must be used with specified buffer' # assert buffer is not None, msg stat = ireceive_array(buffer, source, tag) else: import types # Input check errmsg = 'Source id (%s) must be an integer.' %source assert type(source) == types.IntType, errmsg errmsg = 'Tag %d is reserved by pypar - please use another.'\ % control_tag assert tag != control_tag, errmsg # Either receive or create metadata about object to receive if buffer is None: control_info, source = receive_control_info(source, return_source=True) protocol, typecode, size, shape = control_info else: protocol, typecode, size, shape = create_control_info(buffer, vanilla) # Receive payload data if protocol == 'array': if buffer is None: buffer = zeros(size,typecode) buffer = reshape(buffer, shape) stat = receive_array(buffer, source, tag) elif protocol == 'string': if buffer is None: buffer = ' '*size stat = receive_string(buffer, source, tag) elif protocol == 'vanilla': from cPickle import dumps, loads, UnpicklingError if buffer is None: s = ' '*size else: s = dumps(buffer, protocol=2) s = s + ' '*int(0.1*len(s)) #safety stat = receive_string(s, source, tag) try: buffer = loads(s) #Replace buffer with received result except UnpicklingError, err: raise UnpicklingError(str(err) + " - '%s'" % s) else:
def case_db_to_dict(dbname, varnames, case_sql='', var_sql='', include_errors=False): """ Retrieve the values of specified variables from a sqlite DB containing Case data. Returns a dict containing a list of values for each entry, keyed on variable name. Only data from cases containing ALL of the specified variables will be returned so that all data values with the same index will correspond to the same case. dbname: str The name of the sqlite DB file. varnames: list[str] Iterator of names of variables to be retrieved. case_sql: str (optional) SQL syntax that will be placed in the WHERE clause for Case retrieval. var_sql: str (optional) SQL syntax that will be placed in the WHERE clause for variable retrieval. include_errors: bool (optional) [False] If True, include data from cases that reported an error. """ connection = sqlite3.connect(dbname) vardict = dict([(name, []) for name in varnames]) sql = ["SELECT id FROM cases"] qlist = [] if case_sql: qlist.append(case_sql) if not include_errors: qlist.append("msg = ''") if qlist: sql.append("WHERE %s" % ' AND '.join(qlist)) casecur = connection.cursor() casecur.execute(' '.join(sql)) sql = ["SELECT name, value from casevars WHERE case_id=%s"] vars_added = False for i, name in enumerate(vardict.keys()): if i == 0: sql.append("AND (") else: sql.append("OR") sql.append("name='%s'" % name) vars_added = True if vars_added: sql.append(")") if var_sql: sql.append(" AND %s" % var_sql) combined = ' '.join(sql) varcur = connection.cursor() for case_id in casecur: casedict = {} varcur.execute(combined % case_id) for vname, value in varcur: if not isinstance(value, (float, int, str)): try: value = loads(str(value)) except UnpicklingError as err: raise UnpicklingError("can't unpickle value '%s' from" " database: %s" % (vname, str(err))) casedict[vname] = value if len(casedict) != len(vardict): continue # case doesn't contain a complete set of specified vars, # so skip it to avoid data mismatches for name, value in casedict.items(): vardict[name].append(value) return vardict
def persistent_load(task_id): raise UnpicklingError('message contained references to ' 'external objects: %s' % task_id)
def persistent_load(persid): if persid.startswith(KEY): value = persid[len(KEY):] return value else: raise UnpicklingError('Invalid persistent id')