def _load_flann_model(self): if not self._descr_cache and not self._descr_cache_elem.is_empty(): # Load descriptor cache # - is copied on fork, so only need to load here. self._log.debug("Loading cached descriptors") self._descr_cache = \ cPickle.loads(self._descr_cache_elem.get_bytes()) # Params pickle include the build params + our local state params if self._index_param_elem and not self._index_param_elem.is_empty(): state = cPickle.loads(self._index_param_elem.get_bytes()) self._build_autotune = state['b_autotune'] self._build_target_precision = state['b_target_precision'] self._build_sample_frac = state['b_sample_frac'] self._distance_method = state['distance_method'] self._flann_build_params = state['flann_build_params'] # Load the binary index if self._index_elem and not self._index_elem.is_empty(): # make numpy matrix of descriptor vectors for FLANN pts_array = [d.vector() for d in self._descr_cache] pts_array = numpy.array(pts_array, dtype=pts_array[0].dtype) pyflann.set_distance_type(self._distance_method) self._flann = pyflann.FLANN() tmp_fp = self._index_elem.write_temp() self._flann.load_index(tmp_fp, pts_array) self._index_elem.clean_temp() del pts_array, tmp_fp # Set current PID to the current self._pid = multiprocessing.current_process().pid
def test_pickle(self): a = T.scalar() # the a is for 'anonymous' (un-named). x, s = T.scalars('xs') f = function([x, In(a, value=1.0, name='a'), In(s, value=0.0, update=s+a*x, mutable=True)], s+a*x) try: # Note that here we also test protocol 0 on purpose, since it # should work (even though one should not use it). g = pickle.loads(pickle.dumps(f, protocol=0)) g = pickle.loads(pickle.dumps(f, protocol=-1)) except NotImplementedError as e: if e[0].startswith('DebugMode is not picklable'): return else: raise # if they both return, assume that they return equivalent things. # print [(k,id(k)) for k in f.finder.keys()] # print [(k,id(k)) for k in g.finder.keys()] self.assertFalse(g.container[0].storage is f.container[0].storage) self.assertFalse(g.container[1].storage is f.container[1].storage) self.assertFalse(g.container[2].storage is f.container[2].storage) self.assertFalse(x in g.container) self.assertFalse(x in g.value) self.assertFalse(g.value[1] is f.value[1]) # should not have been copied self.assertFalse(g.value[2] is f.value[2]) # should have been copied because it is mutable. self.assertFalse((g.value[2] != f.value[2]).any()) # its contents should be identical self.assertTrue(f(2, 1) == g(2)) # they should be in sync, default value should be copied. self.assertTrue(f(2, 1) == g(2)) # they should be in sync, default value should be copied. f(1, 2) # put them out of sync self.assertFalse(f(1, 2) == g(1, 2)) # they should not be equal anymore.
def test_rrulewrapper(): r = rrulewrapper(2) try: pickle.loads(pickle.dumps(r)) except RecursionError: print('rrulewrapper pickling test failed') raise
def test_store_update_context(): src.val.nested = 'nested{{src.ccdid}}' src.val['ccdid'] = 2 src['srcdir'] = 'obs{{ src.obsid }}/{{src.nested}}' files['srcdir'] = '{{ src.srcdir }}' src['obsid'] = 123 src.val['ccdid'] = 2 src['ra'] = 1.4343256789 src['ra'].format = '%.4f' files['evt2'] = 'obs{{ src.obsid }}/{{src.nested}}/acis_evt2' src.val.nested = 'nested{{src.ccdid}}' tmp = pickle.dumps(src) tmp2 = pickle.dumps(files) src.clear() files.clear() assert src['ra'].val is None assert files['evt2'].val is None src.update(pickle.loads(tmp)) files.update(pickle.loads(tmp2)) assert str(src['ra']) == '1.4343' assert str(src['srcdir']) == 'obs123/nested2' assert files['srcdir'].rel == 'data/obs123/nested2' assert files.rel.srcdir == 'data/obs123/nested2' assert str(files['evt2.fits']) == 'data/obs123/nested2/acis_evt2.fits'
def test_run_in_runclassmethod(): class C(object): @classmethod def fn(cls, *args, **kwargs): return cls, args, kwargs C.__name__ = 'C_' + str(uuid4()).replace('-', '_') C.__module__ = 'pytest_shutil.run' with patch('pytest_shutil.run.execnet') as execnet: gw = execnet.makegateway.return_value chan = gw.remote_exec.return_value chan.receive.return_value = cPickle.dumps(sentinel.ret) c = C() with patch.object(run, C.__name__, C, create=True): run.run_in_subprocess(c.fn, python='sentinel.python')(ARG, kw=KW) ((s,), _) = chan.send.call_args if sys.version_info < (3, 0, 0): # Class methods are not pickleable in Python 2. assert cPickle.loads(s) == (run._invoke_method, (C, 'fn', ARG), {'kw': KW}) else: # Class methods are pickleable in Python 3. assert cPickle.loads(s) == (c.fn, (ARG,), {'kw': KW}) ((remote_fn,), _) = gw.remote_exec.call_args ((chan.receive.return_value,), _) = chan.send.call_args remote_fn(chan) chan.send.assert_called_with(cPickle.dumps((C, (ARG,), {'kw': KW}), protocol=0))
def test_polar(): ax = plt.subplot(111, polar=True) fig = plt.gcf() result = BytesIO() pf = pickle.dumps(fig) pickle.loads(pf) plt.draw()
def test_unpickle_wrongname(self, universe_n, pickle_str_n): # we change the universe's anchor_name universe_n.anchor_name = "test2" # shouldn't unpickle if no name matches, even if there's a compatible # universe in the unnamed anchor list. with pytest.raises(RuntimeError): cPickle.loads(pickle_str_n)
def test_pickling(self): """intbitset - pickling""" from six.moves import cPickle for set1 in self.sets + [[]]: self.assertEqual(self.intbitset(set1), cPickle.loads(cPickle.dumps(self.intbitset(set1), -1))) for set1 in self.sets + [[]]: self.assertEqual(self.intbitset(set1, trailing_bits=True), cPickle.loads(cPickle.dumps(self.intbitset(set1, trailing_bits=True), -1)))
def test_none_Constant(): """ Tests equals We had an error in the past with unpickling """ o1 = Constant(NoneTypeT(), None, name='NoneConst') o2 = Constant(NoneTypeT(), None, name='NoneConst') assert o1.equals(o2) assert NoneConst.equals(o1) assert o1.equals(NoneConst) assert NoneConst.equals(o2) assert o2.equals(NoneConst) # This trigger equals that returned the wrong answer in the past. import six.moves.cPickle as pickle import theano from theano import tensor x = tensor.vector('x') y = tensor.argmax(x) kwargs = {} # We can't pickle DebugMode if theano.config.mode in ["DebugMode", "DEBUG_MODE"]: kwargs = {'mode': 'FAST_RUN'} f = theano.function([x], [y], **kwargs) pickle.loads(pickle.dumps(f))
def test_added_descriptor_table_caching(self): cache_elem = DataMemoryElement(readonly=False) descrs = [random_descriptor() for _ in range(3)] expected_table = dict((r.uuid(), r) for r in descrs) i = MemoryDescriptorIndex(cache_elem) self.assertTrue(cache_elem.is_empty()) # Should add descriptors to table, caching to writable element. i.add_many_descriptors(descrs) self.assertFalse(cache_elem.is_empty()) self.assertEqual(pickle.loads(i.cache_element.get_bytes()), expected_table) # Changing the internal table (remove, add) it should reflect in # cache new_d = random_descriptor() expected_table[new_d.uuid()] = new_d i.add_descriptor(new_d) self.assertEqual(pickle.loads(i.cache_element.get_bytes()), expected_table) rm_d = list(expected_table.values())[0] del expected_table[rm_d.uuid()] i.remove_descriptor(rm_d.uuid()) self.assertEqual(pickle.loads(i.cache_element.get_bytes()), expected_table)
def set_user_definitions(self, definitions): if definitions: if six.PY2: self.user = pickle.loads(base64.decodestring(definitions.encode('ascii'))) else: self.user = pickle.loads(base64.decodebytes(definitions.encode('ascii'))) else: self.user = {}
def test_unpickle_noanchor(self, universe, pickle_str): # Shouldn't unpickle if the universe is removed from the anchors universe.remove_anchor() # In the complex (parallel) testing environment there's the risk of # other compatible Universes being available for anchoring even after # this one is expressly removed. # assert_raises(RuntimeError, cPickle.loads, pickle_str) with pytest.raises(RuntimeError): cPickle.loads(pickle_str)
def test_serialization(self): e = smqtk.representation.classification_element.memory\ .MemoryClassificationElement('test', 0) e2 = cPickle.loads(cPickle.dumps(e)) self.assertEqual(e, e2) e.set_classification(a=0, b=1) e2 = cPickle.loads(cPickle.dumps(e)) self.assertEqual(e, e2)
def assert_pickle_idempotent(obj): '''Assert that obj does not change (w.r.t. ==) under repeated picklings ''' from six.moves.cPickle import dumps, loads obj1 = loads(dumps(obj)) obj2 = loads(dumps(obj1)) obj3 = loads(dumps(obj2)) assert_equivalent(obj, obj1) assert_equivalent(obj, obj2) assert_equivalent(obj, obj3) assert type(obj) is type(obj3)
def get(self, id): envelope_raw, attempts, delivered_indexes_raw = \ self.redis.hmget(self._get_key(id), 'envelope', 'attempts', 'delivered_indexes') if not envelope_raw: raise KeyError(id) envelope = cPickle.loads(envelope_raw) del envelope_raw if delivered_indexes_raw: delivered_indexes = cPickle.loads(delivered_indexes_raw) self._remove_delivered_rcpts(envelope, delivered_indexes) return envelope, int(attempts or 0)
def assert_sml_written_to_db(self, sml, voter_id): with sqlite3.connect("sqlite_sink.db") as conn: c = conn.cursor() c.execute('SELECT sml FROM smls WHERE voter_id = "' + voter_id + '"') fetched_sml = c.fetchone() if six.PY2: fetched_sml = cPickle.loads(str(fetched_sml[0])) else: fetched_sml = cPickle.loads(fetched_sml[0]) self.assertEqual(len(sml), len(fetched_sml)) self.assertTrue((sml == fetched_sml).all())
def iterdescriptors(self): """ Return an iterator over indexed descriptor element instances. """ r = self.solr.select('%s:%s %s:*' % (self.index_uuid_field, self.index_uuid, self.descriptor_field)) for doc in r.results: yield cPickle.loads(doc[self.descriptor_field]) for _ in range(r.numFound // 10): r = r.next_batch() for doc in r.results: yield cPickle.loads(doc[self.descriptor_field])
def iteritems(self): """ Return an iterator over indexed descriptor key and instance pairs. """ r = self.solr.select('%s:%s %s:* %s:*' % (self.index_uuid_field, self.index_uuid, self.d_uid_field, self.descriptor_field)) for doc in r.results: d = cPickle.loads(doc[self.descriptor_field]) yield d.uuid(), d for _ in range(r.numFound // 10): r = r.next_batch() for doc in r.results: d = cPickle.loads(doc[self.descriptor_field]) yield d.uuid(), d
def get_value(self, key, generator=None, user=None, expires=False): """Returns cache value. If not found and generator function is given, it will call the generator. :param key: Cache key. :param generator: Function to be called to generate a value if `None` is returned. :param expires: If the key is supposed to be with an expiry, don't store it in frappe.local """ original_key = key key = self.make_key(key, user) if key in frappe.local.cache: val = frappe.local.cache[key] else: val = None try: val = self.get(key) except redis.exceptions.ConnectionError: pass if val is not None: val = pickle.loads(val) if not expires: if val is None and generator: val = generator() self.set_value(original_key, val, user=user) else: frappe.local.cache[key] = val return val
def _check_failure_status(self): """ Check the status of command failures. Raise exceptions as necessary The failure status property is used by the various asynchronous command execution threads which interface with the remote browser manager processes. If a failure status is found, the appropriate steps are taken to gracefully close the infrastructure """ self.logger.debug("Checking command failure status indicator...") if self.failure_status: self.logger.debug( "TaskManager failure status set, halting command execution.") self._cleanup_before_fail() if self.failure_status['ErrorType'] == 'ExceedCommandFailureLimit': raise CommandExecutionError( "TaskManager exceeded maximum consecutive command " "execution failures.", self.failure_status['CommandSequence'] ) elif (self.failure_status['ErrorType'] == ("ExceedLaunch" "FailureLimit")): raise CommandExecutionError( "TaskManager failed to launch browser within allowable " "failure limit.", self.failure_status['CommandSequence'] ) if self.failure_status['ErrorType'] == 'CriticalChildException': reraise(*pickle.loads(self.failure_status['Exception']))
def get(self, block=True, timeout=None): """ Retrieves the next item in the queue. Parameters ---------- block: bool, default is True If true, the queue is blocked until it an object is retrieved reaches the timeout. timeout: long The timeout (in seconds) that the method should wait for the queue to return an item. If block is False, time wil ignored. Returns ------- item: object """ if block: item = self._db.blpop(self._key, timeout=timeout) if item: item = item[1] logger.debug("Wait...") else: item = self._db.lpop(self._key) logger.debug("No-wait...") logger.debug("Item: %s" % [item]) if item: return pickle.loads(item) else: raise six.moves.queue.Empty
def _fetch_missing(cls, key): result = {} urls = env.trackerClient.call(GetValueMessage('mutable_dict:%s' % key)) for url in urls: f = urllib.request.urlopen(url) if f.code is not None and f.code != 200: raise IOError('Open %s failed:%s' % (url, f.code)) data = f.read() if len(data) < 4: raise IOError('Transfer %s failed: %s received' % (url, len(data))) length, = struct.unpack('<I', data[:4]) if length != len(data): raise IOError('Transfer %s failed: %s received, %s expected' % (url, len(data), length)) data = cPickle.loads(decompress(data[4:])) for k, v in data.items(): if k in result: r = result[k] if v[1] == r[1]: r0 = r[0] v0 = v[0] merged = r0.value if isinstance(r0, ConflictValues) else [r0] merged += v0.value if isinstance(v0, ConflictValues) else [v0] result[k] = (ConflictValues(merged), r[1]) else: result[k] = v if v[1] > r[1] else r else: result[k] = v return result
def get_server_weights(master_url='localhost:5000'): ''' Retrieve master weights from parameter server ''' request = urllib2.Request('http://{0}/parameters'.format(master_url), headers={'Content-Type': 'application/elephas'}) return pickle.loads(urllib2.urlopen(request).read())
def test_pickle(dcd): dcd.seek(len(dcd) - 1) dump = pickle.dumps(dcd) new_dcd = pickle.loads(dump) assert dcd.fname == new_dcd.fname assert dcd.tell() == new_dcd.tell()
def test_pickle_unpickle_without_reoptimization(): mode = theano.config.mode if mode in ["DEBUG_MODE", "DebugMode"]: mode = "FAST_RUN" x1 = T.fmatrix('x1') x2 = T.fmatrix('x2') x3 = theano.shared(numpy.ones((10, 10), dtype=floatX)) x4 = theano.shared(numpy.ones((10, 10), dtype=floatX)) y = T.sum(T.sum(T.sum(x1**2 + x2) + x3) + x4) updates = OrderedDict() updates[x3] = x3 + 1 updates[x4] = x4 + 1 f = theano.function([x1, x2], y, updates=updates, mode=mode) # now pickle the compiled theano fn string_pkl = pickle.dumps(f, -1) # compute f value in1 = numpy.ones((10, 10), dtype=floatX) in2 = numpy.ones((10, 10), dtype=floatX) # test unpickle without optimization default = theano.config.reoptimize_unpickled_function try: # the default is True theano.config.reoptimize_unpickled_function = False f_ = pickle.loads(string_pkl) assert f(in1, in2) == f_(in1, in2) finally: theano.config.reoptimize_unpickled_function = default
def __delitem__(self, key): if key in self.cache: del self.cache[key] from six.moves.cPickle import loads for item_id, key_pickle, version_pickle in self.db_conn.execute( "select id, key_pickle, version_pickle from data" " where key_hash = ? and version_hash = ?", (hash(key), self.version_hash)): if loads(key_pickle) == key and loads(version_pickle) == self.version: self.db_conn.execute("delete from data where id = ?", (item_id,)) self.commit_countdown -= 1 if self.commit_countdown <= 0: self.commit_countdown = self.commit_interval self.db_conn.commit()
def test_unpickle_named(self): """Test that an AtomGroup can be unpickled (Issue 293)""" newag = cPickle.loads(self.pickle_str_n) # Can unpickle assert_array_equal(self.ag_n.indices, newag.indices) assert_(newag.universe is self.universe_n, "Unpickled AtomGroup on wrong Universe.")
def get(self, key): """ Gets the object specified by key. It will also unserialize the object before returning if it is serialized in memcache with JSON, or if it is pickled and unpickling is allowed. :param key: key :returns: value of the key in memcache """ key = md5hash(key) value = None for (server, fp, sock) in self._get_conns(key): try: with Timeout(self._io_timeout): sock.sendall('get %s\r\n' % key) line = fp.readline().strip().split() while line[0].upper() != 'END': if line[0].upper() == 'VALUE' and line[1] == key: size = int(line[3]) value = fp.read(size) if int(line[2]) & PICKLE_FLAG: if self._allow_unpickle: value = pickle.loads(value) else: value = None elif int(line[2]) & JSON_FLAG: value = json.loads(value) fp.readline() line = fp.readline().strip().split() self._return_conn(server, fp, sock) return value except (Exception, Timeout) as e: self._exception_occurred(server, e, sock=sock, fp=fp)
def __contains__(self, key): if key in self.cache: return True else: from six.moves.cPickle import loads for key_pickle, version_pickle, result_pickle in self.db_conn.execute( "select key_pickle, version_pickle, result_pickle from data" " where key_hash = ? and version_hash = ?", (hash(key), self.version_hash)): if loads(str(key_pickle)) == key \ and loads(str(version_pickle)) == self.version: result = loads(str(result_pickle)) self.cache[key] = result return True return False
def __getitem__(self, key): try: return self.cache[key] except KeyError: from six.moves.cPickle import loads for key_pickle, version_pickle, result_pickle in self.db_conn.execute( "select key_pickle, version_pickle, result_pickle from data" " where key_hash = ? and version_hash = ?", (hash(key), self.version_hash)): if loads(str(key_pickle)) == key \ and loads(str(version_pickle)) == self.version: result = loads(str(result_pickle)) self.cache[key] = result return result raise KeyError(key)
def test_pickle_cpu(self): s = pickle.dumps(self.fs) fs2 = pickle.loads(s) self.check_equal_fs(self.fs, fs2)
def test_pickle_cpu(self): fs2_serialized = pickle.dumps(self.fs2) fs2_loaded = pickle.loads(fs2_serialized) self.assertTrue((self.fs2.b.p.data == fs2_loaded.b.p.data).all()) self.assertTrue( (self.fs2.fs1.a.p.data == fs2_loaded.fs1.a.p.data).all())
def pickle_then_unpickle(obj): """Pickle to a temp file then un-pickle.""" import six.moves.cPickle as pickle return pickle.loads(pickle.dumps(obj))
def _check_output(self): self._assert_expected_item(pickle.loads(self.output.getvalue()))
def deserialise_tracker(self, sender_id, _json): dialogue = pickler.loads(_json) tracker = self.init_tracker(sender_id) tracker.recreate_from_dialogue(dialogue) return tracker
def test_pickling(self): s = Size("10 MiB") self.assertEqual(s, cPickle.loads(cPickle.dumps(s)))
def add_classifier(self): """ Upload a **trained** classifier pickled and encoded in standard base64 encoding, matched with a descriptive label of that classifier's topic. Since all classifiers have only two result classes (positive and negative), the topic of the classifier is encoded in the descriptive label the user applies to the classifier. Below is an example call to this endpoint via the ``requests`` python module, showing how base64 data is sent:: import base64 import requests data_bytes = "Load some content bytes here." requests.post('http://localhost:5000/classifier', data={'bytes_b64': base64.b64encode(data_bytes), 'label': 'some_label'}) With curl on the command line:: $ curl -X POST localhost:5000/classifier -d label=some_label \ --data-urlencode "bytes_b64=$(base64 -w0 /path/to/file)" # If this fails, you may wish to encode the file separately and # use the file reference syntax instead: $ base64 -w0 /path/to/file.pkl > /path/to/file.pkl.b64 $ curl -X POST localhost:5000/classifier -d label=some_label \ --data-urlencode bytes_64@/path/to/file.pkl.b64 To lock this classifier and guard it against deletion, add "lock_label=true":: $ curl -X POST localhost:5000/classifier \ -d "label=some_label" \ -d "lock_label=true" \ --data-urlencode "bytes_b64=$(base64 -w0 /path/to/file.pkl)" Data/Form arguments: bytes_b64 Bytes, in the standard base64 encoding, of the pickled classifier. label Descriptive label to apply to this classifier. This should not conflict with existing classifier labels. lock_label If 'true', disallow deletion of this label. If 'false', allow deletion of this label. Only has an effect if deletion is enabled for this service. (Default: 'false') Possible error codes: 400 May mean one of: - No pickled classifier base64 data or label provided. - Label provided is in conflict with an existing label in the classifier collection. Returns code 201 on success and the message: { label: <str> } """ clfr_b64 = flask.request.values.get('bytes_b64', default=None) label = flask.request.values.get('label', default=None) lock_clfr_str = flask.request.values.get('lock_label', default='false') if clfr_b64 is None or len(clfr_b64) == 0: return make_response_json("No state base64 data provided.", 400) elif label is None or len(label) == 0: return make_response_json("No descriptive label provided.", 400) try: # This can throw a ValueError if lock_clfr is malformed JSON lock_clfr = bool(flask.json.loads(lock_clfr_str)) except JSON_DECODE_EXCEPTION: return make_response_json( "Invalid boolean value for" " 'lock_label'. Was given: '%s'" % lock_clfr_str, 400) # If the given label conflicts with one already in the collection, # fail. if label in self.classifier_collection.labels(): return make_response_json("Label '%s' already exists in" " classifier collection." % label, 400, label=label) clfr = pickle.loads(base64.b64decode(clfr_b64.encode('utf-8'))) try: self.classifier_collection.add_classifier(label, clfr) # If we're allowing deletions, get the lock flag from the form # and set it for this classifier if self.enable_classifier_removal and lock_clfr: self.immutable_labels.add(label) except ValueError: return make_response_json("Data added for label '%s' is not a" " Classifier." % label, 400, label=label) return make_response_json("Uploaded classifier for label '%s'." % label, 201, label=label)
def __copy(d): # Faster than deepcopy - ideally remove the need for deepcopy altogether return cPickle.loads(cPickle.dumps(d, protocol=2))
def update(self, job): """ High-level method that replicates a single partition. :param job: a dict containing info about the partition to be replicated """ self.replication_count += 1 self.logger.increment('partition.update.count.%s' % (job['device'], )) headers = dict(self.default_headers) headers['X-Backend-Storage-Policy-Index'] = int(job['policy']) target_devs_info = set() failure_devs_info = set() begin = time.time() try: hashed, local_hash = tpool_reraise( self._diskfile_mgr._get_hashes, job['path'], do_listdir=(self.replication_count % 10) == 0, reclaim_age=self.reclaim_age) self.suffix_hash += hashed self.logger.update_stats('suffix.hashes', hashed) attempts_left = len(job['nodes']) synced_remote_regions = set() random.shuffle(job['nodes']) nodes = itertools.chain( job['nodes'], job['policy'].object_ring.get_more_nodes( int(job['partition']))) while attempts_left > 0: # If this throws StopIteration it will be caught way below node = next(nodes) target_devs_info.add((node['replication_ip'], node['device'])) attempts_left -= 1 # if we have already synced to this remote region, # don't sync again on this replication pass if node['region'] in synced_remote_regions: continue try: with Timeout(self.http_timeout): resp = http_connect(node['replication_ip'], node['replication_port'], node['device'], job['partition'], 'REPLICATE', '', headers=headers).getresponse() if resp.status == HTTP_INSUFFICIENT_STORAGE: self.logger.error( _('%(ip)s/%(device)s responded' ' as unmounted'), node) attempts_left += 1 failure_devs_info.add( (node['replication_ip'], node['device'])) continue if resp.status != HTTP_OK: self.logger.error( _("Invalid response %(resp)s " "from %(ip)s"), { 'resp': resp.status, 'ip': node['replication_ip'] }) failure_devs_info.add( (node['replication_ip'], node['device'])) continue remote_hash = pickle.loads(resp.read()) del resp suffixes = [ suffix for suffix in local_hash if local_hash[suffix] != remote_hash.get(suffix, -1) ] if not suffixes: self.stats['hashmatch'] += 1 continue hashed, recalc_hash = tpool_reraise( self._diskfile_mgr._get_hashes, job['path'], recalculate=suffixes, reclaim_age=self.reclaim_age) self.logger.update_stats('suffix.hashes', hashed) local_hash = recalc_hash suffixes = [ suffix for suffix in local_hash if local_hash[suffix] != remote_hash.get(suffix, -1) ] self.stats['rsync'] += 1 success, _junk = self.sync(node, job, suffixes) with Timeout(self.http_timeout): conn = http_connect(node['replication_ip'], node['replication_port'], node['device'], job['partition'], 'REPLICATE', '/' + '-'.join(suffixes), headers=headers) conn.getresponse().read() if not success: failure_devs_info.add( (node['replication_ip'], node['device'])) # add only remote region when replicate succeeded if success and node['region'] != job['region']: synced_remote_regions.add(node['region']) self.suffix_sync += len(suffixes) self.logger.update_stats('suffix.syncs', len(suffixes)) except (Exception, Timeout): failure_devs_info.add( (node['replication_ip'], node['device'])) self.logger.exception( _("Error syncing with node: %s") % node) self.suffix_count += len(local_hash) except (Exception, Timeout): failure_devs_info.update(target_devs_info) self.logger.exception(_("Error syncing partition")) finally: self.stats['success'] += len(target_devs_info - failure_devs_info) self._add_failure_stats(failure_devs_info) self.partition_times.append(time.time() - begin) self.logger.timing_since('partition.update.timing', begin)
def loadTestReactor(inputFilePath=TEST_ROOT, customSettings=None, inputFileName="armiRun.yaml"): r""" Loads a test reactor. Can be used in other test modules. Parameters ---------- inputFilePath : str Path to the directory of the armiRun.yaml input file. customSettings : dict with str keys and values of any type For each key in customSettings, the cs which is loaded from the armiRun.yaml will be overwritten to the value given in customSettings for that key. Returns ------- o : Operator r : Reactor """ # TODO: it would be nice to have this be more stream-oriented. Juggling files is # devilishly difficult. global TEST_REACTOR isotopicDepletion.applyDefaultBurnChain() fName = os.path.join(inputFilePath, inputFileName) customSettings = customSettings or {} isPickeledReactor = fName == ARMI_RUN_PATH and customSettings == {} assemblies.resetAssemNumCounter() if isPickeledReactor and TEST_REACTOR: # return test reactor only if no custom settings are needed. o, r, assemNum = cPickle.loads(TEST_REACTOR) assemblies.setAssemNumCounter(assemNum) settings.setMasterCs(o.cs) o.reattach(r, o.cs) return o, r cs = settings.Settings(fName=fName) # Overwrite settings if desired if customSettings: for settingKey, settingVal in customSettings.items(): cs[settingKey] = settingVal if "verbosity" not in customSettings: runLog.setVerbosity("error") settings.setMasterCs(cs) cs["stationaryBlocks"] = [] cs["nCycles"] = 3 o = operators.factory(cs) r = reactors.loadFromCs(cs) o.initializeInterfaces(r) # put some stuff in the SFP too. for a in range(10): a = o.r.blueprints.constructAssem(o.r.core.geomType, o.cs, name="feed fuel") o.r.core.sfp.add(a) o.r.core.regenAssemblyLists() if isPickeledReactor: # cache it for fast load for other future tests # protocol=2 allows for classes with __slots__ but not __getstate__ to be pickled TEST_REACTOR = cPickle.dumps((o, o.r, assemblies.getAssemNum()), protocol=2) return o, o.r
def _get_suffixes_to_sync(self, job, node): """ For SYNC jobs we need to make a remote REPLICATE request to get the remote node's current suffix's hashes and then compare to our local suffix's hashes to decide which suffixes (if any) are out of sync. :param: the job dict, with the keys defined in ``_get_part_jobs`` :param node: the remote node dict :returns: a (possibly empty) list of strings, the suffixes to be synced with the remote node. """ # get hashes from the remote node remote_suffixes = None try: with Timeout(self.http_timeout): resp = http_connect( node['replication_ip'], node['replication_port'], node['device'], job['partition'], 'REPLICATE', '', headers=self.headers).getresponse() if resp.status == HTTP_INSUFFICIENT_STORAGE: self.logger.error( _('%s responded as unmounted'), self._full_path(node, job['partition'], '', job['policy'])) elif resp.status != HTTP_OK: full_path = self._full_path(node, job['partition'], '', job['policy']) self.logger.error( _("Invalid response %(resp)s from %(full_path)s"), {'resp': resp.status, 'full_path': full_path}) else: remote_suffixes = pickle.loads(resp.read()) except (Exception, Timeout): # all exceptions are logged here so that our caller can # safely catch our exception and continue to the next node # without logging self.logger.exception('Unable to get remote suffix hashes ' 'from %r' % self._full_path( node, job['partition'], '', job['policy'])) if remote_suffixes is None: raise SuffixSyncError('Unable to get remote suffix hashes') suffixes = self.get_suffix_delta(job['hashes'], job['frag_index'], remote_suffixes, node['index']) # now recalculate local hashes for suffixes that don't # match so we're comparing the latest local_suff = self._get_hashes(job['policy'], job['path'], recalculate=suffixes) suffixes = self.get_suffix_delta(local_suff, job['frag_index'], remote_suffixes, node['index']) self.suffix_count += len(suffixes) return suffixes
def _run_in_subprocess_remote_fn(channel): from six.moves import cPickle # @UnresolvedImport @Reimport # NOQA fn, args, kwargs = cPickle.loads(channel.receive(None)) channel.send(cPickle.dumps(fn(*args, **kwargs), protocol=0))
def merge(docs, otherenv): # type: (List[unicode], bytes) -> None env = pickle.loads(otherenv) self.env.merge_info_from(docs, env, self.app)
def exercise_bins(): uc = uctbx.unit_cell((11,11,13,90,90,120)) sg_type = sgtbx.space_group_type("P 3 2 1") anomalous_flag = False d_min = 1 m = miller.index_generator(uc, sg_type, anomalous_flag, d_min).to_array() f = flex.double() for i in range(m.size()): f.append(random.random()) n_bins = 10 b = miller.binning(uc, n_bins, 0, d_min) b = miller.binning(uc, n_bins, 0, d_min, 1.e-6) b = miller.binning(uc, n_bins, m) b = miller.binning(uc, n_bins, m, 0) b = miller.binning(uc, n_bins, m, 0, d_min) b = miller.binning(uc, n_bins, m, 0, d_min, 1.e-6) assert b.d_max() == -1 assert approx_equal(b.d_min(), d_min) assert b.bin_d_range(0) == (-1,-1) assert approx_equal(b.bin_d_range(1), (-1,2.1544336)) assert approx_equal(b.bin_d_range(b.n_bins_all()-1), (1,-1)) d_star_sq = 0.5 r = b.bin_d_range(b.get_i_bin(d_star_sq)) d = 1/math.sqrt(d_star_sq) assert r[1] <= d <= r[0] h = (3,4,5) r = b.bin_d_range(b.get_i_bin(h)) assert r[1] <= uc.d(h) <= r[0] # a quick test to excercise d-spacings on fractional Miller indices: assert approx_equal( uc.d((3,4,5)), uc.d_frac((3.001,4,5)), eps=0.001) binning1 = miller.binning(uc, n_bins, m) assert binning1.unit_cell().is_similar_to(uc) assert binning1.n_bins_used() == n_bins assert binning1.limits().size() == n_bins + 1 assert binning1.n_bins_all() == n_bins + 2 s = pickle.dumps(binning1) l = pickle.loads(s) assert str(l.unit_cell()) == "(11, 11, 13, 90, 90, 120)" assert approx_equal(l.limits(), binning1.limits()) # binner1 = miller.ext.binner(binning1, m) assert binner1.miller_indices().id() == m.id() assert binner1.count(binner1.i_bin_d_too_large()) == 0 assert binner1.count(binner1.i_bin_d_too_small()) == 0 counts = binner1.counts() for i_bin in binner1.range_all(): assert binner1.count(i_bin) == counts[i_bin] assert binner1.selection(i_bin).count(True) == counts[i_bin] assert list(binner1.range_all()) == list(range(binner1.n_bins_all())) assert list(binner1.range_used()) == list(range(1, binner1.n_bins_used()+1)) binning2 = miller.binning(uc, n_bins - 2, binning1.bin_d_min(2), binning1.bin_d_min(n_bins)) binner2 = miller.ext.binner(binning2, m) assert tuple(binner1.counts())[1:-1] == tuple(binner2.counts()) array_indices = flex.size_t(range(m.size())) perm_array_indices1 = flex.size_t() perm_array_indices2 = flex.size_t() for i_bin in binner1.range_all(): perm_array_indices1.extend(array_indices.select(binner1.selection(i_bin))) perm_array_indices2.extend(binner1.array_indices(i_bin)) assert perm_array_indices1.size() == m.size() assert perm_array_indices2.size() == m.size() assert tuple(perm_array_indices1) == tuple(perm_array_indices2) b = miller.ext.binner(miller.binning(uc, n_bins, m, 0, d_min), m) assert approx_equal(b.bin_centers(1), (0.23207956, 0.52448148, 0.62711856, 0.70311998, 0.7652538, 0.818567, 0.86566877, 0.90811134, 0.94690405, 0.98274518)) assert approx_equal(b.bin_centers(2), (0.10772184, 0.27871961, 0.39506823, 0.49551249, 0.58642261, 0.67067026, 0.74987684, 0.82507452, 0.89697271, 0.96608584)) assert approx_equal(b.bin_centers(3), (0.050000075, 0.15000023, 0.25000038, 0.35000053, 0.45000068, 0.55000083, 0.65000098, 0.75000113, 0.85000128, 0.95000143)) v = flex.double(range(b.n_bins_used())) i = b.interpolate(v, 0) for i_bin in b.range_used(): assert i.select(b.selection(i_bin)).all_eq(v[i_bin-1]) dss = uc.d_star_sq(m) for d_star_power in (1,2,3): j = b.interpolate(v, d_star_power) x = flex.pow(dss, (d_star_power/2.)) r = flex.linear_correlation(x, j) assert r.is_well_defined() assert approx_equal( r.coefficient(), (0.946401,0.990764,1.0)[d_star_power-1], eps=1.e-4, multiplier=None) # s = pickle.dumps(binner2) l = pickle.loads(s) assert str(l.unit_cell()) == "(11, 11, 13, 90, 90, 120)" assert approx_equal(l.limits(), binner2.limits()) assert l.miller_indices().all_eq(binner2.miller_indices()) assert l.bin_indices().all_eq(binner2.bin_indices()) # limits = flex.random_double(size=10) bng = miller.binning(uc, limits) assert bng.unit_cell().is_similar_to(uc) assert approx_equal(bng.limits(), limits)
class MujocoEngineTest(parameterized.TestCase): def setUp(self): super(MujocoEngineTest, self).setUp() self._physics = engine.Physics.from_xml_path(MODEL_PATH) def _assert_attributes_equal(self, actual_obj, expected_obj, attr_to_compare): for name in attr_to_compare: actual_value = getattr(actual_obj, name) expected_value = getattr(expected_obj, name) try: if isinstance(expected_value, np.ndarray): np.testing.assert_array_equal(actual_value, expected_value) else: self.assertEqual(actual_value, expected_value) except AssertionError as e: raise AssertionError("Attribute '{}' differs from expected value. {}" "".format(name, e.message)) @parameterized.parameters(0, 'cart', u'cart') def testCameraIndexing(self, camera_id): height, width = 480, 640 _ = engine.Camera( self._physics, height, width, camera_id=camera_id) def testDepthRender(self): plane_and_box = """ <mujoco> <worldbody> <geom type="plane" pos="0 0 0" size="2 2 .1"/> <geom type="box" size=".1 .1 .1" pos="0 0 .1"/> <camera name="top" pos="0 0 3"/> </worldbody> </mujoco> """ physics = engine.Physics.from_xml_string(plane_and_box) pixels = physics.render(height=200, width=200, camera_id='top', depth=True) # Nearest pixels should be 2.8m away np.testing.assert_approx_equal(pixels.min(), 2.8, 3) # Furthest pixels should be 3m away (depth is orthographic) np.testing.assert_approx_equal(pixels.max(), 3.0, 3) @parameterized.parameters([True, False]) def testSegmentationRender(self, enable_geom_frame_rendering): box_four_corners = """ <mujoco> <visual> <scale framelength="2"/> </visual> <worldbody> <geom name="box0" type="box" size=".2 .2 .2" pos="-1 1 .1"/> <geom name="box1" type="box" size=".2 .2 .2" pos="1 1 .1"/> <site name="box2" type="box" size=".2 .2 .2" pos="1 -1 .1"/> <site name="box3" type="box" size=".2 .2 .2" pos="-1 -1 .1"/> <camera name="top" pos="0 0 3"/> </worldbody> </mujoco> """ physics = engine.Physics.from_xml_string(box_four_corners) obj_type_geom = enums.mjtObj.mjOBJ_GEOM # Geom object type obj_type_site = enums.mjtObj.mjOBJ_SITE # Site object type obj_type_decor = enums.mjtObj.mjOBJ_UNKNOWN # Decor object type scene_options = wrapper.MjvOption() if enable_geom_frame_rendering: scene_options.frame = wrapper.mjbindings.enums.mjtFrame.mjFRAME_GEOM pixels = physics.render(height=200, width=200, camera_id='top', segmentation=True, scene_option=scene_options) # The pixel indices below were chosen so that toggling the frame decors do # not affect the segmentation results. with self.subTest('Center pixels should have background label'): np.testing.assert_equal(pixels[95:105, 95:105, 0], -1) np.testing.assert_equal(pixels[95:105, 95:105, 1], -1) with self.subTest('Geoms have correct object type'): np.testing.assert_equal(pixels[15:25, 0:10, 1], obj_type_geom) np.testing.assert_equal(pixels[15:25, 190:200, 1], obj_type_geom) with self.subTest('Sites have correct object type'): np.testing.assert_equal(pixels[190:200, 190:200, 1], obj_type_site) np.testing.assert_equal(pixels[190:200, 0:10, 1], obj_type_site) with self.subTest('Geoms have correct object IDs'): np.testing.assert_equal(pixels[15:25, 0:10, 0], physics.model.name2id('box0', obj_type_geom)) np.testing.assert_equal(pixels[15:25, 190:200, 0], physics.model.name2id('box1', obj_type_geom)) with self.subTest('Sites have correct object IDs'): np.testing.assert_equal(pixels[190:200, 190:200, 0], physics.model.name2id('box2', obj_type_site)) np.testing.assert_equal(pixels[190:200, 0:10, 0], physics.model.name2id('box3', obj_type_site)) with self.subTest('Decor elements present if and only if geom frames are ' 'enabled'): contains_decor = np.any(pixels[:, :, 1] == obj_type_decor) self.assertEqual(contains_decor, enable_geom_frame_rendering) def testTextOverlay(self): height, width = 480, 640 overlay = engine.TextOverlay(title='Title', body='Body', style='big', position='bottom right') no_overlay = self._physics.render(height, width, camera_id=0) with_overlay = self._physics.render(height, width, camera_id=0, overlays=[overlay]) self.assertFalse(np.all(no_overlay == with_overlay), msg='Images are identical with and without text overlay.') def testSceneOption(self): height, width = 480, 640 scene_option = wrapper.MjvOption() mjlib.mjv_defaultOption(scene_option.ptr) # Render geoms as semi-transparent. scene_option.flags[enums.mjtVisFlag.mjVIS_TRANSPARENT] = 1 no_scene_option = self._physics.render(height, width, camera_id=0) with_scene_option = self._physics.render(height, width, camera_id=0, scene_option=scene_option) self.assertFalse(np.all(no_scene_option == with_scene_option), msg='Images are identical with and without scene option.') def testRenderFlags(self): height, width = 480, 640 cam = engine.Camera(self._physics, height, width, camera_id=0) cam.scene.flags[enums.mjtRndFlag.mjRND_WIREFRAME] = 1 # Enable wireframe enabled = cam.render().copy() cam.scene.flags[enums.mjtRndFlag.mjRND_WIREFRAME] = 0 # Disable wireframe disabled = cam.render().copy() self.assertFalse( np.all(disabled == enabled), msg='Images are identical regardless of whether wireframe is enabled.') @parameterized.parameters(((0.5, 0.5), (1, 3)), # pole ((0.5, 0.1), (0, 0)), # ground ((0.9, 0.9), (None, None)), # sky ) def testCameraSelection(self, coordinates, expected_selection): height, width = 480, 640 camera = engine.Camera(self._physics, height, width, camera_id=0) # Test for b/63380170: Enabling visualization of body frames adds # "non-model" geoms to the scene. This means that the indices of geoms # within `camera._scene.geoms` don't match the rows of `model.geom_bodyid`. camera.option.frame = enums.mjtFrame.mjFRAME_BODY selected = camera.select(coordinates) self.assertEqual(expected_selection, selected[:2]) @parameterized.parameters( dict(camera_id='cam0', height=200, width=300), dict(camera_id=1, height=300, width=200), dict(camera_id=-1, height=400, width=400), ) def testCameraMatrix(self, camera_id, height, width): """Tests the camera_matrix() method. Creates a model with two cameras and two small geoms. We render the scene with one of the cameras and check that the geom locations, projected into pixel space, are correct, using segmenation rendering. xyz2pixels() shows how the transformation is used. For a description of the camera matrix see https://en.wikipedia.org/wiki/Camera_matrix. Args: camera_id: One of the two cameras. Can be either integer or String. height: The height of the image (pixels). width: The width of the image (pixels). """ def xyz2pixels(x, y, z, camera_matrix): """Transforms from world coordinates to pixel coordinates.""" xs, ys, s = camera_matrix.dot(np.array([x, y, z, 1.0])) return xs/s, ys/s two_geoms_and_two_cameras = """ <mujoco> <visual> <global fovy="55"/> </visual> <worldbody> <light name="top" pos="0 0 1"/> <geom name="red" pos=".2 0 0" size=".005" rgba="1 0 0 1"/> <geom name="green" pos=".2 .2 .1" size=".005" rgba="0 1 0 1"/> <camera name="cam0" pos="1 .5 1" zaxis="1 .5 1" fovy="20"/> <camera name="cam1" pos=".1 .1 1" xyaxes="1 1 0 -1 0 0"/> </worldbody> </mujoco> """ physics = engine.Physics.from_xml_string(two_geoms_and_two_cameras) camera = engine.Camera(physics, width=width, height=height, camera_id=camera_id) camera_matrix = camera.matrix # Get camera matrix. pixels = camera.render(segmentation=True) # Render a segmentation frame. for geom_id in [0, 1]: # Compute the location of the geom in pixel space using the camera matrix. x, y = xyz2pixels(*physics.data.geom_xpos[geom_id], camera_matrix) row = int(round(y)) column = int(round(x)) # Compare segmentation values of nearest pixel to corresponding geom. [obj_id, obj_type] = pixels[row, column, :] self.assertEqual(obj_type, enums.mjtObj.mjOBJ_GEOM) self.assertEqual(obj_id, geom_id) def testMovableCameraSetGetPose(self): height, width = 240, 320 camera = engine.MovableCamera(self._physics, height, width) image = camera.render().copy() pose = camera.get_pose() lookat_offset = np.array([0.01, 0.02, -0.03]) # Would normally pass the new values directly to camera.set_pose instead of # using the namedtuple _replace method, but this makes the asserts at the # end of the test a little cleaner. new_pose = pose._replace(distance=pose.distance * 1.5, lookat=pose.lookat + lookat_offset, azimuth=pose.azimuth + -15, elevation=pose.elevation - 10) camera.set_pose(*new_pose) self.assertEqual(new_pose.distance, camera.get_pose().distance) self.assertEqual(new_pose.azimuth, camera.get_pose().azimuth) self.assertEqual(new_pose.elevation, camera.get_pose().elevation) np.testing.assert_allclose(new_pose.lookat, camera.get_pose().lookat) self.assertFalse(np.all(image == camera.render())) def testRenderExceptions(self): max_width = self._physics.model.vis.global_.offwidth max_height = self._physics.model.vis.global_.offheight max_camid = self._physics.model.ncam - 1 with six.assertRaisesRegex(self, ValueError, 'width'): self._physics.render(max_height, max_width + 1, camera_id=max_camid) with six.assertRaisesRegex(self, ValueError, 'height'): self._physics.render(max_height + 1, max_width, camera_id=max_camid) with six.assertRaisesRegex(self, ValueError, 'camera_id'): self._physics.render(max_height, max_width, camera_id=max_camid + 1) with six.assertRaisesRegex(self, ValueError, 'camera_id'): self._physics.render(max_height, max_width, camera_id=-2) def testPhysicsRenderMethod(self): height, width = 240, 320 image = self._physics.render(height=height, width=width) self.assertEqual(image.shape, (height, width, 3)) depth = self._physics.render(height=height, width=width, depth=True) self.assertEqual(depth.shape, (height, width)) segmentation = self._physics.render(height=height, width=width, segmentation=True) self.assertEqual(segmentation.shape, (height, width, 2)) def testExceptionIfBothDepthAndSegmentation(self): with self.assertRaisesWithLiteralMatch( ValueError, engine._BOTH_SEGMENTATION_AND_DEPTH_ENABLED): self._physics.render(depth=True, segmentation=True) def testRenderFlagOverridesAreNotPersistent(self): camera = engine.Camera(self._physics) first_rgb = camera.render().copy() camera.render(segmentation=True) second_rgb = camera.render().copy() np.testing.assert_array_equal(first_rgb, second_rgb) def testCustomRenderFlags(self): default = self._physics.render() wireframe_string_key = self._physics.render( render_flag_overrides=dict(wireframe=True)) self.assertFalse((default == wireframe_string_key).all()) wireframe_enum_key = self._physics.render( render_flag_overrides={enums.mjtRndFlag.mjRND_WIREFRAME: True}) np.testing.assert_array_equal(wireframe_string_key, wireframe_enum_key) @parameterized.parameters(dict(depth=True), dict(segmentation=True)) def testExceptionIfRenderFlagOverridesAndDepthOrSegmentation(self, **kwargs): with self.assertRaisesWithLiteralMatch( ValueError, engine._RENDER_FLAG_OVERRIDES_NOT_SUPPORTED_FOR_DEPTH_OR_SEGMENTATION): self._physics.render(render_flag_overrides=dict(wireframe=True), **kwargs) def testExceptionIfOverlaysAndDepthOrSegmentation(self): overlay = engine.TextOverlay() with self.assertRaisesWithLiteralMatch( ValueError, engine._OVERLAYS_NOT_SUPPORTED_FOR_DEPTH_OR_SEGMENTATION): self._physics.render(depth=True, overlays=[overlay]) with self.assertRaisesWithLiteralMatch( ValueError, engine._OVERLAYS_NOT_SUPPORTED_FOR_DEPTH_OR_SEGMENTATION): self._physics.render(segmentation=True, overlays=[overlay]) def testNamedViews(self): self.assertEqual((1,), self._physics.control().shape) self.assertEqual((2,), self._physics.position().shape) self.assertEqual((2,), self._physics.velocity().shape) self.assertEqual((0,), self._physics.activation().shape) self.assertEqual((4,), self._physics.state().shape) self.assertEqual(0., self._physics.time()) self.assertEqual(0.01, self._physics.timestep()) def testSetGetPhysicsState(self): physics_state = self._physics.get_state() self._physics.set_state(physics_state) new_physics_state = np.random.random_sample(physics_state.shape) self._physics.set_state(new_physics_state) np.testing.assert_allclose(new_physics_state, self._physics.get_state()) def testSetInvalidPhysicsState(self): badly_shaped_state = np.repeat(self._physics.get_state(), repeats=2) with self.assertRaises(ValueError): self._physics.set_state(badly_shaped_state) def testNamedIndexing(self): self.assertEqual((3,), self._physics.named.data.xpos['cart'].shape) self.assertEqual((2, 3), self._physics.named.data.xpos[['cart', 'pole']].shape) def testReload(self): self._physics.reload_from_xml_path(MODEL_PATH) def testReset(self): self._physics.reset() self.assertEqual(self._physics.data.qpos[1], 0) keyframe_id = 0 self._physics.reset(keyframe_id=keyframe_id) self.assertEqual(self._physics.data.qpos[1], self._physics.model.key_qpos[keyframe_id, 1]) out_of_range = [-1, 3] max_valid = self._physics.model.nkey - 1 for actual in out_of_range: with self.assertRaisesWithLiteralMatch( ValueError, engine._KEYFRAME_ID_OUT_OF_RANGE.format( max_valid=max_valid, actual=actual)): self._physics.reset(keyframe_id=actual) def testLoadAndReloadFromStringWithAssets(self): physics = engine.Physics.from_xml_string( MODEL_WITH_ASSETS, assets=ASSETS) physics.reload_from_xml_string(MODEL_WITH_ASSETS, assets=ASSETS) def testFree(self): def mock_free(obj): return mock.patch.object(obj, 'free', wraps=obj.free) with mock_free(self._physics.model) as mock_free_model: with mock_free(self._physics.data) as mock_free_data: with mock_free(self._physics.contexts.mujoco) as mock_free_mjrcontext: self._physics.free() mock_free_model.assert_called_once() mock_free_data.assert_called_once() mock_free_mjrcontext.assert_called_once() self.assertIsNone(self._physics.model.ptr) self.assertIsNone(self._physics.data.ptr) @parameterized.parameters(*enums.mjtWarning._fields[:-1]) def testDivergenceException(self, warning_name): warning_enum = getattr(enums.mjtWarning, warning_name) with self.assertRaisesWithLiteralMatch( control.PhysicsError, engine._INVALID_PHYSICS_STATE.format(warning_names=warning_name)): with self._physics.check_invalid_state(): self._physics.data.warning[warning_enum].number = 1 # Existing warnings should not raise an exception. with self._physics.check_invalid_state(): pass self._physics.reset() with self._physics.check_invalid_state(): pass @parameterized.parameters(float('inf'), float('nan'), 1e15) def testBadQpos(self, bad_value): with self._physics.reset_context(): self._physics.data.qpos[0] = bad_value with self.assertRaises(control.PhysicsError): with self._physics.check_invalid_state(): mjlib.mj_checkPos(self._physics.model.ptr, self._physics.data.ptr) self._physics.reset() with self._physics.check_invalid_state(): mjlib.mj_checkPos(self._physics.model.ptr, self._physics.data.ptr) def testNanControl(self): with self._physics.reset_context(): pass # Apply the controls. with self.assertRaisesWithLiteralMatch( control.PhysicsError, engine._INVALID_PHYSICS_STATE.format(warning_names='mjWARN_BADCTRL')): with self._physics.check_invalid_state(): self._physics.data.ctrl[0] = float('nan') self._physics.step() def testSuppressPhysicsError(self): bad_value = float('nan') message = engine._INVALID_PHYSICS_STATE.format( warning_names='mjWARN_BADCTRL') def assert_physics_error(): self._physics.data.ctrl[0] = bad_value with self.assertRaisesWithLiteralMatch(control.PhysicsError, message): self._physics.forward() def assert_warning(): self._physics.data.ctrl[0] = bad_value with mock.patch.object(engine.logging, 'warn') as mock_warn: self._physics.forward() mock_warn.assert_called_once_with(message) assert_physics_error() with self._physics.suppress_physics_errors(): assert_warning() with self._physics.suppress_physics_errors(): assert_warning() assert_warning() assert_physics_error() @parameterized.named_parameters( ('_copy', lambda x: x.copy()), ('_pickle_and_unpickle', lambda x: cPickle.loads(cPickle.dumps(x))), ) def testCopyOrPicklePhysics(self, func): for _ in range(10): self._physics.step() physics2 = func(self._physics) self.assertNotEqual(physics2.model.ptr, self._physics.model.ptr) self.assertNotEqual(physics2.data.ptr, self._physics.data.ptr) model_attr_to_compare = ('nnames', 'njmax', 'body_pos', 'geom_quat') self._assert_attributes_equal( physics2.model, self._physics.model, model_attr_to_compare) data_attr_to_compare = ('time', 'energy', 'qpos', 'xpos') self._assert_attributes_equal( physics2.data, self._physics.data, data_attr_to_compare) for _ in range(10): self._physics.step() physics2.step() self._assert_attributes_equal( physics2.model, self._physics.model, model_attr_to_compare) self._assert_attributes_equal( physics2.data, self._physics.data, data_attr_to_compare) def testCopyDataOnly(self): physics2 = self._physics.copy(share_model=True) self.assertEqual(physics2.model.ptr, self._physics.model.ptr) self.assertNotEqual(physics2.data.ptr, self._physics.data.ptr) def testForwardDynamicsUpdatedAfterReset(self): gravity = -9.81 self._physics.model.opt.gravity[2] = gravity with self._physics.reset_context(): pass self.assertAlmostEqual( self._physics.named.data.sensordata['accelerometer'][2], -gravity) def testActuationNotAppliedInAfterReset(self): self._physics.data.ctrl[0] = 1. self._physics.after_reset() # Calls `forward()` with actuation disabled. self.assertEqual(self._physics.data.actuator_force[0], 0.) self._physics.forward() # Call `forward` directly with actuation enabled. self.assertEqual(self._physics.data.actuator_force[0], 1.) def testActionSpec(self): xml = """ <mujoco> <worldbody> <body> <geom type="sphere" size="0.1"/> <joint type="hinge" name="hinge"/> </body> </worldbody> <actuator> <motor joint="hinge" ctrllimited="false"/> <motor joint="hinge" ctrllimited="true" ctrlrange="-1 2"/> </actuator> </mujoco> """ physics = engine.Physics.from_xml_string(xml) spec = engine.action_spec(physics) self.assertEqual(np.float, spec.dtype) np.testing.assert_array_equal(spec.minimum, [-constants.mjMAXVAL, -1.0]) np.testing.assert_array_equal(spec.maximum, [constants.mjMAXVAL, 2.0]) def _check_valid_rotation_matrix(self, data_field_name): name = 'foo' quat = '0.5 0.7 0 0' # Not normalized. xml_string = """ <mujoco> <worldbody> <body name='{name}' quat='{quat}'/> <camera name='{name}' quat='{quat}'/> <geom name='{name}' quat='{quat}' size='0.1'/> <site name='{name}' quat='{quat}' size='0.1'/> </worldbody> </mujoco> """.format(name=name, quat=quat) physics = engine.Physics.from_xml_string(xml_string) rotation_matrix = getattr(physics.named.data, data_field_name)[name] self.assertAlmostEqual(np.linalg.det(rotation_matrix.reshape(3, 3)), 1) @parameterized.parameters(['xmat', 'geom_xmat', 'site_xmat']) def testValidRotationMatrixIfQuatNotNormalizedInXML(self, field_name): self._check_valid_rotation_matrix(field_name) # TODO(b/123918714): Update this once the bug has been fixed in MuJoCo. @unittest.expectedFailure def testValidCameraRotationMatrixIfQuatNotNormalizedInXML(self): self._check_valid_rotation_matrix('cam_xmat')
def _loads(self, value): return pickle.loads(value)
def deserialize(data): return cPickle.loads(data)
def test_pickling(self): dataset = cPickle.loads(cPickle.dumps(self.dataset)) assert_equal(len(dataset.nodes), 1)
def hgetall(self, name): return {key: pickle.loads(value) for key, value in iteritems(super(RedisWrapper, self).hgetall(self.make_key(name)))}
def decode(self, input): try: return pickle.loads(input) except pickle.PickleError as e: raise SessionSerializerException(e)
def test_mnist(self): ds = fuel.load_mnist() m = model.SequentialClassifier(N.Flatten(outdim=2), N.Dense(64, activation=K.relu), N.Dense(10, activation=K.softmax)) m.set_inputs( K.placeholder(shape=(None, 28, 28), name='X', dtype='float32')).set_outputs( K.placeholder(shape=(None, ), name='y', dtype='int32')) # ====== query info ====== # m.path self.assertEqual(m.is_initialized, True) self.assertEqual(m.input_shape, (None, 28, 28)) self.assertEqual(m.output_shape, (None, 10)) # ====== training test ====== # m.set_training_info(learning_rate=0.001, n_epoch=3) m.fit(X=(ds['X_train'], ds['y_train']), X_valid=(ds['X_valid'], ds['y_valid'])) score = m.score(ds['X_test'][:], ds['y_test'][:]) self.assertEqual( score > 0.8, True, msg='Test if the model get reasonable results: %f accuracy' % score) # ====== make prediction and transform test ====== # np.random.seed(12) _ = np.random.rand(8, 28, 28) self.assertEqual(m.transform(_).shape, (8, 10)) self.assertEqual( np.isclose(m.predict_proba(_).sum(-1), 1.).sum() == 8, True) self.assertEqual(len(m.predict(_)), 8) # ====== pickling test ====== # str_old = str(m) p_old = m.get_params(True) m = cPickle.loads(cPickle.dumps(m, protocol=cPickle.HIGHEST_PROTOCOL)) str_new = str(m) p_new = m.get_params(True) # ====== test same configurations ====== # self.assertEqual(str_new, str_old) # ====== test same params ====== # for i, j in p_new.items(): k = p_old[i] for a, b in zip(j, k): self.assertEqual(np.array_equal(a, b), True) # ====== test set params ====== # params = m.get_params(deep=True) params_new = {} for n, p in params.items(): params_new[n] = [ np.random.rand(*i.shape).astype('float32') for i in p ] m.set_params(**params_new) # test if equal new onces for i, j in m.get_params(deep=True).items(): k = params_new[i] for a, b in zip(j, k): self.assertEqual(np.array_equal(a, b), True) # ====== training test ====== # print('Re-train the model second time:') m.fit(X=(ds['X_train'], ds['y_train']), X_valid=(ds['X_valid'], ds['y_valid'])) score = m.score(ds['X_test'][:], ds['y_test'][:]) self.assertEqual( score > 0.8, True, msg='Test if the model get reasonable results: %f accuracy' % score)
def test_data_stream_pickling(self): stream = DataStream(H5PYDataset(self.h5file, which_sets=('train', )), iteration_scheme=SequentialScheme(100, 10)) cPickle.loads(cPickle.dumps(stream)) stream.close()
def set_state(self, state): self._lazy_value = state[0] self._value = pickle.loads(state[1])
def result(self, timeout=None, check_only=False, throw_except=True, storage_handler=None): """ From the python docs: Return the value returned by the call. If the call hasn't yet completed then this method will wait up to timeout seconds. If the call hasn't completed in timeout seconds then a TimeoutError will be raised. timeout can be an int or float.If timeout is not specified or None then there is no limit to the wait time. If the future is cancelled before completing then CancelledError will be raised. If the call raised then this method will raise the same exception. """ if self._state == JobState.new: raise ValueError("job not yet invoked") if self._state == JobState.success: return self._return_val if self._state == JobState.error: if throw_except: raise self._exception else: return None if storage_handler is None: storage_config = wrenconfig.extract_storage_config( wrenconfig.default()) storage_handler = storage.Storage(storage_config) storage_utils.check_storage_path(storage_handler.get_storage_config(), self.storage_path) call_status = storage_handler.get_call_status(self.callset_id, self.call_id) self.status_query_count += 1 ## FIXME implement timeout if timeout is not None: raise NotImplementedError() if check_only is True: if call_status is None: return None while call_status is None: time.sleep(self.GET_RESULT_SLEEP_SECS) call_status = storage_handler.get_call_status( self.callset_id, self.call_id) self.status_query_count += 1 self._invoke_metadata['status_done_timestamp'] = time.time() self._invoke_metadata['status_query_count'] = self.status_query_count self.run_status = call_status # this is the remote status information self.invoke_status = self._invoke_metadata # local status information if call_status['exception'] is not None: # the wrenhandler had an exception exception_str = call_status['exception'] print(call_status) exception_args = call_status['exception_args'] if exception_args[0] == "WRONGVERSION": if throw_except: raise Exception("Pywren version mismatch: remote " + \ "expected version {}, local library is version {}".format( exception_args[2], exception_args[3])) return None elif exception_args[0] == "OUTATIME": if throw_except: raise Exception("process ran out of time") return None else: if throw_except: if 'exception_traceback' in call_status: logger.error(call_status['exception_traceback']) raise Exception(exception_str, *exception_args) return None call_output_time = time.time() call_invoker_result = pickle.loads( storage_handler.get_call_output(self.callset_id, self.call_id)) call_output_time_done = time.time() self._invoke_metadata[ 'download_output_time'] = call_output_time_done - call_output_time self._invoke_metadata[ 'download_output_timestamp'] = call_output_time_done call_success = call_invoker_result['success'] logger.info("ResponseFuture.result() {} {} call_success {}".format( self.callset_id, self.call_id, call_success)) self._call_invoker_result = call_invoker_result if call_success: self._return_val = call_invoker_result['result'] self._state = JobState.success return self._return_val elif throw_except: self._exception = call_invoker_result['result'] self._traceback = (call_invoker_result['exc_type'], call_invoker_result['exc_value'], call_invoker_result['exc_traceback']) self._state = JobState.error if call_invoker_result.get('pickle_fail', False): logging.warning( "there was an error pickling. The original exception: " + \ "{}\nThe pickling exception: {}".format( call_invoker_result['exc_value'], str(call_invoker_result['pickle_exception']))) reraise(Exception, call_invoker_result['exc_value'], call_invoker_result['exc_traceback']) else: # reraise the exception reraise(*self._traceback) else: return None # nothing, don't raise, no value
def test_polar(): ax = plt.subplot(111, polar=True) fig = plt.gcf() pf = pickle.dumps(fig) pickle.loads(pf) plt.draw()
def _fix_recid(recid, logger): """Fix a given recid.""" # logger.info("Upgrading record %s:" % recid) # 1) moving docname and type to the relation with bibrec bibrec_docs = run_sql( """select id_bibdoc, type from bibrec_bibdoc where id_bibrec=%s""", (recid, )) are_equal = True for docid_str in bibrec_docs: docid = str(docid_str[0]) doctype = str(docid_str[1]) # logger.info("Upgrading document %s:" % (docid, )) res2 = run_sql("select docname, more_info from bibdoc where id=%s", (docid, )) if not res2: logger.error( ("Error when migrating document %s attached to the " "record %s: can not retrieve from the bibdoc table ") % (docid, recid)) else: docname = str(res2[0][0]) run_sql( """update bibrec_bibdoc set docname=%%s where id_bibrec=%s and id_bibdoc=%s""" % (str(recid), docid), (docname, )) run_sql("update bibdoc set doctype=%%s where id=%s" % (docid, ), (doctype, )) # 2) moving moreinfo to the new moreinfo structures (default namespace) if res2[0][1]: minfo = cPickle.loads(res2[0][1]) # 2a migrating descriptions->version->format new_value = cPickle.dumps(minfo.get('descriptions', {})) run_sql( """INSERT INTO bibdocmoreinfo (id_bibdoc, namespace, data_key, data_value) VALUES (%s, %s, %s, %s)""", (str(docid), "", "descriptions", new_value)) # 2b migrating comments->version->format new_value = cPickle.dumps(minfo.get('comments', {})) run_sql( """INSERT INTO bibdocmoreinfo (id_bibdoc, namespace, data_key, data_value) VALUES (%s, %s, %s, %s)""", (str(docid), "", "comments", new_value)) # 2c migrating flags->flagname->version->format new_value = cPickle.dumps(minfo.get('flags', {})) run_sql( """INSERT INTO bibdocmoreinfo (id_bibdoc, namespace, data_key, data_value) VALUES (%s, %s, %s, %s)""", (str(docid), "", "flags", new_value)) # 3) Verify the correctness of moreinfo transformations try: descriptions = cPickle.loads( run_sql( """SELECT data_value FROM bibdocmoreinfo WHERE id_bibdoc=%s AND namespace=%s AND data_key=%s""", (str(docid), '', 'descriptions'))[0][0]) for version in minfo.get('descriptions', {}): for docformat in minfo['descriptions'][version]: v1 = descriptions[version][docformat] v2 = minfo['descriptions'][version][docformat] if v1 != v2: are_equal = False logger.info(("ERROR: Document %s: Expected " "description %s and got %s") % (str(docid), str(v2), str(v1))) except Exception as e: logger.info( ("ERROR: Document %s: Problem with retrieving " "descriptions: %s MoreInfo: %s Descriptions: %s") % (str(docid), str(e), str(minfo), str(descriptions))) try: comments = cPickle.loads( run_sql( """SELECT data_value FROM bibdocmoreinfo WHERE id_bibdoc=%s AND namespace=%s AND data_key=%s""", (str(docid), '', 'comments'))[0][0]) for version in minfo.get('comments', {}): for docformat in minfo['comments'][version]: v1 = comments[version][docformat] v2 = minfo['comments'][version][docformat] if v1 != v2: are_equal = False logger.info(("ERROR: Document %s: Expected " "comment %s and got %s") % (str(docid), str(v2), str(v1))) except Exception as e: logger.info(("ERROR: Document %s: Problem with retrieving " "comments: %s MoreInfo: %s Comments: %s") % (str(docid), str(e), str(minfo), str(comments))) try: flags = cPickle.loads( run_sql( """SELECT data_value FROM bibdocmoreinfo WHERE id_bibdoc=%s AND namespace=%s AND data_key=%s""", (str(docid), '', 'flags'))[0][0]) for flagname in minfo.get('flags', {}): for version in minfo['flags'][flagname]: for docformat in minfo['flags'][flagname][version]: if minfo['flags'][flagname][version][docformat]: are_equal = are_equal and \ (docformat in flags[flagname][version]) if not (docformat in flags[flagname][version]): logger.info(("ERROR: Document %s: " "Expected %s") % (str(docid), str(minfo))) except Exception as e: logger.info(("ERROR: Document %s: Problem with retrieving " "flags. %s MoreInfo: %s flags: %s") % (str(docid), str(e), str(minfo), str(flags))) if not are_equal: logger.info( ("Failed to move MoreInfo structures from old " "database to the new one docid: %s") % (str(docid), )) return are_equal
def get_extra_data(self): """Get extra data saved to the object.""" return cPickle.loads(base64.b64decode(self._extra_data))
def get_data(self): """Get data saved in the object.""" return cPickle.loads(base64.b64decode(self._data))
def save_caption_vectors_products(self, train_ratio=0.8): target, one_hot_targets, n_target = get_one_hot_targets(self.category_path) train_list = [] with open(self.parse_data_path) as cap_f: for i, line in enumerate(cap_f): row = line.strip().split('\t') asin = row[0] categories = row[1] title = row[2] imgid = asin+'.jpg' onehot_cate = one_hot_encode_str_lbl(categories, target, one_hot_targets) train_list.append((imgid, onehot_cate, title)) if i % self.n_log_print == 0: print(i, train_list[-1]) chunk_size = self.chunk_size datasize = len(train_list) self.logger.info('data size: %d' % datasize) n_chunk = datasize // chunk_size + 1 chunk_list = [] for index in range(n_chunk): start_index = index * chunk_size end_index = start_index + chunk_size chunk = train_list[start_index:end_index] chunk_list.append((index, chunk)) for chunk in chunk_list: index = chunk[0] chunk = chunk[1] st = time.time() temp_list = [] for i, (key, cate_vec, title) in enumerate(chunk): title_vec = self.parser.text2vec(title) temp_list.append((key, cate_vec, title_vec)) chunk_path = (os.path.join(self.temp_dir_path, 'products_tv_{}.pkl'.format(index))) f_out = open(chunk_path, 'wb') p = cPickle.Pickler(f_out) p.dump(temp_list) p.clear_memo() f_out.close() self.logger.info("%s done: %d" % (chunk_path, time.time() - st)) if not os.path.isdir(self.train_dir_path): os.makedirs(self.train_dir_path) all_train = train_ratio >= 1.0 all_dev = train_ratio == 0.0 train_indices, train_size = self.get_train_indices(datasize, train_ratio) dev_size = datasize - train_size if all_dev: train_size = 1 dev_size = datasize if all_train: dev_size = 1 train_size = datasize data_fout = h5py.File(os.path.join(self.train_dir_path, 'data.h5py'), 'w') train = data_fout.create_group('train') dev = data_fout.create_group('dev') self.create_dataset(train, train_size, n_target) self.create_dataset(dev, dev_size, n_target) sample_idx = 0 dataset = {'train': train, 'dev': dev} num_samples = {'train': 0, 'dev': 0} chunk = {'train': self.init_chunk(chunk_size, n_target), 'dev': self.init_chunk(chunk_size, n_target)} # make h5file for input_chunk_idx in range(n_chunk): path = os.path.join(self.temp_dir_path, 'products_tv_{}.pkl'.format(input_chunk_idx)) print('processing %s ...' % path) data = cPickle.loads(open(path, 'rb').read()) for data_idx, (img_idx, cate_vec, title_vec) in enumerate(data): is_train = train_indices[sample_idx + data_idx] if all_dev: is_train = False if all_train: is_train = True c = chunk['train'] if is_train else chunk['dev'] idx = c['num'] c['docvec'][idx] = title_vec c['cate'][idx] = cate_vec c['asin'].append(np.string_(img_idx)) c['num'] += 1 for t in ['train', 'dev']: if chunk[t]['num'] >= chunk_size: self.copy_chunk(dataset[t], chunk[t], num_samples[t]) num_samples[t] += chunk[t]['num'] chunk[t] = self.init_chunk(chunk_size, n_target) sample_idx += len(data) for t in ['train', 'dev']: if chunk[t]['num'] > 0: self.copy_chunk(dataset[t], chunk[t], num_samples[t]) num_samples[t] += chunk[t]['num'] for div in ['train', 'dev']: ds = dataset[div] size = num_samples[div] ds['cate'].resize((size, n_target)) ds['docvec'].resize((size, self.doc_vec_size)) data_fout.close()
def track_event(category=None, action=None, label=None): if not settings.analytics.getboolean('enabled'): return anonymousConfig = Config() anonymousConfig.anonimize_ip_address = True tracker = Tracker('UA-138214134-3', 'none', conf=anonymousConfig) try: if settings.analytics.visitor: visitor = pickle.loads( codecs.decode(settings.analytics.visitor.encode(), "base64")) if visitor.user_agent is None: visitor.user_agent = os.environ.get("SZ_USER_AGENT") if visitor.unique_id > int(0x7fffffff): visitor.unique_id = random.randint(0, 0x7fffffff) except: visitor = Visitor() visitor.unique_id = random.randint(0, 0x7fffffff) session = Session() event = Event(category=category, action=action, label=label, value=1) tracker.add_custom_variable( CustomVariable(index=1, name='BazarrVersion', value=os.environ["BAZARR_VERSION"], scope=1)) tracker.add_custom_variable( CustomVariable(index=2, name='PythonVersion', value=platform.python_version(), scope=1)) if settings.general.getboolean('use_sonarr'): tracker.add_custom_variable( CustomVariable(index=3, name='SonarrVersion', value=sonarr_version, scope=1)) else: tracker.add_custom_variable( CustomVariable(index=3, name='SonarrVersion', value='unused', scope=1)) if settings.general.getboolean('use_radarr'): tracker.add_custom_variable( CustomVariable(index=4, name='RadarrVersion', value=radarr_version, scope=1)) else: tracker.add_custom_variable( CustomVariable(index=4, name='RadarrVersion', value='unused', scope=1)) tracker.add_custom_variable( CustomVariable(index=5, name='OSVersion', value=platform.platform(), scope=1)) try: tracker.track_event(event, session, visitor) except: logging.debug("BAZARR unable to track event.") pass else: settings.analytics.visitor = codecs.encode(pickle.dumps(visitor), "base64").decode() with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle: settings.write(handle)