def test_copying(self): # Check that counters are copyable, deepcopyable, picklable, and #have a repr/eval round-trip words = Counter('which witch had which witches wrist watch'.split()) update_test = Counter() update_test.update(words) for i, dup in enumerate([ words.copy(), copy.copy(words), copy.deepcopy(words), pickle.loads(pickle.dumps(words, 0)), pickle.loads(pickle.dumps(words, 1)), pickle.loads(pickle.dumps(words, 2)), pickle.loads(pickle.dumps(words, -1)), cPickle.loads(cPickle.dumps(words, 0)), cPickle.loads(cPickle.dumps(words, 1)), cPickle.loads(cPickle.dumps(words, 2)), cPickle.loads(cPickle.dumps(words, -1)), eval(repr(words)), update_test, Counter(words), ]): msg = (i, dup, words) self.assertTrue(dup is not words) self.assertEquals(dup, words) self.assertEquals(len(dup), len(words)) self.assertEquals(type(dup), type(words))
def unpicklable(target_dict, key): try: dumps(target_dict[key]) except PicklingError: return ("{} can't be pickled. If it is a function, try defining "+\ "it in the module space rather than in a class or closure.").\ format(key)
def test_pickle(): import pickle # classification obj = tree.DecisionTreeClassifier() obj.fit(iris.data, iris.target) score = obj.score(iris.data, iris.target) s = pickle.dumps(obj) obj2 = pickle.loads(s) assert_equal(type(obj2), obj.__class__) score2 = obj2.score(iris.data, iris.target) assert score == score2, "Failed to generate same score " + \ " after pickling (classification) " # regression obj = tree.DecisionTreeRegressor() obj.fit(boston.data, boston.target) score = obj.score(boston.data, boston.target) s = pickle.dumps(obj) obj2 = pickle.loads(s) assert_equal(type(obj2), obj.__class__) score2 = obj2.score(boston.data, boston.target) assert score == score2, "Failed to generate same score " + \ " after pickling (regression) "
def add_scores(self, space, scores, latest, unfinished): """ Store partial results of the calculation. :param space: SearchSpace object explored during calculations. :param scores: List of tuples (precision, sensitivity, f1). :param latest: Highest index of a description pair for which the results have been calculated. :param unfinished: List of tuples describing ranges of indices of descriptor pairs for which the results haven't been calculated yet, indices < latest. """ session = self._session_factory() space_obj = session.query(SearchOperation).filter_by(space_descr=str(space)).first() if not space_obj: space_obj = SearchOperation(space_descr=str(space), space=pickle.dumps(space), enabled=True, done=False) session.add(space_obj) session.commit() space_obj.latest = latest space_obj.unfinished = pickle.dumps(unfinished) scores_obj = SearchSpaceScore(space_id=space_obj.id) scores_obj.scores = pickle.dumps(scores) session.add(scores_obj) session.commit() session.close()
def test_executor(self): executor = MyExecutor() driver = MockExecutorDriver(executor) executorInfo = mesos_pb2.ExecutorInfo() executorInfo.executor_id.value = "test-id" executorInfo.data = marshal.dumps(("./", os.getcwd(), sys.path, 8, "", "", 1, {'DPARK_HAS_DFS':'False', 'WORKDIR':'/tmp/xxxxx'})) frameworkInfo = mesos_pb2.FrameworkInfo() frameworkInfo.id.value = "test" slaveInfo = mesos_pb2.SlaveInfo() slaveInfo.id.value = "test-slave" slaveInfo.hostname = socket.gethostname() executor.registered(driver, executorInfo, frameworkInfo, slaveInfo) assert executor.init_args task = mesos_pb2.TaskInfo() task.name = 'test-task' task.task_id.value = '1' task.slave_id.value = 'test-slave' task.data = pickle.dumps((TestTask(1), 1), -1) executor.launchTask(driver, task) task.task_id.value = '2' task.data = pickle.dumps((TestTask(2), 1), -1) executor.launchTask(driver, task) executor.frameworkMessage(driver, 'data') executor.killTask(driver, task.task_id)
def test_is_pickable(self): m = DefaultsModel() m.save() try: pickle.dumps(m) except TypeError as e: self.fail('pickle of DefaultsModel failed: %s' % e)
def test_cached_list(self): # Set up the test data. users = User.objects.all()[:10] user_cache = TestUserCachedList(users) self.assertEqual([user.pk for user in users], [user.pk for user in user_cache]) # Force it through the pickle cycle. user_cache = pickle.loads(pickle.dumps(user_cache)) self.assertEqual([user.pk for user in users], [user.pk for user in user_cache]) # The pickle size is greatly reduced. While making this test, it went # from 6377 bytes to 201 bytes. To avoid a brittle test, just check # that it's less that half the size. normal_pickle_size = len(pickle.dumps(users)) improved_pickle_size = len(pickle.dumps(user_cache)) self.assertTrue(improved_pickle_size < normal_pickle_size / 2.0) # Force it through the cache cycle. cache_key = 'apncore.cache.tests.test_cached_list' user_cache.cache(cache_key) user_cache = cache.get(cache_key) self.assertEqual([user.pk for user in users], [user.pk for user in user_cache]) # Delete the cached items, forcing the class to rebuild them. # The main list must be retrieved again to test unpacking its items. item_cache_keys = list(user_cache.make_cache_keys([user.pk for user in users])) cache.delete_many(item_cache_keys) user_cache = cache.get(cache_key) self.assertEqual([user.pk for user in users], [user.pk for user in user_cache])
def CreateTable(self, table, columns): """CreateTable(table, columns) - Create a new table in the database. raises TableDBError if it already exists or for other DB errors. """ raise isinstance(columns, list) or AssertionError txn = None try: if contains_metastrings(table): raise ValueError('bad table name: contains reserved metastrings') for column in columns: if contains_metastrings(column): raise ValueError('bad column name: contains reserved metastrings') columnlist_key = _columns_key(table) if getattr(self.db, 'has_key')(columnlist_key): raise TableAlreadyExists, 'table already exists' txn = self.env.txn_begin() getattr(self.db, 'put_bytes', self.db.put)(columnlist_key, pickle.dumps(columns, 1), txn=txn) tablelist = pickle.loads(getattr(self.db, 'get_bytes', self.db.get)(_table_names_key, txn=txn, flags=db.DB_RMW)) tablelist.append(table) self.db.delete(_table_names_key, txn=txn) getattr(self.db, 'put_bytes', self.db.put)(_table_names_key, pickle.dumps(tablelist, 1), txn=txn) txn.commit() txn = None except db.DBError as dberror: if txn: txn.abort() if sys.version_info < (2, 6): raise TableDBError, dberror[1] else: raise TableDBError, dberror.args[1] return
def test_copy_pickle(self): d = Deque('abc') e = d.__copy__() self.assertEqual(type(d), type(e)) self.assertEqual(list(d), list(e)) e = Deque(d) self.assertEqual(type(d), type(e)) self.assertEqual(list(d), list(e)) for proto in range(pickle.HIGHEST_PROTOCOL + 1): s = pickle.dumps(d, proto) e = pickle.loads(s) self.assertNotEqual(id(d), id(e)) self.assertEqual(type(d), type(e)) self.assertEqual(list(d), list(e)) d = Deque('abcde', maxlen=4) e = d.__copy__() self.assertEqual(type(d), type(e)) self.assertEqual(list(d), list(e)) e = Deque(d) self.assertEqual(type(d), type(e)) self.assertEqual(list(d), list(e)) for proto in range(pickle.HIGHEST_PROTOCOL + 1): s = pickle.dumps(d, proto) e = pickle.loads(s) self.assertNotEqual(id(d), id(e)) self.assertEqual(type(d), type(e)) self.assertEqual(list(d), list(e))
def test_make_proxy_disc(self): abc = DiscreteVariable("abc", values="abc", ordered=True) abc1 = abc.make_proxy() abc2 = abc1.make_proxy() self.assertIs(abc.master, abc) self.assertIs(abc1.master, abc) self.assertIs(abc2.master, abc) self.assertEqual(abc, abc1) self.assertEqual(abc, abc2) self.assertEqual(abc1, abc2) self.assertEqual(hash(abc), hash(abc1)) self.assertEqual(hash(abc1), hash(abc2)) abcx = DiscreteVariable("abc", values="abc", ordered=True) self.assertNotEqual(abc, abcx) abc1p = pickle.loads(pickle.dumps(abc1)) self.assertIs(abc1p.master, abc) self.assertEqual(abc1p, abc) abcp, abc1p, abc2p = pickle.loads(pickle.dumps((abc, abc1, abc2))) self.assertIs(abcp.master, abcp.master) self.assertIs(abc1p.master, abcp.master) self.assertIs(abc2p.master, abcp.master) self.assertEqual(abcp, abc1p) self.assertEqual(abcp, abc2p) self.assertEqual(abc1p, abc2p)
def request(self, *args, **kwargs): """Maintains the existing api for Session.request. Used by all of the higher level methods, e.g. Session.get. The background_callback param allows you to do some processing on the response in the background, e.g. call resp.json() so that json parsing happens in the background thread. """ if self.session: func = self.session.request else: # avoid calling super to not break pickled method func = partial(Session.request, self) background_callback = kwargs.pop('background_callback', None) if background_callback: func = partial(wrap, self, func, background_callback) if isinstance(self.executor, ProcessPoolExecutor): # verify function can be pickled try: dumps(func) except (TypeError, PickleError): raise RuntimeError(PICKLE_ERROR) return self.executor.submit(func, *args, **kwargs)
def on_recv_data(self, message): parsed_msg = util.ws_parse( message, trading_date = self.trading_date, to_dict=True ) # 更新redis中行情内容 for data in parsed_msg: try: if data["data_type"] == "orders": self.publish( data = pickle.dumps(data), channel_name = "dHydra.SinaL2." + data["symbol"] + ".orders" ) elif data["data_type"] == "quotation": self.publish( data = pickle.dumps(data), channel_name = "dHydra.SinaL2." + data["symbol"] + ".quotation" ) elif data["data_type"] == "transaction": self.publish( data = pickle.dumps(data), channel_name = "dHydra.SinaL2." + data["symbol"] + ".transaction" ) except Exception as e: self.logger.error(e) self.publish(pickle.dumps(parsed_msg))
def testDatabaseFixes(self): # Hack the pickle to make it refer to a timezone abbreviation # that does not match anything. The unpickler should be able # to repair this case tz = pytz.timezone('Australia/Melbourne') p = pickle.dumps(tz) tzname = tz._tzname hacked_p = p.replace(_byte_string(tzname), _byte_string('???')) self.assertNotEqual(p, hacked_p) unpickled_tz = pickle.loads(hacked_p) self.assertTrue(tz is unpickled_tz) # Simulate a database correction. In this case, the incorrect # data will continue to be used. p = pickle.dumps(tz) new_utcoffset = tz._utcoffset.seconds + 42 # Python 3 introduced a new pickle protocol where numbers are stored in # hexadecimal representation. Here we extract the pickle # representation of the number for the current Python version. old_pickle_pattern = pickle.dumps(tz._utcoffset.seconds)[3:-1] new_pickle_pattern = pickle.dumps(new_utcoffset)[3:-1] hacked_p = p.replace(old_pickle_pattern, new_pickle_pattern) self.assertNotEqual(p, hacked_p) unpickled_tz = pickle.loads(hacked_p) self.assertEqual(unpickled_tz._utcoffset.seconds, new_utcoffset) self.assertTrue(tz is not unpickled_tz)
def test01(self): tabname = "test01" colname = 'cool numbers' try: self.tdb.Drop(tabname) except dbtables.TableDBError: pass self.tdb.CreateTable(tabname, [colname]) import sys if sys.version_info[0] < 3 : self.tdb.Insert(tabname, {colname: pickle.dumps(3.14159, 1)}) else : self.tdb.Insert(tabname, {colname: pickle.dumps(3.14159, 1).decode("iso8859-1")}) # 8 bits if verbose: self.tdb._db_print() values = self.tdb.Select( tabname, [colname], conditions={colname: None}) import sys if sys.version_info[0] < 3 : colval = pickle.loads(values[0][colname]) else : colval = pickle.loads(bytes(values[0][colname], "iso8859-1")) self.assert_(colval > 3.141) self.assert_(colval < 3.142)
def http_method(method_name): if request.method == 'GET': return render_template('xmlrpc-methods.html', methods = current_app.wl_server_methods) if method_name not in current_app.wl_server_methods: return "Method name not supported", 404 server_instance = current_app.wl_server_instance raw_data = request.get_data() args = pickle.loads(raw_data) try: if method_name == 'test_me': result = args[0] else: method = getattr(server_instance, 'do_%s' % method_name) result = method(*args) except: exc_type, exc_instance, _ = sys.exc_info() remote_exc_type = _get_type_name(exc_type) log.error(__name__, 'Error on %s' % method_name) log.error_exc(__name__) return pickle.dumps({ 'is_error' : True, 'error_type' : remote_exc_type, 'error_args' : exc_instance.args, }) else: return pickle.dumps({ 'result' : result })
def setAttrs(self, attrs, replace=False): if replace: self.attributes = dumps(attrs) return a = loads(self.attributes) a.update(attrs) self.attributes = dumps(a)
def _test_marshalling(self): seq_code = "434TTT" shot_code = "8324" class_name = "TestCommand" marshaller = Marshaller() marshaller.set_class( class_name ) marshaller.set_option("asset_code", "chr102") marshaller.add_arg(seq_code) marshaller.add_arg(shot_code) pickle1 = pickle.dumps(marshaller) marshalled = marshaller.get_marshalled() # create a new marshaller marshaller2 = Marshaller() marshalled2 = marshaller2.get_from_marshalled(marshalled) pickle2 = pickle.dumps(marshaller) self.assertEquals(pickle1, pickle2)
def test_unpicklable(self): self.make_reactors() import pickle with self.assertRaises(NotImplementedError): pickle.dumps(self.r1) with self.assertRaises(NotImplementedError): pickle.dumps(self.net)
def _get_pickling_errors(cls, obj, seen=None): """ Return any errors that would be raised if `obj' is pickled Courtesy of koffie @ http://stackoverflow.com/a/7218986/109897 """ if seen == None: seen = [] try: state = obj.__getstate__() except AttributeError: return if state == None: return if isinstance(state, tuple): if not isinstance(state[0], dict): state = state[1] else: state = state[0].update(state[1]) result = {} for i in state: try: pickle.dumps(state[i], protocol=2) except pickle.PicklingError: if not state[i] in seen: seen.append(state[i]) result[i] = cls._get_pickling_errors(state[i], seen) return result
def initiate(main_port, tracker_addr, uname, pwd): sock = create_sock(main_port) sock.connect((tracker_addr["ip"], tracker_addr["port"])) sock.send(pickle.dumps({"cat" : "get", "item" : "peerlist"})) r = sock.recv(1024) peerlist = pickle.loads(r) sock.close() print(peerlist) modpath = input("Enter path of module : ") if os.path.isfile(modpath) == True: command = "python3 " + modpath + " initiate" threading.Thread(target = caller, args = (command, ), daemon = True).start() else: initiate(main_port, tracker_addr, uname, pwd) time.sleep(3) sock = create_sock(main_port) sock.connect(("127.0.0.1", 54321)) sock.send(pickle.dumps("data")) data = pickle.loads(transfer.receiver(sock)) sock.close() print("Data received : ") print(data) results = initiate2(main_port, tracker_addr, uname, pwd, modpath, data) sock = create_sock() sock.connect(("127.0.0.1", 54321)) transfer.sender(sock, pickle.dumps(results)) sock.close()
def sendTask(task): connection = pika.BlockingConnection(pika.ConnectionParameters( host=REMOTE_IP)) channel = connection.channel() channel.queue_declare(queue=TASK_DESTINATION, durable=True) wow = {'Name': 'Zara', 'Age': 7, 'Class': 'First'}; sendable = pickle.dumps(wow) myJob1 = Job(0, 5, [1,2,3,4,5]) myJob2 = Job(0, 5, [1,2,3,4,6]) message = pickle.dumps(myJob1) channel.basic_publish(exchange='', routing_key=TASK_DESTINATION, body=message, properties=pika.BasicProperties( delivery_mode = 2, # make message persistent )) message = pickle.dumps(myJob2) channel.basic_publish(exchange='', routing_key=TASK_DESTINATION, body=message, properties=pika.BasicProperties( delivery_mode = 2, # make message persistent )) print(" [TASK] Sent %r" % message) connection.close()
def reader(): rd = socket(AF_INET, SOCK_DGRAM) rd.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) rd.setsockopt(SOL_SOCKET, SO_BROADCAST, 1) rd.bind(('', PORT)) while True: msg, (addr, _) = rd.recvfrom(MAX) data = pickle.loads(msg) if data[:6] == '^rqst&': sd.sendto(pickle.dumps('^rply&:'+name), (addr, PORT)) if addr not in users: users[addr] = data[7:] print '[{0}] {1} joined to chat room'.format(addr, users[addr]) elif data[:6] == '^rply&': if addr not in users: users[addr] = data[7:] elif data[:6] == '^mssg&': if addr not in users: sd.sendto(pickle.dumps('^rqst&:'+name), (my_bcast, PORT)) print '[{0}] Anonim: '.format(addr) + data[7:] else: print '[{0}] {1}: '.format(addr, users[addr]) + data[7:] elif data[:6] == '^exit&': if addr == my_ip: rd.close() return print "[{0}] {1} exit from the chat room".format(addr, users[addr]) del users[addr]
def test_deprecated_class(): orig_A = TA.__bases__[0] # The only thing that should be different about the new class # is __doc__, __init__, __bases__ and __subclasshook__. # and __init_subclass__ for Python 3.6+. for x in dir(orig_A): if x not in ('__doc__', '__init__', '__bases__', '__dict__', '__subclasshook__', '__init_subclass__'): assert getattr(TA, x) == getattr(orig_A, x) with catch_warnings(AstropyDeprecationWarning) as w: TA() assert len(w) == 1 if TA.__doc__ is not None: assert 'function' not in TA.__doc__ assert 'deprecated' in TA.__doc__ assert 'function' not in TA.__init__.__doc__ assert 'deprecated' in TA.__init__.__doc__ # Make sure the object is picklable pickle.dumps(TA) with catch_warnings(NewDeprecationWarning) as w: TC() assert len(w) == 1 assert w[0].category == NewDeprecationWarning
def submitFinalStage(self, stage, jobId): stage.pendingTasks = [] partitionsToCompute = [] if stage.isShuffle: partitionsToCompute = [i for i in range(stage.numPartitions) if not stage.outputLocs[i]] else: job = stage.resultOfJob partitionsToCompute = [i for i in range(job.numPartitions) if not job.finished[i]] print "Target partitions: " + str(partitionsToCompute) print "Start running stage" + str(stage.id) + " (" + str(stage.rdd.id) + ")" self.runningStages.add(stage) #print self.runningStages taskBinary = '' if stage.isShuffle: taskBinary = pickle.dumps([stage.rdd, stage.shuffleDep]) else: taskBinary = pickle.dumps([stage.rdd, stage.resultOfJob.func]) tasks = [] if stage.isShuffle: for id in partitionsToCompute: if not stage.outputLocs[id]: tasks.append(ShuffleTask(stage.id, taskBinary, id)) else: job = stage.resultOfJob for id in partitionsToCompute: if not job.finished[id]: p = job.partitions[id] #same thing tasks.append(ResultTask(stage.id, taskBinary, p, id)) if len(tasks) > 0: stage.pendingTasks.extend(tasks) print "Task set of stage " + str(stage.id) + " (" + str(stage.rdd.id) + ") has been submitted" self.taskScheduler.submitTasks(TaskSet(tasks, stage.id, stage.jobId))
def test_pickling(self): # Create a template response. The context is # known to be unpicklable (e.g., a function). response = SimpleTemplateResponse('first/test.html', { 'value': 123, 'fn': datetime.now, }) with self.assertRaises(ContentNotRenderedError): pickle.dumps(response) # But if we render the response, we can pickle it. response.render() pickled_response = pickle.dumps(response) unpickled_response = pickle.loads(pickled_response) self.assertEqual(unpickled_response.content, response.content) self.assertEqual(unpickled_response['content-type'], response['content-type']) self.assertEqual(unpickled_response.status_code, response.status_code) # ...and the unpickled response doesn't have the # template-related attributes, so it can't be re-rendered template_attrs = ('template_name', 'context_data', '_post_render_callbacks') for attr in template_attrs: self.assertFalse(hasattr(unpickled_response, attr)) # ...and requesting any of those attributes raises an exception for attr in template_attrs: with self.assertRaises(AttributeError): getattr(unpickled_response, attr)
def test_copy_pickle(self): d = Deque('abc') e = d.__copy__() self.assertEqual(type(d), type(e)) self.assertEqual(list(d), list(e)) e = Deque(d) self.assertEqual(type(d), type(e)) self.assertEqual(list(d), list(e)) s = pickle.dumps(d) e = pickle.loads(s) self.assertNotEqual(id(d), id(e)) self.assertEqual(type(d), type(e)) self.assertEqual(list(d), list(e)) d = Deque('abcde', maxlen=4) e = d.__copy__() self.assertEqual(type(d), type(e)) self.assertEqual(list(d), list(e)) e = Deque(d) self.assertEqual(type(d), type(e)) self.assertEqual(list(d), list(e)) s = pickle.dumps(d) e = pickle.loads(s) self.assertNotEqual(id(d), id(e)) self.assertEqual(type(d), type(e)) self.assertEqual(list(d), list(e))
def handle_GETNAME(self, name): # Handle duplicate name if userManager.has_user(name): d = {"packet_type": "message", "data": {"message_type": "error_newname_taken"}} self.transport.write(pickle.dumps(d)) return # Handle spaces in name if not name_validate(name): d = {"packet_type": "message", "data": {"message_type": "error_newname_invalid"}} self.transport.write(pickle.dumps(d)) return # Name is Valid, Add to Document self.user = User(name, self) userManager.add_user(self.user) self.state = "CHAT" d = { "packet_type": "message", "data": {"message_type": "connect_success", "name": name, "collaborators": userManager.all_users_to_json()}, } if userManager.is_multi(): d["data"]["buffer"] = self.factory.buff self.transport.write(pickle.dumps(d)) print 'User "' + self.user.name + '" Connected' # Alert other Collaborators of new user d = {"packet_type": "message", "data": {"message_type": "user_connected", "user": self.user.to_json()}} self.user.broadcast_packet(d)
def test_pickle_parameters(self): # test that we can pickle a Parameters object p = Parameters() p.add('a', 10, True, 0, 100) p.add('b', 10, True, 0, 100, 'a * sin(1)') p.update_constraints() p._asteval.symtable['abc'] = '2 * 3.142' pkl = pickle.dumps(p, -1) q = pickle.loads(pkl) q.update_constraints() assert_(p == q) assert_(not p is q) # now test if the asteval machinery survived assert_(q._asteval.symtable['abc'] == '2 * 3.142') # check that unpickling of Parameters is not affected by expr that # refer to Parameter that are added later on. In the following # example var_0.expr refers to var_1, which is a Parameter later # on in the Parameters OrderedDict. p = Parameters() p.add('var_0', value=1) p.add('var_1', value=2) p['var_0'].expr = 'var_1' pkl = pickle.dumps(p) q = pickle.loads(pkl)
def test_pickle(self): vlist = _create_variable_list(3) ctuple = matrix_constraint( numpy.array([[0,0,0],[0,0,0]]), x=vlist) self.assertTrue((ctuple.lb == -numpy.inf).all()) self.assertTrue((ctuple.ub == numpy.inf).all()) self.assertTrue((ctuple.equality == False).all()) self.assertEqual(ctuple.parent, None) ctuple_up = pickle.loads( pickle.dumps(ctuple)) self.assertTrue((ctuple_up.lb == -numpy.inf).all()) self.assertTrue((ctuple_up.ub == numpy.inf).all()) self.assertTrue((ctuple_up.equality == False).all()) self.assertEqual(ctuple_up.parent, None) b = block() b.ctuple = ctuple self.assertIs(ctuple.parent, b) bup = pickle.loads( pickle.dumps(b)) ctuple_up = bup.ctuple self.assertTrue((ctuple_up.lb == -numpy.inf).all()) self.assertTrue((ctuple_up.ub == numpy.inf).all()) self.assertTrue((ctuple_up.equality == False).all()) self.assertIs(ctuple_up.parent, bup)
def func(*args, **kwargs): args_list = map(lambda k: (k, kwargs[k]), kwargs.keys()) args_key = str(pickle.dumps(sorted(list(args_list)))) if debug: print(">> CACHE:ARGS:", [kwargs]) cache_key = "{namespace}:{hash}".format( namespace=namespace or function.__name__, hash=hashlib.md5(args_key.encode("utf-8")).hexdigest() ) try: cache_result = cache.get(cache_key) except Exception: cache_result = None # If the result was None, we need to evaluate the function if not cache_result: if debug: print(">> CACHE:MISS: ", cache_key) func_result = function(*args, **kwargs) # Save the result to the cache cache.setex(cache_key, expiry, pickle.dumps(func_result)) else: if debug: print(">> CACHE:HIT: ", cache_key) func_result = pickle.loads(cache_result) return func_result
def testPickle(self): testvec = Vec2d(5, .3) testvec_str = pickle.dumps(testvec) loaded_vec = pickle.loads(testvec_str) self.assertEquals(testvec, loaded_vec)
def test_pickling_exceptions(): exc = System.Exception("test") dumped = pickle.dumps(exc) loaded = pickle.loads(dumped) assert exc.args == loaded.args
def extract_features_from_model(base_model, model_name, feature_shape): label_encoder = None # loop over the data splits for split in (config.TRAIN, config.TEST, config.VAL): # grab all image paths in the current split p = os.path.sep.join([config.MODEL_DATASET_PATH, split]) imagePaths = list(paths.list_images(p)) # randomly shuffle the image paths and then extract the class labels # from the file paths. It is more efficient to shuffle the classes # now, instead of during the training; random.shuffle(imagePaths) # get the labels in the same order as the random image paths # path/dataset/training/nonfood/0_123.jpb # index of -2 references 'nonfood' labels = [imagePath.split(os.path.sep)[-2] for imagePath in imagePaths] # if the label encoder is None, create it if label_encoder is None: label_encoder = LabelEncoder() label_encoder.fit(labels) # open the output CSV file for writing Path(config.BASE_CSV_PATH.format(model_name)).mkdir(parents=True, exist_ok=True) csvPath = os.path.sep.join( [config.BASE_CSV_PATH.format(model_name), f"{split}.csv"]) csv = open(csvPath, "w") # loop over the images in batches that match the batchsize # for the model # will feed the image through the model in batches # to get the resulting vector for (b, i) in enumerate(range(0, len(imagePaths), config.BATCH_SIZE)): # extract the batch of images and labels, then initialize the # list of actual images that will be passed through the network # for feature extraction logger.info( f"Processing batch {b+1}/{int(np.ceil(len(imagePaths)/float(config.BATCH_SIZE)))}" ) batchPaths = imagePaths[i:i + config.BATCH_SIZE] batchLabels = label_encoder.transform(labels[i:i + config.BATCH_SIZE]) batchImages = [] for imagePath in batchPaths: # load the input image using the keras helpt utility # while ensuring the image is resized to 224x224 pixels image = load_img(imagePath, target_size=(224, 224)) image = img_to_array(image) # preprocess the image by: # 1 - expanding the dimensions because the model expects and array of array of image values # and what image currently is, is a single array image = np.expand_dims(image, axis=0) # 2 - subracting the mean RGB pixel intensity from the ImageNet dataset image = imagenet_utils.preprocess_input(image) # add the image to the batch collection batchImages.append(image) # at this point we are ready to pass the image through the model network to extract the # features. which in this case is an array/vector of size: 7*7*512 # pass the images through the network nad use the outputs as # our actual features, then reshape the features into a flattened volume batchImages = np.vstack(batchImages) # recall our base_model has the front FCN layer REMOVED so we are getting the output # of the convolutional network. features = base_model.predict(batchImages, batch_size=config.BATCH_SIZE) # reshape features into an array of array features = features.reshape((features.shape[0], feature_shape)) # loop over the class labels and extracted features for (label, vec) in zip(batchLabels, features): # construct a row that exists of the class label and extracted features vec = ",".join([str(v) for v in vec]) csv.write(f"{label},{vec}\n") # close file csv.close() Path(config.LE_PATH.format(model_name)).mkdir(parents=True, exist_ok=True) f = open(config.LE_FILE.format(model_name), "wb") f.write(pickle.dumps(label_encoder)) f.close()
def pack(value): return pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
def encryptObj(key, obj, path): byteObj = pickle.dumps(obj) encryptedObj = encrypt(key, byteObj, encode=False) with open(path, "wb") as file: pickle.dump(encryptedObj, file)
def test_pickle(self): pj = PointJacobi(curve=CurveFp(23, 1, 1, 1), x=2, y=3, z=1, order=1) self.assertEqual(pickle.loads(pickle.dumps(pj)), pj)
def receiver(p, neighbours, capacity): election_ended = 0 global turn, chosen_node, num_acks, parent, elid, is_sender while True: #print('MESSAGE QUEUE:') #print(rd.queue[0]) print ('\nwaiting to receive message') data = sock.recv(1024) if turn != 1: turn = 1 message = pickle.loads(data) message_type = message['TYPE'] message_pid = message['PID'] message_elid = message['ELID'] message_sender = ('127.0.0.1', 10000 + int(message_pid)) print('MESSAGE TYPE: %s' % message_type) if message_type == 'election': print ('RECEIVED ELECTION FROM %s' % message_pid) print ('CURRENT ELID: %s MESSAGE ELID: %s' % (elid, message_elid)) if int(message_elid) > int(elid): print('ASSIGNING MY PARENT TO %s' % message_pid) message = { 'TYPE': 'election', 'PID': p, 'ELID': message_elid } parent = message_pid elid = message_elid for i in filter(lambda x: int(x) != parent, neighbours): # error print ('SENDING TO %s, MY PARENT IS %s' % (i, parent)) sent = sock.sendto(pickle.dumps(message), ('127.0.0.1', 10000 + int(i))) time.sleep(3) num_acks = 0 is_sender = 0 elif message_pid != parent and is_sender != 1: print('I HAVE A PARENT') message = { 'TYPE': 'ack', 'PID': p, 'ELID': elid, 'NODE': p, 'CAPACITY': capacity } sent = sock.sendto(pickle.dumps(message), message_sender) elif message_type == 'info' and election_ended != 1: # Election ended message_node = message['NODE'] message_capacity = message['CAPACITY'] message = { 'TYPE': 'info', 'PID': p, 'ELID': elid, 'NODE': message_node, 'CAPACITY': message_capacity} print ('THE NODE %s WITH CAPACITY %s HAS BEEN ELECTED' % (message_node, message_capacity)) for i in filter(lambda x: int(x) != int(message_pid), neighbours): print ('SENDING TO %s, MY INFORMANT IS %s' % (i, message_pid)) sent = sock.sendto(pickle.dumps(message), ('127.0.0.1', 10000 + int(i))) chosen_node['PID'] = message_node chosen_node['CAPACITY'] = message_capacity election_ended = 1 elif message_type == 'ack' and int(message_elid) == int(elid): print ('RECEIVED ACK FROM %s' % message_pid) num_acks = num_acks + 1 message_node = message['NODE'] message_capacity = message['CAPACITY'] if int(message_capacity) > int(chosen_node['CAPACITY']): chosen_node['PID'] = message_node chosen_node['CAPACITY'] = message_capacity if num_acks == len(neighbours) - 1 and election_ended != 1 and is_sender != 1: print ('RECEIVED ALL ACKS FROM NEIGHBOURS') print ('SENDING TO PARENT NODE %s' % parent) message = { 'TYPE': 'ack', 'PID': p, 'ELID': elid, 'NODE': chosen_node['PID'], 'CAPACITY': chosen_node['CAPACITY']} sent = sock.sendto(pickle.dumps(message), ('127.0.0.1', 10000 + int(parent))) time.sleep(random.random() * 5)
def _encode_items(self, a , b): return sqlite.Binary(pickle.dumps(a, -1)), sqlite.Binary(pickle.dumps(b, -1))
def cluster_distributions(output_file, tiff_cluster, tiff_built, tiff_night_light, tiff_population, cities_csv): #LOADING OF THE CLUSTERS MAP tiff_file = gdal.Open(tiff_cluster) arr_img = tiff_file.ReadAsArray() arr_img = arr_img.astype(np.int) tiff_file = None #close it band = None #close it #DEFINITION OF THE DIFFERENT COORDINATES OF CLUSTERS arr_img = np.where(arr_img == -1, 0, arr_img) arr_img = np.where(arr_img == -999, 1, arr_img) clusters_list = cm.get_all_urban_cluster(arr_img) #LOADING OF THE DIFFERENT OUTPUTS tiff_file = gdal.Open(tiff_built) arr_built = tiff_file.ReadAsArray() arr_built = np.where(arr_built == -999, 0, arr_built) tiff_file = gdal.Open(tiff_night_light) arr_night_light = tiff_file.ReadAsArray() arr_night_light = np.where(arr_night_light == -999, 0, arr_night_light) tiff_file = gdal.Open(tiff_population) arr_population = tiff_file.ReadAsArray() tiff_file = None arr_population = np.where(arr_population == -999, 0, arr_population) #CREATION OF DISTRIBUTIONS FOR EACH CLUSTERS clusters = [] for c in clusters_list: if len(c) > 1: cluster = {} cluster["coordinates"] = c built_density = [] night_light_density = [] population_density = [] for x, y in c: built_density.append(arr_built[x, y]) night_light_density.append(arr_night_light[x, y]) population_density.append(arr_population[x, y]) cluster["built density"] = built_density cluster["night light density"] = night_light_density cluster["population density"] = population_density clusters.append(cluster) #NAMING OF CLUSTERS #Load a geo matrix from a tiff tiff_file = gdal.Open(tiff_cluster) geo_matrix = tiff_file.GetGeoTransform() #open the csv that stores the different cities and their coordinates #Notice that the csv should be decreasinly ordered df = pd.read_csv(cities_csv, header=0) for c in clusters: start = time.time() for _, row in df.iterrows(): city_coordinates = set() city_coordinates.add( world_to_pixel(geo_matrix, row["long"], row["lat"])) if len(c["coordinates"].intersection(city_coordinates)) > 0: clusters[clusters.index(c)]["name"] = row["com_nom"] clusters[clusters.index(c)]["country"] = row["country"] print("OK") break #some clusters do not match with city if "name" not in c: clusters[clusters.index(c)]["name"] = "NO WHERE" clusters[clusters.index(c)]["country"] = "NO WHERE" print("NO WHERE") end = time.time() print(end - start) # output_file should have extension .pkl with open(output_file, "wb") as f: f.write(pickle.dumps(clusters)) return clusters
def save(self): "保存" with open('db.pkl', 'wb') as f: f.write(pickle.dumps(self.memo_list)) print("文件保存成功")
def test_record_not_pickleable(self): r = Record(R_A, (42, )) with self.assertRaises(Exception): pickle.dumps(r)
def trainKeras(En, A, Cl, A_test, Cl_test, Root): import tensorflow as tf # Use this to restrict GPU memory allocation in TF opts = tf.GPUOptions(per_process_gpu_memory_fraction=sysDef.fractionGPUmemory) conf = tf.ConfigProto(gpu_options=opts) #conf.gpu_options.allow_growth = True if kerasDef.useTFKeras: import tensorflow.keras as keras #tf.keras tf.Session(config=conf) else: import keras # pure keras from keras.backend.tensorflow_backend import set_session set_session(tf.Session(config=conf)) from sklearn import preprocessing from tensorflow.contrib.learn.python.learn import monitors as monitor_lib tb_directory = "keras_" + str(len(kerasDef.hidden_layers))+"HL_"+str(kerasDef.hidden_layers[0]) model_directory = "." if kerasDef.regressor: model_name = model_directory+"/keras_regressor_"+str(len(kerasDef.hidden_layers))+"HL_"+str(kerasDef.hidden_layers[0])+".hd5" else: model_name = model_directory+"/keras_"+str(len(kerasDef.hidden_layers))+"HL_"+str(kerasDef.hidden_layers[0])+".hd5" model_le = model_directory+"/keras_model_le.pkl" if kerasDef.alwaysRetrain == False: print(" Training model saved in: ", model_name, "\n") else: kerasDef.alwaysImprove = False print(" Training model not saved\n") #********************************************** ''' Initialize Estimator and training data ''' #********************************************** print(' Preprocessing data and classes for Keras\n') totA = np.vstack((A, A_test)) totCl = np.append(Cl, Cl_test) if kerasDef.regressor: Cl2 = np.copy(Cl) Cl2_test = np.copy(Cl_test) le = None else: numTotClasses = np.unique(totCl).size le = preprocessing.LabelEncoder() totCl2 = le.fit_transform(totCl) Cl2 = le.transform(Cl) Cl2_test = le.transform(Cl_test) totCl2 = keras.utils.to_categorical(totCl2, num_classes=np.unique(totCl).size) Cl2 = keras.utils.to_categorical(Cl2, num_classes=np.unique(totCl).size+1) Cl2_test = keras.utils.to_categorical(Cl2_test, num_classes=np.unique(totCl).size+1) print(" Label Encoder saved in:", model_le) with open(model_le, 'ab') as f: f.write(pickle.dumps(le)) if kerasDef.fullBatch == True: batch_size = A.shape[0] else: batch_size = kerasDef.batchSize printParamKeras(A) if kerasDef.alwaysImprove == True or os.path.exists(model_name) is False: model = keras.models.Sequential() for numLayers in kerasDef.hidden_layers: model.add(keras.layers.Dense(numLayers, activation = kerasDef.activation_function, input_dim=A.shape[1], kernel_regularizer=keras.regularizers.l2(kerasDef.l2_reg_strength))) model.add(keras.layers.Dropout(kerasDef.dropout_perc)) if kerasDef.regressor: model.add(keras.layers.Dense(1)) model.compile(loss='mse', optimizer=kerasDef.optimizer, metrics=['mae']) else: model.add(keras.layers.Dense(np.unique(totCl).size+1, activation = 'softmax')) model.compile(loss='categorical_crossentropy', optimizer=kerasDef.optimizer, metrics=['accuracy']) tbLog = keras.callbacks.TensorBoard(log_dir=tb_directory, histogram_freq=kerasDef.tbHistogramFreq, batch_size=batch_size, write_graph=True, write_grads=True, write_images=True) #tbLog.set_model(model) tbLogs = [tbLog] log = model.fit(A, Cl2, epochs=kerasDef.trainingSteps, batch_size=batch_size, callbacks = tbLogs, verbose = 2, validation_data=(A_test, Cl2_test)) loss = np.asarray(log.history['loss']) val_loss = np.asarray(log.history['val_loss']) if kerasDef.regressor: accuracy = None val_acc = None else: accuracy = np.asarray(log.history['acc']) val_acc = np.asarray(log.history['val_acc']) model.save(model_name) if kerasDef.plotModel == True: from keras.utils import plot_model keras.utils.plot_model(model, to_file=model_directory+'/keras_MLP_model.png', show_shapes=True) import matplotlib.pyplot as plt plt.figure(tight_layout=True) plotInd = int(len(kerasDef.hidden_layers))*100+11 visibleX = True for layer in model.layers: try: w_layer = layer.get_weights()[0] ax = plt.subplot(plotInd) newX = np.arange(En[0], En[-1], (En[-1]-En[0])/w_layer.shape[0]) plt.plot(En, np.interp(En, newX, w_layer[:,0]), label=layer.get_config()['name']) plt.legend(loc='upper right') plt.setp(ax.get_xticklabels(), visible=visibleX) visibleX = False plotInd +=1 except: pass plt.xlabel('Raman shift [1/cm]') plt.legend(loc='upper right') plt.savefig('keras_weights_MLP' + '.png', dpi = 160, format = 'png') # Save plot printModelKeras(model) print("\n Number of spectra = ",A.shape[0]) print(" Number of points in each spectra = ", A.shape[1]) if kerasDef.regressor == False: print(" Number unique classes (training): ", np.unique(Cl).size) print(" Number unique classes (validation):", np.unique(Cl_test).size) print(" Number unique classes (total): ", np.unique(totCl).size) printParamKeras(A) printTrainSummary(accuracy, loss, val_acc, val_loss) else: print(" Retreaving training model from: ", model_name,"\n") model = keras.models.load_model(model_name) printModelKeras(model) printParamKeras(A) score = model.evaluate(A_test, Cl2_test, batch_size=batch_size) printEvalSummary(model_name, score) return model, le
def _encode_item(self, item): return sqlite.Binary(pickle.dumps(item, -1))