def create_plugin(self, ctx, plugin_obj): new_plugin = serializer.deserialize_entity(ctx, plugin_obj) LOG.debug('Creating plugin %s', new_plugin.name) new_plugin.code = cpickle.dumps(new_plugin.code, 0).decode() new_plugin.create() return serializer.serialize_entity(ctx, new_plugin)
def _serialize_dict(cls, data): ''' serializes a dictionary :param data: data to serialize ''' return b64encode(zlib.compress(cPickle.dumps(data, protocol=2))).decode()
def train(ai, config): loaded_files = [] x = config.iterations i = len(glob.glob(config.data.model_location+"*.h5")) loaded_files, _ = load_games(ai, loaded_files, config) while(x != 0): if i > config.iter3: ai.update_lr(config.learning_rate3) elif i > config.iter2: ai.update_lr(config.learning_rate2) else: ai.update_lr(config.learning_rate1) loaded_files, diff = load_games(ai, loaded_files, config) total_diff = diff start = time() print("Iteration %04d"%i) end = config.min_new_game_files if i> 0 else config.min_game_file util.print_progress_bar(0, end, start=start) while(total_diff < end): if diff > 0: total_diff += diff util.print_progress_bar(total_diff, end, start=start) sleep(5) loaded_files, diff = load_games(ai, loaded_files, config) util.print_progress_bar(end, end, start=start) print("Training for %d batches on %d samples" % (config.batches_per_iter, len(ai.buffer.buffer))) start = time() history = ai.train_batches(config.batch_size, config.batches_per_iter, config.verbose) for val in history.history.keys(): print("%s: %0.4f" % (val, history.history[val][-1])) if i % config.save_model_cycles == 0: ai.save("%smodel_%04d.h5" % (config.data.model_location, i)) file = open("%shist_%04d.pickle" % (config.data.history_location, i), 'wb') pickle.dump(pickle.dumps(history.history), file) file.close() print("Iteration Time: %0.2f" % (time()-start)) x -= 1 i += 1
def populate(cls): if cls.fonts is not None: return start = time.time() from cube.constants.application import config_directory fonts_dir = os.path.join(config_directory(), "fonts") if not os.path.exists(fonts_dir): os.makedirs(fonts_dir) fonts_file = os.path.join(fonts_dir, "fonts.lst") if os.path.exists(fonts_file): cube.debug("Loading fonts from cache file '%s'" % fonts_file) try: with open(fonts_file, 'rb') as f: cls.fonts = pickle.loads(f.read()) except Exception as e: cube.warn("font cache file '%s' is not valid, it will be removed:" % fonts_file, e) os.unlink(fonts_file) else: cube.info("Finding fonts on your system, this may take a while...") cls.fonts = {} for font_dir in cls.font_directories(): for root, dirs, files in os.walk(font_dir): for f in files: path = os.path.join(root, f) if font.is_valid(path): try: cls.fonts[path] = font.get_infos(path) except: cube.error("ignoring font file", path) if not cls.fonts: raise Exception("Couldn't find any font on your system !") cube.info(len(cls.fonts), "font infos fetched in %f seconds" % (time.time() - start)) cube.debug("Saving fonts into cache file '%s'" % fonts_file) with open(fonts_file, 'wb') as f: f.write(pickle.dumps(cls.fonts))
for i in range(len(grads)): grads[i] += per_grads[i] for i in range(len(grads)): grads[i] /= BATCH_SIZE loss /= BATCH_SIZE train_loss += loss for i in range(len(Clip_bound)): Clip_bound[i] /= BATCH_SIZE grads_noise = Add_noise(grads, Clip_bound) client.publish("mapa_grads/" + CLIENT_ID, cPickle.dumps(grads_noise), 2) if step % TEST_NUM == 0: print(step) total = 0 correct = 0 correct_pad = 0 test_loss = 0 man_file1 = open( RESULT_ROOT + '[' + str(EDGE_NAME) + ']' + '[Mapa-Accuracy]', 'w') for batch_idx, (test_x, test_y) in enumerate(test_loader): if batch_idx < test_idx: output = model(test_x) pred_y = torch.max(output, 1)[1].data.numpy()
def serialize(self, generic_object): byte_str = pickle.dumps(generic_object) return byte_str.hex()
def __conform__(self, protocol): if protocol is sqlite3.PrepareProtocol: # return cpickle.dumps(self.data, 1) return sqlite3.Binary(cpickle.dumps(self.data, -1))
def pickleIT(list): with open("dictionaries_list.pkl", "wb") as f: f.write(cPickle.dumps(list))
def __call__(self, *args, **kwds): str = cPickle.dumps(args, 1) + cPickle.dumps(kwds, 1) if not str in self.memo: self.memo[str] = self.fn(*args, **kwds) return self.memo[str]
#client.on_connect = on_connect #client.on_disconnect=on_disconnect #client.on_log = on_log #client.on_message = on_message client.on_publish = on_publish face_cascade = cv.CascadeClassifier( '/usr/share/OpenCV/haarcascades/haarcascade_frontalface_default.xml') # 1 should correspond to /dev/video1 , your USB camera. The 0 is reserved for the TX2 onboard camera cap = cv.VideoCapture(1) while (True): # Capture frame-by-frame ret, frame = cap.read() # We don't use the color information, so might as well save space gray = cv.cvtColor(frame, cv.COLOR_BGR2GRAY) # face detection and other logic goes here faces = face_cascade.detectMultiScale(gray, 1.3, 5) for (x, y, w, h) in faces: crop_img = gray[y:y + h, x:x + w] client.connect(broker) client.loop_start() #client.subscribe("pictures/faces",1) client.publish("pictures/faces", pk.dumps(crop_img)) time.sleep(5) client.loop_stop() client.disconnect() time.sleep(5)
def compress_object(data, level=9): if data is None: return 'null' return base64.b64encode(zlib.compress(cPickle.dumps(data), level)).decode('utf-8')
def save(self, filename): file = open(filename, 'wb') pickle.dump(pickle.dumps(self.buffer), file) file.close()
def get(self, exp_id): """ Simulate your experiment on a simple model The model that is drawn from is: y = -(x - c)**2 + c2 + rnorm(mu,var) Currently there is no context. Make sure that the action of your experiment results in: {"x" : x} This is how the model currently expects your action to be formulated. This might become more flexible later on. +--------------------------------------------------------------------+ | Example | +====================================================================+ |http://example.com/eval/5/simulate?key=XXX&N=10&c=5&c2=10&mu=0&var=1| +--------------------------------------------------------------------+ :param int exp_id: Experiment ID as specified in the url :param string key: The key corresponding to the experiment :param int N: The number of simulation draws :param int c: The size of the parabola :param int c2: The height of the parabola :param int mu: The mean of the noise on the model :param int var: The variance of the noise on the model :param string log_stats: Flag for logging the results in the database :returns: A JSON of the form: {"simulate":"success"} :raises AuthError: 401 Invalid Key """ key = self.get_argument("key", default = False) # Number of draws N = int(self.get_argument("N", default = 1000)) log_stats = self.get_argument("log_stats", default = True) # Parameterset for the simulator c = float(self.get_argument("c", default = 5)) c2 = float(self.get_argument("c2", default = 10)) mu = float(self.get_argument("mu", default = 0)) var = float(self.get_argument("var", default = .1)) if not key: self.set_status(401) self.write("Key not given") return __EXP__ = Experiment(exp_id, key) rewards = np.array([0]) reward_over_time = np.array([]) regret = np.array([0]) if __EXP__.is_valid(): for i in range(N): # Generate context context = {} # Get action action = __EXP__.run_action_code(context) # Generate reward y = -(action["x"] - c)**2 + c2 + np.random.normal(mu, var) #y = 15 + 8*action["x"] + 10*action["x"]**2 + np.random.normal(mu, var) reward = {"y" : y} # Set reward __EXP__.run_reward_code(context, action, reward) # Save stats rewards = np.append(rewards, y) tmp_rot = (rewards[-1] + y) / (i+1) reward_over_time = np.append(reward_over_time, tmp_rot) regret = np.append(regret, (regret[-1] + (c2 - y))) #self.write("n = {}, Regret is: {}, reward = {} <br>".format(i,regret[-1], rewards[-1])) # Now save the data together with a timestamp in the logs # To read out the Numpy array data out again, use array = # pickle.loads(record['feature']) # FOR FUTURE, the json_tricks package might be interesting if log_stats == True: print("Logging data") __EXP__.log_data({ "type" : "evaluation", "time" : int(time.time()), "experiment" : exp_id, "N" : N, "c" : c, "c2" : c2, "rewards" : Binary(_pickle.dumps(rewards, protocol = 2), subtype = 128), "reward_over_time" : Binary(_pickle.dumps(reward_over_time, protocol = 2), subtype = 128), "regret" : Binary(_pickle.dumps(regret, protocol = 2), subtype = 128) }) self.write(json.dumps({'simulate':'success','experiment':exp_id})) else: self.set_status(401) self.write("Key is not valid for this experiment") return
% COUNTER) #print (tabulate(filter(lambda k,v : np.asarray(v).size==1, stats.items()))) #pylint: disable=W0110 print( tabulate( filter(lambda x: np.asarray(x[1]).size == 1, stats.items()))) #pylint: disable=W0110 # Store to hdf5 if args.use_hdf: for (stat, val) in stats.items(): if np.asarray(val).ndim == 0: diagnostics[stat].append(val) else: assert val.ndim == 1 diagnostics[stat].extend(val) if args.snapshot_every and ((COUNTER % args.snapshot_every == 0) or (COUNTER == args.n_iter)): hdf['/agent_snapshots/%0.4i' % COUNTER] = np.array( cPickle.dumps(agent, -1)) # Plot if args.plot: animate_rollout(env, agent, min(500, args.timestep_limit)) run_policy_gradient_algorithm(env, agent, callback=callback, usercfg=cfg) if args.use_hdf: hdf['env_id'] = env_spec.id try: hdf['env'] = np.array(cPickle.dumps(env, -1)) except Exception: print("failed to pickle env") #pylint: disable=W0703 env.close()
def dumps(data): if _USE_COMPRESS: return lz4.frame.compress(cPickle.dumps(data)) else: return cPickle.dumps(data)
def dumps(obj): return pkl.dumps(obj)
def _writeAttributesToFile(self, attribute_ids): with open('attributes_dict.txt', 'wb') as file: file.write(pickle.dumps(attribute_ids))
def quick_deepcopy(dictionary): return cPickle.loads(cPickle.dumps(dictionary, -1))
per_grads.append(params.grad.data) per_grads = Clip(per_grads, Clip_bound) for i in range(len(grads)): grads[i] += per_grads[i] for i in range(len(grads)): grads[i] /= BATCH_SIZE loss /= BATCH_SIZE train_loss += loss grads_noise = Add_noise(grads, Clip_bound / BATCH_SIZE) client.publish("fixdp_grads/" + CLIENT_ID, cPickle.dumps(grads_noise), 2) if step % TEST_NUM == 0: print(step) total = 0 correct = 0 correct_pad = 0 test_loss = 0 man_file1 = open(RESULT_ROOT + '[' + str(EDGE_NAME) + ']' + '[FixedDP-Accuracy]', 'w') for batch_idx, (test_x, test_y) in enumerate(test_loader): if batch_idx < test_idx: output = model(test_x) pred_y = torch.max(output, 1)[1].data.numpy() test_y = test_y.view(-1) test_loss += nn.CrossEntropyLoss()(output, test_y).item()
def deepcopy(self, model): return cPickle.loads(cPickle.dumps(model, -1))
def __deepcopy__(self, memo={}): obj = self.__class__(doc=pickle.loads(pickle.dumps(self.copy())), gen_skel=self.gen_skel, mongokat_collection=self.mongokat_collection, fetched_fields=self._fetched_fields) obj.__dict__ = self.__dict__.copy() return obj
def deepcopy(obj): return cPickle.loads(cPickle.dumps(obj))
def server_thread(self, clientSocket): try: print('\nHandling client connection. . .') # get the request from browser data = b'' while True: part = clientSocket.recv(128) data += part if len(part) < 128: data += part break info = pickle.loads(data) # str_data = data.decode() # the connected client's IP addr h, p = clientSocket.getpeername() if info['type'] not in ['IMAGE', 'INTERNET_MSG']: print('\nMessage from client: ') print(info) else: print("Message from client: IMAGE FILE") req_type = info['type'] if req_type == 'TEST_CONNECT': return_data = {'THIS CONTENT DOES NOT MATTER': -1} clientSocket.sendall(pickle.dumps(return_data)) if req_type == 'KEEP_ALIVE': # update the time which we have last seen this client client_info = { 'IP': h, 'PORT_NO': info['port'], 'nickname': info['nickname'], 'local_ip': info['local_ip'], 'mode': info['mode'] } self.update_peer(client_info) if req_type == 'REQUEST_PEER_DICT': # respond with directory of all active clients self.send_list_of_all_peers_to_peer(clientSocket) if req_type == 'QUIT': for key in self.clients: print(key) # remove client from peers dict index = h + ':' + str(info['port']) print(self.clients) # try to delete the user's mailbox - should succeed if on INTERNET mode try: del self.mailboxes[index] print('Removed ' + index + '\'s mailbox due to QUIT command. . .') except: pass try: # this should succeed for INTERNET or LAN mode. . . del self.clients[index] print('Removed ' + index + ' due to QUIT command. . .') except Exception as e: # user must be hosting server in separate window - try deleting peer at index of their local IP local_ip_index = info['local_ip'] + ':' + str(info['port']) del self.clients[local_ip_index] print('Removed ' + local_ip_index + ' due to QUIT command. . .') finally: pass if req_type == "MSG_CHECK": # check if the peer has any messages waiting # assume that the peer is not on the same LAN as the server local_ip = info['local_ip'] ip = h port = info['port'] if ip == local_ip: # operating in internet mode on LAN for some reason - refer to peer using its local IP index = local_ip + ':' + str(port) else: # truly operating over the internet - refer to peer using IP not local IP index = ip + ':' + str(port) if self.mailboxes.get(index) is not None: return_data = {'data': []} for tup in self.mailboxes[index]: print("\nRETURNING IMAGE\n") return_data['data'].append((tup[0], tup[1])) self.mailboxes[index] = [ ] # messages will be sent to user, so remove them from central server data = pickle.dumps(return_data) clientSocket.sendall(data) else: return # take no further action if req_type == "INTERNET_MSG": # message is to be sent to clients over network ip = h local_ip = info['local_ip'] port = info['port'] sender = info['sender'] png = info['data'] if ip == local_ip: # operating in internet mode on LAN for some reason but continue index = local_ip + ':' + str(port) else: # truly operating over the internet - use IP not Local IP index = ip + ':' + str(port) for key in self.mailboxes: if key != index: # don't send to yourself print(key, index) self.mailboxes[key].append((sender, png)) except Exception as e: print("EXCEPTION IN SERVER THREAD. . .") print(e) finally: clientSocket.close()
def generator(): inputs = imgs_file_list targets = list(zip(objs_info_list, mask_list)) assert len(inputs) == len(targets) for _input, _target in zip(inputs, targets): yield _input.encode('utf-8'), cPickle.dumps(_target)
list_of_number_of_shared_molecules = [] #set of the ingredient from the "rows" set1 = row["set_molecules"] #name of the ingredient from the "rows" ingredient_1 = row["ingredient"] #starting a dictionary entry with a value of an empty dict flavor_matrix_df[ingredient_1] = {} #iterate through the "columns" of ingredients for index, row in flavorDB_pandas.iterrows(): #set of the ingredient from the "columns" of ingredients set2 = row["set_molecules"] #nome of the ingredient from the "columns" of ingredients ingredient_2 = row["ingredient"] #The molecules that are shared between the two sets shared_molecules = set1.intersection(set2) #access the dictionary of a dictionary from 1st ingredient #set the value as the number of shared molecules flavor_matrix_df[ingredient_1][ingredient_2] = len(shared_molecules) with open('./data/ingredients/flavor_matrix_dict.pickle', 'wb') as file: file.write(pickle.dumps(flavor_matrix_df)) file.close() #Celebratory print statement print("we did it!")
print(len(data1['images'])) print(len(data1['labels'])) data = split_data(data1, ratio=ratio) print('train', data['train']['size']) print('valid', data['valid']['size']) print('test', data['test']['size']) return data if __name__ == '__main__': data = get_data(shape=(540, 540, 1), ratio=(6, 1, 3)) # add_pickle dump = pickle.dumps(data) print('dump.pickle') GZIP = True if GZIP: with gzip.open('dump.gz', 'wb') as f: f.write(dump) print('gzip dump was written') else: with open('dump.pickle', 'wb') as f: pickle.dump(dump, f, protocol=4) print('dump was written')
spectrum=spectrum[spectrum[:,1] > 10] for i in scan_to_lines[scan_number]: mz_low = skyinfo['obs_mz'].values[i] - (10.0/skyinfo['charge'].values[i]) mz_high = skyinfo['obs_mz'].values[i] + (60.0/skyinfo['charge'].values[i]) try: output_scans[i].append(spectrum[(mz_low < spectrum[:,0]) & (spectrum[:,0] < mz_high)]) except: print (i, output_scans[i], mz_low, mz_high) print (spectrum) print (spectrum[(mz_low < spectrum[:,0]) & (spectrum[:,0] < mz_high)]) sys.exit(0) if len(output_scans[i]) == scans_per_line[i]: hdx_time = mzml[23:-5] name = skyinfo.iloc[i]['name'] #RT_start = skyinfo.iloc[i]['RT_'+str(sys.argv[1])] RT_lo = int(RT_matches[name][i][0]) RT_hi = int(RT_matches[name][i][1]) keep_drift_times = drift_times[(drift_times >= dt_lbounds[i]) & (drift_times <= dt_ubounds[i]) & (scan_times <= ret_ubounds[i]) & (scan_times >= ret_lbounds[i])] keep_scan_times = scan_times[(drift_times >= dt_lbounds[i]) & (drift_times <= dt_ubounds[i]) & (scan_times <= ret_ubounds[RT_hi]) & (scan_times >= ret_lbounds[RT_lo])] output = [sorted(set(keep_scan_times)), sorted(set(keep_drift_times)), output_scans[i]] with open(str(i)+"_"+hdx_time+"_"+name+".cpickle.zlib", 'wb') as file: file.write(zlib.compress(cpickle.dumps(output))) print (scan_number, process.memory_info().rss / (1024*1024*1024), 'presave') output_scans[i] = [] print (scan_number, process.memory_info().rss / (1024*1024*1024), 'savedisk') if len(scan_to_lines[scan_number]) > 0: cur_lengths = np.array([len(output_scans[i]) for i in scan_to_lines[scan_number]]) target_lengths = np.array([scans_per_line[i] for i in scan_to_lines[scan_number]])
province_map['香港'] = sheng elif sheng == '澳门特别行政区': province_map['澳门'] = sheng r = redis.Redis(host='192.168.0.148', port=6379, password='******', db=0) area_map = r.get("area_map") city_map = r.get("city_map") province_area_map = r.get("province_area_map") province_map = r.get("province_map") # defaultdict 反序列化和序列化 https://mlog.club/article/1456963 # 自定义defaultdict 序列化和反序列化 https://www.coder.work/article/356672 if not area_map or not city_map or not province_area_map or not province_map: area_map, city_map, province_area_map, province_map, latlng = _data_from_csv( ) r.set("area_map", cPickle.dumps(area_map)) r.set("city_map", cPickle.dumps(city_map)) r.set("province_area_map", cPickle.dumps(province_area_map)) r.set("province_map", cPickle.dumps(province_map)) else: area_map = cPickle.loads(area_map) city_map = cPickle.loads(city_map) province_area_map = cPickle.loads(province_area_map) province_map = cPickle.loads(province_map) # print(area_map) # print(city_map) # print(province_area_map) # print(province_map) # 直辖市
def generator(): """TF Dataset generator.""" assert len(train_img_paths_list) == len(train_targets_list) for _input, _target in zip(train_img_paths_list, train_targets_list): yield _input.encode('utf-8'), cPickle.dumps(_target)
from socket import * import _pickle as pickle serverName = "localhost" serverPort = 12001 selectHero = "" clientSocket = socket(AF_INET, SOCK_STREAM) clientSocket.connect((serverName, serverPort)) waiting = 0 while 1: # before hero selection while not selectHero: hero = input('choose your hero :') clientSocket.send( pickle.dumps({ 'user': selectHero, 'command': ['select-hero', hero] })) response = pickle.loads(clientSocket.recv(2048)) selectHero = response['user'] if not selectHero: print("{} is unavailable".format(hero)) else: print("You are selecting '{}'".format(hero)) data = response['data'] print(data) # waiting for server response if waiting: clientSocket.settimeout(5) try: response = pickle.loads(clientSocket.recv(2048)) if response['status'] == 'death':
def threaded_client(conn, _id): """ runs in a new thread for each player connected to the server :param con: ip address of connection :param _id: int :return: None """ global connections, players current_id = _id # recieve a name from the client data = conn.recv(16) name = data.decode("utf-8") print("[LOG]", name, "connected to the server.") # Setup properties for each new player color = colors[current_id % len(colors)] x, y = get_start_location(players) players[current_id] = { "x": x, "y": y, "color": color, "name": name, "direction": 0.0 } # direction in Radians # pickle data and send initial info to clients conn.send(str.encode(str(current_id))) print("send", current_id) # server will recieve basic commands from client # it will send back all of the other clients info ''' commands start with: move jump get id - returns id of client ''' while True: try: # Recieve data from client data = conn.recv(64) if not data: break data = data.decode("utf-8") #print("[DATA] Recieved", data, "from client id:", current_id) # look for specific commands from recieved data if data.split(" ")[0] == "move": split_data = data.split(" ") x = int(split_data[1]) y = int(split_data[2]) direction = float(split_data[3]) players[current_id]["x"] = x players[current_id]["y"] = y players[current_id]["direction"] = direction player_collision(players) send_data = pickle.dumps((players)) if data.split(" ")[0] == "shoot": print(data) split_data = data.split(" ") x = int(split_data[1]) y = int(split_data[2]) dir_x = float(split_data[3]) dir_y = float(split_data[4]) owner_id = int(split_data[5]) shots.append({ "x": x, "y": y, "dir_x": dir_x, "dir_y": dir_y, "owner_id": owner_id }) shots_collision(shots, players, owner_id) send_data = pickle.dumps((shots)) elif data.split(" ")[0] == "id": send_data = str.encode( str(current_id)) # if user requests id then send it elif data.split(" ")[0] == "jump": send_data = pickle.dumps((players)) else: # any other command just send back list of players send_data = pickle.dumps((players)) # send data back to clients conn.send(send_data) except Exception as e: print(e) break # if an exception has been reached disconnect client time.sleep(0.001) # When user disconnects print("[DISCONNECT] Name:", name, ", Client Id:", current_id, "disconnected") connections -= 1 try: del players[current_id] # remove client information from players list except: pass #MAch schöner, wenn man getroffen wird conn.close() # close connection
def process_cursor(cursor): with open('agg-d-std.json') as f: std = json.load(f) client = MongoClient("localhost", 27017) db = client.dwh with tqdm(total=cursor[0], desc="Progress{}".format(cursor[2]), position=cursor[2], leave=True) as pbar: for doc in db.agg_d.find( { "date": { "$gte": datetime.datetime(year=2012, month=1, day=1) }, "target": { "$lte": 250000 } }, no_cursor_timeout=True).skip(cursor[1]).limit(cursor[0]): time_filter = { "$and": [{ "date": { "$gte": doc["date"] - relativedelta(days=365) } }, { "date": { "$lt": doc["date"] } }] } if db.agg_d.find({"id": doc["id"], **time_filter}).count() < 50: pbar.update(1) continue record = { "x": np.empty((365, 5)), "y": (doc["target"] - std["mean"]) / std["std"] } for i in range(365): date = doc["date"] - relativedelta(days=365 - i) wday = int(date.strftime("%w")) + 1 record["x"][i][0] = -std["mean"] / std["std"] record["x"][i][1] = np.sin(2 * np.pi * wday / 7) record["x"][i][2] = np.cos(2 * np.pi * wday / 7) record["x"][i][3] = np.sin(2 * np.pi * date.month / 12) record["x"][i][4] = np.cos(2 * np.pi * date.month / 12) for day in db.agg_d.find({"id": doc["id"], **time_filter}): index = 365 - (doc["date"] - day["date"]).days record["x"][index][0] = (day["target"] - std["mean"]) / std["std"] wday = int(day["date"].strftime("%w")) + 1 record["x"][index][1] = np.sin(2 * np.pi * wday / 7) record["x"][index][2] = np.cos(2 * np.pi * wday / 7) record["x"][index][3] = np.sin(2 * np.pi * doc["month"] / 12) record["x"][index][4] = np.cos(2 * np.pi * doc["month"] / 12) record["x"] = Binary(_pickle.dumps(record["x"].tolist())) if np.random.rand() < TRAIN_RATIO: db.train.insert(record) else: db.test.insert(record) pbar.update(1)
def run_parallelism_analysis(nmin_reps=3, nmin = 2, FDR = 0.05, n_nonsyn_min=50): # pass nest list with frequency, coverage of major, coverage of minor, taxon #output_to_keep = ['INS', 'DEL', 'SNP', 'SUB'] to_keep_samples = get_breseq_samples_to_keep() to_keep_taxa = get_breseq_taxa_to_keep() p_star_dict = {} G_score_list = [] for taxon in to_keep_taxa: print(taxon) effective_gene_lengths, effective_gene_lengths_syn, Lsyn, Lnon, substitution_specific_synonymous_fraction = lt.calculate_synonymous_nonsynonymous_target_sizes(taxon) taxon_sites = [] taxon_samples = [ x for x in to_keep_samples if x.startswith(taxon) ] sites_to_remove = get_sites_to_remove(taxon) # keep insertion, deletions, and nonsynonymous SNPs # get size_dict gene_count_dict = {} gene_count_syn_dict = {} #print(sites_to_remove) for taxon_sample in taxon_samples: for i, line in enumerate(open(lt.get_path() + '/data/breseq/annotated/' + taxon_sample + '.gd', 'r')): line_split = line.strip().split('\t') if line_split[0] == '#=GENOME_DIFF': continue if (line_split[3] + '_' + line_split[4] in sites_to_remove): continue if (line_split[0] not in output_to_keep): #or ('frequency' in line_split[6]) or (line_split[3] + '_' + line_split[4] in sites_to_remove): continue if line_split[0] == 'SNP': if [s for s in line_split if 'snp_type=' in s][0].split('=')[1] == 'nonsynonymous': locus_tag = [s for s in line_split if 'locus_tag=' in s][0].split('=')[1] frequency = float([s for s in line_split if 'frequency=' in s][0].split('=')[1]) if ';' in locus_tag: for locus_tag_j in locus_tag.split(';'): if locus_tag_j not in gene_count_dict: gene_count_dict[locus_tag_j] = {} gene_count_dict[locus_tag_j]['freqs'] = [] gene_count_dict[locus_tag_j]['n_mut'] = 0 gene_count_dict[locus_tag_j]['n_mut'] += 1 gene_count_dict[locus_tag_j]['freqs'].append(frequency) else: if locus_tag not in gene_count_dict: #gene_count_dict[locus_tag] = 1 gene_count_dict[locus_tag] = {} gene_count_dict[locus_tag]['freqs'] = [] gene_count_dict[locus_tag]['n_mut'] = 0 gene_count_dict[locus_tag]['n_mut'] += 1 gene_count_dict[locus_tag]['freqs'].append(frequency) elif [s for s in line_split if 'snp_type=' in s][0].split('=')[1] == 'synonymous': locus_tag = [s for s in line_split if 'locus_tag=' in s][0].split('=')[1] frequency = float([s for s in line_split if 'frequency=' in s][0].split('=')[1]) if ';' in locus_tag: for locus_tag_j in locus_tag.split(';'): if locus_tag_j not in gene_count_syn_dict: gene_count_syn_dict[locus_tag_j] = {} gene_count_syn_dict[locus_tag_j]['freqs'] = [] gene_count_syn_dict[locus_tag_j]['n_mut'] = 0 gene_count_syn_dict[locus_tag_j]['n_mut'] += 1 gene_count_syn_dict[locus_tag_j]['freqs'].append(frequency) else: if locus_tag not in gene_count_syn_dict: gene_count_syn_dict[locus_tag] = {} gene_count_syn_dict[locus_tag]['freqs'] = [] gene_count_syn_dict[locus_tag]['n_mut'] = 0 gene_count_syn_dict[locus_tag]['n_mut'] += 1 gene_count_syn_dict[locus_tag]['freqs'].append(frequency) else: continue else: if len([s for s in line_split if 'gene_position=coding' in s]) >= 1: locus_tag = [s for s in line_split if 'locus_tag=' in s][0].split('=')[1] frequency = float([s for s in line_split if 'frequency=' in s][0].split('=')[1]) if ';' in locus_tag: for locus_tag_j in locus_tag.split(';'): if locus_tag_j not in gene_count_dict: gene_count_dict[locus_tag_j] = {} gene_count_dict[locus_tag_j]['freqs'] = [] gene_count_dict[locus_tag_j]['n_mut'] = 0 gene_count_dict[locus_tag_j]['freqs'].append(frequency) gene_count_dict[locus_tag_j]['n_mut'] += 1 else: if locus_tag not in gene_count_dict: #gene_count_dict[locus_tag] = 1 gene_count_dict[locus_tag] = {} gene_count_dict[locus_tag]['freqs'] = [] gene_count_dict[locus_tag]['n_mut'] = 0 gene_count_dict[locus_tag]['freqs'].append(frequency) gene_count_dict[locus_tag]['n_mut'] += 1 gene_parallelism_statistics = {} for gene_i, length_i in effective_gene_lengths.items(): gene_parallelism_statistics[gene_i] = {} gene_parallelism_statistics[gene_i]['length'] = length_i gene_parallelism_statistics[gene_i]['observed'] = 0 gene_parallelism_statistics[gene_i]['multiplicity'] = 0 gene_parallelism_statistics_syn = {} for gene_i, length_i in effective_gene_lengths_syn.items(): gene_parallelism_statistics_syn[gene_i] = {} gene_parallelism_statistics_syn[gene_i]['length'] = length_i gene_parallelism_statistics_syn[gene_i]['observed'] = 0 gene_parallelism_statistics_syn[gene_i]['multiplicity'] = 0 # save number of mutations for multiplicity for locus_tag_i, locus_tag_i_dict in gene_count_dict.items(): gene_parallelism_statistics[locus_tag_i]['observed'] = locus_tag_i_dict['n_mut'] gene_parallelism_statistics[locus_tag_i]['mean_freq'] = np.mean(locus_tag_i_dict['freqs']) # same thing for synonymous for locus_tag_i, locus_tag_i_dict in gene_count_syn_dict.items(): gene_parallelism_statistics_syn[locus_tag_i]['observed'] = locus_tag_i_dict['n_mut'] gene_parallelism_statistics_syn[locus_tag_i]['mean_freq'] = np.mean(locus_tag_i_dict['freqs']) L_mean = np.mean(list(effective_gene_lengths.values())) L_tot = sum(list(effective_gene_lengths.values())) n_tot = sum([ x['n_mut'] for x in gene_count_dict.values() ]) # don't include taxa with less than 20 mutations print("N_total = " + str(n_tot)) if n_tot < n_nonsyn_min: continue # go back over and calculate multiplicity for locus_tag_i in gene_parallelism_statistics.keys(): # double check the measurements from this gene_parallelism_statistics[locus_tag_i]['multiplicity'] = gene_parallelism_statistics[locus_tag_i]['observed'] *1.0/ effective_gene_lengths[locus_tag_i] * L_mean gene_parallelism_statistics[locus_tag_i]['expected'] = n_tot*gene_parallelism_statistics[locus_tag_i]['length']/L_tot # get multiplicity for synonymous mutations L_mean_syn = np.mean(list(effective_gene_lengths_syn.values())) L_tot_syn = sum(list(effective_gene_lengths_syn.values())) n_tot_syn = sum([ x['n_mut'] for x in gene_count_syn_dict.values() ]) # go back over and calculate multiplicity for locus_tag_i in gene_parallelism_statistics_syn.keys(): # double check the measurements from this gene_parallelism_statistics_syn[locus_tag_i]['multiplicity'] = gene_parallelism_statistics_syn[locus_tag_i]['observed'] *1.0/ effective_gene_lengths_syn[locus_tag_i] * L_mean_syn gene_parallelism_statistics_syn[locus_tag_i]['expected'] = n_tot_syn*gene_parallelism_statistics_syn[locus_tag_i]['length']/L_tot_syn pooled_multiplicities = np.array([gene_parallelism_statistics[gene_name]['multiplicity'] for gene_name in gene_parallelism_statistics.keys() if gene_parallelism_statistics[gene_name]['multiplicity'] >=1]) pooled_multiplicities.sort() pooled_tupe_multiplicities = np.array([(gene_parallelism_statistics[gene_name]['multiplicity'], gene_parallelism_statistics[gene_name]['observed']) for gene_name in gene_parallelism_statistics.keys() if gene_parallelism_statistics[gene_name]['multiplicity'] >=1]) pooled_tupe_multiplicities = sorted(pooled_tupe_multiplicities, key=lambda x: x[0]) pooled_tupe_multiplicities_x = [i[0] for i in pooled_tupe_multiplicities] pooled_tupe_multiplicities_y = [i[1] for i in pooled_tupe_multiplicities] pooled_tupe_multiplicities_y = [sum(pooled_tupe_multiplicities_y[i:]) / sum(pooled_tupe_multiplicities_y) for i in range(len(pooled_tupe_multiplicities_y))] null_multiplicity_survival = lt.NullGeneMultiplicitySurvivalFunction.from_parallelism_statistics( gene_parallelism_statistics ) #observed_ms_test, observed_multiplicity_survival_test = lt.calculate_unnormalized_survival_from_vector(pooled_multiplicities) null_multiplicity_survival_copy = null_multiplicity_survival(pooled_multiplicities) null_multiplicity_survival_copy = [sum(null_multiplicity_survival_copy[i:]) / sum(null_multiplicity_survival_copy) for i in range(len(null_multiplicity_survival_copy)) ] #threshold_idx = numpy.nonzero((null_multiplicity_survival(observed_ms)*1.0/observed_multiplicity_survival)<FDR)[0][0] mult_survival_dict = {'Mult': pooled_multiplicities, 'Obs_fract': pooled_tupe_multiplicities_y, 'Null_fract': null_multiplicity_survival_copy} mult_survival_df = pd.DataFrame(mult_survival_dict) mult_survival_df_out = lt.get_path() + '/data/breseq/mult_survival_curves/' + taxon + '.txt' mult_survival_df.to_csv(mult_survival_df_out, sep = '\t', index = True) # get likelihood score and null test observed_G, pvalue = lt.calculate_total_parallelism(gene_parallelism_statistics) G_score_list.append((taxon, observed_G, pvalue)) print(observed_G, pvalue) if pvalue >= 0.05: continue # Give each gene a p-value, get distribution gene_logpvalues = lt.calculate_parallelism_logpvalues(gene_parallelism_statistics) pooled_pvalues = [] for gene_name in gene_logpvalues.keys(): if (gene_parallelism_statistics[gene_name]['observed']>= nmin) and (float(gene_logpvalues[gene_name]) >= 0): pooled_pvalues.append( gene_logpvalues[gene_name] ) pooled_pvalues = np.array(pooled_pvalues) pooled_pvalues.sort() if len(pooled_pvalues) == 0: continue null_pvalue_survival = lt.NullGeneLogpSurvivalFunction.from_parallelism_statistics( gene_parallelism_statistics, nmin=nmin) observed_ps, observed_pvalue_survival = lt.calculate_unnormalized_survival_from_vector(pooled_pvalues, min_x=-4) # Pvalue version # remove negative minus log p values. neg_p_idx = np.where(observed_ps>=0) observed_ps_copy = observed_ps[neg_p_idx] observed_pvalue_survival_copy = observed_pvalue_survival[neg_p_idx] pvalue_pass_threshold = np.nonzero(null_pvalue_survival(observed_ps_copy)*1.0/observed_pvalue_survival_copy<FDR)[0] if len(pvalue_pass_threshold) == 0: continue threshold_idx = pvalue_pass_threshold[0] pstar = observed_ps_copy[threshold_idx] # lowest value where this is true num_significant = observed_pvalue_survival[threshold_idx] # make it log base 10 logpvalues_dict = {'P_value': observed_ps/math.log(10), 'Obs_num': observed_pvalue_survival, 'Null_num': null_pvalue_survival(observed_ps)} logpvalues_df = pd.DataFrame(logpvalues_dict) logpvalues_df_out = lt.get_path() + '/data/breseq/logpvalues/' + taxon + '.txt' logpvalues_df.to_csv(logpvalues_df_out, sep = '\t', index = True) p_star_dict[taxon] = (num_significant, pstar/math.log(10)) output_mult_gene_filename = lt.get_path() + '/data/breseq/mult_genes_nonsyn_sig/' + taxon + '.txt' output_mult_gene = open(output_mult_gene_filename,"w") output_mult_gene.write(",".join(["Gene", "Length", "Observed", "Expected", "Multiplicity", "-log10(P)"])) for gene_name in sorted(gene_parallelism_statistics, key=lambda x: gene_parallelism_statistics.get(x)['observed'],reverse=True): if gene_logpvalues[gene_name] >= pstar and gene_parallelism_statistics[gene_name]['observed']>=nmin: output_mult_gene.write("\n") # log base 10 transform the p-values here as well output_mult_gene.write("%s, %0.1f, %d, %0.2f, %0.2f, %g" % (gene_name, gene_parallelism_statistics[gene_name]['length'], gene_parallelism_statistics[gene_name]['observed'], gene_parallelism_statistics[gene_name]['expected'], gene_parallelism_statistics[gene_name]['multiplicity'], abs(gene_logpvalues[gene_name])/math.log(10) )) output_mult_gene.close() output_mult_syn_filename = lt.get_path() + '/data/breseq/mult_genes_all/' + taxon + '.txt' output_mult_syn = open(output_mult_syn_filename,"w") output_mult_syn.write(",".join(["Gene", "mult", "mult_syn", "mean_freq", "mean_freq_syn"])) for locus_tag_i in gene_parallelism_statistics.keys(): mult_i = gene_parallelism_statistics[locus_tag_i]['multiplicity'] mult_i_syn = gene_parallelism_statistics_syn[locus_tag_i]['multiplicity'] if (mult_i > 0) and (mult_i_syn > 0): freq_i = gene_parallelism_statistics[locus_tag_i]['mean_freq'] freq_i_syn = gene_parallelism_statistics_syn[locus_tag_i]['mean_freq'] output_mult_syn.write("\n") output_mult_syn.write("%s, %f, %f, %f, %f" % (locus_tag_i, mult_i, mult_i_syn, freq_i, freq_i_syn)) output_mult_syn.close() G_score_list_p_vales = [i[2] for i in G_score_list] reject, pvals_corrected, alphacSidak, alphacBonf = mt.multipletests(G_score_list_p_vales, alpha=0.05, method='fdr_bh') total_parallelism_path = lt.get_path() + '/data/breseq/total_parallelism.txt' total_parallelism = open(total_parallelism_path,"w") total_parallelism.write("\t".join(["Taxon", "G_score", "p_value", "p_value_BH"])) for i in range(len(pvals_corrected)): taxon_i = G_score_list[i][0] G_score_i = G_score_list[i][1] p_value_i = G_score_list[i][2] pvals_corrected_i = pvals_corrected[i] total_parallelism.write("\n") total_parallelism.write("\t".join([taxon_i, str(G_score_i), str(p_value_i), str(pvals_corrected_i)])) total_parallelism.close() with open(lt.get_path() + '/data/breseq/p_star.txt', 'wb') as file: file.write(pickle.dumps(p_star_dict)) # use `pickle.loads` to do the reverse
def save(data, path): serialized = cPickle.dumps(data) with gzip.open(path, 'wb', compresslevel=1) as file_object: file_object.write(serialized)
def threaded_client(conn, _id): global connections, players, dots, nxt, started current_id = _id game_time = 0 # receive a name from the client data = conn.recv(16) name = data.decode("utf-8") print("[LOG]", name, "connected to the server") # properties for new players color = colors[current_id] x, y = get_start_location(players) players[current_id] = { "x": x, "y": y, "color": color, "score": 0, "name": name } #pickle data and send initial info to clients conn.send(str.encode(str(current_id))) while True: if start: game_time = round(time.time() - start_time) # if the game time passes the round time, the game will stop if game_time >= ROUND_TIME: start = False else: if game_time // MASS_LOSS_TIME == nxt: nxt += 1 release_mass(players) print(f"[GAME] {name}'s Mass depleting") try: # receive data from client data = conn.recv(32) if not data: break data = data.decode("utf-8") print("[DATA] Received", data, "from client id", current_id) #look for specific commands from received data if data.split(" ")[0] == "move": split_data = data.split(" ") x = int(split_data[1]) y = int(split_data[2]) players[current_id]["x"] = x players[current_id]["y"] = y if start: check_collision(players, dots) player_collision(players) if len(dots) < 150: create_dots(dots, random.randrange(100, 150)) print("[GAME] Generating more Dots") send_data = pickle.dumps((dots, players, game_time)) else: send_data = pickle.dumps((dots, players, game_time)) # send data back to clients conn.send(send_data) except Exception as e: print(e) break # if exception, disconnect client time.sleep(0.001) # when user disconnects print("[DISCONNECT] Name:", name, " Client Id:", current_id, " disconnected") connections -= 1 del players[current_id] conn.close()
def init(self): df = pd.read_csv(self.fpath) return self.redis.set(ct.CALENDAR_INFO, _pickle.dumps(df, 2))
if __name__ == "__main__": loads = marshal.loads dumps = marshal.dumps #test cases ht = HashTable('/dev/shm/test.HashTable', 1024, 1) #set ht['a'] = b'1' ht.set('b', b'2') # c = {'hello': 'world'} c = [0,1] ht.setobj('c', c) #get print(ht['b'] == b'2') print(ht['c'] == marshal.dumps(c)) print(ht.getobj('c') == c) print(ht.get('d') == None) try: ht['d'] print(False) except: print(True) #contains print(('c' in ht) == True) print(('d' in ht) == False) #del del ht['c'] print(ht.get('c') == None)
if DEBUG_MIDI: debug_state.enable_midi() ic.enable() else: ic.disable() ic.configureOutput(includeContext=True) ## MAIN try: # NB: this is slow AF resp = mpx1_control_tree.make_flat(inport, outport, DEVICE_ID) with open("control_tree_flat.pickle", "wb") as f: f.write(pickle.dumps(resp)) # control_tree = mpx1_control_tree.make(inport, outport, DEVICE_ID) # pprint(control_tree['desc']['label']) # pprint(control_tree['children'][0]['desc']['label']) # pprint(control_tree['children'][0]['children'][2]['desc']['label']) # pprint(control_tree['children'][0]['children'][2]['children'][1]['desc']['label']) # pprint(control_tree['children'][0]['children'][2]['children'][1]['children'][2]['desc']['label']) except KeyboardInterrupt as e: cleanup_mido() ## CLEANUP cleanup_mido()