def plotMessagesPerDay(messagesList): firstDay = messagesList[0]['dateTime'].date() lastDay = messagesList[-1]['dateTime'].date() #create a dict with all days daysDict = {} day = firstDay while (day <= lastDay): daysDict[day] = 0 day = day + timedelta(days=1) #sum number of messages in each day for message in messagesList: daysDict[message['dateTime'].date()] += 1 #convert to lists days, numMessages = zip(*daysDict.items()) sort_together([days, numMessages]) print("Ploting group activity on time") print("Average: %f messages/day"%np.mean(numMessages)) maxIndex = np.argmax(numMessages) maxMessages = numMessages[maxIndex] maxDay = days[maxIndex] print("Day of maximum: %s with %d messages"%(str(maxDay), maxMessages)) plt.plot(days, numMessages) plt.xticks(rotation=45) plt.title("Messages/day") plt.show()
def refer(self, so, slot_name, slot_pos, template, max_refs, was_already_seen): refs_1st = self.ref_db['1st'].get(so, set()) refs_1st.add(self.fallback(so)) refs_2nd = self.ref_db['2nd'].get(so, set()) slot = '{{{}}}'.format(f'{slot_name}-{slot_pos}') def score_reg(r): ref_id = text_to_id(r) text = template.template_text.replace(slot, ref_id) score = self.ref_lm.score(text) self.logger.debug(f'{score:.3f} -> {ref_id} -> {text}') return score if not was_already_seen or not refs_2nd: refs = refs_1st else: refs = refs_2nd scores = [score_reg(r) for r in refs] scores, sorted_refs = sort_together([scores, refs], reverse=True) return scores[:max_refs], sorted_refs[:max_refs]
def select_sentence_aggregation(self, dp, n_triples): sas = list(partitions(dp)) sas_scores = self.sa_scorer(sas, n_triples) sas = sort_together([sas_scores, sas], reverse=True)[1] return sas
def select_discourse_planning(self, entry, n_triples): dps = list(permutations(entry.triples)) dps_scores = self.dp_scorer(dps, n_triples) dps = sort_together([dps_scores, dps], reverse=True)[1] return dps
def topListByAverageMessageSize(messagesList, nBest): #create a dict with all members membersDict = {} for message in messagesList: member = message['member'] text = message['text'] if member not in membersDict.keys(): membersDict[member] = [1, len(text)] else: membersDict[member][0] += 1 membersDict[member][1] += len(text) #convert to lists members, messagesData = zip(*membersDict.items()) #calculate messages average size: averageSizes = [] for i in range(len(messagesData)): averageSizes.append(float(messagesData[i][1])/ messagesData[i][0]) averageSizes, members = sort_together([averageSizes, members], reverse=True) nBest = min(nBest, len(members)) print("-----------------------------------------") print("Top %d by average message size:"%nBest) print("-----------------------------------------") for i in range(nBest): print('%s: %1.1f characters'%(members[i], averageSizes[i])) print("Average of all members: %f characters"%np.mean(averageSizes))
def evolve(self): # pop = population pop = self.population # Population ranking fitenss_pop = [self.fitness(individual) for individual in pop] fitenss_pop, pop = sort_together([fitenss_pop, pop]) self.keep_if_best_result(fitenss_pop[0], pop[0]) # Get top half of the population to go to the next generation old_count = int(self.d.p_size / 2) new_pop = [v for v in pop[0:old_count]] # Generate second half of the populaiton by combining parents next to # each other for i in range(0, self.d.p_size, 2): male = pop[i] female = pop[i + 1] child = [] for i in range(len(male)): min_ = min([male[i], female[i]]) max_ = max([female[i], male[i]]) v = np.random.uniform(min_, max_) child.append(v) new_pop.append(child) new_pop = self.mutate_population(new_pop, self.d.mutate, self.d.v_min, self.d.v_max) self.population = new_pop
def select_templates(self, sa): tss = [] for sa_part in sa: ts = self.template_db.select(sa_part) if not ts: return [] tss.append(ts) all_ts = [] all_scores = [] for ts, sa_part in zip(tss, sa): scores = [self.score_template(t, sa_part) for t in ts] scores, ts = sort_together([scores, ts], reverse=True) all_ts.append(ts[:self.max_tems]) all_scores.append(scores[:self.max_tems]) top_combs = top_combinations(all_scores, self.max_refs) top_template_combs = [[all_ts[i][ix_] for i, ix_ in enumerate(ix)] for ix in top_combs] self.logger.debug('Template Selection - top combs: {}'.format('\n'.join(' ;; '.join(t.template_text for t in tems) for tems in top_template_combs))) return top_template_combs
def get_corridors(self): """Return list of corridor fields, associated list of possible actions and associated list of action unicode symbols. Sorted by assorting field index.""" # Assign columns from left to right (always 3). Take into account special cases. cols, col_actions, col_action_symbols = [], [], [] for i in range(3): col_start_idx = [0, self.grid_size[1] // 2, self.grid_size[1] - 1] cols += np.arange(col_start_idx[i], np.prod(self.grid_size), self.grid_size[1]).tolist() col_i_actions = [(0, 2)] * self.grid_size[0] col_i_action_symbols = [8597] * self.grid_size[0] if i == 0: # first column col_i_actions[0], col_i_action_symbols[0] = ( 1, 2), 8600 # UL corner for j in range(self.n_shelve_units // 2): col_i_actions[(j + 1) * 3], col_i_action_symbols[(j + 1) * 3] = ( 0, 1, 2), 8614 # intermediate corners col_i_actions[-1], col_i_action_symbols[-1] = ( 0, 1), 8599 # BL corner if i == 1: # second column col_i_actions[0], col_i_action_symbols[0] = ( 1, 2, 3), 8615 # upper intermediate corner for j in range(self.n_shelve_units // 2): col_i_actions[(j + 1) * 3], col_i_action_symbols[(j + 1) * 3] = ( 0, 1, 2, 3), 8623 # intermediate corners col_i_actions[-1], col_i_action_symbols[-1] = ( 0, 1, 3), 8613 # bottom intermediate corner (start) if i == 2: # second column col_i_actions[0], col_i_action_symbols[0] = ( 2, 3), 8601 # UR corner for j in range(self.n_shelve_units // 2): col_i_actions[(j + 1) * 3], col_i_action_symbols[(j + 1) * 3] = ( 0, 2, 3), 8612 # intermediate corners col_i_actions[-1], col_i_action_symbols[-1] = ( 0, 3), 8598 # BR corner col_actions += col_i_actions col_action_symbols += col_i_action_symbols # Assign rows, leave out columns rows = [] row_starts = range( 0, 3 * self.grid_size[1] * (self.n_shelve_units // 2 + 1), 3 * self.grid_size[1]) for i in range(self.n_shelve_units // 2 + 1): row_i = list( range(row_starts[i] + 1, row_starts[i] + self.grid_size[1] - 1)) rows += row_i[:len(row_i) // 2] + row_i[len(row_i) // 2 + 1:] row_actions = [(1, 3)] * len(rows) row_action_symbols = [8596] * len(rows) # Collect all, sort by index and return return sort_together([ cols + rows, col_actions + row_actions, col_action_symbols + row_action_symbols ])
def lowestEigs(M, k = 10): E, V = eigsh(M.tocsc(), k = k , which = 'SA',maxiter=10000000) psi = [] for i in range(len(V[0,:])): psi.append((np.transpose(np.matrix(V[:,i])))) E, psi = sort_together([E, psi]) return E, psi
def sort_population_by_grade(self): fitenss_population = [ self.fitness(individual) for individual in self.population ] fitenss_population, population = sort_together( [fitenss_population, self.population]) self.keep_if_best_result(fitenss_population[0], population[0]) self.population = population
def select_sa(self, dp): sas = list(partitions(dp)) sas_scores = self.sa_scorer(sas) sas_scores, sas = sort_together([sas_scores, sas], reverse=True) self.logger.debug('Sentence Aggregation: {}'.format('\n'.join(f'{score:.3f} -> {sa}' for score, sa in zip(sas_scores, sas)))) return [[tuple(sa_part) for sa_part in sa] for sa in sas[:self.max_sa]]
def select_dp(self, entry): dps = list(permutations(entry.triples)) dps_scores = self.dp_scorer(dps) dps_scores, dps = sort_together([dps_scores, dps], reverse=True) self.logger.debug('Discourse Planning: {}'.format('\n'.join(f'{score:.3f} -> {dp}' for score, dp in zip(dps_scores, dps)))) return dps[:self.max_dp]
def _sortBoxesByArea(boxes): ratios = [] boxesByArea = [] for box in boxes: ratios.append(box.ratio_of_image) boxesByAreaSmToLg = sort_together([ratios, boxes])[1] boxesByAreaLgToSm = list(reversed(boxesByAreaSmToLg)) return boxesByAreaLgToSm
def evolve(self): population = self.population count = self.d.p_size * 2 # Sort the population by fitness fitenss_population = [ self.fitness(individual) for individual in population ] fitenss_population, population = sort_together( [fitenss_population, population]) self.keep_if_best_result(fitenss_population[0], population[0]) # Find the max fitness and invert all values to find percentages max_value = fitenss_population[-1] fitenss_population = [(max_value + 1 - v) for v in fitenss_population] # Generate new positions rand = np.random.uniform(0, sum(fitenss_population), count) rand.sort() # Generate new positions by cumulative addition of the weights and if # the random number is withing individual range, the individual is added # for reproduction. IMPORTANT: random numbers must be sorted reproduction = [] cc = 0 i = 0 for j in range(len(fitenss_population)): cc += fitenss_population[j] while i < len(rand) and cc >= rand[i]: reproduction.append(population[j]) i += 1 # Create new generation from new parents random.shuffle(reproduction) new_population = [] for i in range(0, count, 2): male = reproduction[i] female = reproduction[i + 1] child = [] for i in range(len(male)): min_ = min([male[i], female[i]]) max_ = max([female[i], male[i]]) v = np.random.uniform(min_, max_) child.append(v) new_population.append(child) # Mutate new_population = self.mutate_population(new_population, self.d.mutate, self.d.v_min, self.d.v_max) self.population = new_population
def ask_question(self, Question): tokenize_question = self.tokenizer.texts_to_sequences([Question]) tokenize_question = pad_sequences(tokenize_question, self.max_question_length) result = self.model.predict(tokenize_question) max_position = result[0].argmax() list_column = list(User.Y.keys()) list_result = result.tolist()[0] print(sort_together([list_result, list_column])[1]) return [list_column, list_result]
def evolve(self): # pop = population pop = self.population # Sort population by fitness fitenss_pop = [self.fitness(individual) for individual in pop] fitenss_pop, pop = sort_together([fitenss_pop, pop]) self.keep_if_best_result(fitenss_pop[0], pop[0]) # Generate random values in range of 0 to total sum of all weights ps = self.d.p_size count = ps * 2 total_sum = (ps * (ps + 1)) / 2 rand = np.random.randint(0, total_sum, count) # Generate new positions by cumulative addition of the weights and if # the random number is withing individual range, the individual is added # for reproduction. IMPORTANT: random numbers must be sorted rand.sort() reproduction = [] cc = 0 i = 0 for j in range(ps): cc += (ps - j) while i < len(rand) and cc >= rand[i]: reproduction.append(pop[j]) i += 1 # Create offsprings random.shuffle(reproduction) new_population = [] for i in range(0, count, 2): male = reproduction[i] female = reproduction[i + 1] child = [] for i in range(len(male)): min_ = min([male[i], female[i]]) max_ = max([female[i], male[i]]) v = np.random.uniform(min_, max_) child.append(v) new_population.append(child) # Mutate new_population = self.mutate_population(new_population, self.d.mutate, self.d.v_min, self.d.v_max) self.population = new_population
def Cluster(G): print(len(G)) from more_itertools import sort_together d = Distribution(G) x = [item[0] for item in d.items()] y = [item[1] for item in d.items()] sum_y = sum(y) y_d = [i / sum_y for i in y] x, y_d = sort_together([x, y_d]) c_results = ([x, y, y_d]) plt.plot(c_results[0], c_results[2], 'bo') plt.show() plt.hist(c_results[0], density=True) plt.show()
def acc_markov(scorer, test_dp_db): from more_itertools import sort_together from itertools import permutations tp = 0 for ts, sorteds_ts in test_dp_db: all_perms = list(permutations(ts)) scores = scorer(all_perms) best_score_perm = sort_together([scores, all_perms], reverse=True)[1][0] if best_score_perm in sorteds_ts: tp += 1 return tp / len(test_dp_db)
def n_lowest_score(self, n, networks, score_history_list, network_history_list, verbose=False): networks_sorted = networks.copy() scores = [] for network in networks: score_ = self.score(network, verbose=verbose) scores.append(score_) x, y = sort_together([scores, networks_sorted]) score_history_list += list(x) network_history_list += list(y) y = list(y[:n]) return y
def order(b_base): b = b_base[0] truth = [] sumtrue = [] elements = [] n = b_base[0] for i in range(len(b_base)): b = And(b, b_base[i]) b = list(b.atoms()) table = list(itertools.product([False, True], repeat=len(b))) for i in range(len(b_base)): truth.append([]) for j in range(len(table)): if i == 0: elements.append([]) for k in range(len(table[j])): if k == 0: n = b_base[i].subs(b[k], table[j][k]) else: n = n.subs(b[k], table[j][k]) if i == 0: if table[j][k] == False: elements[j].append(Not(b[k])) else: elements[j].append(b[k]) truth[i].append(n) if i == len(b_base) - 1: for i, k in enumerate(truth): for j, item in enumerate(k): if truth[i][j] == True: truth[i][j] = 1 else: truth[i][j] = 0 for z in zip(*truth): sumtrue.append(sum(z)) m = b + table Z = sort_together([sumtrue, elements])[1] Z = Z[::-1] return Z, sorted(sumtrue, reverse=True)
def topListByNumberOfMessages(messagesList, nBest): #create a dict with all members membersDict = {} for message in messagesList: member = message['member'] if member not in membersDict.keys(): membersDict[member] = 1 else: membersDict[member] += 1 #convert to lists members, numMessages = zip(*membersDict.items()) numMessages, members = sort_together([numMessages, members], reverse=True) nBest = min(nBest, len(members)) print("--------------------------------") print("Top %d by number of messages:"%nBest) print("--------------------------------") for i in range(nBest): print('%s: %d messages'%(members[i], numMessages[i])) print("Average of all members: %f messages"%np.mean(numMessages))
def topListByNumberOfCharacters(messagesList, nBest): #create a dict with all members membersDict = {} for message in messagesList: member = message['member'] text = message['text'] if member not in membersDict.keys(): membersDict[member] = len(text) else: membersDict[member] += len(text) #convert to lists members, numChars = zip(*membersDict.items()) numChars, members = sort_together([numChars, members], reverse=True) nBest = min(nBest, len(members)) print("--------------------------------------------") print("Top %d by total number of typed characters:"%nBest) print("--------------------------------------------") for i in range(nBest): print('%s: %d characters'%(members[i], numChars[i])) print("Average of all members: %f characters"%np.mean(numChars))
def acc_ltr(test_dp_db): from pretrained_models import ltr_lasso_dp_scorer from more_itertools import sort_together from itertools import permutations m = ltr_lasso_dp_scorer(('train', 'dev')) tp = 0 for ts, sorteds_ts in test_dp_db: all_perms = list(permutations(ts)) scores = m(all_perms, len(ts)) best_score_perm = sort_together([scores, all_perms], reverse=True)[1][0] if best_score_perm in sorteds_ts: tp += 1 return tp / len(test_dp_db)
def image_pass_places(input_img): global model_places, graph_places, places_labels (latitude, longitude, date, time) = exif_extract_information(input_img) img_target_size = (224, 224) img = prepare_image(input_img, img_target_size) with graph_places.as_default(): features = model_places.predict(img) cat_list = [] prob_list = [] for x in range(0, 162): cat_list.append(places_labels[x]) prob_list.append(features[0, x]) sorted_list = sort_together([prob_list, cat_list], reverse=True) prob_list = sorted_list[0] cat_list = sorted_list[1] top_5_cat = cat_list[0:5] jsonString = [] for x in top_5_cat: x = re.sub(r'[^a-zA-Z ]+', ' ', x) x = x.title() jsonString.append(x) return_dict = \ { "tags": jsonString, "latitude": latitude, "longitude": longitude, "date": date, "time": time, } return return_dict
def topListWords(messagesList, wordsToRemove, nBest): #create a dict with all members wordsDict = {} for message in messagesList: text = message['text'] words = text.split() for word in words: word = word.lower() if word not in wordsToRemove: if word not in wordsDict.keys(): wordsDict[word] = 1 else: wordsDict[word] += 1 #convert to lists words, occurrences = zip(*wordsDict.items()) occurrences, words = sort_together([occurrences, words], reverse=True) nBest = min(nBest, len(words)) print("-------------------------") print("Top %d used words:"%nBest) print("-------------------------") for i in range(nBest): print('%s: %d occurrences'%(words[i], occurrences[i]))
def create_mating_pool(fitness, population, to_choose): cumulative_fitness = sum(fitness) fitness = list(map(lambda x: x / cumulative_fitness, fitness)) fitness, population = sort_together([fitness, population], reverse=True) mating_pool = [] indexes = [] for i in range(to_choose): p = np.random.uniform(0, 1) for j in range(len(fitness)): if sum(fitness[:j + 1]) >= p: if j not in indexes: mating_pool.append(population[j]) indexes.append(j) while len(mating_pool) <= i: index_to_append = random.randint(0, len(fitness) - 1) if index_to_append not in indexes: indexes.append(index_to_append) mating_pool.append(population[index_to_append]) return mating_pool
def get_state(model, device, pool_dataset, train_dataset): pool_embeddings = get_model_embeddings(model, device, pool_dataset) pool_predictions = get_model_predictions(model, device, pool_dataset) train_embeddings = get_model_embeddings(model, device, train_dataset) train_predictions = get_model_predictions(model, device, train_dataset) if prop.ADD_GRADIENT_EMBEDDING: gradient_embeddings = get_model_gradient_embeddings( model, device, pool_dataset) gradient_embeddings_flat = gradient_embeddings.flatten() lab_emb = torch.mean(train_embeddings, dim=0) if prop.ADD_POOL_MEAN_EMB: pool_emb = torch.mean(pool_embeddings, dim=0) train_label_statistics = torch.bincount( train_dataset.tensors[1]).float() / len(train_dataset) # train predictions statistics. if predictions missing, fill up list with 0 train_pred_lab_unique_cnts = np.unique(train_predictions, return_counts=True) for i in range(0, prop.NUM_CLASSES): if i not in train_pred_lab_unique_cnts[0]: new_tuple = (np.concatenate((train_pred_lab_unique_cnts[0], [i])), np.concatenate((train_pred_lab_unique_cnts[1], [0]))) train_pred_lab_unique_cnts = new_tuple sorted_uniques_cnts = sort_together( [train_pred_lab_unique_cnts[0], train_pred_lab_unique_cnts[1]]) train_pred_label_statistics = torch.Tensor(sorted_uniques_cnts[1] / sum(sorted_uniques_cnts[1])) state = [] if prop.ADD_POOL_MEAN_EMB: for ind, sample_emb in enumerate(pool_embeddings): state.append( torch.cat([ lab_emb, pool_emb, sample_emb, train_label_statistics, train_pred_label_statistics, get_one_hot(pool_predictions[ind]) ])) if prop.ADD_GRADIENT_EMBEDDING and not prop.ADD_PREDICTIONS and not prop.ADD_POOL_MEAN_EMB and prop.ARBITRARY_CLASSES: for ind, sample_emb in enumerate(pool_embeddings): state.append(torch.cat([lab_emb, gradient_embeddings[ind]])) if prop.ADD_GRADIENT_EMBEDDING and prop.ADD_PREDICTIONS and not prop.ADD_POOL_MEAN_EMB and not prop.ARBITRARY_CLASSES: for ind, sample_emb in enumerate(pool_embeddings): state.append( torch.cat([ lab_emb, sample_emb, train_label_statistics, train_pred_label_statistics, gradient_embeddings[ind], get_one_hot(pool_predictions[ind]) ])) if prop.ADD_GRADIENT_EMBEDDING and not prop.ADD_PREDICTIONS and not prop.ADD_POOL_MEAN_EMB and not prop.ARBITRARY_CLASSES: for ind, sample_emb in enumerate(pool_embeddings): state.append( torch.cat([ lab_emb, train_label_statistics, train_pred_label_statistics, gradient_embeddings[ind] ])) if prop.ADD_PREDICTIONS and not prop.ADD_GRADIENT_EMBEDDING and not prop.ADD_POOL_MEAN_EMB and not prop.ARBITRARY_CLASSES: for ind, sample_emb in enumerate(pool_embeddings): state.append( torch.cat([ lab_emb, sample_emb, train_label_statistics, train_pred_label_statistics, get_one_hot(pool_predictions[ind]) ])) return torch.stack(state)
with open(filename, 'w') as f: json.dump(data, f, indent=4) def loadJSON(filename): with open(filename, "r") as json_file: return json.load(json_file) data = loadJSON("afgekeurde_voortuigen.json") #gebrekken = loadJSON("gebrekken.json") name = [] length = [] for a in data: if len(data[a]) > 200: name.append(a) length.append(len(data[a])) values = sort_together([length, name], reverse=True) x = np.arange(len(values[0])) fig, ax = plt.subplots() ax.bar(values[1], values[0]) ax.set_xticks(x) ax.set_xticklabels(values[1], rotation=(45)) plt.show()
h = 18 F_H = 0.0981 M_nosum.append(-F_H * h) xtab.append(h) D = 4.2 gamma = ma.atan(d / D) F_brz = ((Mtot - e * Fe - h * F_H) / d) Ay = -F_brz / ma.tan(gamma) Az = Fe - F - H + F_brz - L #%%-----------------Adding point moments to moment diagram------------------------ from more_itertools import sort_together sort = sort_together([xtab, M_nosum]) #sorting points together by location xtab_sorted = sort[0] Mtab_sorted = sort[1] xtab_fin = [] Mtab_fin = [] Mtot2 = 0 for i in range(0, len(xtab_sorted)): Mtot2 = Mtot2 + Mtab_sorted[i] Mtab_fin.append(Mtot2) #print(Mtab_sorted) #print(Mtab_fin) plt.plot(xtab_sorted, Mtab_fin)
# A_V values av = np.linspace(0, 0.5, 20) # filter files filtfiles = glob.glob(iodir + "data/filters/*") filtfiles.sort() # limb darkening files from Castelli and Kurucz ldfiles = glob.glob(iodir + "data/limbdark/*.pck*") # metallicities of these files Z = [] for f in ldfiles: zstr = os.path.basename(f)[1:4].replace('m', '-').replace('p', '') Z.append(float(zstr[:-1] + '.' + zstr[-1])) Z, ldfiles = sort_together([Z, ldfiles]) lam, _, _ = ld.getlamTg(ldfiles[0]) # wavelengths in the Kurucz files ut.setlam(lam) # set the wavelength array for filtering and reddening ldlist = [] # list of limbdarkening objects for i in tqdm(range(len(ldfiles))): lam, T, g = ld.getlamTg(ldfiles[i]) # get the intensity array from this file I = ld.getI(ldfiles[i], lam, T, g) # initialize a limb darkening object l = ld.LimbDark(lam, T, g, Z[i]) # filter and redden the object, return band x reddening intensities Ib = l.filter(I, filtfiles, a_v=av) # fit polynomials for I(mu) l.fit(Ib, bounds)