def plotMessagesPerDay(messagesList):
firstDay = messagesList[0]['dateTime'].date()
lastDay = messagesList[-1]['dateTime'].date()
#create a dict with all days
daysDict = {}
day = firstDay
while (day <= lastDay):
daysDict[day] = 0
day = day + timedelta(days=1)
#sum number of messages in each day
for message in messagesList:
daysDict[message['dateTime'].date()] += 1
#convert to lists
days, numMessages = zip(*daysDict.items())
sort_together([days, numMessages])
print("Ploting group activity on time")
print("Average: %f messages/day"%np.mean(numMessages))
maxIndex = np.argmax(numMessages)
maxMessages = numMessages[maxIndex]
maxDay = days[maxIndex]
print("Day of maximum: %s with %d messages"%(str(maxDay), maxMessages))
plt.plot(days, numMessages)
plt.xticks(rotation=45)
plt.title("Messages/day")
plt.show()
def refer(self, so, slot_name, slot_pos, template, max_refs,
was_already_seen):
refs_1st = self.ref_db['1st'].get(so, set())
refs_1st.add(self.fallback(so))
refs_2nd = self.ref_db['2nd'].get(so, set())
slot = '{{{}}}'.format(f'{slot_name}-{slot_pos}')
def score_reg(r):
ref_id = text_to_id(r)
text = template.template_text.replace(slot, ref_id)
score = self.ref_lm.score(text)
self.logger.debug(f'{score:.3f} -> {ref_id} -> {text}')
return score
if not was_already_seen or not refs_2nd:
refs = refs_1st
else:
refs = refs_2nd
scores = [score_reg(r) for r in refs]
scores, sorted_refs = sort_together([scores, refs], reverse=True)
return scores[:max_refs], sorted_refs[:max_refs]
def select_sentence_aggregation(self, dp, n_triples):
sas = list(partitions(dp))
sas_scores = self.sa_scorer(sas, n_triples)
sas = sort_together([sas_scores, sas], reverse=True)[1]
return sas
def select_discourse_planning(self, entry, n_triples):
dps = list(permutations(entry.triples))
dps_scores = self.dp_scorer(dps, n_triples)
dps = sort_together([dps_scores, dps], reverse=True)[1]
return dps
def topListByAverageMessageSize(messagesList, nBest):
#create a dict with all members
membersDict = {}
for message in messagesList:
member = message['member']
text = message['text']
if member not in membersDict.keys():
membersDict[member] = [1, len(text)]
else:
membersDict[member][0] += 1
membersDict[member][1] += len(text)
#convert to lists
members, messagesData = zip(*membersDict.items())
#calculate messages average size:
averageSizes = []
for i in range(len(messagesData)):
averageSizes.append(float(messagesData[i][1])/ messagesData[i][0])
averageSizes, members = sort_together([averageSizes, members], reverse=True)
nBest = min(nBest, len(members))
print("-----------------------------------------")
print("Top %d by average message size:"%nBest)
print("-----------------------------------------")
for i in range(nBest):
print('%s: %1.1f characters'%(members[i], averageSizes[i]))
print("Average of all members: %f characters"%np.mean(averageSizes))
def evolve(self):
# pop = population
pop = self.population
# Population ranking
fitenss_pop = [self.fitness(individual) for individual in pop]
fitenss_pop, pop = sort_together([fitenss_pop, pop])
self.keep_if_best_result(fitenss_pop[0], pop[0])
# Get top half of the population to go to the next generation
old_count = int(self.d.p_size / 2)
new_pop = [v for v in pop[0:old_count]]
# Generate second half of the populaiton by combining parents next to
# each other
for i in range(0, self.d.p_size, 2):
male = pop[i]
female = pop[i + 1]
child = []
for i in range(len(male)):
min_ = min([male[i], female[i]])
max_ = max([female[i], male[i]])
v = np.random.uniform(min_, max_)
child.append(v)
new_pop.append(child)
new_pop = self.mutate_population(new_pop, self.d.mutate, self.d.v_min,
self.d.v_max)
self.population = new_pop
def select_templates(self, sa):
tss = []
for sa_part in sa:
ts = self.template_db.select(sa_part)
if not ts:
return []
tss.append(ts)
all_ts = []
all_scores = []
for ts, sa_part in zip(tss, sa):
scores = [self.score_template(t, sa_part) for t in ts]
scores, ts = sort_together([scores, ts], reverse=True)
all_ts.append(ts[:self.max_tems])
all_scores.append(scores[:self.max_tems])
top_combs = top_combinations(all_scores, self.max_refs)
top_template_combs = [[all_ts[i][ix_] for i, ix_ in enumerate(ix)] for ix in top_combs]
self.logger.debug('Template Selection - top combs: {}'.format('\n'.join(' ;; '.join(t.template_text for t in tems) for tems in top_template_combs)))
return top_template_combs
def get_corridors(self):
"""Return list of corridor fields, associated list of possible actions and associated list of action unicode
symbols. Sorted by assorting field index."""
# Assign columns from left to right (always 3). Take into account special cases.
cols, col_actions, col_action_symbols = [], [], []
for i in range(3):
col_start_idx = [0, self.grid_size[1] // 2, self.grid_size[1] - 1]
cols += np.arange(col_start_idx[i], np.prod(self.grid_size),
self.grid_size[1]).tolist()
col_i_actions = [(0, 2)] * self.grid_size[0]
col_i_action_symbols = [8597] * self.grid_size[0]
if i == 0: # first column
col_i_actions[0], col_i_action_symbols[0] = (
1, 2), 8600 # UL corner
for j in range(self.n_shelve_units // 2):
col_i_actions[(j + 1) *
3], col_i_action_symbols[(j + 1) * 3] = (
0, 1, 2), 8614 # intermediate corners
col_i_actions[-1], col_i_action_symbols[-1] = (
0, 1), 8599 # BL corner
if i == 1: # second column
col_i_actions[0], col_i_action_symbols[0] = (
1, 2, 3), 8615 # upper intermediate corner
for j in range(self.n_shelve_units // 2):
col_i_actions[(j + 1) *
3], col_i_action_symbols[(j + 1) * 3] = (
0, 1, 2, 3), 8623 # intermediate corners
col_i_actions[-1], col_i_action_symbols[-1] = (
0, 1, 3), 8613 # bottom intermediate corner (start)
if i == 2: # second column
col_i_actions[0], col_i_action_symbols[0] = (
2, 3), 8601 # UR corner
for j in range(self.n_shelve_units // 2):
col_i_actions[(j + 1) *
3], col_i_action_symbols[(j + 1) * 3] = (
0, 2, 3), 8612 # intermediate corners
col_i_actions[-1], col_i_action_symbols[-1] = (
0, 3), 8598 # BR corner
col_actions += col_i_actions
col_action_symbols += col_i_action_symbols
# Assign rows, leave out columns
rows = []
row_starts = range(
0, 3 * self.grid_size[1] * (self.n_shelve_units // 2 + 1),
3 * self.grid_size[1])
for i in range(self.n_shelve_units // 2 + 1):
row_i = list(
range(row_starts[i] + 1,
row_starts[i] + self.grid_size[1] - 1))
rows += row_i[:len(row_i) // 2] + row_i[len(row_i) // 2 + 1:]
row_actions = [(1, 3)] * len(rows)
row_action_symbols = [8596] * len(rows)
# Collect all, sort by index and return
return sort_together([
cols + rows, col_actions + row_actions,
col_action_symbols + row_action_symbols
])
def lowestEigs(M, k = 10):
E, V = eigsh(M.tocsc(), k = k , which = 'SA',maxiter=10000000)
psi = []
for i in range(len(V[0,:])):
psi.append((np.transpose(np.matrix(V[:,i]))))
E, psi = sort_together([E, psi])
return E, psi
def sort_population_by_grade(self):
fitenss_population = [
self.fitness(individual) for individual in self.population
]
fitenss_population, population = sort_together(
[fitenss_population, self.population])
self.keep_if_best_result(fitenss_population[0], population[0])
self.population = population
def select_sa(self, dp):
sas = list(partitions(dp))
sas_scores = self.sa_scorer(sas)
sas_scores, sas = sort_together([sas_scores, sas], reverse=True)
self.logger.debug('Sentence Aggregation: {}'.format('\n'.join(f'{score:.3f} -> {sa}' for score, sa in zip(sas_scores, sas))))
return [[tuple(sa_part) for sa_part in sa] for sa in sas[:self.max_sa]]
def select_dp(self, entry):
dps = list(permutations(entry.triples))
dps_scores = self.dp_scorer(dps)
dps_scores, dps = sort_together([dps_scores, dps], reverse=True)
self.logger.debug('Discourse Planning: {}'.format('\n'.join(f'{score:.3f} -> {dp}' for score, dp in zip(dps_scores, dps))))
return dps[:self.max_dp]
def _sortBoxesByArea(boxes):
ratios = []
boxesByArea = []
for box in boxes:
ratios.append(box.ratio_of_image)
boxesByAreaSmToLg = sort_together([ratios, boxes])[1]
boxesByAreaLgToSm = list(reversed(boxesByAreaSmToLg))
return boxesByAreaLgToSm
def evolve(self):
population = self.population
count = self.d.p_size * 2
# Sort the population by fitness
fitenss_population = [
self.fitness(individual) for individual in population
]
fitenss_population, population = sort_together(
[fitenss_population, population])
self.keep_if_best_result(fitenss_population[0], population[0])
# Find the max fitness and invert all values to find percentages
max_value = fitenss_population[-1]
fitenss_population = [(max_value + 1 - v) for v in fitenss_population]
# Generate new positions
rand = np.random.uniform(0, sum(fitenss_population), count)
rand.sort()
# Generate new positions by cumulative addition of the weights and if
# the random number is withing individual range, the individual is added
# for reproduction. IMPORTANT: random numbers must be sorted
reproduction = []
cc = 0
i = 0
for j in range(len(fitenss_population)):
cc += fitenss_population[j]
while i < len(rand) and cc >= rand[i]:
reproduction.append(population[j])
i += 1
# Create new generation from new parents
random.shuffle(reproduction)
new_population = []
for i in range(0, count, 2):
male = reproduction[i]
female = reproduction[i + 1]
child = []
for i in range(len(male)):
min_ = min([male[i], female[i]])
max_ = max([female[i], male[i]])
v = np.random.uniform(min_, max_)
child.append(v)
new_population.append(child)
# Mutate
new_population = self.mutate_population(new_population, self.d.mutate,
self.d.v_min, self.d.v_max)
self.population = new_population
def ask_question(self, Question):
tokenize_question = self.tokenizer.texts_to_sequences([Question])
tokenize_question = pad_sequences(tokenize_question,
self.max_question_length)
result = self.model.predict(tokenize_question)
max_position = result[0].argmax()
list_column = list(User.Y.keys())
list_result = result.tolist()[0]
print(sort_together([list_result, list_column])[1])
return [list_column, list_result]
def evolve(self):
# pop = population
pop = self.population
# Sort population by fitness
fitenss_pop = [self.fitness(individual) for individual in pop]
fitenss_pop, pop = sort_together([fitenss_pop, pop])
self.keep_if_best_result(fitenss_pop[0], pop[0])
# Generate random values in range of 0 to total sum of all weights
ps = self.d.p_size
count = ps * 2
total_sum = (ps * (ps + 1)) / 2
rand = np.random.randint(0, total_sum, count)
# Generate new positions by cumulative addition of the weights and if
# the random number is withing individual range, the individual is added
# for reproduction. IMPORTANT: random numbers must be sorted
rand.sort()
reproduction = []
cc = 0
i = 0
for j in range(ps):
cc += (ps - j)
while i < len(rand) and cc >= rand[i]:
reproduction.append(pop[j])
i += 1
# Create offsprings
random.shuffle(reproduction)
new_population = []
for i in range(0, count, 2):
male = reproduction[i]
female = reproduction[i + 1]
child = []
for i in range(len(male)):
min_ = min([male[i], female[i]])
max_ = max([female[i], male[i]])
v = np.random.uniform(min_, max_)
child.append(v)
new_population.append(child)
# Mutate
new_population = self.mutate_population(new_population, self.d.mutate,
self.d.v_min, self.d.v_max)
self.population = new_population
def Cluster(G):
print(len(G))
from more_itertools import sort_together
d = Distribution(G)
x = [item[0] for item in d.items()]
y = [item[1] for item in d.items()]
sum_y = sum(y)
y_d = [i / sum_y for i in y]
x, y_d = sort_together([x, y_d])
c_results = ([x, y, y_d])
plt.plot(c_results[0], c_results[2], 'bo')
plt.show()
plt.hist(c_results[0], density=True)
plt.show()
def acc_markov(scorer, test_dp_db):
from more_itertools import sort_together
from itertools import permutations
tp = 0
for ts, sorteds_ts in test_dp_db:
all_perms = list(permutations(ts))
scores = scorer(all_perms)
best_score_perm = sort_together([scores, all_perms],
reverse=True)[1][0]
if best_score_perm in sorteds_ts:
tp += 1
return tp / len(test_dp_db)
def n_lowest_score(self,
n,
networks,
score_history_list,
network_history_list,
verbose=False):
networks_sorted = networks.copy()
scores = []
for network in networks:
score_ = self.score(network, verbose=verbose)
scores.append(score_)
x, y = sort_together([scores, networks_sorted])
score_history_list += list(x)
network_history_list += list(y)
y = list(y[:n])
return y
def order(b_base):
b = b_base[0]
truth = []
sumtrue = []
elements = []
n = b_base[0]
for i in range(len(b_base)):
b = And(b, b_base[i])
b = list(b.atoms())
table = list(itertools.product([False, True], repeat=len(b)))
for i in range(len(b_base)):
truth.append([])
for j in range(len(table)):
if i == 0:
elements.append([])
for k in range(len(table[j])):
if k == 0:
n = b_base[i].subs(b[k], table[j][k])
else:
n = n.subs(b[k], table[j][k])
if i == 0:
if table[j][k] == False:
elements[j].append(Not(b[k]))
else:
elements[j].append(b[k])
truth[i].append(n)
if i == len(b_base) - 1:
for i, k in enumerate(truth):
for j, item in enumerate(k):
if truth[i][j] == True:
truth[i][j] = 1
else:
truth[i][j] = 0
for z in zip(*truth):
sumtrue.append(sum(z))
m = b + table
Z = sort_together([sumtrue, elements])[1]
Z = Z[::-1]
return Z, sorted(sumtrue, reverse=True)
def topListByNumberOfMessages(messagesList, nBest):
#create a dict with all members
membersDict = {}
for message in messagesList:
member = message['member']
if member not in membersDict.keys():
membersDict[member] = 1
else:
membersDict[member] += 1
#convert to lists
members, numMessages = zip(*membersDict.items())
numMessages, members = sort_together([numMessages, members], reverse=True)
nBest = min(nBest, len(members))
print("--------------------------------")
print("Top %d by number of messages:"%nBest)
print("--------------------------------")
for i in range(nBest):
print('%s: %d messages'%(members[i], numMessages[i]))
print("Average of all members: %f messages"%np.mean(numMessages))
def topListByNumberOfCharacters(messagesList, nBest):
#create a dict with all members
membersDict = {}
for message in messagesList:
member = message['member']
text = message['text']
if member not in membersDict.keys():
membersDict[member] = len(text)
else:
membersDict[member] += len(text)
#convert to lists
members, numChars = zip(*membersDict.items())
numChars, members = sort_together([numChars, members], reverse=True)
nBest = min(nBest, len(members))
print("--------------------------------------------")
print("Top %d by total number of typed characters:"%nBest)
print("--------------------------------------------")
for i in range(nBest):
print('%s: %d characters'%(members[i], numChars[i]))
print("Average of all members: %f characters"%np.mean(numChars))
def acc_ltr(test_dp_db):
from pretrained_models import ltr_lasso_dp_scorer
from more_itertools import sort_together
from itertools import permutations
m = ltr_lasso_dp_scorer(('train', 'dev'))
tp = 0
for ts, sorteds_ts in test_dp_db:
all_perms = list(permutations(ts))
scores = m(all_perms, len(ts))
best_score_perm = sort_together([scores, all_perms],
reverse=True)[1][0]
if best_score_perm in sorteds_ts:
tp += 1
return tp / len(test_dp_db)
def image_pass_places(input_img):
global model_places, graph_places, places_labels
(latitude, longitude, date, time) = exif_extract_information(input_img)
img_target_size = (224, 224)
img = prepare_image(input_img, img_target_size)
with graph_places.as_default():
features = model_places.predict(img)
cat_list = []
prob_list = []
for x in range(0, 162):
cat_list.append(places_labels[x])
prob_list.append(features[0, x])
sorted_list = sort_together([prob_list, cat_list], reverse=True)
prob_list = sorted_list[0]
cat_list = sorted_list[1]
top_5_cat = cat_list[0:5]
jsonString = []
for x in top_5_cat:
x = re.sub(r'[^a-zA-Z ]+', ' ', x)
x = x.title()
jsonString.append(x)
return_dict = \
{
"tags": jsonString,
"latitude": latitude,
"longitude": longitude,
"date": date,
"time": time,
}
return return_dict
def topListWords(messagesList, wordsToRemove, nBest):
#create a dict with all members
wordsDict = {}
for message in messagesList:
text = message['text']
words = text.split()
for word in words:
word = word.lower()
if word not in wordsToRemove:
if word not in wordsDict.keys():
wordsDict[word] = 1
else:
wordsDict[word] += 1
#convert to lists
words, occurrences = zip(*wordsDict.items())
occurrences, words = sort_together([occurrences, words], reverse=True)
nBest = min(nBest, len(words))
print("-------------------------")
print("Top %d used words:"%nBest)
print("-------------------------")
for i in range(nBest):
print('%s: %d occurrences'%(words[i], occurrences[i]))
def create_mating_pool(fitness, population, to_choose):
cumulative_fitness = sum(fitness)
fitness = list(map(lambda x: x / cumulative_fitness, fitness))
fitness, population = sort_together([fitness, population], reverse=True)
mating_pool = []
indexes = []
for i in range(to_choose):
p = np.random.uniform(0, 1)
for j in range(len(fitness)):
if sum(fitness[:j + 1]) >= p:
if j not in indexes:
mating_pool.append(population[j])
indexes.append(j)
while len(mating_pool) <= i:
index_to_append = random.randint(0, len(fitness) - 1)
if index_to_append not in indexes:
indexes.append(index_to_append)
mating_pool.append(population[index_to_append])
return mating_pool
def get_state(model, device, pool_dataset, train_dataset):
pool_embeddings = get_model_embeddings(model, device, pool_dataset)
pool_predictions = get_model_predictions(model, device, pool_dataset)
train_embeddings = get_model_embeddings(model, device, train_dataset)
train_predictions = get_model_predictions(model, device, train_dataset)
if prop.ADD_GRADIENT_EMBEDDING:
gradient_embeddings = get_model_gradient_embeddings(
model, device, pool_dataset)
gradient_embeddings_flat = gradient_embeddings.flatten()
lab_emb = torch.mean(train_embeddings, dim=0)
if prop.ADD_POOL_MEAN_EMB:
pool_emb = torch.mean(pool_embeddings, dim=0)
train_label_statistics = torch.bincount(
train_dataset.tensors[1]).float() / len(train_dataset)
# train predictions statistics. if predictions missing, fill up list with 0
train_pred_lab_unique_cnts = np.unique(train_predictions,
return_counts=True)
for i in range(0, prop.NUM_CLASSES):
if i not in train_pred_lab_unique_cnts[0]:
new_tuple = (np.concatenate((train_pred_lab_unique_cnts[0], [i])),
np.concatenate((train_pred_lab_unique_cnts[1], [0])))
train_pred_lab_unique_cnts = new_tuple
sorted_uniques_cnts = sort_together(
[train_pred_lab_unique_cnts[0], train_pred_lab_unique_cnts[1]])
train_pred_label_statistics = torch.Tensor(sorted_uniques_cnts[1] /
sum(sorted_uniques_cnts[1]))
state = []
if prop.ADD_POOL_MEAN_EMB:
for ind, sample_emb in enumerate(pool_embeddings):
state.append(
torch.cat([
lab_emb, pool_emb, sample_emb, train_label_statistics,
train_pred_label_statistics,
get_one_hot(pool_predictions[ind])
]))
if prop.ADD_GRADIENT_EMBEDDING and not prop.ADD_PREDICTIONS and not prop.ADD_POOL_MEAN_EMB and prop.ARBITRARY_CLASSES:
for ind, sample_emb in enumerate(pool_embeddings):
state.append(torch.cat([lab_emb, gradient_embeddings[ind]]))
if prop.ADD_GRADIENT_EMBEDDING and prop.ADD_PREDICTIONS and not prop.ADD_POOL_MEAN_EMB and not prop.ARBITRARY_CLASSES:
for ind, sample_emb in enumerate(pool_embeddings):
state.append(
torch.cat([
lab_emb, sample_emb, train_label_statistics,
train_pred_label_statistics, gradient_embeddings[ind],
get_one_hot(pool_predictions[ind])
]))
if prop.ADD_GRADIENT_EMBEDDING and not prop.ADD_PREDICTIONS and not prop.ADD_POOL_MEAN_EMB and not prop.ARBITRARY_CLASSES:
for ind, sample_emb in enumerate(pool_embeddings):
state.append(
torch.cat([
lab_emb, train_label_statistics,
train_pred_label_statistics, gradient_embeddings[ind]
]))
if prop.ADD_PREDICTIONS and not prop.ADD_GRADIENT_EMBEDDING and not prop.ADD_POOL_MEAN_EMB and not prop.ARBITRARY_CLASSES:
for ind, sample_emb in enumerate(pool_embeddings):
state.append(
torch.cat([
lab_emb, sample_emb, train_label_statistics,
train_pred_label_statistics,
get_one_hot(pool_predictions[ind])
]))
return torch.stack(state)
with open(filename, 'w') as f:
json.dump(data, f, indent=4)
def loadJSON(filename):
with open(filename, "r") as json_file:
return json.load(json_file)
data = loadJSON("afgekeurde_voortuigen.json")
#gebrekken = loadJSON("gebrekken.json")
name = []
length = []
for a in data:
if len(data[a]) > 200:
name.append(a)
length.append(len(data[a]))
values = sort_together([length, name], reverse=True)
x = np.arange(len(values[0]))
fig, ax = plt.subplots()
ax.bar(values[1], values[0])
ax.set_xticks(x)
ax.set_xticklabels(values[1], rotation=(45))
plt.show()
h = 18
F_H = 0.0981
M_nosum.append(-F_H * h)
xtab.append(h)
D = 4.2
gamma = ma.atan(d / D)
F_brz = ((Mtot - e * Fe - h * F_H) / d)
Ay = -F_brz / ma.tan(gamma)
Az = Fe - F - H + F_brz - L
#%%-----------------Adding point moments to moment diagram------------------------
from more_itertools import sort_together
sort = sort_together([xtab, M_nosum]) #sorting points together by location
xtab_sorted = sort[0]
Mtab_sorted = sort[1]
xtab_fin = []
Mtab_fin = []
Mtot2 = 0
for i in range(0, len(xtab_sorted)):
Mtot2 = Mtot2 + Mtab_sorted[i]
Mtab_fin.append(Mtot2)
#print(Mtab_sorted)
#print(Mtab_fin)
plt.plot(xtab_sorted, Mtab_fin)
# A_V values
av = np.linspace(0, 0.5, 20)
# filter files
filtfiles = glob.glob(iodir + "data/filters/*")
filtfiles.sort()
# limb darkening files from Castelli and Kurucz
ldfiles = glob.glob(iodir + "data/limbdark/*.pck*")
# metallicities of these files
Z = []
for f in ldfiles:
zstr = os.path.basename(f)[1:4].replace('m', '-').replace('p', '')
Z.append(float(zstr[:-1] + '.' + zstr[-1]))
Z, ldfiles = sort_together([Z, ldfiles])
lam, _, _ = ld.getlamTg(ldfiles[0]) # wavelengths in the Kurucz files
ut.setlam(lam) # set the wavelength array for filtering and reddening
ldlist = [] # list of limbdarkening objects
for i in tqdm(range(len(ldfiles))):
lam, T, g = ld.getlamTg(ldfiles[i])
# get the intensity array from this file
I = ld.getI(ldfiles[i], lam, T, g)
# initialize a limb darkening object
l = ld.LimbDark(lam, T, g, Z[i])
# filter and redden the object, return band x reddening intensities
Ib = l.filter(I, filtfiles, a_v=av)
# fit polynomials for I(mu)
l.fit(Ib, bounds)
