def candiate_urls():
for (item, property, id) in page_statements(page_title):
if property != "P9586":
continue
if not id or id in skip_ids:
continue
url = "https://tv.apple.com/us/movie/{}".format(id)
yield (url, id)
for url in shuffled(appletv.fetch_new_sitemap_urls())[0:250]:
(type, id) = parseurl(url)
if type != "movie":
continue
if not id or id in skip_ids:
continue
yield (url, id)
for index_url in shuffled(appletv.fetch_sitemap_index_urls())[0:250]:
for url in shuffled(appletv.fetch_sitemap_index(index_url)):
(type, id) = parseurl(url)
if type != "movie":
continue
if not id or id in skip_ids:
continue
yield (url, id)
def random(cls, elizo):
""" generate random element from available.
Generate useful element, depending on drop_useless_chance
` elizo - elements in zone, list of Element instances
"""
Logger.debug("Element.random: elizo=%s available_elnames=%s", elizo,
cls.available_elnames)
all_elnames = [x.elname for x in elizo]
green_elnames = [x.elname for x in elizo if x.activated]
white_elnames = [x.elname for x in elizo if not x.activated]
# first - check if we can just drop enything
if random() < defs.drop_useless_chance:
elname = choice(list(cls.available_elnames))
Logger.debug("elements: appear pure random element")
return Element(elname)
Logger.debug("second")
# try to drop element E (which is not in zone) which combined with GREEN elements in zone will give
# element R which is new
for x in shuffled(set(cls.available_elnames) - set(white_elnames)):
#iterate over all availables except those which lay just by wizard
# (to not duplicate them)
if cls.is_useful(x,
with_elnames=green_elnames,
avoid=cls.available_elnames):
return Element(elname=x)
Logger.debug("third")
# try to drop element E (which is not in zone) which combined with ANY elements in zone will give
# element R which is new
for x in shuffled(set(cls.available_elnames) - set(white_elnames)):
#iterate over all availables except those which lay just by wizard
if cls.is_useful(x,
with_elnames=all_elnames,
avoid=cls.available_elnames):
return Element(elname=x)
# Nothing useful, drop random
ret = choice(list(cls.available_elnames))
Logger.debug("fourth nothing useful, drop pure random(%s)", ret)
return Element(ret)
def _get_elements_weighted_random(self):
"""
Elements are shuffled semi-randomly.
The `position` attribute of each element act as a weight for the randomization.
Elements that are "heavier" are more likely to be at the beginning of the list.
"""
return shuffled(self._get_elements(), weight=lambda e: e.position)
def init_population(pop_number, gene_pool, state_length):
""" initialize population """
population = []
for i in range(pop_number):
population.append(utils.shuffled(gene_pool))
return population
def init_population(pop_number, gene_pool, state_length):
"""initialize population"""
population = []
for i in range(pop_number):
population.append(utils.shuffled(gene_pool))
return population
def _get_elements_cluster_random(self):
"""
Elements are grouped according to their `position` attribute.
Each group is then shuffled randomly.
"""
qs = self.elements.order_by("position")
grouped = groupby(qs, key=lambda e: e.position)
return chain.from_iterable(shuffled(elements) for _, elements in grouped)
def _get_elements_cluster_random(self):
"""
Elements are grouped according to their `position` attribute.
Each group is then shuffled randomly.
"""
qs = self.elements.order_by('position')
grouped = groupby(qs, key=lambda e: e.position)
return chain.from_iterable(shuffled(elements) for _, elements in grouped)
def run(self):
right = cards = total = 0
while True:
cards = cards or shuffled(copy(self.cards))
percent = (right / total * 100.0) if total else 0
stat = status % (right, total, percent)
right += int(cards.pop().draw(stat))
total += 1
def run(self):
right = cards = total = 0
while True:
cards = cards or shuffled(copy(self.cards))
percent = (right/total*100.0) if total else 0
stat = status % (right, total, percent)
right += int( cards.pop().draw(stat) )
total += 1
def open_files(self,target_file,input_file,index_file):
input = h5py.File(input_file,'r')
target = np.load(target_file)
index_data = np.load(index_file)
if self.istrain:
index_data = shuffled([a for a in index_data if a[1] not in self.val_list and a[1]!=0])
else:
index_data = [a for a in index_data if a[1] in self.val_list and a[1]!=0]
if self.calc_vid not None:
index_data = [a for a in index_data if a[1]==self.calc_vid]
def open_files(self, target_file, index_file):
target = np.load(target_file)
index_data = np.load(index_file)
if self.istrain:
index_data = shuffled(
[a for a in index_data if a[1] not in self.val_list and a[1] != 0])
else:
index_data = [a for a in index_data if a[1]
in self.val_list and a[1] != 0]
print len(index_data)
return index_data, target
def evolve():
population = [rand_term_guesser(2) for i in range(POPSIZE)]
for iteration in itertools.count(0):
sequents = exp2.random_statements(max_depth=3,
allow_dups=True,
num_stmts=NUMBER_SEQUENTS,
connectives=None)
fitnesses = [calc_fitness(tg, sequents) for tg in population]
crossover_base = [tournament(population, fitnesses) for i in range(2 * POPSIZE)]
indices = pairs(utils.shuffled(range(2 * POPSIZE)))
population = [x for i, j in indices for x in crossover(crossover_base[i], crossover_base[j])]
# print(population)
for i in range(POPSIZE):
population[i] = mutate(population[i])
print('POPULATION:', [string(x) for x in population])
print('FITNESSES:', fitnesses)
print('CROSSOVERB:', [string(x) for x in crossover_base])
print('INDICES:', indices)
def find_start_bag(td: TreeDecomposition,
history: Counter = None,
debug=False):
if USING_COMPLEXITY_WIDTH: # pick bag with highest complexity
complexities = compute_complexities(td, DOMAIN_SIZES)
if CW_TRAV_STRAT == "max":
bag_order = sorted(complexities,
key=complexities.get,
reverse=True)
elif CW_TRAV_STRAT == "max-min":
bag_order = sorted(complexities,
key=complexities.get,
reverse=not CW_TARGET_REACHED)
else: # max-rand or tw-max-rand
if CW_TARGET_REACHED:
bag_order = shuffled(complexities.keys())
else:
bag_order = sorted(complexities,
key=complexities.get,
reverse=True)
mincount = min(history[bag_id] for bag_id in bag_order)
for bag_id in bag_order:
if history[bag_id] == mincount:
return bag_id
# maxbagidx, _ = max(complexities.items(), key=itemgetter(1))
# if history[maxbagidx] == 0:
# return maxbagidx
# else: # cw target already met, return random bag
# print("randomly picking bag:", end="")
# return pick(td.bags.keys())
elif TRAV_STRAT == "random": # randomly pick a bag
return pick(td.bags.keys())
else: # pick bag with least count and earliest in order
if TRAV_STRAT == "post":
trav_order = list(nx.dfs_postorder_nodes(td.decomp))
elif TRAV_STRAT == "pre":
trav_order = list(nx.dfs_postorder_nodes(td.decomp))
else:
raise ValueError(f"invalid traversal strategy {TRAV_STRAT}")
mincount = min(history.values())
# todo[opt]: avoid retraversing every time
for bag_id in trav_order:
if history[bag_id] == mincount:
return bag_id
def train_als(self, X_data, optimizer, epochs=1000):
"""
Use alt. least-squares to find the CP/Tucker decomposition of tensor `X`.
"""
X_var = tf.Variable(X_data)
loss_op, train_ops = self.get_train_ops(X_var, optimizer)
init_op = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init_op)
for e in trange(epochs):
for alt, train_op in enumerate(shuffled(train_ops)):
_, loss = sess.run([train_op, loss_op],
feed_dict={X_var: X_data})
_log.debug('[%3d:%3d] loss: %.5f' % (e, alt, loss))
print 'final loss: %.5f' % loss
return sess.run(self.X)
def train_als_early(self,
X_data,
optimizer,
epochs=1000,
stop_freq=50,
stop_thresh=1e-10):
"""
ALS with early stopping.
"""
X_var = tf.Variable(X_data)
loss_op, train_ops = self.get_train_ops(X_var, optimizer)
fit_prev = 0.0
init_op = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init_op)
for e in trange(epochs):
for alt, train_op in enumerate(shuffled(train_ops)):
_, loss = sess.run([train_op, loss_op],
feed_dict={X_var: X_data})
_log.debug('[%3d:%3d] loss: %.5f' % (e, alt, loss))
if e % stop_freq:
X_predict = sess.run(self.X)
fit = get_fit(X_data, X_predict)
fit_change = abs(fit - fit_prev)
if fit_change < stop_thresh and e > 0:
print 'Stopping early, fit_change: %.10f' % (
fit_change)
break
fit_prev = fit
print 'final loss: %.5f\nfinal fit %.5f' % (loss, fit)
return sess.run(self.X)
def renew_integrity(self, scale: list):
''' checks the duration of the bar and adds or remove notes if the duration do not match '''
# calculate total duration
total_duration = 0
for note in self.notes:
total_duration += note.duration
duration_delta = self.duration - total_duration
while duration_delta != 0:
for note in shuffled(self.notes):
if duration_delta < 0: # we try to reduce bar's duration
# By removing notes
self.notes.remove(note)
duration_delta += note.duration
else: # we try to increse bar's duration
# By adding notes
new_duration = None
for duration in NOTE_DURATIONS:
if duration == duration_delta:
new_duration = duration_delta
break
if not new_duration:
new_duration = random.choice(NOTE_DURATIONS)
new_tone = random.choice(scale) # this could be better, using a relative tone
new_note = Note(new_tone, new_duration)
self.notes.append(new_note)
duration_delta -= new_duration
def hooi(self, X_data, epochs=100):
"""
HOOI: Higher-Order Orthogonal Iteration
"""
X_var = tf.Variable(X_data)
init_op = tf.global_variables_initializer()
svd_ops = [None] * self.order
with tf.Session() as sess:
sess.run(init_op)
for e in trange(epochs):
# Set up orthogonal iteration operators
for n in range(self.order):
Y = self.get_ortho_iter(X_var, n)
_, u, _ = tf.svd(unfold_tf(Y, n), name='svd%3d' % n)
svd_ops[n] = tf.assign(self.U[n], u[:, :self.ranks[n]])
for n in shuffled(trange(self.order)):
sess.run([svd_ops[n]], feed_dict={X_var: X_data})
X_predict = sess.run(self.X)
fit = get_fit(X_data, X_predict)
_log.debug('[%3d-U%3d] fit: %.5f' % (e, n, fit))
# Compute new core tensor value G
core_op = self.get_core_op(X_var)
sess.run([core_op], feed_dict={X_var: X_data})
# Log final fit
X_predict = sess.run(self.X)
fit = get_fit(X_data, X_predict)
_log.debug('[G] fit: %.5f' % fit)
return X_predict
def hosvd(self, X_data):
"""
HOSVD
"""
X_var = tf.Variable(X_data)
init_op = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init_op)
for n in shuffled(trange(self.order)):
_, u, _ = tf.svd(unfold_tf(X_var, n), name='svd%3d' % n)
# Set U[n] to the first ranks[n] left-singular values of X
new_U = tf.transpose(u[:self.ranks[n]])
svd_op = tf.assign(self.U[n], new_U)
# Run SVD and assign new variable U[n]
sess.run([svd_op], feed_dict={X_var: X_data})
# Log fit after training nth component
X_predict = sess.run(self.X)
fit = get_fit(X_data, X_predict)
_log.debug('[U%3d] fit: %.5f' % (n, fit))
# Compute new core tensor value G
core_op = self.get_core_op(X_var)
sess.run([core_op], feed_dict={X_var: X_data})
# Log final fit
X_predict = sess.run(self.X)
fit = get_fit(X_data, X_predict)
_log.debug('[G] fit: %.5f' % fit)
return X_predict
def col(): return shuffled([cmp(), cmp(), 0]) # Random color for cmap
cmap = ColorMap(col(), col())
def _get_elements_random(self):
"""
Elements are simply shuffled randomly.
"""
return shuffled(self.elements.all())
def _get_elements_random(self):
"""
Elements are simply shuffled randomly.
"""
return shuffled(self.elements.all())
def _to_db(self, database):
'''Common method to export data for all the relational databases using DB-API'''
cursor = database.cursor()
logging.info('creating the associations table ...')
cursor.execute('CREATE TABLE associations (%s)' % ', '.join('group_%i INTEGER' % i for i in xrange(len(self._fragments))))
cursor.execute('CREATE UNIQUE INDEX i_associations ON associations (%s)' % ', '.join('group_%i' % i for i in xrange(len(self._fragments))))
cursor.executemany('INSERT INTO associations VALUES (%s)' % placeholders(len(self._fragments)), ifilter(None, (self._associations.values())))
# create the schema table that will be used by the querier to know where the attributes are
cursor.execute('CREATE TABLE schema (attribute TEXT PRIMARY KEY, typ TEXT, fragment INTEGER)')
for fragment_id, fragment in enumerate(self._fragments):
# select will extract only the data included in the fragment
select = itemgetter(*fragment)
attributes = select(self._table.attributes)
data = (select(self._table[row]) + (group,) for group, rows in self._associations.get_groups(fragment_id).items() for row in shuffled(rows))
logging.info('creating the fragment_%i table ...' % fragment_id)
cursor.execute('CREATE TABLE fragment_{0} ({1}, group_{0} INTEGER)'.format(fragment_id, ', '.join('%s %s' % attr for attr in attributes)))
cursor.execute('CREATE INDEX i_fragment_{0} ON fragment_{0} (group_{0})'.format(fragment_id))
cursor.executemany('INSERT INTO fragment_%i VALUES (%s)' % (fragment_id, placeholders(len(fragment) + 1)), data)
cursor.executemany('INSERT INTO schema VALUES (?, ?, ?)', ((attr[0], attr[1], fragment_id) for attr in attributes))
database.commit()
return database
