def candiate_urls(): for (item, property, id) in page_statements(page_title): if property != "P9586": continue if not id or id in skip_ids: continue url = "https://tv.apple.com/us/movie/{}".format(id) yield (url, id) for url in shuffled(appletv.fetch_new_sitemap_urls())[0:250]: (type, id) = parseurl(url) if type != "movie": continue if not id or id in skip_ids: continue yield (url, id) for index_url in shuffled(appletv.fetch_sitemap_index_urls())[0:250]: for url in shuffled(appletv.fetch_sitemap_index(index_url)): (type, id) = parseurl(url) if type != "movie": continue if not id or id in skip_ids: continue yield (url, id)
def random(cls, elizo): """ generate random element from available. Generate useful element, depending on drop_useless_chance ` elizo - elements in zone, list of Element instances """ Logger.debug("Element.random: elizo=%s available_elnames=%s", elizo, cls.available_elnames) all_elnames = [x.elname for x in elizo] green_elnames = [x.elname for x in elizo if x.activated] white_elnames = [x.elname for x in elizo if not x.activated] # first - check if we can just drop enything if random() < defs.drop_useless_chance: elname = choice(list(cls.available_elnames)) Logger.debug("elements: appear pure random element") return Element(elname) Logger.debug("second") # try to drop element E (which is not in zone) which combined with GREEN elements in zone will give # element R which is new for x in shuffled(set(cls.available_elnames) - set(white_elnames)): #iterate over all availables except those which lay just by wizard # (to not duplicate them) if cls.is_useful(x, with_elnames=green_elnames, avoid=cls.available_elnames): return Element(elname=x) Logger.debug("third") # try to drop element E (which is not in zone) which combined with ANY elements in zone will give # element R which is new for x in shuffled(set(cls.available_elnames) - set(white_elnames)): #iterate over all availables except those which lay just by wizard if cls.is_useful(x, with_elnames=all_elnames, avoid=cls.available_elnames): return Element(elname=x) # Nothing useful, drop random ret = choice(list(cls.available_elnames)) Logger.debug("fourth nothing useful, drop pure random(%s)", ret) return Element(ret)
def _get_elements_weighted_random(self): """ Elements are shuffled semi-randomly. The `position` attribute of each element act as a weight for the randomization. Elements that are "heavier" are more likely to be at the beginning of the list. """ return shuffled(self._get_elements(), weight=lambda e: e.position)
def init_population(pop_number, gene_pool, state_length): """ initialize population """ population = [] for i in range(pop_number): population.append(utils.shuffled(gene_pool)) return population
def init_population(pop_number, gene_pool, state_length): """initialize population""" population = [] for i in range(pop_number): population.append(utils.shuffled(gene_pool)) return population
def _get_elements_cluster_random(self): """ Elements are grouped according to their `position` attribute. Each group is then shuffled randomly. """ qs = self.elements.order_by("position") grouped = groupby(qs, key=lambda e: e.position) return chain.from_iterable(shuffled(elements) for _, elements in grouped)
def _get_elements_cluster_random(self): """ Elements are grouped according to their `position` attribute. Each group is then shuffled randomly. """ qs = self.elements.order_by('position') grouped = groupby(qs, key=lambda e: e.position) return chain.from_iterable(shuffled(elements) for _, elements in grouped)
def run(self): right = cards = total = 0 while True: cards = cards or shuffled(copy(self.cards)) percent = (right / total * 100.0) if total else 0 stat = status % (right, total, percent) right += int(cards.pop().draw(stat)) total += 1
def run(self): right = cards = total = 0 while True: cards = cards or shuffled(copy(self.cards)) percent = (right/total*100.0) if total else 0 stat = status % (right, total, percent) right += int( cards.pop().draw(stat) ) total += 1
def open_files(self,target_file,input_file,index_file): input = h5py.File(input_file,'r') target = np.load(target_file) index_data = np.load(index_file) if self.istrain: index_data = shuffled([a for a in index_data if a[1] not in self.val_list and a[1]!=0]) else: index_data = [a for a in index_data if a[1] in self.val_list and a[1]!=0] if self.calc_vid not None: index_data = [a for a in index_data if a[1]==self.calc_vid]
def open_files(self, target_file, index_file): target = np.load(target_file) index_data = np.load(index_file) if self.istrain: index_data = shuffled( [a for a in index_data if a[1] not in self.val_list and a[1] != 0]) else: index_data = [a for a in index_data if a[1] in self.val_list and a[1] != 0] print len(index_data) return index_data, target
def evolve(): population = [rand_term_guesser(2) for i in range(POPSIZE)] for iteration in itertools.count(0): sequents = exp2.random_statements(max_depth=3, allow_dups=True, num_stmts=NUMBER_SEQUENTS, connectives=None) fitnesses = [calc_fitness(tg, sequents) for tg in population] crossover_base = [tournament(population, fitnesses) for i in range(2 * POPSIZE)] indices = pairs(utils.shuffled(range(2 * POPSIZE))) population = [x for i, j in indices for x in crossover(crossover_base[i], crossover_base[j])] # print(population) for i in range(POPSIZE): population[i] = mutate(population[i]) print('POPULATION:', [string(x) for x in population]) print('FITNESSES:', fitnesses) print('CROSSOVERB:', [string(x) for x in crossover_base]) print('INDICES:', indices)
def find_start_bag(td: TreeDecomposition, history: Counter = None, debug=False): if USING_COMPLEXITY_WIDTH: # pick bag with highest complexity complexities = compute_complexities(td, DOMAIN_SIZES) if CW_TRAV_STRAT == "max": bag_order = sorted(complexities, key=complexities.get, reverse=True) elif CW_TRAV_STRAT == "max-min": bag_order = sorted(complexities, key=complexities.get, reverse=not CW_TARGET_REACHED) else: # max-rand or tw-max-rand if CW_TARGET_REACHED: bag_order = shuffled(complexities.keys()) else: bag_order = sorted(complexities, key=complexities.get, reverse=True) mincount = min(history[bag_id] for bag_id in bag_order) for bag_id in bag_order: if history[bag_id] == mincount: return bag_id # maxbagidx, _ = max(complexities.items(), key=itemgetter(1)) # if history[maxbagidx] == 0: # return maxbagidx # else: # cw target already met, return random bag # print("randomly picking bag:", end="") # return pick(td.bags.keys()) elif TRAV_STRAT == "random": # randomly pick a bag return pick(td.bags.keys()) else: # pick bag with least count and earliest in order if TRAV_STRAT == "post": trav_order = list(nx.dfs_postorder_nodes(td.decomp)) elif TRAV_STRAT == "pre": trav_order = list(nx.dfs_postorder_nodes(td.decomp)) else: raise ValueError(f"invalid traversal strategy {TRAV_STRAT}") mincount = min(history.values()) # todo[opt]: avoid retraversing every time for bag_id in trav_order: if history[bag_id] == mincount: return bag_id
def train_als(self, X_data, optimizer, epochs=1000): """ Use alt. least-squares to find the CP/Tucker decomposition of tensor `X`. """ X_var = tf.Variable(X_data) loss_op, train_ops = self.get_train_ops(X_var, optimizer) init_op = tf.global_variables_initializer() with tf.Session() as sess: sess.run(init_op) for e in trange(epochs): for alt, train_op in enumerate(shuffled(train_ops)): _, loss = sess.run([train_op, loss_op], feed_dict={X_var: X_data}) _log.debug('[%3d:%3d] loss: %.5f' % (e, alt, loss)) print 'final loss: %.5f' % loss return sess.run(self.X)
def train_als_early(self, X_data, optimizer, epochs=1000, stop_freq=50, stop_thresh=1e-10): """ ALS with early stopping. """ X_var = tf.Variable(X_data) loss_op, train_ops = self.get_train_ops(X_var, optimizer) fit_prev = 0.0 init_op = tf.global_variables_initializer() with tf.Session() as sess: sess.run(init_op) for e in trange(epochs): for alt, train_op in enumerate(shuffled(train_ops)): _, loss = sess.run([train_op, loss_op], feed_dict={X_var: X_data}) _log.debug('[%3d:%3d] loss: %.5f' % (e, alt, loss)) if e % stop_freq: X_predict = sess.run(self.X) fit = get_fit(X_data, X_predict) fit_change = abs(fit - fit_prev) if fit_change < stop_thresh and e > 0: print 'Stopping early, fit_change: %.10f' % ( fit_change) break fit_prev = fit print 'final loss: %.5f\nfinal fit %.5f' % (loss, fit) return sess.run(self.X)
def renew_integrity(self, scale: list): ''' checks the duration of the bar and adds or remove notes if the duration do not match ''' # calculate total duration total_duration = 0 for note in self.notes: total_duration += note.duration duration_delta = self.duration - total_duration while duration_delta != 0: for note in shuffled(self.notes): if duration_delta < 0: # we try to reduce bar's duration # By removing notes self.notes.remove(note) duration_delta += note.duration else: # we try to increse bar's duration # By adding notes new_duration = None for duration in NOTE_DURATIONS: if duration == duration_delta: new_duration = duration_delta break if not new_duration: new_duration = random.choice(NOTE_DURATIONS) new_tone = random.choice(scale) # this could be better, using a relative tone new_note = Note(new_tone, new_duration) self.notes.append(new_note) duration_delta -= new_duration
def hooi(self, X_data, epochs=100): """ HOOI: Higher-Order Orthogonal Iteration """ X_var = tf.Variable(X_data) init_op = tf.global_variables_initializer() svd_ops = [None] * self.order with tf.Session() as sess: sess.run(init_op) for e in trange(epochs): # Set up orthogonal iteration operators for n in range(self.order): Y = self.get_ortho_iter(X_var, n) _, u, _ = tf.svd(unfold_tf(Y, n), name='svd%3d' % n) svd_ops[n] = tf.assign(self.U[n], u[:, :self.ranks[n]]) for n in shuffled(trange(self.order)): sess.run([svd_ops[n]], feed_dict={X_var: X_data}) X_predict = sess.run(self.X) fit = get_fit(X_data, X_predict) _log.debug('[%3d-U%3d] fit: %.5f' % (e, n, fit)) # Compute new core tensor value G core_op = self.get_core_op(X_var) sess.run([core_op], feed_dict={X_var: X_data}) # Log final fit X_predict = sess.run(self.X) fit = get_fit(X_data, X_predict) _log.debug('[G] fit: %.5f' % fit) return X_predict
def hosvd(self, X_data): """ HOSVD """ X_var = tf.Variable(X_data) init_op = tf.global_variables_initializer() with tf.Session() as sess: sess.run(init_op) for n in shuffled(trange(self.order)): _, u, _ = tf.svd(unfold_tf(X_var, n), name='svd%3d' % n) # Set U[n] to the first ranks[n] left-singular values of X new_U = tf.transpose(u[:self.ranks[n]]) svd_op = tf.assign(self.U[n], new_U) # Run SVD and assign new variable U[n] sess.run([svd_op], feed_dict={X_var: X_data}) # Log fit after training nth component X_predict = sess.run(self.X) fit = get_fit(X_data, X_predict) _log.debug('[U%3d] fit: %.5f' % (n, fit)) # Compute new core tensor value G core_op = self.get_core_op(X_var) sess.run([core_op], feed_dict={X_var: X_data}) # Log final fit X_predict = sess.run(self.X) fit = get_fit(X_data, X_predict) _log.debug('[G] fit: %.5f' % fit) return X_predict
def col(): return shuffled([cmp(), cmp(), 0]) # Random color for cmap cmap = ColorMap(col(), col())
def _get_elements_random(self): """ Elements are simply shuffled randomly. """ return shuffled(self.elements.all())
def _to_db(self, database): '''Common method to export data for all the relational databases using DB-API''' cursor = database.cursor() logging.info('creating the associations table ...') cursor.execute('CREATE TABLE associations (%s)' % ', '.join('group_%i INTEGER' % i for i in xrange(len(self._fragments)))) cursor.execute('CREATE UNIQUE INDEX i_associations ON associations (%s)' % ', '.join('group_%i' % i for i in xrange(len(self._fragments)))) cursor.executemany('INSERT INTO associations VALUES (%s)' % placeholders(len(self._fragments)), ifilter(None, (self._associations.values()))) # create the schema table that will be used by the querier to know where the attributes are cursor.execute('CREATE TABLE schema (attribute TEXT PRIMARY KEY, typ TEXT, fragment INTEGER)') for fragment_id, fragment in enumerate(self._fragments): # select will extract only the data included in the fragment select = itemgetter(*fragment) attributes = select(self._table.attributes) data = (select(self._table[row]) + (group,) for group, rows in self._associations.get_groups(fragment_id).items() for row in shuffled(rows)) logging.info('creating the fragment_%i table ...' % fragment_id) cursor.execute('CREATE TABLE fragment_{0} ({1}, group_{0} INTEGER)'.format(fragment_id, ', '.join('%s %s' % attr for attr in attributes))) cursor.execute('CREATE INDEX i_fragment_{0} ON fragment_{0} (group_{0})'.format(fragment_id)) cursor.executemany('INSERT INTO fragment_%i VALUES (%s)' % (fragment_id, placeholders(len(fragment) + 1)), data) cursor.executemany('INSERT INTO schema VALUES (?, ?, ?)', ((attr[0], attr[1], fragment_id) for attr in attributes)) database.commit() return database