def evaluate_cosine_similarity(evaluated_sentences, reference_sentences): evaluated_words = tuple(chain(*(s.words for s in evaluated_sentences))) reference_words = tuple(chain(*(s.words for s in reference_sentences))) evaluated_model = TfDocumentModel(evaluated_words) reference_model = TfDocumentModel(reference_words) return cosine_similarity(evaluated_model, reference_model)
def present_value(prefix, arg): """Present a whole value that is either added or deleted. Calls out to other formatters for cells, outputs, and multiline strings. Uses pprint.pformat, otherwise. """ # TODO: improve pretty-print of arbitrary values? if isinstance(arg, dict): if 'cell_type' in arg: return present_cell(prefix, arg) elif 'output_type' in arg: return present_output(prefix, arg) elif isinstance(arg, list) and arg: first = arg[0] if isinstance(first, dict): if 'cell_type' in first: return chain(*[ present_cell(prefix + ' ', cell) for cell in arg ]) elif 'output_type' in first: return chain(*[ present_output(prefix + ' ', out) for out in arg ]) elif isinstance(arg, string_types): return present_multiline_string(prefix, arg) lines = pprint.pformat(arg).splitlines() return [prefix + line for line in lines]
def window(iter, pre_size=1, post_size=1): """ Given an iterable, return a new iterable which yields triples of (pre, item, post), where pre and post are the items preceeding and following the item (or None if no such item is appropriate). pre and post will always be pre_size and post_size in length. >>> example = window(range(10), pre_size=2) >>> pre, item, post = next(example) >>> pre (None, None) >>> post (1,) >>> next(example) ((None, 0), 1, (2,)) >>> list(example)[-1] ((7, 8), 9, (None,)) """ pre_iter, iter = itertools.tee(iter) pre_iter = itertools.chain((None,) * pre_size, pre_iter) pre_iter = nwise(pre_iter, pre_size) post_iter, iter = itertools.tee(iter) post_iter = itertools.chain(post_iter, (None,) * post_size) post_iter = nwise(post_iter, post_size) next(post_iter, None) return six.moves.zip(pre_iter, iter, post_iter)
def nsmallest(n, iterable, key=None): """Find the n smallest elements in a dataset. Equivalent to: sorted(iterable, key=key)[:n] """ # Short-cut for n==1 is to use min() when len(iterable)>0 if n == 1: it = iter(iterable) head = list(islice(it, 1)) if not head: return [] if key is None: return [min(chain(head, it))] return [min(chain(head, it), key=key)] # When n>=size, it's faster to use sorted() try: size = len(iterable) except (TypeError, AttributeError): pass else: if n >= size: return sorted(iterable, key=key)[:n] # When key is none, use simpler decoration if key is None: it = izip(iterable, count()) # decorate result = _nsmallest(n, it) return map(itemgetter(0), result) # undecorate # General case, slowest method in1, in2 = tee(iterable) it = izip(imap(key, in1), count(), in2) # decorate result = _nsmallest(n, it) return map(itemgetter(2), result) # undecorate
def translate_selection(self, x, y, x0, y0): x, y = 10*round(x/10), 10*round(y/10) # survey conditions for frame in self: for point in chain(frame[0], frame[1]): if point[2]: # do any of the points fall on another point? if not frame.can_fall(point[0] + x - x0, point[1] + y - y0): return for frame in self: for point in chain(frame[0], frame[1]): if point[2]: point[:2] = [point[0] + x - x0, point[1] + y - y0] # check y alignment if frame[0][0][1] != frame[1][0][1]: # determine which should move if frame[0][0][2]: flip = 1 else: flip = 0 frame[flip][0][1] = frame[not flip][0][1] if frame[0][-1][1] != frame[1][-1][1]: # determine which should move if frame[0][-1][2]: flip = 1 else: flip = 0 frame[flip][-1][1] = frame[not flip][-1][1]
def test_attr_label_matrix_and_data(self): w = self.widget # Don't run the MDS optimization to save time and to prevent the # widget be in a blocking state when trying to send the next signal w.start = Mock() # Data and matrix data = Table("zoo") dist = Euclidean(data) self.send_signal(w.Inputs.distances, dist) self.send_signal(w.Inputs.data, data) self.assertTrue(set(chain(data.domain.variables, data.domain.metas)) < set(w.controls.attr_label.model())) # Has data, but receives a signal without data: has to keep the label self.send_signal(w.Inputs.distances, None) self.assertTrue(set(chain(data.domain.variables, data.domain.metas)) < set(w.controls.attr_label.model())) # Has matrix without data, and loses the data: remove the label self.send_signal(w.Inputs.data, None) self.assertEqual(list(w.controls.attr_label.model()), [None]) # Has matrix without data, receives data: add attrs to combo, select self.send_signal(w.Inputs.data, data) self.assertTrue(set(chain(data.domain.variables, data.domain.metas)) < set(w.controls.attr_label.model()))
def dependencies(self, stack): """Return the Resource objects in given stack on which this depends.""" def path(section): return '.'.join([self.name, section]) def get_resource(res_name): if res_name not in stack: raise exception.InvalidTemplateReference(resource=res_name, key=self.name) return stack[res_name] def strict_func_deps(data, datapath): return six.moves.filter( lambda r: getattr(r, 'strict_dependency', True), function.dependencies(data, datapath)) explicit_depends = [] if self._depends is None else self._depends prop_deps = strict_func_deps(self._properties, path('Properties')) metadata_deps = strict_func_deps(self._metadata, path('Metadata')) # (ricolin) External resource should not depend on any other resources. # This operation is not allowed for now. if self.external_id(): if explicit_depends: raise exception.InvalidExternalResourceDependency( external_id=self.external_id(), resource_type=self.resource_type ) return itertools.chain() return itertools.chain((get_resource(dep) for dep in explicit_depends), prop_deps, metadata_deps)
def _write_serial(self, path, data, metadata): from mpi4py import MPI comm, rank, root = get_comm_rank_root() if rank != root: for tag, buf in enumerate(data): comm.Send(buf.copy(), root, tag) else: # Recv all of the non-local data MPI.Prequest.Startall(self._mpi_rreqs) MPI.Prequest.Waitall(self._mpi_rreqs) # Combine local and MPI data names = it.chain(self._loc_names, self._mpi_names) dats = it.chain(data, self._mpi_rbufs) # Convert any metadata to ASCII metadata = {k: np.array(v, dtype='S') for k, v in metadata.items()} # Create the output dictionary outdict = dict(zip(names, dats), **metadata) with h5py.File(path, 'w') as h5file: for k, v in outdict.items(): h5file[k] = v
def populate_more_bopomofo_index(): for (bopomofo, flags) in bopomofo_index: # populate hsu bopomofo index correct = bopomofo matches = itertools.chain(handle_rules(bopomofo, hsu_correct), handle_special_rules(bopomofo, hsu_correct_special)) for wrong in matches: newflags = '|'.join((flags, 'HSU_CORRECT')) hsu_bopomofo_index.append((wrong, newflags, correct)) # populate eten26 bopomofo index matches = itertools.chain(handle_rules(bopomofo, eten26_correct), handle_special_rules(bopomofo, eten26_correct_special)) for wrong in matches: newflags = '|'.join((flags, 'ETEN26_CORRECT')) eten26_bopomofo_index.append((wrong, newflags, correct)) for (bopomofo, flags) in bopomofo_index: correct = bopomofo # remove duplicate items if bopomofo not in [x[0] for x in hsu_bopomofo_index]: hsu_bopomofo_index.append((bopomofo, flags, correct)) if bopomofo not in [x[0] for x in eten26_bopomofo_index]: eten26_bopomofo_index.append((bopomofo, flags, correct))
def delplot(self, *names): """ Removes the named sub-plots. """ # This process involves removing the plots, then checking the index range # and value range for leftover datasources, and removing those if necessary. # Remove all the renderers from us (container) and create a set of the # datasources that we might have to remove from the ranges deleted_sources = set() for renderer in itertools.chain(*[self.plots.pop(name) for name in names]): self.remove(renderer) deleted_sources.add(renderer.index) deleted_sources.add(renderer.value) # Cull the candidate list of sources to remove by checking the other plots sources_in_use = set() for p in itertools.chain(*self.plots.values()): sources_in_use.add(p.index) sources_in_use.add(p.value) unused_sources = deleted_sources - sources_in_use - set([None]) # Remove the unused sources from all ranges for source in unused_sources: if source.index_dimension == "scalar": # Try both index and range, it doesn't hurt self.index_range.remove(source) self.value_range.remove(source) elif source.index_dimension == "image": self.range2d.remove(source) else: warnings.warn("Couldn't remove datasource from datarange.") return
def view_notification(request): user = request.user unread_a_notif = NotiAnswer.objects.filter(to_user=user).filter(unread=1).order_by('-timestamp') unread_v_notif = NotiVote.objects.filter(to_user=user).filter(unread=1).order_by('-timestamp') unread_f_notif = NotiFollow.objects.filter(to_user=user).filter(unread=1).order_by('-timestamp') unread_c_notif = NotiComment.objects.filter(to_user=user).filter(unread=1).order_by('-timestamp') unread_ask_notif = NotiAsk.objects.filter(to_user=user).filter(unread=1).order_by('-timestamp') read_a_notif = NotiAnswer.objects.filter(to_user=user).filter(unread=0).order_by('-timestamp') read_v_notif = NotiVote.objects.filter(to_user=user).filter(unread=0).order_by('-timestamp') read_f_notif = NotiFollow.objects.filter(to_user=user).filter(unread=0).order_by('-timestamp') read_c_notif = NotiComment.objects.filter(to_user=user).filter(unread=0).order_by('-timestamp') read_ask_notif = NotiAsk.objects.filter(to_user=user).filter(unread=0).order_by('-timestamp') unread_num_notif = len(unread_a_notif) + len(unread_v_notif) + len(unread_f_notif) + len(unread_c_notif) + len(unread_ask_notif) read_notif = list(chain(read_a_notif, read_v_notif, read_f_notif, read_c_notif, read_ask_notif))[:10] notifications = list(chain(unread_a_notif, unread_v_notif, unread_f_notif, unread_c_notif, unread_ask_notif, read_notif)) # Delete notifications older than 40 days days_to_delete = 60 for notif in notifications: created = notif.timestamp.replace(tzinfo=None) if (datetime.now() + timedelta(days=-days_to_delete)) > created: print "apagar %s, para %s" % (notif.kind, notif.from_user) context = { 'num_notif': unread_num_notif, 'notifications': notifications, } return render(request, 'notifications/view_all.html', context)
def if_clause(value): main = chain(value.cond, value.if_cmds) rest = value.else_cmds if isinstance(rest, tuple) and rest[0] == "elif": return chain(main, if_clause(rest[1])) else: return chain(main, rest)
def update_memory_all(self): #Update the memory used in each node DFS=[self.root] DFS1=[] dist=0 depth=0 while DFS: cur_node=DFS.pop() DFS1.append(cur_node) for k in chain(cur_node.kiddir,cur_node.kidfile): DFS.append(k) k.update_dist(cur_node.dist+1) while DFS1: n=DFS1.pop() if os.path.isdir(n.name): result=0 depth=0 for k in chain(n.kiddir,n.kidfile): result+=k.memsize if k.depth>depth: depth=k.depth depth+=1 n.update_memsize(result) n.update_depth(depth) self.depth=self.root.depth
def simple_signature(ambiguous_word, pos=None, stem=True, \ hyperhypo=True, stop=True): """ Returns a synsets_signatures dictionary that includes signature words of a sense from its: (i) definition (ii) example sentences (iii) hypernyms and hyponyms """ synsets_signatures = {} for ss in wn.synsets(ambiguous_word): # If POS is specified. if pos and str(ss.pos()) is not pos: continue signature = [] # Includes definition. signature+= ss.definition().split() # Includes examples signature+= list(chain(*[i.split() for i in ss.examples()])) # Includes lemma_names. signature+= ss.lemma_names() # Optional: includes lemma_names of hypernyms and hyponyms. if hyperhypo == True: signature+= list(chain(*[i.lemma_names() for i \ in ss.hypernyms()+ss.hyponyms()])) # Optional: removes stopwords. if stop == True: signature = [i for i in signature if i not in stopwords.words('english')] # Matching exact words causes sparsity, so optional matching for stems. if stem == True: signature = [porter.stem(i) for i in signature] synsets_signatures[ss] = signature return synsets_signatures
def case_clause(value): # Element 0 of each item in the case is the list of patterns, and # Element 1 of each item in the case is the list of commands to be # executed when that pattern matches. words = chain(*[item[0] for item in value.items]) cmds = chain(*[item[1] for item in value.items]) return cmds, words
def make_cycle(n): xadj = list(range(0, 2 * n + 1, 2)) adjncy = list( itertools.chain.from_iterable( zip(itertools.chain([n - 1], range(n - 1)), itertools.chain(range(1, n), [0])))) return xadj, adjncy
def create_words_bigrams_scores(): posdata = tp.seg_fil_senti_excel("./Machine-learning-features/seniment review set/pos_review.xlsx", 1, 1) negdata = tp.seg_fil_senti_excel("./Machine-learning-features/seniment review set/neg_review.xlsx", 1, 1) posWords = list(itertools.chain(*posdata)) negWords = list(itertools.chain(*negdata)) bigram_finder = BigramCollocationFinder.from_words(posWords) bigram_finder = BigramCollocationFinder.from_words(negWords) posBigrams = bigram_finder.nbest(BigramAssocMeasures.chi_sq, 5000) negBigrams = bigram_finder.nbest(BigramAssocMeasures.chi_sq, 5000) pos = posWords + posBigrams neg = negWords + negBigrams word_fd = FreqDist() cond_word_fd = ConditionalFreqDist() for word in pos: word_fd[word]+=1 cond_word_fd['pos'][word]+=1 for word in neg: word_fd[word]+=1 cond_word_fd['neg'][word]+=1 pos_word_count = cond_word_fd['pos'].N() neg_word_count = cond_word_fd['neg'].N() total_word_count = pos_word_count + neg_word_count word_scores = {} for word, freq in word_fd.iteritems(): pos_score = BigramAssocMeasures.chi_sq(cond_word_fd['pos'][word], (freq, pos_word_count), total_word_count) neg_score = BigramAssocMeasures.chi_sq(cond_word_fd['neg'][word], (freq, neg_word_count), total_word_count) word_scores[word] = pos_score + neg_score return word_scores
def is_variant_of(self, other): ''' Two requests are loosely equal (or variants) if: - They have the same URL - They have the same HTTP method - They have the same parameter names - The values for each parameter have the same type (int / string) :return: True if self and other are variants. ''' dc = self._dc odc = other._dc if (self._method == other._method and self._url == other._url and dc.keys() == odc.keys()): for vself, vother in izip_longest( chain(*dc.values()), chain(*odc.values()), fillvalue=None ): if None in (vself, vother) or \ vself.isdigit() != vother.isdigit(): return False return True return False
def serialize_session_timetable(self, session_, without_blocks=False, strip_empty_days=False): event_tz = self.event.tzinfo timetable = {} if session_.blocks: start_dt = min(chain((b.start_dt for b in session_.blocks), (self.event.start_dt,))).astimezone(event_tz) end_dt = max(chain((b.end_dt for b in session_.blocks), (self.event.end_dt,))).astimezone(event_tz) else: start_dt = self.event.start_dt_local end_dt = self.event.end_dt_local for day in iterdays(start_dt, end_dt): timetable[day.strftime('%Y%m%d')] = {} for block in session_.blocks: block_entry = block.timetable_entry if not block_entry: continue date_key = block_entry.start_dt.astimezone(event_tz).strftime('%Y%m%d') entries = block_entry.children if without_blocks else [block_entry] for entry in entries: if not entry.can_view(self.user): continue entry_key = self._get_entry_key(entry) timetable[date_key][entry_key] = self.serialize_timetable_entry(entry, load_children=True) if strip_empty_days: timetable = self._strip_empty_days(timetable) return timetable
def main(): header, responses = parse_csv() df = DataFrame(data=responses, columns=header) print('Reason I Attended') reason_i_attended = list(chain(*df['Reason I Attended'].tolist())) pprint(Counter(reason_i_attended).most_common()) print('---') print('Role') role = list(chain(*df['Role'].tolist())) pprint(Counter(role).most_common()) print('---') print('DevOps Experience (years)') pprint(Counter(df['DevOps Experience'].tolist()).most_common()) print('---') print('Valuable Learnings') valuable_learnings = list(filter(None, list(chain(*df['Valuable Learnings'].tolist())))) pprint(valuable_learnings) print('---') print('Challenges') challenges = list(filter(None, list(chain(*df['Challenges'].tolist())))) pprint(challenges)
def list_opts(): return [ ('DEFAULT', itertools.chain(magnum.api.auth.AUTH_OPTS, magnum.common.paths.PATH_OPTS, magnum.common.utils.UTILS_OPTS, magnum.common.rpc_service.periodic_opts, magnum.common.service.service_opts, )), ('api', magnum.api.app.API_SERVICE_OPTS), ('bay', magnum.conductor.template_definition.template_def_opts), ('conductor', magnum.conductor.config.SERVICE_OPTS), ('database', magnum.db.sql_opts), ('docker', magnum.common.docker_utils.docker_opts), ('magnum_client', magnum.common.clients.magnum_client_opts), ('heat_client', magnum.common.clients.heat_client_opts), ('glance_client', magnum.common.clients.glance_client_opts), ('barbican_client', magnum.common.clients.barbican_client_opts), ('nova_client', magnum.common.clients.nova_client_opts), ('x509', magnum.common.x509.config.x509_opts), ('bay_heat', magnum.conductor.handlers.bay_conductor.bay_heat_opts), ('certificates', itertools.chain(magnum.common.cert_manager.cert_manager_opts, local_cert_manager.local_cert_manager_opts, )), ('baymodel', magnum.api.validation.baymodel_opts), ]
def plot_gate(self, circ_plot, gate_idx): min_wire = int(min(chain(self.controls, self.targets))) max_wire = int(max(chain(self.controls, self.targets))) circ_plot.control_line(gate_idx, min_wire, max_wire) for c in self.controls: circ_plot.control_point(gate_idx, int(c)) self.gate.plot_gate(circ_plot, gate_idx)
def cat(self,argv): # Get the master's state. state = self.hit_endpoint(argv["--addr"], "/state") if state is None: print ("Could not connect to Master") return # Build a dict from agent ID to agents. agents = {} for agent in state['slaves']: agents[agent['id']] = agent def cat(agent, task): for data in self.read(agent, task, argv["<file>"]): sys.stdout.write(data) for framework in itertools.chain(state['frameworks'], state['completed_frameworks']): if framework['id'] == argv["<framework-ID>"]: for task in itertools.chain(framework['tasks'], framework['completed_tasks']): if (task['id'] == argv["<task-ID>"]): cat(agents[task['slave_id']], task) sys.exit(0) print('No task found!')
def merge(self, others, merge_conditions, common_ancestor=None): if not others: return False self.merged_from.extend(h for h in others) self.merge_conditions = merge_conditions # we must fix this in order to get # correct results when using constraints_since() self.parent = common_ancestor if common_ancestor is not None else self.parent self.recent_events = [e.recent_events for e in itertools.chain([self], others) if not isinstance(e, SimActionConstraint) ] # rebuild recent constraints recent_constraints = [ h.constraints_since(common_ancestor) for h in itertools.chain([self], others) ] combined_constraint = self.state.solver.Or( *[ self.state.solver.simplify(self.state.solver.And(*history_constraints)) for history_constraints in recent_constraints ] ) self.recent_events.append(SimActionConstraint(self.state, combined_constraint)) # hard to say what we should do with these others list of things... #self.recent_bbl_addrs = [e.recent_bbl_addrs for e in itertools.chain([self], others)] #self.recent_ins_addrs = [e.recent_ins_addrs for e in itertools.chain([self], others)] #self.recent_stack_actions = [e.recent_stack_actions for e in itertools.chain([self], others)] return True
def navigate(self, dest): """Find a path from current location to the given destination.""" # Make our node objects for A*. rospy.loginfo("POSE: %f, %f" %(self.pose.x, self.pose.y)) start = Landmark(name="START", ltype="END", x=self.pose.x, y=self.pose.y) goal = Landmark(name="GOAL", ltype="END", x=dest.x, y=dest.y) # Find visible neighbors of start and goal to add as edges to the graph. landmarks = imap(lambda pair: pair[0], self.landmark_graph.itervalues()) # Here we manually add the goal to the candidate landmarks for start # to allow for navigation directly to the goal location. start_zone = self.find_nearest_visibles(start, chain(landmarks, [goal])) # We have to add start here for the empty zone check below. landmarks = imap(lambda pair: pair[0], self.landmark_graph.itervalues()) goal_zone = self.find_nearest_visibles(goal, chain(landmarks, [start])) rospy.loginfo("Start zone: %s" % (", ".join(l.name for l in start_zone))) rospy.loginfo("Goal zone: %s" % (", ".join(l.name for l in goal_zone))) if not start_zone or not goal_zone: rospy.logerr("Cannot connect start and goal! A* failed.") return []; # A* functions. is_goal = lambda node: node.name == "GOAL" heuristic = lambda node: point_distance(node, goal) def neighbors(node): if node.name == "START": return start_zone nbrs = self.landmark_graph[node.name][1] if node in goal_zone: # Intentionally use list concat to make a copy of the list. # If we just modify nbrs, it will modify the original graph. return nbrs + [goal] return nbrs return a_star(start, is_goal, neighbors, point_distance, heuristic)
def create_word_scores(self): [posWords, negWords] = self.getAllWords() posWords = list(itertools.chain(*posWords)) negWords = list(itertools.chain(*negWords)) word_fd = FreqDist() cond_word_fd = ConditionalFreqDist() for word in posWords: word_fd.inc(word) cond_word_fd['pos'].inc(word) for word in negWords: word_fd.inc(word) cond_word_fd['neg'].inc(word) pos_word_count = cond_word_fd['pos'].N() neg_word_count = cond_word_fd['neg'].N() total_word_count = pos_word_count + neg_word_count log("Total number of words: %d" % total_word_count) word_scores = {} for word, freq in word_fd.iteritems(): pos_score = BigramAssocMeasures.chi_sq(cond_word_fd['pos'][word], (freq, pos_word_count), total_word_count) neg_score = BigramAssocMeasures.chi_sq(cond_word_fd['neg'][word], (freq, neg_word_count), total_word_count) word_scores[word] = pos_score + neg_score return word_scores
def _allocate_to_hkvms(self, vm, hkvms): # Check some relevant errors iter_hkvm = iter(hkvms) try: not_empty_hkvm_group = itertools.chain([next(iter_hkvm)], iter_hkvm) except StopIteration: raise ValueError('Empty group or selection') try: hkvm_ok = filter(lambda hkvm: hkvm.last_status == 'OK', not_empty_hkvm_group) hkvm_status_ok = itertools.chain([next(hkvm_ok)], hkvm_ok) except StopIteration: raise ValueError('All HKVM in the selection are failing') iter_weight = self._iter_hkvm_weight(hkvm_status_ok, vm) hkvm, weight = max(iter_weight, key=operator.itemgetter(1)) if weight <= 0: # All weight are below zero: no hkvm can fit raise ValueError('No HKVM can fit the size of the VM') else: hkvm.memory -= vm.memory hkvm.disk -= vm.disk vm.hkvm = hkvm vm.status = 'TO_CREATE' self.logger.info('\'{}\' goes to \'{}\''.format(vm, hkvm)) vm.save()
def update_controls(self): if self.data is None and getattr(self.matrix, 'axis', 1) == 0: # Column-wise distances attr = "Attribute names" self.labelvar_model[:] = ["No labels", attr] self.shapevar_model[:] = ["Same shape", attr] self.colorvar_model[:] = ["Same solor", attr] self.color_value = attr self.shape_value = attr else: # initialize the graph state from data domain = self.data.domain all_vars = list(filter_visible(domain.variables + domain.metas)) cd_vars = [var for var in all_vars if var.is_primitive()] disc_vars = [var for var in all_vars if var.is_discrete] cont_vars = [var for var in all_vars if var.is_continuous] shape_vars = [var for var in disc_vars if len(var.values) <= len(ScatterPlotItem.Symbols) - 1] self.colorvar_model[:] = chain(["Same color"], [self.colorvar_model.Separator] if cd_vars else [], cd_vars) self.shapevar_model[:] = chain(["Same shape"], [self.shapevar_model.Separator] if shape_vars else [], shape_vars) self.sizevar_model[:] = chain(["Same size", "Stress"], [self.sizevar_model.Separator] if cont_vars else [], cont_vars) self.labelvar_model[:] = chain(["No labels"], [self.labelvar_model.Separator] if all_vars else [], all_vars) if domain.class_var is not None: self.color_value = domain.class_var.name
def draw_shot(self, dt=0): """Draw the cannon and shot (in a layer above the level markings) """ if not self.parent: return self.canvas.after.clear() with self.canvas.after: # Shot if self.target: # Fade out the shot dirung 1/4 the time beteen shots self.shot_opacity -= dt * self.interval * 4 # Draw shot & schedule next draw if self.shot_opacity > 0: Color(1, 0, 0, self.shot_opacity) Line(points=itertools.chain(self.center, self.target.pos)) Clock.schedule_once(self.draw_shot) # Set cannon direction self.direction = numpy.arctan2( self.target.pos[1] - self.center[1], self.target.pos[0] - self.center[0], ) # Draw the cannon (if at least lv. 1) Color(0, 0, 0) if self.level: center = self.center direction = self.direction cell_size = self.parent.cell_size LineWidth(self.level) Line(points=itertools.chain(self.center, ( float(center[0] + numpy.cos(direction) * cell_size), float(center[1] + numpy.sin(direction) * cell_size), ))) LineWidth(1)
def render_unicode_table(rows, **kwds): """ Unicode version of above. Such code repetition! """ banner = kwds.get('banner') footer = kwds.get('footer') output = kwds.get('output', sys.stdout) balign = kwds.get('balign', unicode.center) formats = kwds.get('formats') special = kwds.get('special') rows = list(rows) if not formats: formats = lambda: chain((unicode.ljust,), repeat(unicode.rjust)) cols = len(rows[0]) paddings = [ max([len(unicode(r[i])) for r in rows]) + 2 for i in range(cols) ] length = sum(paddings) + cols strip = u'+%s+' % (u'-' * (length-1)) out = list() if banner: lines = iterable(banner) banner = [ strip ] + \ [ u'|%s|' % balign(l, length-1) for l in lines ] + \ [ strip, ] out.append(u'\n'.join(banner)) rows.insert(1, [ u'-', ] * cols) out += [ u'\n'.join([ k + u'|'.join([ fmt(unicode(column), padding, ( special if column == special else fill )) for (column, fmt, padding) in zip(row, fmts(), paddings) ]) + k for (row, fmts, fill, k) in zip( rows, chain( repeat(lambda: repeat(unicode.center,), 1), repeat(formats,) ), chain((u' ',), repeat(u'-', 1), repeat(u' ')), chain((u'|', u'+'), repeat(u'|')) ) ] + [strip,]) ] if footer: footers = iterable(footer) footer = [ strip ] + \ [ u'|%s|' % balign(f, length-1) for f in footers ] + \ [ strip, u'' ] out.append(u'\n'.join(footer)) l = u'\n'.join(out) if l[-1] != u'\n': l = l + u'\n' output.write(l)
md5 = hashlib.md5() md5.update('how to use md5 in'.encode('utf-8')) md5.update('python hashlib?'.encode('utf-8')) print(md5.hexdigest()) sha1 = hashlib.sha1() sha1.update('how to use sha1 in'.encode('utf-8')) sha1.update('python haslib?'.encode('utf-8')) print(sha1.hexdigest()) #itertools #Python的内建模块itertools提供了非常有用的用于操作迭代对象的函数 import itertools natuals = itertools.count(1) #无限迭代器 cs = itertools.cycle('ABC') ns = itertools.repeat('A', 3) for n in ns: print(n) #通常我们会通过takewhile()等函数根据条件判断来截取出一个有限的序列 ps = itertools.takewhile(lambda x: x <= 10, natuals) print(ps) print(list(ps)) #chain chain()可以把一组迭代对象串联起来,形成一个更大的迭代器 for c in itertools.chain('ABC', 'DEF'): print(c) #groupby groupby()把迭代器中相邻的重复元素挑出来放在一起 for key, group in itertools.groupby('AaaBbbCcc', lambda c: c.upper()): print(key, list(group))
def all_user_dictionaries(self): return chain(self.active_user_dictionaries, self.inactive_user_dictionaries)
# Events that are standardised. class _Events(enum.Enum): """zaza.events as an enum for type safety.""" # Core events START_TEST = "start" COMMENT = "comment" EXCEPTION = "exception-in-span" END_TEST = "end" # Enums can't be extended, so we use this little trick. Events = enum.Enum('Events', [(i.name, i.value) for i in itertools.chain(NotifyEvents, _Events)]) class Span(enum.Enum): """Span possibilities.""" BEFORE = "before" WITHIN = "within" AFTER = "after" # Meaning of the fields: # 'collection' - the collection (loosely measurement in InfluxDB parlance) # 'timestamp' - the timestamp # 'event' - the event (from :class:`Events` above) # 'span' - either before, after, or absent - used for spans
def edit_config(self, command): for cmd in chain([b'configure'], to_list(command)): self.send_command(cmd)
def reduce_columns(columns, *clauses, **kw): r"""given a list of columns, return a 'reduced' set based on natural equivalents. the set is reduced to the smallest list of columns which have no natural equivalent present in the list. A "natural equivalent" means that two columns will ultimately represent the same value because they are related by a foreign key. \*clauses is an optional list of join clauses which will be traversed to further identify columns that are "equivalent". \**kw may specify 'ignore_nonexistent_tables' to ignore foreign keys whose tables are not yet configured, or columns that aren't yet present. This function is primarily used to determine the most minimal "primary key" from a selectable, by reducing the set of primary key columns present in the selectable to just those that are not repeated. """ ignore_nonexistent_tables = kw.pop("ignore_nonexistent_tables", False) only_synonyms = kw.pop("only_synonyms", False) columns = util.ordered_column_set(columns) omit = util.column_set() for col in columns: for fk in chain(*[c.foreign_keys for c in col.proxy_set]): for c in columns: if c is col: continue try: fk_col = fk.column except exc.NoReferencedColumnError: # TODO: add specific coverage here # to test/sql/test_selectable ReduceTest if ignore_nonexistent_tables: continue else: raise except exc.NoReferencedTableError: # TODO: add specific coverage here # to test/sql/test_selectable ReduceTest if ignore_nonexistent_tables: continue else: raise if fk_col.shares_lineage(c) and (not only_synonyms or c.name == col.name): omit.add(col) break if clauses: def visit_binary(binary): if binary.operator == operators.eq: cols = util.column_set( chain(*[c.proxy_set for c in columns.difference(omit)])) if binary.left in cols and binary.right in cols: for c in reversed(columns): if c.shares_lineage( binary.right) and (not only_synonyms or c.name == binary.left.name): omit.add(c) break for clause in clauses: if clause is not None: visitors.traverse(clause, {}, {"binary": visit_binary}) return ColumnSet(columns.difference(omit))
def analyze_data(shops: dict): return list(chain(*shops.values()))
def save_as_pdf_pages(plots, filename=None, path=None, verbose=True, **kwargs): """ Save multiple :class:`ggplot` objects to a PDF file, one per page. Parameters ---------- plots : collection or generator of :class:`ggplot` Plot objects to write to file. `plots` may be either a collection such as a :py:class:`list` or :py:class:`set`: >>> base_plot = ggplot(…) >>> plots = [base_plot + ggtitle('%d of 3' % i) for i in range(1, 3)] >>> save_as_pdf_pages(plots) or, a generator that yields :class:`ggplot` objects: >>> def myplots(): >>> for i in range(1, 3): >>> yield ggplot(…) + ggtitle('%d of 3' % i) >>> save_as_pdf_pages(myplots()) filename : :py:class:`str`, optional File name to write the plot to. If not specified, a name like “plotnine-save-<hash>.pdf” is used. path : :py:class:`str`, optional Path to save plot to (if you just want to set path and not filename). verbose : :py:class:`bool` If ``True``, print the saving information. kwargs : :py:class:`dict` Additional arguments to pass to :py:meth:`matplotlib.figure.Figure.savefig`. Notes ----- Using pandas' :meth:`~pandas.DataFrame.groupby` methods, tidy data can be “faceted” across pages: >>> from plotnine.data import mtcars >>> def facet_pages(column) >>> base_plot = [ >>> aes(x='wt', y='mpg', label='name'), >>> geom_text(), >>> ] >>> for label, group_data in mtcars.groupby(column): >>> yield ggplot(group_data) + base_plot + ggtitle(label) >>> save_as_pdf_pages(facet_pages('cyl')) Unlike :meth:`ggplot.save`, :meth:`save_as_pdf_pages` does not process arguments for `height` or `width`. To set the figure size, add :class:`~plotnine.themes.themeable.figure_size` to the theme for some or all of the objects in `plots`: >>> plot = ggplot(…) >>> # The following are equivalent >>> plot.save('filename.pdf', height=6, width=8) >>> save_as_pdf_pages([plot + theme(figure_size=(8, 6))]) """ from itertools import chain from matplotlib.backends.backend_pdf import PdfPages from six import raise_from # as in ggplot.save() fig_kwargs = {'bbox_inches': 'tight'} fig_kwargs.update(kwargs) figure = [None] # If plots is already an iterator, this is a no-op; otherwise convert a # list, etc. to an iterator plots = iter(plots) peek = [] # filename, depends on the object if filename is None: # Take the first element from the iterator, store it, and use it to # generate a file name peek = [next(plots)] filename = peek[0]._save_filename('pdf') if path: filename = os.path.join(path, filename) if verbose: warn('Filename: {}'.format(filename)) with PdfPages(filename) as pdf: # Re-add the first element to the iterator, if it was removed for plot in chain(peek, plots): try: fig = figure[0] = plot.draw() # as in ggplot.save() facecolor = fig.get_facecolor() edgecolor = fig.get_edgecolor() if edgecolor: fig_kwargs['facecolor'] = facecolor if edgecolor: fig_kwargs['edgecolor'] = edgecolor fig_kwargs['frameon'] = True # Save as a page in the PDF file pdf.savefig(figure[0], **fig_kwargs) except AttributeError as err: raise_from( TypeError('non-ggplot object of %s: %s' % (type(plot), plot)), err) except Exception as err: raise finally: # Close the figure whether or not there was an exception, to # conserve memory when plotting a large number of pages figure[0] and plt.close(figure[0])
def action(): dict_extention = { 'audio_extention': ['.mp3', '.m4a', '.wav', '.cda', '.aif', '.mid', '.midi', '.mpa', '.ogg', '.wpl', '.wma'], 'video_extention': ['.mp4', '.mkv', '.MKV', '.flv', '.webm', '.vob', '.gif', '.avi', '.wmv', '.mpeg', '.3gp'], 'document_extention': ['.txt', '.pdf', '.doc', '.html', '.ppt'], 'image_extention': ['.jpg', '.gif', '.png', '.jpeg', '.ico', '.ps', '.psd', '.tif', '.tiff', '.bmp', '.ai'], 'programming_extentions': ['.py', '.js', '.java', '.swift', '.class', '.c', '.html''.xml', '.css', '.php', '.rss', '.xhtml', '.asp', '.aspx', '.cer', '.cfm', '.cgi', '.pl', '.htm', '.jsp', '.part'], 'compressed_extentions': ['.7z', '.arj', '.deb', '.pkg', '.rar', '.rpm', '.tar', '.gz', '.z', '.zip'], 'executabelFile_extentions': ['.apk', '.bat', '.bin ', '.pl', '.com ', '.exe', '.wsf', '.msi', '.jar', '.gadget'], 'font_extentions': ['.fnt', '.otf', '.fon', '.ttf'], } walkfiles = [] walkroot = Dictionary() old_folder_path = folderpath.get() for root, directory, filename in os.walk(old_folder_path): walkfiles.append(filename) walkroot.add(root, filename) merge = list(itertools.chain(*walkfiles)) if len(merge) == 0: messagebox.showerror("Path Not Found", "The path is not given or is not available.") else: pass def file_finder(merge_list, file_extentions): files = [] for file in merge_list: for extentions in file_extentions: if file.endswith(extentions): files.append(file) return files new_folder_path_list = [] for type, extentions in dict_extention.items(): new_folder_name = (str(type.split("_")[0])).capitalize() + "Files" new_folder_path = os.path.join(old_folder_path, new_folder_name) new_folder_path_list.append(new_folder_path) if os.path.exists(new_folder_path): pass else: os.mkdir(new_folder_path) old_file_path_list = [] for i, j in walkroot.items(): for k in j: old_file_path = os.path.join(i, k) old_file_path_list.append(old_file_path) for file in file_finder(merge, extentions): for old in old_file_path_list: try: shutil.move(old, old_folder_path) except: continue for file in file_finder(merge, extentions): old_file_path = os.path.join(old_folder_path, file) new_file_path = os.path.join(new_folder_path, file) shutil.move(old_file_path, new_file_path) walkroot2 = Dictionary() for root, directory, file in os.walk(old_folder_path): walkroot2.add(root, file) for i in reversed(walkroot2.keys()): if len(os.listdir(i)) == 0: # print(f"DELETED --- {i}") os.rmdir(i) else: # print(f"NOT DELETED---{i}") continue folderpath.delete(0, END)
idx2ner = [ 'PER_NAM', 'ORG_NAM', 'GPE_NAM', 'LOC_NAM', 'FAC_NAM', 'PER_NOM', 'ORG_NOM', 'GPE_NOM', 'LOC_NOM', 'FAC_NOM', 'O' ] if not os.path.exists( valid_file ): # load 10% KBP test data source = imap( lambda x: x[1], ifilter( lambda x : x[0] % 10 >= 9, enumerate( imap( lambda x: x[:4], LoadED( config.data_path + '/%s-eval-parsed' % config.language ) ) ) ) ) # load 5% iflytek data if args.iflytek: source = chain( source, imap( lambda x: x[1], ifilter( lambda x : 90 <= x[0] % 100 < 95, enumerate( imap( lambda x: x[:4], LoadED( 'iflytek-clean-%s' % config.language ) ) ) ) ) ) # istantiate a batch constructor valid = batch_constructor( source, numericizer1, numericizer2, gazetteer = kbp_gazetteer, alpha = config.word_alpha, window = config.n_window, n_label_type = config.n_label_type, language = config.language ) logger.info( 'valid: ' + str(valid) ) if config.language != 'spa' and not os.path.exists( test_file ): source = imap( lambda x: x[1], ifilter( lambda x: x[0] % 100 >= 95,
def sat(val): return Clauses(max(abs(v) for v in chain(*val))).sat(val)
def DEFAULTPOLICY(root,state): while state.terminal()==False: state=state.next_state() MAX_STEPS = len([tile for tile in list(chain(*root.state.board)) if tile!='X']) return state.reward(MAX_STEPS)
def application_factory(name='public'): if name not in ('admin', 'public'): raise RuntimeError('Application name (for base_url lookup) must be ' 'either `admin` or `public`.') # NOTE(morgan): The Flask App actually dispatches nothing until we migrate # some routers to Flask-Blueprints, it is simply a placeholder. app = flask.Flask(name) app.after_request(_add_vary_x_auth_token_header) # NOTE(morgan): Configure the Flask Environment for our needs. app.config.update( # We want to bubble up Flask Exceptions (for now) PROPAGATE_EXCEPTIONS=True) # TODO(morgan): Convert Subsystems over to Flask-Native, for now, we simply # dispatch to another "application" [e.g "keystone"] # NOTE(morgan): as each router is converted to flask-native blueprint, # remove from this list. WARNING ORDER MATTERS; ordered dict used to # ensure sane ordering of the routers in the legacy-dispatch model. dispatch_map = collections.OrderedDict() # Load in Healthcheck and map it to /healthcheck hc_app = healthcheck.Healthcheck.app_factory( {}, oslo_config_project='keystone') dispatch_map['/healthcheck'] = hc_app # More legacy code to instantiate all the magic for the dispatchers. # The move to blueprints (FLASK) will allow this to be eliminated. _routers = [] sub_routers = [] mapper = routes.Mapper() for api_routers in ALL_API_ROUTERS: moved_found = [ pfx for pfx in getattr(api_routers, '_path_prefixes', []) if pfx in _MOVED_API_PREFIXES ] if moved_found: raise RuntimeError('An API Router is trying to register path ' 'prefix(s) `%(pfx)s` that is handled by the ' 'native Flask app. Keystone cannot ' 'start.' % {'pfx': ', '.join([p for p in moved_found])}) routers_instance = api_routers.Routers() _routers.append(routers_instance) routers_instance.append_v3_routers(mapper, sub_routers) # TODO(morgan): Remove "API version registration". For now this is kept # for ease of conversion (minimal changes) keystone.api.discovery.register_version('v3') # NOTE(morgan): We add in all the keystone.api blueprints here, this # replaces (as they are implemented) the legacy dispatcher work. for api in keystone.api.__apis__: for api_bp in api.APIs: api_bp.instantiate_and_register_to_app(app) # Build and construct the dispatching for the Legacy dispatching model sub_routers.append(_ComposibleRouterStub(_routers)) legacy_dispatcher = keystone_wsgi.ComposingRouter(mapper, sub_routers) for pfx in itertools.chain( *[rtr.Routers._path_prefixes for rtr in ALL_API_ROUTERS]): dispatch_map['/v3/%s' % pfx] = legacy_dispatcher app.wsgi_app = KeystoneDispatcherMiddleware(app.wsgi_app, dispatch_map) return app
def remove(self,board,tileIndexs): # tileIndex = [(i,j)...] new_board = [tile if (i//self.board_size,i%self.board_size) not in tileIndexs else 'X' for i,tile in enumerate(list(chain(*board)))] new_board = [new_board[j*self.board_size:j*self.board_size + self.board_size] for j in range(self.board_size)] return new_board
def __init__(self,board,color={},move_state='',steps=0,pre_move='',side_panalty=0,move_board=''): self.board = board self.move_board = move_board self.board_size = len(board) self.steps = steps self.color = color self.pre_move = pre_move self.side_panalty = side_panalty self.move_state = move_state next_move_state = 'ply_move' if self.move_state == 'AI_move' else 'AI_move' self.moves = [(i//self.board_size,i%self.board_size)for i,tile in enumerate(list(chain(*board))) if tile == color[next_move_state]] # possible moves
def get_content(cls, abspath, start=None, end=None): data = super(StaticHandler, cls).get_content(abspath, start=start, end=end) return itertools.chain(data, [cls.get_append(abspath)])
def __hash__(self): boardString = ' '.join(list(chain(*self.board))) return int(hashlib.md5(boardString.encode('utf-8')).hexdigest(),16)
def test_cp2_reg_init_unused_hwregs(self): '''Test that all cap hwregs that aren't defined in the spec are None''' for t in self.MIPS.threads.values(): for i in itertools.chain(range(2, 8), range(9, 22), range(24, 28)): self.assertIsNone(t.cp2_hwregs[i], msg="Hwreg " + str(i) + " should not exist")
userFirst = True if input('user go first or not(Y/N):') == 'Y' else False #get / set colors colors = list({tile for tile in board[0]}) ply_colors = initPlyColor(colors,userFirst) # ply_colors={'ply_move' = 'R', 'AI_move' = 'B'} #Game loop initState = State(board,color=ply_colors,move_state='AI_move'if userFirst else 'ply_move') terminal = False pre_state = '' final_board = '' while not terminal: if not pre_state: pre_state = initState firstLoopFunc = plyLoop if userFirst else AILoop secondLoopFunc = AILoop if userFirst else plyLoop pre_state,final_board = firstLoopFunc(pre_state) if final_board : break pre_state,final_board = secondLoopFunc(pre_state) if final_board : break remainSet = list({tile for tile in list(chain(*final_board)) if tile!= 'X'}) message = 'user win' if (len(remainSet) == 1 and remainSet[0] == ply_colors['AI_move']) else 'AI win' if (len(remainSet) == 1 and remainSet[0] == ply_colors['ply_move']) else 'draw' print(message)
def asList(self): return list(chain(*(self.lookup[i] for i in self.indexes)))
########################################### # calculating class weights since this is an imbalanced dataset y_collect = df.select("sponsoring_country").groupBy( "sponsoring_country").count().collect() unique_y = [x["sponsoring_country"] for x in y_collect] total_y = sum([x["count"] for x in y_collect]) unique_y_count = len(y_collect) bin_count = [x["count"] for x in y_collect] class_weights_spark = { i: ii for i, ii in zip(unique_y, total_y / (unique_y_count * np.array(bin_count))) } # print(class_weights_spark) # {0.0: 5.0, 1.0: 0.5555555555555556} mapping_expr = f.create_map( [f.lit(x) for x in chain(*class_weights_spark.items())]) df = df.withColumn("weight", mapping_expr.getItem(f.col("sponsoring_country"))) # numerically labelling the sponsoring country stringIndexer = StringIndexer(inputCol="sponsoring_country", outputCol="label") model = stringIndexer.fit(df) df = model.transform(df) label_decoder = sorted(set([ (i[0], i[1]) for i in df.select(df.sponsoring_country, df.label).collect() ]), key=lambda x: x[0]) label_decoder # train/test split train, test = df[df['is_validation'] == False], df[df['is_validation'] == True]
def checkout(self, url, ref=None, dir_path=None, recursive=False, submodules=True, submodule_update_force=False, keep_paths=None, step_suffix=None, curl_trace_file=None, can_fail_build=True, set_got_revision=False, remote_name=None, display_fetch_size=None, file_name=None, submodule_update_recursive=True, use_git_cache=False): """Performs a full git checkout and returns sha1 of checked out revision. Args: url (str): url of remote repo to use as upstream ref (str): ref to fetch and check out dir_path (Path): optional directory to clone into recursive (bool): whether to recursively fetch submodules or not submodules (bool): whether to sync and update submodules or not submodule_update_force (bool): whether to update submodules with --force keep_paths (iterable of strings): paths to ignore during git-clean; paths are gitignore-style patterns relative to checkout_path. step_suffix (str): suffix to add to a each step name curl_trace_file (Path): if not None, dump GIT_CURL_VERBOSE=1 trace to that file. Useful for debugging git issue reproducible only on bots. It has a side effect of all stderr output of 'git fetch' going to that file. can_fail_build (bool): if False, ignore errors during fetch or checkout. set_got_revision (bool): if True, resolves HEAD and sets got_revision property. remote_name (str): name of the git remote to use display_fetch_size (bool): if True, run `git count-objects` before and after fetch and display delta. Adds two more steps. Defaults to False. file_name (str): optional path to a single file to checkout. submodule_update_recursive (bool): if True, updates submodules recursively. use_git_cache (bool): if True, git cache will be used for this checkout. WARNING, this is EXPERIMENTAL!!! This wasn't tested with: * submodules * since origin url is modified to a local path, may cause problem with scripts that do "git fetch origin" or "git push origin". * arbitrary refs such refs/whatever/not-fetched-by-default-to-cache Returns: If the checkout was successful, this returns the commit hash of the checked-out-repo. Otherwise this returns None. """ retVal = None # TODO(robertocn): Break this function and refactor calls to it. # The problem is that there are way too many unrealated use cases for # it, and the function's signature is getting unwieldy and its body # unreadable. display_fetch_size = display_fetch_size or False if not dir_path: dir_path = url.rsplit('/', 1)[-1] if dir_path.endswith('.git'): # ex: https://host/foobar.git dir_path = dir_path[:-len('.git')] # ex: ssh://host:repo/foobar/.git dir_path = dir_path or dir_path.rsplit('/', 1)[-1] dir_path = self.m.path['start_dir'].join(dir_path) if 'checkout' not in self.m.path: self.m.path['checkout'] = dir_path git_setup_args = ['--path', dir_path, '--url', url] if remote_name: git_setup_args += ['--remote', remote_name] else: remote_name = 'origin' if self.m.platform.is_win: self.ensure_win_git_tooling() git_setup_args += [ '--git_cmd_path', self.package_repo_resource('git.bat')] step_suffix = '' if step_suffix is None else ' (%s)' % step_suffix self.m.python( 'git setup%s' % step_suffix, self.resource('git_setup.py'), git_setup_args) # Some of the commands below require depot_tools to be in PATH. path = self.m.path.pathsep.join([ str(self.package_repo_resource()), '%(PATH)s']) with self.m.step.context({'cwd': dir_path}): if use_git_cache: with self.m.step.context({'env': {'PATH': path}}): self('retry', 'cache', 'populate', '-c', self.m.infra_paths.default_git_cache_dir, url, name='populate cache', can_fail_build=can_fail_build) dir_cmd = self( 'cache', 'exists', '--quiet', '--cache-dir', self.m.infra_paths.default_git_cache_dir, url, can_fail_build=can_fail_build, stdout=self.m.raw_io.output(), step_test_data=lambda: self.m.raw_io.test_api.stream_output('mirror_dir')) mirror_dir = dir_cmd.stdout.strip() self('remote', 'set-url', 'origin', mirror_dir, can_fail_build=can_fail_build) # There are five kinds of refs we can be handed: # 0) None. In this case, we default to properties['branch']. # 1) A 40-character SHA1 hash. # 2) A fully-qualifed arbitrary ref, e.g. 'refs/foo/bar/baz'. # 3) A fully qualified branch name, e.g. 'refs/heads/master'. # Chop off 'refs/heads' and now it matches case (4). # 4) A branch name, e.g. 'master'. # Note that 'FETCH_HEAD' can be many things (and therefore not a valid # checkout target) if many refs are fetched, but we only explicitly fetch # one ref here, so this is safe. fetch_args = [] if not ref: # Case 0 fetch_remote = remote_name fetch_ref = self.m.properties.get('branch') or 'master' checkout_ref = 'FETCH_HEAD' elif self._GIT_HASH_RE.match(ref): # Case 1. fetch_remote = remote_name fetch_ref = '' checkout_ref = ref elif ref.startswith('refs/heads/'): # Case 3. fetch_remote = remote_name fetch_ref = ref[len('refs/heads/'):] checkout_ref = 'FETCH_HEAD' else: # Cases 2 and 4. fetch_remote = remote_name fetch_ref = ref checkout_ref = 'FETCH_HEAD' fetch_args = [x for x in (fetch_remote, fetch_ref) if x] if recursive: fetch_args.append('--recurse-submodules') fetch_env = {'PATH': path} fetch_stderr = None if curl_trace_file: fetch_env['GIT_CURL_VERBOSE'] = '1' fetch_stderr = self.m.raw_io.output(leak_to=curl_trace_file) fetch_step_name = 'git fetch%s' % step_suffix if display_fetch_size: count_objects_before_fetch = self.count_objects( name='count-objects before %s' % fetch_step_name, step_test_data=lambda: self.m.raw_io.test_api.stream_output( self.test_api.count_objects_output(1000))) with self.m.step.context({'env': fetch_env}): self('retry', 'fetch', *fetch_args, name=fetch_step_name, stderr=fetch_stderr, can_fail_build=can_fail_build) if display_fetch_size: self.count_objects( name='count-objects after %s' % fetch_step_name, previous_result=count_objects_before_fetch, step_test_data=lambda: self.m.raw_io.test_api.stream_output( self.test_api.count_objects_output(2000))) if file_name: self('checkout', '-f', checkout_ref, '--', file_name, name='git checkout%s' % step_suffix, can_fail_build=can_fail_build) else: self('checkout', '-f', checkout_ref, name='git checkout%s' % step_suffix, can_fail_build=can_fail_build) rev_parse_step = self('rev-parse', 'HEAD', name='read revision', stdout=self.m.raw_io.output(), can_fail_build=False, step_test_data=lambda: self.m.raw_io.test_api.stream_output('deadbeef')) if rev_parse_step.presentation.status == 'SUCCESS': sha = rev_parse_step.stdout.strip() retVal = sha rev_parse_step.presentation.step_text = "<br/>checked out %r<br/>" % sha if set_got_revision: rev_parse_step.presentation.properties['got_revision'] = sha clean_args = list(itertools.chain( *[('-e', path) for path in keep_paths or []])) self('clean', '-f', '-d', '-x', *clean_args, name='git clean%s' % step_suffix, can_fail_build=can_fail_build) if submodules: self('submodule', 'sync', name='submodule sync%s' % step_suffix, can_fail_build=can_fail_build) submodule_update = ['submodule', 'update', '--init'] if submodule_update_recursive: submodule_update.append('--recursive') if submodule_update_force: submodule_update.append('--force') self(*submodule_update, name='submodule update%s' % step_suffix, can_fail_build=can_fail_build) return retVal
def __iter__(self): for value in chain(*(self.lookup[i] for i in self.indexes)): yield value
# -------------------------------------------------------------------------- all_words = set([PAD_WORD]) wordchar2vector_path = os.path.join(data_folder,'wordchar2vector.dat') print( 'Loading the wordchar2vector model {}'.format(wordchar2vector_path) ) wc2v = gensim.models.KeyedVectors.load_word2vec_format(wordchar2vector_path, binary=False) wc2v_dims = len(wc2v.syn0[0]) print('wc2v_dims={0}'.format(wc2v_dims)) for word in wc2v.vocab: all_words.add(word) word2id = dict( [(c,i) for i,c in enumerate( itertools.chain([PAD_WORD], filter(lambda z:z!=PAD_WORD, all_words)))] ) nb_words = len(all_words) print('nb_words={}'.format(nb_words)) # -------------------------------------------------------------------------- #w2v_path = '/home/eek/polygon/w2v/w2v.CBOW=0_WIN=5_DIM=32.txt' w2v_path = '/home/eek/polygon/w2v/w2v.CBOW=0_WIN=5_DIM=48.txt' #w2v_path = '/home/eek/polygon/WordSDR2/sdr.dat' #w2v_path = '/home/eek/polygon/w2v/w2v.CBOW=0_WIN=5_DIM=128.txt' #w2v_path = r'f:\Word2Vec\word_vectors_cbow=1_win=5_dim=32.txt' print( 'Loading the w2v model {}'.format(w2v_path) ) w2v = gensim.models.KeyedVectors.load_word2vec_format(w2v_path, binary=False) w2v_dims = len(w2v.syn0[0]) print('w2v_dims={0}'.format(w2v_dims))
def asdict(ghcn_source, antarc_source): print "Load GHCN records" return dict((record.uid, record) for record in itertools.chain(ghcn_source, antarc_source))
def index(): if request.method == 'POST': searchString = request.form['content'].replace(" ","") try: USR = "******" PWD = "Test#12345" # Connection URL #CONNECTION_URL = f"mongodb+srv://{USR}:{PWD}@cluster0.zvs9q.mongodb.net/{DB_NAME}?retryWrites=true&w=majority" CONNECTION_URL = "mongodb://localhost:27017/" # opening a connection to Mongo # opening a connection to Mongo try: # Establish a connection with mongoDB dbConn = pymongo.MongoClient(CONNECTION_URL) print("Connected successfully!!!") except: print("Could not connect to MongoDB") # Enter into collection Name dataBase = dbConn["Web_Scraping_Project"] # Specifiy a Database Name collection = dataBase[searchString].find({}) # check collection count if collection.count() > 500: # if there is a collection with searched keyword and it has records in it return render_template('results1.html',reviews=collection) # show the results to user else: flipkart_url = "https://www.flipkart.com/search?q=" + searchString uClient = uReq(flipkart_url) flipkartPage = uClient.read() uClient.close() # Create a Collection Name collection = dataBase[searchString] review_soup = bs(flipkartPage, 'html.parser') pagesize_content = review_soup.find("div", {"class": "_2MImiq"}) pagesize_content.prettify() pagesize_info = pagesize_content.span.contents[0] res = [int(i) for i in pagesize_info.split() if i.isdigit()] page_size = res[-1] page_size = 1 print(page_size) for n in list(range(page_size)): print("Entering into page:", n + 1) product_scrap_data = [] # Create the list to store the final result print("Pagesize****************:",n) #Navigate the required page page_url = "https://www.flipkart.com/search?q="+searchString+"+&%3Bas=on&%3Bas-show=on&%3Botracker=AS_Query_HistoryAutoSuggest_1_2_na_na_na&%3Botracker1=AS_Query_HistoryAutoSuggest_1_2_na_na_na&%3Bas-pos=1&%3Bas-type=HISTORY&%3BsuggestionId=samsung+&%3BrequestId=3deaa522-51bf-4a70-a7fc-d86c0a81128d&%3Bpage=4&page="+str(n) eachpage = requests.get(page_url) eachpages = bs(eachpage.content,'html.parser') #Extratc the product size from the page container = eachpages.find('span',attrs={"class":'_10Ermr'}) page_product_start_count=(int(container.text.split(' ')[1]))-1 page_product_end_count = (int(container.text.split(' ')[3])) products_per_page = page_product_end_count-page_product_start_count print("products_per_page:",products_per_page) #Extratc the product list from that page pre_product_name_list = ['None'] * products_per_page product_name_list = [] try: for container in eachpages.findAll('a',href=True,attrs={"class":'_1fQZEK'}): product_name_list.append(container.get('href').split('/')[1]) except: product_name_list.append("Product name is not available") pre_product_name_list[:len(product_name_list)] = product_name_list[:min(len(pre_product_name_list),len(product_name_list))] #print("product_name_list",len(pre_product_name_list)) product_name = pre_product_name_list #print(product_name) #Extarct the product rating pre_product_rating_list = ['None'] * products_per_page product_rating_list = [] try: for container in eachpages.findAll('div',attrs={"class":'_3LWZlK'}): product_rating_list.append(container.text) except: product_rating_list.append("Product rating is not available") #Consider only poduct rating from the main list pre_product_rating_list[:len(product_rating_list)] = product_rating_list[:min(len(pre_product_rating_list),len(product_rating_list))] product_rating= pre_product_rating_list #print("product_rating",len(product_rating)) #print(product_rating) #Extarct the rating count & review count pre_rating_counts_list = ['None'] * products_per_page pre_review_counts_list = ['None'] * products_per_page rating_counts_list = [] review_counts_list = [] try: for container in eachpages.findAll('span',attrs={"class":'_2_R_DZ'}): rating_counts_list.append((container.text.replace('\xa0','').split('&'))[0]) review_counts_list.append((container.text.replace('\xa0','').split('&'))[1]) except: review_counts_list.append("Product review count is not available") rating_counts_list.append("Product rating count is not available") pre_rating_counts_list[:len(rating_counts_list)] = rating_counts_list[:min(len(pre_rating_counts_list),len(rating_counts_list))] pre_review_counts_list[:len(review_counts_list)] = review_counts_list[:min(len(pre_review_counts_list),len(review_counts_list))] #print("rating_counts_list",len(pre_rating_counts_list)) #print("review_counts_list",len(pre_review_counts_list)) rating_counts = pre_rating_counts_list review_counts = pre_rating_counts_list #print(rating_counts) #print(review_counts) #Extarct the product description pre_product_description_list = ['None'] * products_per_page product_description_list = [] try: for container in eachpages.findAll('ul',attrs={"class":'_1xgFaf'}): product_description_list.append(container.text) except: product_description_list.append("Product description is not available") pre_product_description_list[:len(product_description_list)] = product_description_list[:min(len(pre_product_description_list),len(product_description_list))] #print(len(pre_product_description_list)) product_description = pre_product_description_list #print(product_description) #Extarct the offer price pre_offer_price_list = ['None'] * products_per_page offer_price_list =[] try: for container in eachpages.findAll('div',attrs={"class":'_30jeq3 _1_WHN1'}): offer_price_list.append(container.text) except: offer_price_list.appned("Product offer_price is not available") pre_offer_price_list[:len(offer_price_list)] = offer_price_list[:min(len(pre_offer_price_list),len(offer_price_list))] #print(len(pre_offer_price_list)) offer_price = pre_offer_price_list #print(offer_price) #Extarct the actual price pre_actual_price_list = ['None'] * products_per_page actual_price_list = [] try: for container in eachpages.findAll('div',attrs={"class":'_3I9_wc _27UcVY'}): actual_price_list.append(container.text) actual_price_list = actual_price_list[0:page_product_count[0]] except: actual_price_list.append("Product actual_price is not available") pre_actual_price_list[:len(actual_price_list)] = actual_price_list[:min(len(pre_actual_price_list),len(actual_price_list))] #print("length of price list",len(pre_actual_price_list)) #print(actual_price_list) actual_price = pre_actual_price_list #print(actual_price) #Extarct the offer percentage pre_offer_percentage_list = ['None'] * products_per_page offer_percentage_list = [] try: for container in eachpages.findAll('div',attrs={"class":'_3Ay6Sb'}): offer_percentage_list.append(container.text) offer_percentage_list = offer_percentage_list[0:page_product_count[0]] except: offer_percentage_list.append("Product offer_percentage is not available") pre_offer_percentage_list[:len(offer_percentage_list)] = offer_percentage_list[:min(len(pre_offer_percentage_list),len(offer_percentage_list))] #print(len(pre_offer_percentage_list) offer_percentage = pre_offer_percentage_list #print(offer_percentage) # Extarct the top level class content container = eachpages.findAll("div", {"class": "_1YokD2 _3Mn1Gg"}) # print(len(raw_bigboxes)) # Navigate to product list section raw_bigboxes = container[1].findAll("div", {"class": "_1AtVbE col-12-12"}) # print(len(bigboxes)) # Eliminate the last two uncessary product info link bigboxes = raw_bigboxes[: len(raw_bigboxes) - 2] # print(len(bigboxes)) review_list = [] for i in list(range(products_per_page)): print("Entering into product", i + 1) box = bigboxes[i] productLink = "https://www.flipkart.com" + box.div.div.div.a['href'] prodlink_info = requests.get(productLink) prod_detail_info_html = bs(prodlink_info.content,'html.parser') # Function to extarct the product info def product_info_finder(tag_name, class_name, required_info): try: for container in prod_detail_info_html.find(tag_name, attrs={"class": class_name}): return container.text except: return required_info + " data is not available" # Extarct the offer difference offer_difference = (product_info_finder('div', '_1V_ZGU', 'offer_difference')) # print(offer_difference) # Extarct the avaliable offer avaliable_offer = (product_info_finder('div', '_3TT44I', 'avaliable_offer')) # Extarct the warrenty info warranty_info = (product_info_finder('div', 'XcYV4g', 'warranty_info')) # print(warranty_info) # Extarct the highlights_info highlights_info = (product_info_finder('div', '_2418kt', 'highlights_info')) # print(highlights_info) # Extarct the easy payment info easy_payment_info = (product_info_finder('div', '_250Jnj', 'easy_payment_info')) # print(easy_payment_info) # Extarct the seller info seller_info = (product_info_finder('div', '_1RLviY', 'seller_info')) # print(seller_info) # Extarct the description_info description_info = (product_info_finder('div', '_2o-xpa', 'description_info')) # print(description_info) # Extarct the product specification product_spec = [] try: for containers in prod_detail_info_html.findAll('div', attrs={"class": '_3k-BhJ'}): summary = containers.findAll('table', attrs={"class": '_14cfVK'}) for items in summary: rows = items.findAll('tr') product_spec.append( [':'.join([td.findNext(text=True) for td in tr.findAll("td")]) for tr in rows]) except: product_spec.append("product_specification is not available") product_spec_jointed = itertools.chain(*product_spec) # Join the list item into single string product_specification = (' '.join(product_spec_jointed)) # print(product_specification) # function to extract the review details def review_info_finder(tag_name, class_name, required_info): try: commentbox = prod_detail_info_html.find('div', {'class': "_16PBlm"}) tag_result = commentbox.find(tag_name, attrs={"class": class_name}) return tag_result.text except: return required_info + " data is not available" # Extarct the Reviewer name reviewer_name = (review_info_finder('p', '_2sc7ZR _2V5EHH', 'reviewer_name')) # print(reviewer_name) # Extarct the reviewer_rating reviewer_rating = (review_info_finder('div', '_3LWZlK _1BLPMq', 'reviewer_rating')) # print(reviewer_rating) # Extract the comment_heading comment_heading = (review_info_finder('p', '_2-N8zT', 'comment_heading')) # print(comment_heading) # Extarct the review_comments review_comments = (review_info_finder('div', 't-ZTKy', 'review_comments')) # print(review_comments) # Extract the thumps up try: commentbox = prod_detail_info_html.find('div', {'class': "_16PBlm"}) thumsup = (commentbox.div.div.find_all('span', {'_3c3Px5'})[0].text) except: thumsup = 'Thumsup data is not avaliable' # print(thumsup) # Extarct the thumpsdown try: commentbox = prod_detail_info_html.find('div', {'class': "_16PBlm"}) thumsdown = (commentbox.div.div.find_all('span', {'_3c3Px5'})[1].text) except: thumsdown = 'Thumsdown data is not avaliable' # print(thumsdown) mydict = {"Product_Name": product_name[n], "Product_Rating": product_rating[n], "Rating_Counts": rating_counts[n], "Review_Counts": review_counts[n], "Product_Description": product_description[n], "Offer_Price": offer_price[n], "Actual_Price": actual_price[n], "Offer_Percentage": offer_percentage[n], "Offer_Difference": offer_difference, "Avaliable_Offer": avaliable_offer, "Warranty_Info": warranty_info, "Highlights_Info": highlights_info, "Easy_Payment_Info": easy_payment_info, "Seller_Info": seller_info, "Description_Info": description_info, "Product_Specification": product_specification, "Reviewer_Name": reviewer_name, "Reviewer_Rating": reviewer_rating, "Comment_Heading": comment_heading, "Review_Comments": review_comments, "Thums-Up": thumsup, "Thums_Down": thumsdown} review_list.append(mydict) table = dataBase[searchString] # print(db.list_collection_names()) col_exists = searchString in dataBase.list_collection_names() if col_exists == True: table = dataBase[searchString] table.drop # drop the existing collection table = dataBase[searchString] # creating a collection with the same name as search string. Tables and Collections are analogous. time.sleep(1) table.insert_many(review_list) # insertig the dictionary containing the rview comments to the collection time.sleep(1) return render_template('results1.html', reviews=review_list) except: return 'something is wrong' # return render_template('results.html') else: return render_template('index.html')
def rpad_wordseq(words, n): return list( itertools.chain( words, itertools.repeat(PAD_WORD, n-len(words)) ) )
def build_vocabulary(cutoff, *texts): combined_texts = chain(*texts) return NgramModelVocabulary(cutoff, combined_texts)
def enrich_report_with_guidelines(scan_report: Report, guidelines: Dict[str, str]) -> None: for record in itertools.chain(scan_report.failed_checks, scan_report.passed_checks, scan_report.skipped_checks): if record.check_id in guidelines: record.set_guideline(guidelines[record.check_id])
class Test_expand_operator: # Conceptually, a lot of these tests are complete duplicates of # `TestGateExpansion`, except that they explicitly target an underlying # function in `qutip_qip.operations.gates` which (as of 2020-03-01) is not # called in the majority of cases---it appears to be newer than the # surrounding code, but the surrounding code wasn't updated. @pytest.mark.parametrize( 'permutation', tuple(itertools.chain(*[ itertools.permutations(range(k)) for k in [2, 3, 4] ])), ids=_permutation_id) def test_permutation_without_expansion(self, permutation): base = qutip.tensor([qutip.rand_unitary(2) for _ in permutation]) test = gates.expand_operator(base, N=len(permutation), targets=permutation) expected = base.permute(_apply_permutation(permutation)) np.testing.assert_allclose(test.full(), expected.full(), atol=1e-15) @pytest.mark.parametrize('n_targets', range(1, 5)) def test_general_qubit_expansion(self, n_targets): # Test all permutations with the given number of targets. n_qubits = 5 operation = qutip.rand_unitary(2**n_targets, dims=[[2]*n_targets]*2) for targets in itertools.permutations(range(n_qubits), n_targets): expected = _tensor_with_entanglement([qutip.qeye(2)] * n_qubits, operation, targets) test = gates.expand_operator(operation, n_qubits, targets) np.testing.assert_allclose(test.full(), expected.full(), atol=1e-15) def test_cnot_explicit(self): test = gates.expand_operator(gates.cnot(), 3, [2, 0]).full() expected = np.array([[1, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 1], [0, 0, 0, 0, 1, 0, 0, 0], [0, 1, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 1, 0], [0, 0, 0, 1, 0, 0, 0, 0]]) np.testing.assert_allclose(test, expected, atol=1e-15) def test_cyclic_permutation(self): operators = [qutip.sigmax(), qutip.sigmaz()] test = gates.expand_operator(qutip.tensor(*operators), N=3, targets=[0, 1], cyclic_permutation=True) base_expected = qutip.tensor(*operators, qutip.qeye(2)) expected = [base_expected.permute(x) for x in [[0, 1, 2], [1, 2, 0], [2, 0, 1]]] assert len(expected) == len(test) for element in expected: assert element in test @pytest.mark.parametrize('dimensions', [ pytest.param([3, 4, 5], id="standard"), pytest.param([3, 3, 4, 4, 2], id="standard"), pytest.param([1, 2, 3], id="1D space"), ]) def test_non_qubit_systems(self, dimensions): n_qubits = len(dimensions) for targets in itertools.permutations(range(n_qubits), 2): operators = [qutip.rand_unitary(dimension) if n in targets else qutip.qeye(dimension) for n, dimension in enumerate(dimensions)] expected = qutip.tensor(*operators) base_test = qutip.tensor(*[operators[x] for x in targets]) test = gates.expand_operator(base_test, N=n_qubits, targets=targets, dims=dimensions) assert test.dims == expected.dims np.testing.assert_allclose(test.full(), expected.full())
def _extract_drug_info(drug_xml: ElementTree.Element) -> Mapping[str, Any]: """Extract information from an XML element representing a drug.""" # assert drug_xml.tag == f'{ns}drug' row = { 'type': drug_xml.get('type'), 'drugbank_id': drug_xml.findtext(f"{ns}drugbank-id[@primary='true']"), 'cas_number': drug_xml.findtext(f"{ns}cas-number"), 'name': drug_xml.findtext(f"{ns}name"), 'description': drug_xml.findtext(f"{ns}description"), 'groups': [ group.text for group in drug_xml.findall(f"{ns}groups/{ns}group") ], 'atc_codes': [ code.get('code') for code in drug_xml.findall(f"{ns}atc-codes/{ns}atc-code") ], 'categories': [ { 'name': x.findtext(f'{ns}category'), 'mesh_id': x.findtext(f'{ns}mesh-id'), } for x in drug_xml.findall(f"{ns}categories/{ns}category") ], 'patents': [ { 'patent_id': x.findtext(f'{ns}number'), 'country': x.findtext(f'{ns}country'), 'approved': datetime.datetime.strptime(x.findtext(f'{ns}approved'), '%Y-%m-%d'), 'expires': datetime.datetime.strptime(x.findtext(f'{ns}expires'), '%Y-%m-%d'), 'pediatric_extension': x.findtext(f'{ns}pediatric-extension') != 'false', } for x in drug_xml.findall(f"{ns}patents/{ns}patent") ], 'xrefs': [ { 'resource': x.findtext(f'{ns}resource'), 'identifier': x.findtext(f'{ns}identifier'), } for x in drug_xml.findall(f"{ns}external-identifiers/{ns}external-identifier") ], 'inchi': drug_xml.findtext(inchi_template), 'inchikey': drug_xml.findtext(inchikey_template), 'smiles': drug_xml.findtext(smiles_template), } # Add drug aliases aliases = { elem.text.strip() for elem in itt.chain( drug_xml.findall(f"{ns}international-brands/{ns}international-brand"), drug_xml.findall(f"{ns}synonyms/{ns}synonym[@language='English']"), drug_xml.findall(f"{ns}international-brands/{ns}international-brand"), drug_xml.findall(f"{ns}products/{ns}product/{ns}name"), ) if elem.text.strip() } aliases.add(row['name']) row['aliases'] = aliases row['protein_interactions'] = [] row['protein_group_interactions'] = [] for category, protein in _iterate_protein_stuff(drug_xml): target_row = _extract_protein_info(category, protein) if not target_row: continue row['protein_interactions'].append(target_row) return row