def evaluate_cosine_similarity(evaluated_sentences, reference_sentences):
evaluated_words = tuple(chain(*(s.words for s in evaluated_sentences)))
reference_words = tuple(chain(*(s.words for s in reference_sentences)))
evaluated_model = TfDocumentModel(evaluated_words)
reference_model = TfDocumentModel(reference_words)
return cosine_similarity(evaluated_model, reference_model)
def present_value(prefix, arg):
"""Present a whole value that is either added or deleted.
Calls out to other formatters for cells, outputs, and multiline strings.
Uses pprint.pformat, otherwise.
"""
# TODO: improve pretty-print of arbitrary values?
if isinstance(arg, dict):
if 'cell_type' in arg:
return present_cell(prefix, arg)
elif 'output_type' in arg:
return present_output(prefix, arg)
elif isinstance(arg, list) and arg:
first = arg[0]
if isinstance(first, dict):
if 'cell_type' in first:
return chain(*[ present_cell(prefix + ' ', cell) for cell in arg ])
elif 'output_type' in first:
return chain(*[ present_output(prefix + ' ', out) for out in arg ])
elif isinstance(arg, string_types):
return present_multiline_string(prefix, arg)
lines = pprint.pformat(arg).splitlines()
return [prefix + line for line in lines]
def window(iter, pre_size=1, post_size=1): """ Given an iterable, return a new iterable which yields triples of (pre, item, post), where pre and post are the items preceeding and following the item (or None if no such item is appropriate). pre and post will always be pre_size and post_size in length. >>> example = window(range(10), pre_size=2) >>> pre, item, post = next(example) >>> pre (None, None) >>> post (1,) >>> next(example) ((None, 0), 1, (2,)) >>> list(example)[-1] ((7, 8), 9, (None,)) """ pre_iter, iter = itertools.tee(iter) pre_iter = itertools.chain((None,) * pre_size, pre_iter) pre_iter = nwise(pre_iter, pre_size) post_iter, iter = itertools.tee(iter) post_iter = itertools.chain(post_iter, (None,) * post_size) post_iter = nwise(post_iter, post_size) next(post_iter, None) return six.moves.zip(pre_iter, iter, post_iter)
def nsmallest(n, iterable, key=None):
"""Find the n smallest elements in a dataset.
Equivalent to: sorted(iterable, key=key)[:n]
"""
# Short-cut for n==1 is to use min() when len(iterable)>0
if n == 1:
it = iter(iterable)
head = list(islice(it, 1))
if not head:
return []
if key is None:
return [min(chain(head, it))]
return [min(chain(head, it), key=key)]
# When n>=size, it's faster to use sorted()
try:
size = len(iterable)
except (TypeError, AttributeError):
pass
else:
if n >= size:
return sorted(iterable, key=key)[:n]
# When key is none, use simpler decoration
if key is None:
it = izip(iterable, count()) # decorate
result = _nsmallest(n, it)
return map(itemgetter(0), result) # undecorate
# General case, slowest method
in1, in2 = tee(iterable)
it = izip(imap(key, in1), count(), in2) # decorate
result = _nsmallest(n, it)
return map(itemgetter(2), result) # undecorate
def translate_selection(self, x, y, x0, y0):
x, y = 10*round(x/10), 10*round(y/10)
# survey conditions
for frame in self:
for point in chain(frame[0], frame[1]):
if point[2]:
# do any of the points fall on another point?
if not frame.can_fall(point[0] + x - x0, point[1] + y - y0):
return
for frame in self:
for point in chain(frame[0], frame[1]):
if point[2]:
point[:2] = [point[0] + x - x0, point[1] + y - y0]
# check y alignment
if frame[0][0][1] != frame[1][0][1]:
# determine which should move
if frame[0][0][2]:
flip = 1
else:
flip = 0
frame[flip][0][1] = frame[not flip][0][1]
if frame[0][-1][1] != frame[1][-1][1]:
# determine which should move
if frame[0][-1][2]:
flip = 1
else:
flip = 0
frame[flip][-1][1] = frame[not flip][-1][1]
def test_attr_label_matrix_and_data(self):
w = self.widget
# Don't run the MDS optimization to save time and to prevent the
# widget be in a blocking state when trying to send the next signal
w.start = Mock()
# Data and matrix
data = Table("zoo")
dist = Euclidean(data)
self.send_signal(w.Inputs.distances, dist)
self.send_signal(w.Inputs.data, data)
self.assertTrue(set(chain(data.domain.variables, data.domain.metas))
< set(w.controls.attr_label.model()))
# Has data, but receives a signal without data: has to keep the label
self.send_signal(w.Inputs.distances, None)
self.assertTrue(set(chain(data.domain.variables, data.domain.metas))
< set(w.controls.attr_label.model()))
# Has matrix without data, and loses the data: remove the label
self.send_signal(w.Inputs.data, None)
self.assertEqual(list(w.controls.attr_label.model()), [None])
# Has matrix without data, receives data: add attrs to combo, select
self.send_signal(w.Inputs.data, data)
self.assertTrue(set(chain(data.domain.variables, data.domain.metas))
< set(w.controls.attr_label.model()))
def dependencies(self, stack):
"""Return the Resource objects in given stack on which this depends."""
def path(section):
return '.'.join([self.name, section])
def get_resource(res_name):
if res_name not in stack:
raise exception.InvalidTemplateReference(resource=res_name,
key=self.name)
return stack[res_name]
def strict_func_deps(data, datapath):
return six.moves.filter(
lambda r: getattr(r, 'strict_dependency', True),
function.dependencies(data, datapath))
explicit_depends = [] if self._depends is None else self._depends
prop_deps = strict_func_deps(self._properties, path('Properties'))
metadata_deps = strict_func_deps(self._metadata, path('Metadata'))
# (ricolin) External resource should not depend on any other resources.
# This operation is not allowed for now.
if self.external_id():
if explicit_depends:
raise exception.InvalidExternalResourceDependency(
external_id=self.external_id(),
resource_type=self.resource_type
)
return itertools.chain()
return itertools.chain((get_resource(dep) for dep in explicit_depends),
prop_deps, metadata_deps)
def _write_serial(self, path, data, metadata):
from mpi4py import MPI
comm, rank, root = get_comm_rank_root()
if rank != root:
for tag, buf in enumerate(data):
comm.Send(buf.copy(), root, tag)
else:
# Recv all of the non-local data
MPI.Prequest.Startall(self._mpi_rreqs)
MPI.Prequest.Waitall(self._mpi_rreqs)
# Combine local and MPI data
names = it.chain(self._loc_names, self._mpi_names)
dats = it.chain(data, self._mpi_rbufs)
# Convert any metadata to ASCII
metadata = {k: np.array(v, dtype='S')
for k, v in metadata.items()}
# Create the output dictionary
outdict = dict(zip(names, dats), **metadata)
with h5py.File(path, 'w') as h5file:
for k, v in outdict.items():
h5file[k] = v
def populate_more_bopomofo_index():
for (bopomofo, flags) in bopomofo_index:
# populate hsu bopomofo index
correct = bopomofo
matches = itertools.chain(handle_rules(bopomofo, hsu_correct),
handle_special_rules(bopomofo, hsu_correct_special))
for wrong in matches:
newflags = '|'.join((flags, 'HSU_CORRECT'))
hsu_bopomofo_index.append((wrong, newflags, correct))
# populate eten26 bopomofo index
matches = itertools.chain(handle_rules(bopomofo, eten26_correct),
handle_special_rules(bopomofo, eten26_correct_special))
for wrong in matches:
newflags = '|'.join((flags, 'ETEN26_CORRECT'))
eten26_bopomofo_index.append((wrong, newflags, correct))
for (bopomofo, flags) in bopomofo_index:
correct = bopomofo
# remove duplicate items
if bopomofo not in [x[0] for x in hsu_bopomofo_index]:
hsu_bopomofo_index.append((bopomofo, flags, correct))
if bopomofo not in [x[0] for x in eten26_bopomofo_index]:
eten26_bopomofo_index.append((bopomofo, flags, correct))
def delplot(self, *names):
""" Removes the named sub-plots. """
# This process involves removing the plots, then checking the index range
# and value range for leftover datasources, and removing those if necessary.
# Remove all the renderers from us (container) and create a set of the
# datasources that we might have to remove from the ranges
deleted_sources = set()
for renderer in itertools.chain(*[self.plots.pop(name) for name in names]):
self.remove(renderer)
deleted_sources.add(renderer.index)
deleted_sources.add(renderer.value)
# Cull the candidate list of sources to remove by checking the other plots
sources_in_use = set()
for p in itertools.chain(*self.plots.values()):
sources_in_use.add(p.index)
sources_in_use.add(p.value)
unused_sources = deleted_sources - sources_in_use - set([None])
# Remove the unused sources from all ranges
for source in unused_sources:
if source.index_dimension == "scalar":
# Try both index and range, it doesn't hurt
self.index_range.remove(source)
self.value_range.remove(source)
elif source.index_dimension == "image":
self.range2d.remove(source)
else:
warnings.warn("Couldn't remove datasource from datarange.")
return
def view_notification(request):
user = request.user
unread_a_notif = NotiAnswer.objects.filter(to_user=user).filter(unread=1).order_by('-timestamp')
unread_v_notif = NotiVote.objects.filter(to_user=user).filter(unread=1).order_by('-timestamp')
unread_f_notif = NotiFollow.objects.filter(to_user=user).filter(unread=1).order_by('-timestamp')
unread_c_notif = NotiComment.objects.filter(to_user=user).filter(unread=1).order_by('-timestamp')
unread_ask_notif = NotiAsk.objects.filter(to_user=user).filter(unread=1).order_by('-timestamp')
read_a_notif = NotiAnswer.objects.filter(to_user=user).filter(unread=0).order_by('-timestamp')
read_v_notif = NotiVote.objects.filter(to_user=user).filter(unread=0).order_by('-timestamp')
read_f_notif = NotiFollow.objects.filter(to_user=user).filter(unread=0).order_by('-timestamp')
read_c_notif = NotiComment.objects.filter(to_user=user).filter(unread=0).order_by('-timestamp')
read_ask_notif = NotiAsk.objects.filter(to_user=user).filter(unread=0).order_by('-timestamp')
unread_num_notif = len(unread_a_notif) + len(unread_v_notif) + len(unread_f_notif) + len(unread_c_notif) + len(unread_ask_notif)
read_notif = list(chain(read_a_notif, read_v_notif, read_f_notif, read_c_notif, read_ask_notif))[:10]
notifications = list(chain(unread_a_notif, unread_v_notif, unread_f_notif, unread_c_notif, unread_ask_notif, read_notif))
# Delete notifications older than 40 days
days_to_delete = 60
for notif in notifications:
created = notif.timestamp.replace(tzinfo=None)
if (datetime.now() + timedelta(days=-days_to_delete)) > created:
print "apagar %s, para %s" % (notif.kind, notif.from_user)
context = {
'num_notif': unread_num_notif,
'notifications': notifications,
}
return render(request, 'notifications/view_all.html', context)
def if_clause(value):
main = chain(value.cond, value.if_cmds)
rest = value.else_cmds
if isinstance(rest, tuple) and rest[0] == "elif":
return chain(main, if_clause(rest[1]))
else:
return chain(main, rest)
def update_memory_all(self): #Update the memory used in each node DFS=[self.root] DFS1=[] dist=0 depth=0 while DFS: cur_node=DFS.pop() DFS1.append(cur_node) for k in chain(cur_node.kiddir,cur_node.kidfile): DFS.append(k) k.update_dist(cur_node.dist+1) while DFS1: n=DFS1.pop() if os.path.isdir(n.name): result=0 depth=0 for k in chain(n.kiddir,n.kidfile): result+=k.memsize if k.depth>depth: depth=k.depth depth+=1 n.update_memsize(result) n.update_depth(depth) self.depth=self.root.depth
def simple_signature(ambiguous_word, pos=None, stem=True, \
hyperhypo=True, stop=True):
"""
Returns a synsets_signatures dictionary that includes signature words of a
sense from its:
(i) definition
(ii) example sentences
(iii) hypernyms and hyponyms
"""
synsets_signatures = {}
for ss in wn.synsets(ambiguous_word):
# If POS is specified.
if pos and str(ss.pos()) is not pos:
continue
signature = []
# Includes definition.
signature+= ss.definition().split()
# Includes examples
signature+= list(chain(*[i.split() for i in ss.examples()]))
# Includes lemma_names.
signature+= ss.lemma_names()
# Optional: includes lemma_names of hypernyms and hyponyms.
if hyperhypo == True:
signature+= list(chain(*[i.lemma_names() for i \
in ss.hypernyms()+ss.hyponyms()]))
# Optional: removes stopwords.
if stop == True:
signature = [i for i in signature if i not in stopwords.words('english')]
# Matching exact words causes sparsity, so optional matching for stems.
if stem == True:
signature = [porter.stem(i) for i in signature]
synsets_signatures[ss] = signature
return synsets_signatures
def case_clause(value):
# Element 0 of each item in the case is the list of patterns, and
# Element 1 of each item in the case is the list of commands to be
# executed when that pattern matches.
words = chain(*[item[0] for item in value.items])
cmds = chain(*[item[1] for item in value.items])
return cmds, words
def make_cycle(n):
xadj = list(range(0, 2 * n + 1, 2))
adjncy = list(
itertools.chain.from_iterable(
zip(itertools.chain([n - 1], range(n - 1)),
itertools.chain(range(1, n), [0]))))
return xadj, adjncy
def create_words_bigrams_scores():
posdata = tp.seg_fil_senti_excel("./Machine-learning-features/seniment review set/pos_review.xlsx", 1, 1)
negdata = tp.seg_fil_senti_excel("./Machine-learning-features/seniment review set/neg_review.xlsx", 1, 1)
posWords = list(itertools.chain(*posdata))
negWords = list(itertools.chain(*negdata))
bigram_finder = BigramCollocationFinder.from_words(posWords)
bigram_finder = BigramCollocationFinder.from_words(negWords)
posBigrams = bigram_finder.nbest(BigramAssocMeasures.chi_sq, 5000)
negBigrams = bigram_finder.nbest(BigramAssocMeasures.chi_sq, 5000)
pos = posWords + posBigrams
neg = negWords + negBigrams
word_fd = FreqDist()
cond_word_fd = ConditionalFreqDist()
for word in pos:
word_fd[word]+=1
cond_word_fd['pos'][word]+=1
for word in neg:
word_fd[word]+=1
cond_word_fd['neg'][word]+=1
pos_word_count = cond_word_fd['pos'].N()
neg_word_count = cond_word_fd['neg'].N()
total_word_count = pos_word_count + neg_word_count
word_scores = {}
for word, freq in word_fd.iteritems():
pos_score = BigramAssocMeasures.chi_sq(cond_word_fd['pos'][word], (freq, pos_word_count), total_word_count)
neg_score = BigramAssocMeasures.chi_sq(cond_word_fd['neg'][word], (freq, neg_word_count), total_word_count)
word_scores[word] = pos_score + neg_score
return word_scores
def is_variant_of(self, other):
'''
Two requests are loosely equal (or variants) if:
- They have the same URL
- They have the same HTTP method
- They have the same parameter names
- The values for each parameter have the same type (int / string)
:return: True if self and other are variants.
'''
dc = self._dc
odc = other._dc
if (self._method == other._method and
self._url == other._url and
dc.keys() == odc.keys()):
for vself, vother in izip_longest(
chain(*dc.values()),
chain(*odc.values()),
fillvalue=None
):
if None in (vself, vother) or \
vself.isdigit() != vother.isdigit():
return False
return True
return False
def serialize_session_timetable(self, session_, without_blocks=False, strip_empty_days=False):
event_tz = self.event.tzinfo
timetable = {}
if session_.blocks:
start_dt = min(chain((b.start_dt for b in session_.blocks), (self.event.start_dt,))).astimezone(event_tz)
end_dt = max(chain((b.end_dt for b in session_.blocks), (self.event.end_dt,))).astimezone(event_tz)
else:
start_dt = self.event.start_dt_local
end_dt = self.event.end_dt_local
for day in iterdays(start_dt, end_dt):
timetable[day.strftime('%Y%m%d')] = {}
for block in session_.blocks:
block_entry = block.timetable_entry
if not block_entry:
continue
date_key = block_entry.start_dt.astimezone(event_tz).strftime('%Y%m%d')
entries = block_entry.children if without_blocks else [block_entry]
for entry in entries:
if not entry.can_view(self.user):
continue
entry_key = self._get_entry_key(entry)
timetable[date_key][entry_key] = self.serialize_timetable_entry(entry, load_children=True)
if strip_empty_days:
timetable = self._strip_empty_days(timetable)
return timetable
def main():
header, responses = parse_csv()
df = DataFrame(data=responses, columns=header)
print('Reason I Attended')
reason_i_attended = list(chain(*df['Reason I Attended'].tolist()))
pprint(Counter(reason_i_attended).most_common())
print('---')
print('Role')
role = list(chain(*df['Role'].tolist()))
pprint(Counter(role).most_common())
print('---')
print('DevOps Experience (years)')
pprint(Counter(df['DevOps Experience'].tolist()).most_common())
print('---')
print('Valuable Learnings')
valuable_learnings = list(filter(None, list(chain(*df['Valuable Learnings'].tolist()))))
pprint(valuable_learnings)
print('---')
print('Challenges')
challenges = list(filter(None, list(chain(*df['Challenges'].tolist()))))
pprint(challenges)
def list_opts():
return [
('DEFAULT',
itertools.chain(magnum.api.auth.AUTH_OPTS,
magnum.common.paths.PATH_OPTS,
magnum.common.utils.UTILS_OPTS,
magnum.common.rpc_service.periodic_opts,
magnum.common.service.service_opts,
)),
('api', magnum.api.app.API_SERVICE_OPTS),
('bay', magnum.conductor.template_definition.template_def_opts),
('conductor', magnum.conductor.config.SERVICE_OPTS),
('database', magnum.db.sql_opts),
('docker', magnum.common.docker_utils.docker_opts),
('magnum_client', magnum.common.clients.magnum_client_opts),
('heat_client', magnum.common.clients.heat_client_opts),
('glance_client', magnum.common.clients.glance_client_opts),
('barbican_client', magnum.common.clients.barbican_client_opts),
('nova_client', magnum.common.clients.nova_client_opts),
('x509', magnum.common.x509.config.x509_opts),
('bay_heat', magnum.conductor.handlers.bay_conductor.bay_heat_opts),
('certificates',
itertools.chain(magnum.common.cert_manager.cert_manager_opts,
local_cert_manager.local_cert_manager_opts,
)),
('baymodel', magnum.api.validation.baymodel_opts),
]
def plot_gate(self, circ_plot, gate_idx):
min_wire = int(min(chain(self.controls, self.targets)))
max_wire = int(max(chain(self.controls, self.targets)))
circ_plot.control_line(gate_idx, min_wire, max_wire)
for c in self.controls:
circ_plot.control_point(gate_idx, int(c))
self.gate.plot_gate(circ_plot, gate_idx)
def cat(self,argv):
# Get the master's state.
state = self.hit_endpoint(argv["--addr"], "/state")
if state is None:
print ("Could not connect to Master")
return
# Build a dict from agent ID to agents.
agents = {}
for agent in state['slaves']:
agents[agent['id']] = agent
def cat(agent, task):
for data in self.read(agent, task, argv["<file>"]):
sys.stdout.write(data)
for framework in itertools.chain(state['frameworks'],
state['completed_frameworks']):
if framework['id'] == argv["<framework-ID>"]:
for task in itertools.chain(framework['tasks'],
framework['completed_tasks']):
if (task['id'] == argv["<task-ID>"]):
cat(agents[task['slave_id']], task)
sys.exit(0)
print('No task found!')
def merge(self, others, merge_conditions, common_ancestor=None):
if not others:
return False
self.merged_from.extend(h for h in others)
self.merge_conditions = merge_conditions
# we must fix this in order to get
# correct results when using constraints_since()
self.parent = common_ancestor if common_ancestor is not None else self.parent
self.recent_events = [e.recent_events for e in itertools.chain([self], others)
if not isinstance(e, SimActionConstraint)
]
# rebuild recent constraints
recent_constraints = [ h.constraints_since(common_ancestor) for h in itertools.chain([self], others) ]
combined_constraint = self.state.solver.Or(
*[ self.state.solver.simplify(self.state.solver.And(*history_constraints)) for history_constraints in recent_constraints ]
)
self.recent_events.append(SimActionConstraint(self.state, combined_constraint))
# hard to say what we should do with these others list of things...
#self.recent_bbl_addrs = [e.recent_bbl_addrs for e in itertools.chain([self], others)]
#self.recent_ins_addrs = [e.recent_ins_addrs for e in itertools.chain([self], others)]
#self.recent_stack_actions = [e.recent_stack_actions for e in itertools.chain([self], others)]
return True
def navigate(self, dest):
"""Find a path from current location to the given destination."""
# Make our node objects for A*.
rospy.loginfo("POSE: %f, %f" %(self.pose.x, self.pose.y))
start = Landmark(name="START", ltype="END", x=self.pose.x, y=self.pose.y)
goal = Landmark(name="GOAL", ltype="END", x=dest.x, y=dest.y)
# Find visible neighbors of start and goal to add as edges to the graph.
landmarks = imap(lambda pair: pair[0], self.landmark_graph.itervalues())
# Here we manually add the goal to the candidate landmarks for start
# to allow for navigation directly to the goal location.
start_zone = self.find_nearest_visibles(start, chain(landmarks, [goal]))
# We have to add start here for the empty zone check below.
landmarks = imap(lambda pair: pair[0], self.landmark_graph.itervalues())
goal_zone = self.find_nearest_visibles(goal, chain(landmarks, [start]))
rospy.loginfo("Start zone: %s" % (", ".join(l.name for l in start_zone)))
rospy.loginfo("Goal zone: %s" % (", ".join(l.name for l in goal_zone)))
if not start_zone or not goal_zone:
rospy.logerr("Cannot connect start and goal! A* failed.")
return [];
# A* functions.
is_goal = lambda node: node.name == "GOAL"
heuristic = lambda node: point_distance(node, goal)
def neighbors(node):
if node.name == "START":
return start_zone
nbrs = self.landmark_graph[node.name][1]
if node in goal_zone:
# Intentionally use list concat to make a copy of the list.
# If we just modify nbrs, it will modify the original graph.
return nbrs + [goal]
return nbrs
return a_star(start, is_goal, neighbors, point_distance, heuristic)
def create_word_scores(self):
[posWords, negWords] = self.getAllWords()
posWords = list(itertools.chain(*posWords))
negWords = list(itertools.chain(*negWords))
word_fd = FreqDist()
cond_word_fd = ConditionalFreqDist()
for word in posWords:
word_fd.inc(word)
cond_word_fd['pos'].inc(word)
for word in negWords:
word_fd.inc(word)
cond_word_fd['neg'].inc(word)
pos_word_count = cond_word_fd['pos'].N()
neg_word_count = cond_word_fd['neg'].N()
total_word_count = pos_word_count + neg_word_count
log("Total number of words: %d" % total_word_count)
word_scores = {}
for word, freq in word_fd.iteritems():
pos_score = BigramAssocMeasures.chi_sq(cond_word_fd['pos'][word], (freq, pos_word_count), total_word_count)
neg_score = BigramAssocMeasures.chi_sq(cond_word_fd['neg'][word], (freq, neg_word_count), total_word_count)
word_scores[word] = pos_score + neg_score
return word_scores
def _allocate_to_hkvms(self, vm, hkvms):
# Check some relevant errors
iter_hkvm = iter(hkvms)
try:
not_empty_hkvm_group = itertools.chain([next(iter_hkvm)],
iter_hkvm)
except StopIteration:
raise ValueError('Empty group or selection')
try:
hkvm_ok = filter(lambda hkvm: hkvm.last_status == 'OK',
not_empty_hkvm_group)
hkvm_status_ok = itertools.chain([next(hkvm_ok)], hkvm_ok)
except StopIteration:
raise ValueError('All HKVM in the selection are failing')
iter_weight = self._iter_hkvm_weight(hkvm_status_ok, vm)
hkvm, weight = max(iter_weight, key=operator.itemgetter(1))
if weight <= 0:
# All weight are below zero: no hkvm can fit
raise ValueError('No HKVM can fit the size of the VM')
else:
hkvm.memory -= vm.memory
hkvm.disk -= vm.disk
vm.hkvm = hkvm
vm.status = 'TO_CREATE'
self.logger.info('\'{}\' goes to \'{}\''.format(vm, hkvm))
vm.save()
def update_controls(self):
if self.data is None and getattr(self.matrix, 'axis', 1) == 0:
# Column-wise distances
attr = "Attribute names"
self.labelvar_model[:] = ["No labels", attr]
self.shapevar_model[:] = ["Same shape", attr]
self.colorvar_model[:] = ["Same solor", attr]
self.color_value = attr
self.shape_value = attr
else:
# initialize the graph state from data
domain = self.data.domain
all_vars = list(filter_visible(domain.variables + domain.metas))
cd_vars = [var for var in all_vars if var.is_primitive()]
disc_vars = [var for var in all_vars if var.is_discrete]
cont_vars = [var for var in all_vars if var.is_continuous]
shape_vars = [var for var in disc_vars
if len(var.values) <= len(ScatterPlotItem.Symbols) - 1]
self.colorvar_model[:] = chain(["Same color"],
[self.colorvar_model.Separator] if cd_vars else [],
cd_vars)
self.shapevar_model[:] = chain(["Same shape"],
[self.shapevar_model.Separator] if shape_vars else [],
shape_vars)
self.sizevar_model[:] = chain(["Same size", "Stress"],
[self.sizevar_model.Separator] if cont_vars else [],
cont_vars)
self.labelvar_model[:] = chain(["No labels"],
[self.labelvar_model.Separator] if all_vars else [],
all_vars)
if domain.class_var is not None:
self.color_value = domain.class_var.name
def draw_shot(self, dt=0):
"""Draw the cannon and shot (in a layer above the level markings)
"""
if not self.parent:
return
self.canvas.after.clear()
with self.canvas.after:
# Shot
if self.target:
# Fade out the shot dirung 1/4 the time beteen shots
self.shot_opacity -= dt * self.interval * 4
# Draw shot & schedule next draw
if self.shot_opacity > 0:
Color(1, 0, 0, self.shot_opacity)
Line(points=itertools.chain(self.center, self.target.pos))
Clock.schedule_once(self.draw_shot)
# Set cannon direction
self.direction = numpy.arctan2(
self.target.pos[1] - self.center[1],
self.target.pos[0] - self.center[0],
)
# Draw the cannon (if at least lv. 1)
Color(0, 0, 0)
if self.level:
center = self.center
direction = self.direction
cell_size = self.parent.cell_size
LineWidth(self.level)
Line(points=itertools.chain(self.center, (
float(center[0] + numpy.cos(direction) * cell_size),
float(center[1] + numpy.sin(direction) * cell_size),
)))
LineWidth(1)
def render_unicode_table(rows, **kwds):
"""
Unicode version of above. Such code repetition!
"""
banner = kwds.get('banner')
footer = kwds.get('footer')
output = kwds.get('output', sys.stdout)
balign = kwds.get('balign', unicode.center)
formats = kwds.get('formats')
special = kwds.get('special')
rows = list(rows)
if not formats:
formats = lambda: chain((unicode.ljust,), repeat(unicode.rjust))
cols = len(rows[0])
paddings = [
max([len(unicode(r[i])) for r in rows]) + 2
for i in range(cols)
]
length = sum(paddings) + cols
strip = u'+%s+' % (u'-' * (length-1))
out = list()
if banner:
lines = iterable(banner)
banner = [ strip ] + \
[ u'|%s|' % balign(l, length-1) for l in lines ] + \
[ strip, ]
out.append(u'\n'.join(banner))
rows.insert(1, [ u'-', ] * cols)
out += [
u'\n'.join([
k + u'|'.join([
fmt(unicode(column), padding, (
special if column == special else fill
)) for (column, fmt, padding) in zip(row, fmts(), paddings)
]) + k for (row, fmts, fill, k) in zip(
rows,
chain(
repeat(lambda: repeat(unicode.center,), 1),
repeat(formats,)
),
chain((u' ',), repeat(u'-', 1), repeat(u' ')),
chain((u'|', u'+'), repeat(u'|'))
)
] + [strip,])
]
if footer:
footers = iterable(footer)
footer = [ strip ] + \
[ u'|%s|' % balign(f, length-1) for f in footers ] + \
[ strip, u'' ]
out.append(u'\n'.join(footer))
l = u'\n'.join(out)
if l[-1] != u'\n':
l = l + u'\n'
output.write(l)
md5 = hashlib.md5()
md5.update('how to use md5 in'.encode('utf-8'))
md5.update('python hashlib?'.encode('utf-8'))
print(md5.hexdigest())
sha1 = hashlib.sha1()
sha1.update('how to use sha1 in'.encode('utf-8'))
sha1.update('python haslib?'.encode('utf-8'))
print(sha1.hexdigest())
#itertools
#Python的内建模块itertools提供了非常有用的用于操作迭代对象的函数
import itertools
natuals = itertools.count(1) #无限迭代器
cs = itertools.cycle('ABC')
ns = itertools.repeat('A', 3)
for n in ns:
print(n)
#通常我们会通过takewhile()等函数根据条件判断来截取出一个有限的序列
ps = itertools.takewhile(lambda x: x <= 10, natuals)
print(ps)
print(list(ps))
#chain chain()可以把一组迭代对象串联起来，形成一个更大的迭代器
for c in itertools.chain('ABC', 'DEF'):
print(c)
#groupby groupby()把迭代器中相邻的重复元素挑出来放在一起
for key, group in itertools.groupby('AaaBbbCcc', lambda c: c.upper()):
print(key, list(group))
def all_user_dictionaries(self):
return chain(self.active_user_dictionaries, self.inactive_user_dictionaries)
# Events that are standardised.
class _Events(enum.Enum):
"""zaza.events as an enum for type safety."""
# Core events
START_TEST = "start"
COMMENT = "comment"
EXCEPTION = "exception-in-span"
END_TEST = "end"
# Enums can't be extended, so we use this little trick.
Events = enum.Enum('Events', [(i.name, i.value)
for i in itertools.chain(NotifyEvents, _Events)])
class Span(enum.Enum):
"""Span possibilities."""
BEFORE = "before"
WITHIN = "within"
AFTER = "after"
# Meaning of the fields:
# 'collection' - the collection (loosely measurement in InfluxDB parlance)
# 'timestamp' - the timestamp
# 'event' - the event (from :class:`Events` above)
# 'span' - either before, after, or absent - used for spans
def edit_config(self, command):
for cmd in chain([b'configure'], to_list(command)):
self.send_command(cmd)
def reduce_columns(columns, *clauses, **kw):
r"""given a list of columns, return a 'reduced' set based on natural
equivalents.
the set is reduced to the smallest list of columns which have no natural
equivalent present in the list. A "natural equivalent" means that two
columns will ultimately represent the same value because they are related
by a foreign key.
\*clauses is an optional list of join clauses which will be traversed
to further identify columns that are "equivalent".
\**kw may specify 'ignore_nonexistent_tables' to ignore foreign keys
whose tables are not yet configured, or columns that aren't yet present.
This function is primarily used to determine the most minimal "primary
key" from a selectable, by reducing the set of primary key columns present
in the selectable to just those that are not repeated.
"""
ignore_nonexistent_tables = kw.pop("ignore_nonexistent_tables", False)
only_synonyms = kw.pop("only_synonyms", False)
columns = util.ordered_column_set(columns)
omit = util.column_set()
for col in columns:
for fk in chain(*[c.foreign_keys for c in col.proxy_set]):
for c in columns:
if c is col:
continue
try:
fk_col = fk.column
except exc.NoReferencedColumnError:
# TODO: add specific coverage here
# to test/sql/test_selectable ReduceTest
if ignore_nonexistent_tables:
continue
else:
raise
except exc.NoReferencedTableError:
# TODO: add specific coverage here
# to test/sql/test_selectable ReduceTest
if ignore_nonexistent_tables:
continue
else:
raise
if fk_col.shares_lineage(c) and (not only_synonyms
or c.name == col.name):
omit.add(col)
break
if clauses:
def visit_binary(binary):
if binary.operator == operators.eq:
cols = util.column_set(
chain(*[c.proxy_set for c in columns.difference(omit)]))
if binary.left in cols and binary.right in cols:
for c in reversed(columns):
if c.shares_lineage(
binary.right) and (not only_synonyms or c.name
== binary.left.name):
omit.add(c)
break
for clause in clauses:
if clause is not None:
visitors.traverse(clause, {}, {"binary": visit_binary})
return ColumnSet(columns.difference(omit))
def analyze_data(shops: dict):
return list(chain(*shops.values()))
def save_as_pdf_pages(plots, filename=None, path=None, verbose=True, **kwargs):
"""
Save multiple :class:`ggplot` objects to a PDF file, one per page.
Parameters
----------
plots : collection or generator of :class:`ggplot`
Plot objects to write to file. `plots` may be either a
collection such as a :py:class:`list` or :py:class:`set`:
>>> base_plot = ggplot(…)
>>> plots = [base_plot + ggtitle('%d of 3' % i) for i in range(1, 3)]
>>> save_as_pdf_pages(plots)
or, a generator that yields :class:`ggplot` objects:
>>> def myplots():
>>> for i in range(1, 3):
>>> yield ggplot(…) + ggtitle('%d of 3' % i)
>>> save_as_pdf_pages(myplots())
filename : :py:class:`str`, optional
File name to write the plot to. If not specified, a name
like “plotnine-save-<hash>.pdf” is used.
path : :py:class:`str`, optional
Path to save plot to (if you just want to set path and
not filename).
verbose : :py:class:`bool`
If ``True``, print the saving information.
kwargs : :py:class:`dict`
Additional arguments to pass to
:py:meth:`matplotlib.figure.Figure.savefig`.
Notes
-----
Using pandas' :meth:`~pandas.DataFrame.groupby` methods, tidy data
can be “faceted” across pages:
>>> from plotnine.data import mtcars
>>> def facet_pages(column)
>>> base_plot = [
>>> aes(x='wt', y='mpg', label='name'),
>>> geom_text(),
>>> ]
>>> for label, group_data in mtcars.groupby(column):
>>> yield ggplot(group_data) + base_plot + ggtitle(label)
>>> save_as_pdf_pages(facet_pages('cyl'))
Unlike :meth:`ggplot.save`, :meth:`save_as_pdf_pages` does not
process arguments for `height` or `width`. To set the figure size,
add :class:`~plotnine.themes.themeable.figure_size` to the theme
for some or all of the objects in `plots`:
>>> plot = ggplot(…)
>>> # The following are equivalent
>>> plot.save('filename.pdf', height=6, width=8)
>>> save_as_pdf_pages([plot + theme(figure_size=(8, 6))])
"""
from itertools import chain
from matplotlib.backends.backend_pdf import PdfPages
from six import raise_from
# as in ggplot.save()
fig_kwargs = {'bbox_inches': 'tight'}
fig_kwargs.update(kwargs)
figure = [None]
# If plots is already an iterator, this is a no-op; otherwise convert a
# list, etc. to an iterator
plots = iter(plots)
peek = []
# filename, depends on the object
if filename is None:
# Take the first element from the iterator, store it, and use it to
# generate a file name
peek = [next(plots)]
filename = peek[0]._save_filename('pdf')
if path:
filename = os.path.join(path, filename)
if verbose:
warn('Filename: {}'.format(filename))
with PdfPages(filename) as pdf:
# Re-add the first element to the iterator, if it was removed
for plot in chain(peek, plots):
try:
fig = figure[0] = plot.draw()
# as in ggplot.save()
facecolor = fig.get_facecolor()
edgecolor = fig.get_edgecolor()
if edgecolor:
fig_kwargs['facecolor'] = facecolor
if edgecolor:
fig_kwargs['edgecolor'] = edgecolor
fig_kwargs['frameon'] = True
# Save as a page in the PDF file
pdf.savefig(figure[0], **fig_kwargs)
except AttributeError as err:
raise_from(
TypeError('non-ggplot object of %s: %s' %
(type(plot), plot)), err)
except Exception as err:
raise
finally:
# Close the figure whether or not there was an exception, to
# conserve memory when plotting a large number of pages
figure[0] and plt.close(figure[0])
def action():
dict_extention = {
'audio_extention': ['.mp3', '.m4a', '.wav', '.cda', '.aif', '.mid', '.midi', '.mpa', '.ogg', '.wpl', '.wma'],
'video_extention': ['.mp4', '.mkv', '.MKV', '.flv', '.webm', '.vob', '.gif', '.avi', '.wmv', '.mpeg', '.3gp'],
'document_extention': ['.txt', '.pdf', '.doc', '.html', '.ppt'],
'image_extention': ['.jpg', '.gif', '.png', '.jpeg', '.ico', '.ps', '.psd', '.tif', '.tiff', '.bmp', '.ai'],
'programming_extentions': ['.py', '.js', '.java', '.swift', '.class', '.c', '.html''.xml', '.css', '.php',
'.rss', '.xhtml', '.asp', '.aspx', '.cer', '.cfm', '.cgi', '.pl', '.htm', '.jsp',
'.part'],
'compressed_extentions': ['.7z', '.arj', '.deb', '.pkg', '.rar', '.rpm', '.tar', '.gz', '.z', '.zip'],
'executabelFile_extentions': ['.apk', '.bat', '.bin ', '.pl', '.com ', '.exe', '.wsf', '.msi', '.jar',
'.gadget'],
'font_extentions': ['.fnt', '.otf', '.fon', '.ttf'],
}
walkfiles = []
walkroot = Dictionary()
old_folder_path = folderpath.get()
for root, directory, filename in os.walk(old_folder_path):
walkfiles.append(filename)
walkroot.add(root, filename)
merge = list(itertools.chain(*walkfiles))
if len(merge) == 0:
messagebox.showerror("Path Not Found", "The path is not given or is not available.")
else:
pass
def file_finder(merge_list, file_extentions):
files = []
for file in merge_list:
for extentions in file_extentions:
if file.endswith(extentions):
files.append(file)
return files
new_folder_path_list = []
for type, extentions in dict_extention.items():
new_folder_name = (str(type.split("_")[0])).capitalize() + "Files"
new_folder_path = os.path.join(old_folder_path, new_folder_name)
new_folder_path_list.append(new_folder_path)
if os.path.exists(new_folder_path):
pass
else:
os.mkdir(new_folder_path)
old_file_path_list = []
for i, j in walkroot.items():
for k in j:
old_file_path = os.path.join(i, k)
old_file_path_list.append(old_file_path)
for file in file_finder(merge, extentions):
for old in old_file_path_list:
try:
shutil.move(old, old_folder_path)
except:
continue
for file in file_finder(merge, extentions):
old_file_path = os.path.join(old_folder_path, file)
new_file_path = os.path.join(new_folder_path, file)
shutil.move(old_file_path, new_file_path)
walkroot2 = Dictionary()
for root, directory, file in os.walk(old_folder_path):
walkroot2.add(root, file)
for i in reversed(walkroot2.keys()):
if len(os.listdir(i)) == 0:
# print(f"DELETED --- {i}")
os.rmdir(i)
else:
# print(f"NOT DELETED---{i}")
continue
folderpath.delete(0, END)
idx2ner = [ 'PER_NAM', 'ORG_NAM', 'GPE_NAM', 'LOC_NAM', 'FAC_NAM',
'PER_NOM', 'ORG_NOM', 'GPE_NOM', 'LOC_NOM', 'FAC_NOM',
'O' ]
if not os.path.exists( valid_file ):
# load 10% KBP test data
source = imap( lambda x: x[1],
ifilter( lambda x : x[0] % 10 >= 9,
enumerate( imap( lambda x: x[:4],
LoadED( config.data_path + '/%s-eval-parsed' % config.language ) ) ) ) )
# load 5% iflytek data
if args.iflytek:
source = chain( source,
imap( lambda x: x[1],
ifilter( lambda x : 90 <= x[0] % 100 < 95,
enumerate( imap( lambda x: x[:4],
LoadED( 'iflytek-clean-%s' % config.language ) ) ) ) ) )
# istantiate a batch constructor
valid = batch_constructor( source,
numericizer1, numericizer2, gazetteer = kbp_gazetteer,
alpha = config.word_alpha, window = config.n_window,
n_label_type = config.n_label_type,
language = config.language )
logger.info( 'valid: ' + str(valid) )
if config.language != 'spa' and not os.path.exists( test_file ):
source = imap( lambda x: x[1],
ifilter( lambda x: x[0] % 100 >= 95,
def sat(val):
return Clauses(max(abs(v) for v in chain(*val))).sat(val)
def DEFAULTPOLICY(root,state):
while state.terminal()==False:
state=state.next_state()
MAX_STEPS = len([tile for tile in list(chain(*root.state.board)) if tile!='X'])
return state.reward(MAX_STEPS)
def application_factory(name='public'):
if name not in ('admin', 'public'):
raise RuntimeError('Application name (for base_url lookup) must be '
'either `admin` or `public`.')
# NOTE(morgan): The Flask App actually dispatches nothing until we migrate
# some routers to Flask-Blueprints, it is simply a placeholder.
app = flask.Flask(name)
app.after_request(_add_vary_x_auth_token_header)
# NOTE(morgan): Configure the Flask Environment for our needs.
app.config.update(
# We want to bubble up Flask Exceptions (for now)
PROPAGATE_EXCEPTIONS=True)
# TODO(morgan): Convert Subsystems over to Flask-Native, for now, we simply
# dispatch to another "application" [e.g "keystone"]
# NOTE(morgan): as each router is converted to flask-native blueprint,
# remove from this list. WARNING ORDER MATTERS; ordered dict used to
# ensure sane ordering of the routers in the legacy-dispatch model.
dispatch_map = collections.OrderedDict()
# Load in Healthcheck and map it to /healthcheck
hc_app = healthcheck.Healthcheck.app_factory(
{}, oslo_config_project='keystone')
dispatch_map['/healthcheck'] = hc_app
# More legacy code to instantiate all the magic for the dispatchers.
# The move to blueprints (FLASK) will allow this to be eliminated.
_routers = []
sub_routers = []
mapper = routes.Mapper()
for api_routers in ALL_API_ROUTERS:
moved_found = [
pfx for pfx in getattr(api_routers, '_path_prefixes', [])
if pfx in _MOVED_API_PREFIXES
]
if moved_found:
raise RuntimeError('An API Router is trying to register path '
'prefix(s) `%(pfx)s` that is handled by the '
'native Flask app. Keystone cannot '
'start.' %
{'pfx': ', '.join([p for p in moved_found])})
routers_instance = api_routers.Routers()
_routers.append(routers_instance)
routers_instance.append_v3_routers(mapper, sub_routers)
# TODO(morgan): Remove "API version registration". For now this is kept
# for ease of conversion (minimal changes)
keystone.api.discovery.register_version('v3')
# NOTE(morgan): We add in all the keystone.api blueprints here, this
# replaces (as they are implemented) the legacy dispatcher work.
for api in keystone.api.__apis__:
for api_bp in api.APIs:
api_bp.instantiate_and_register_to_app(app)
# Build and construct the dispatching for the Legacy dispatching model
sub_routers.append(_ComposibleRouterStub(_routers))
legacy_dispatcher = keystone_wsgi.ComposingRouter(mapper, sub_routers)
for pfx in itertools.chain(
*[rtr.Routers._path_prefixes for rtr in ALL_API_ROUTERS]):
dispatch_map['/v3/%s' % pfx] = legacy_dispatcher
app.wsgi_app = KeystoneDispatcherMiddleware(app.wsgi_app, dispatch_map)
return app
def remove(self,board,tileIndexs):
# tileIndex = [(i,j)...]
new_board = [tile if (i//self.board_size,i%self.board_size) not in tileIndexs else 'X' for i,tile in enumerate(list(chain(*board)))]
new_board = [new_board[j*self.board_size:j*self.board_size + self.board_size] for j in range(self.board_size)]
return new_board
def __init__(self,board,color={},move_state='',steps=0,pre_move='',side_panalty=0,move_board=''):
self.board = board
self.move_board = move_board
self.board_size = len(board)
self.steps = steps
self.color = color
self.pre_move = pre_move
self.side_panalty = side_panalty
self.move_state = move_state
next_move_state = 'ply_move' if self.move_state == 'AI_move' else 'AI_move'
self.moves = [(i//self.board_size,i%self.board_size)for i,tile in enumerate(list(chain(*board))) if tile == color[next_move_state]] # possible moves
def get_content(cls, abspath, start=None, end=None):
data = super(StaticHandler, cls).get_content(abspath, start=start, end=end)
return itertools.chain(data, [cls.get_append(abspath)])
def __hash__(self):
boardString = ' '.join(list(chain(*self.board)))
return int(hashlib.md5(boardString.encode('utf-8')).hexdigest(),16)
def test_cp2_reg_init_unused_hwregs(self):
'''Test that all cap hwregs that aren't defined in the spec are None'''
for t in self.MIPS.threads.values():
for i in itertools.chain(range(2, 8), range(9, 22), range(24, 28)):
self.assertIsNone(t.cp2_hwregs[i],
msg="Hwreg " + str(i) + " should not exist")
userFirst = True if input('user go first or not(Y/N):') == 'Y' else False
#get / set colors
colors = list({tile for tile in board[0]})
ply_colors = initPlyColor(colors,userFirst) # ply_colors={'ply_move' = 'R', 'AI_move' = 'B'}
#Game loop
initState = State(board,color=ply_colors,move_state='AI_move'if userFirst else 'ply_move')
terminal = False
pre_state = ''
final_board = ''
while not terminal:
if not pre_state:
pre_state = initState
firstLoopFunc = plyLoop if userFirst else AILoop
secondLoopFunc = AILoop if userFirst else plyLoop
pre_state,final_board = firstLoopFunc(pre_state)
if final_board : break
pre_state,final_board = secondLoopFunc(pre_state)
if final_board : break
remainSet = list({tile for tile in list(chain(*final_board)) if tile!= 'X'})
message = 'user win' if (len(remainSet) == 1 and remainSet[0] == ply_colors['AI_move']) else 'AI win' if (len(remainSet) == 1 and remainSet[0] == ply_colors['ply_move']) else 'draw'
print(message)
def asList(self):
return list(chain(*(self.lookup[i] for i in self.indexes)))
###########################################
# calculating class weights since this is an imbalanced dataset
y_collect = df.select("sponsoring_country").groupBy(
"sponsoring_country").count().collect()
unique_y = [x["sponsoring_country"] for x in y_collect]
total_y = sum([x["count"] for x in y_collect])
unique_y_count = len(y_collect)
bin_count = [x["count"] for x in y_collect]
class_weights_spark = {
i: ii
for i, ii in zip(unique_y, total_y /
(unique_y_count * np.array(bin_count)))
} # print(class_weights_spark) # {0.0: 5.0, 1.0: 0.5555555555555556}
mapping_expr = f.create_map(
[f.lit(x) for x in chain(*class_weights_spark.items())])
df = df.withColumn("weight", mapping_expr.getItem(f.col("sponsoring_country")))
# numerically labelling the sponsoring country
stringIndexer = StringIndexer(inputCol="sponsoring_country", outputCol="label")
model = stringIndexer.fit(df)
df = model.transform(df)
label_decoder = sorted(set([
(i[0], i[1]) for i in df.select(df.sponsoring_country, df.label).collect()
]),
key=lambda x: x[0])
label_decoder
# train/test split
train, test = df[df['is_validation'] == False], df[df['is_validation'] == True]
def checkout(self, url, ref=None, dir_path=None, recursive=False,
submodules=True, submodule_update_force=False,
keep_paths=None, step_suffix=None,
curl_trace_file=None, can_fail_build=True,
set_got_revision=False, remote_name=None,
display_fetch_size=None, file_name=None,
submodule_update_recursive=True,
use_git_cache=False):
"""Performs a full git checkout and returns sha1 of checked out revision.
Args:
url (str): url of remote repo to use as upstream
ref (str): ref to fetch and check out
dir_path (Path): optional directory to clone into
recursive (bool): whether to recursively fetch submodules or not
submodules (bool): whether to sync and update submodules or not
submodule_update_force (bool): whether to update submodules with --force
keep_paths (iterable of strings): paths to ignore during git-clean;
paths are gitignore-style patterns relative to checkout_path.
step_suffix (str): suffix to add to a each step name
curl_trace_file (Path): if not None, dump GIT_CURL_VERBOSE=1 trace to that
file. Useful for debugging git issue reproducible only on bots. It has
a side effect of all stderr output of 'git fetch' going to that file.
can_fail_build (bool): if False, ignore errors during fetch or checkout.
set_got_revision (bool): if True, resolves HEAD and sets got_revision
property.
remote_name (str): name of the git remote to use
display_fetch_size (bool): if True, run `git count-objects` before and
after fetch and display delta. Adds two more steps. Defaults to False.
file_name (str): optional path to a single file to checkout.
submodule_update_recursive (bool): if True, updates submodules
recursively.
use_git_cache (bool): if True, git cache will be used for this checkout.
WARNING, this is EXPERIMENTAL!!! This wasn't tested with:
* submodules
* since origin url is modified
to a local path, may cause problem with scripts that do
"git fetch origin" or "git push origin".
* arbitrary refs such refs/whatever/not-fetched-by-default-to-cache
Returns: If the checkout was successful, this returns the commit hash of
the checked-out-repo. Otherwise this returns None.
"""
retVal = None
# TODO(robertocn): Break this function and refactor calls to it.
# The problem is that there are way too many unrealated use cases for
# it, and the function's signature is getting unwieldy and its body
# unreadable.
display_fetch_size = display_fetch_size or False
if not dir_path:
dir_path = url.rsplit('/', 1)[-1]
if dir_path.endswith('.git'): # ex: https://host/foobar.git
dir_path = dir_path[:-len('.git')]
# ex: ssh://host:repo/foobar/.git
dir_path = dir_path or dir_path.rsplit('/', 1)[-1]
dir_path = self.m.path['start_dir'].join(dir_path)
if 'checkout' not in self.m.path:
self.m.path['checkout'] = dir_path
git_setup_args = ['--path', dir_path, '--url', url]
if remote_name:
git_setup_args += ['--remote', remote_name]
else:
remote_name = 'origin'
if self.m.platform.is_win:
self.ensure_win_git_tooling()
git_setup_args += [
'--git_cmd_path', self.package_repo_resource('git.bat')]
step_suffix = '' if step_suffix is None else ' (%s)' % step_suffix
self.m.python(
'git setup%s' % step_suffix,
self.resource('git_setup.py'),
git_setup_args)
# Some of the commands below require depot_tools to be in PATH.
path = self.m.path.pathsep.join([
str(self.package_repo_resource()), '%(PATH)s'])
with self.m.step.context({'cwd': dir_path}):
if use_git_cache:
with self.m.step.context({'env': {'PATH': path}}):
self('retry', 'cache', 'populate', '-c',
self.m.infra_paths.default_git_cache_dir, url,
name='populate cache',
can_fail_build=can_fail_build)
dir_cmd = self(
'cache', 'exists', '--quiet',
'--cache-dir', self.m.infra_paths.default_git_cache_dir, url,
can_fail_build=can_fail_build,
stdout=self.m.raw_io.output(),
step_test_data=lambda:
self.m.raw_io.test_api.stream_output('mirror_dir'))
mirror_dir = dir_cmd.stdout.strip()
self('remote', 'set-url', 'origin', mirror_dir,
can_fail_build=can_fail_build)
# There are five kinds of refs we can be handed:
# 0) None. In this case, we default to properties['branch'].
# 1) A 40-character SHA1 hash.
# 2) A fully-qualifed arbitrary ref, e.g. 'refs/foo/bar/baz'.
# 3) A fully qualified branch name, e.g. 'refs/heads/master'.
# Chop off 'refs/heads' and now it matches case (4).
# 4) A branch name, e.g. 'master'.
# Note that 'FETCH_HEAD' can be many things (and therefore not a valid
# checkout target) if many refs are fetched, but we only explicitly fetch
# one ref here, so this is safe.
fetch_args = []
if not ref: # Case 0
fetch_remote = remote_name
fetch_ref = self.m.properties.get('branch') or 'master'
checkout_ref = 'FETCH_HEAD'
elif self._GIT_HASH_RE.match(ref): # Case 1.
fetch_remote = remote_name
fetch_ref = ''
checkout_ref = ref
elif ref.startswith('refs/heads/'): # Case 3.
fetch_remote = remote_name
fetch_ref = ref[len('refs/heads/'):]
checkout_ref = 'FETCH_HEAD'
else: # Cases 2 and 4.
fetch_remote = remote_name
fetch_ref = ref
checkout_ref = 'FETCH_HEAD'
fetch_args = [x for x in (fetch_remote, fetch_ref) if x]
if recursive:
fetch_args.append('--recurse-submodules')
fetch_env = {'PATH': path}
fetch_stderr = None
if curl_trace_file:
fetch_env['GIT_CURL_VERBOSE'] = '1'
fetch_stderr = self.m.raw_io.output(leak_to=curl_trace_file)
fetch_step_name = 'git fetch%s' % step_suffix
if display_fetch_size:
count_objects_before_fetch = self.count_objects(
name='count-objects before %s' % fetch_step_name,
step_test_data=lambda: self.m.raw_io.test_api.stream_output(
self.test_api.count_objects_output(1000)))
with self.m.step.context({'env': fetch_env}):
self('retry', 'fetch', *fetch_args,
name=fetch_step_name,
stderr=fetch_stderr,
can_fail_build=can_fail_build)
if display_fetch_size:
self.count_objects(
name='count-objects after %s' % fetch_step_name,
previous_result=count_objects_before_fetch,
step_test_data=lambda: self.m.raw_io.test_api.stream_output(
self.test_api.count_objects_output(2000)))
if file_name:
self('checkout', '-f', checkout_ref, '--', file_name,
name='git checkout%s' % step_suffix,
can_fail_build=can_fail_build)
else:
self('checkout', '-f', checkout_ref,
name='git checkout%s' % step_suffix,
can_fail_build=can_fail_build)
rev_parse_step = self('rev-parse', 'HEAD',
name='read revision',
stdout=self.m.raw_io.output(),
can_fail_build=False,
step_test_data=lambda:
self.m.raw_io.test_api.stream_output('deadbeef'))
if rev_parse_step.presentation.status == 'SUCCESS':
sha = rev_parse_step.stdout.strip()
retVal = sha
rev_parse_step.presentation.step_text = "<br/>checked out %r<br/>" % sha
if set_got_revision:
rev_parse_step.presentation.properties['got_revision'] = sha
clean_args = list(itertools.chain(
*[('-e', path) for path in keep_paths or []]))
self('clean', '-f', '-d', '-x', *clean_args,
name='git clean%s' % step_suffix,
can_fail_build=can_fail_build)
if submodules:
self('submodule', 'sync',
name='submodule sync%s' % step_suffix,
can_fail_build=can_fail_build)
submodule_update = ['submodule', 'update', '--init']
if submodule_update_recursive:
submodule_update.append('--recursive')
if submodule_update_force:
submodule_update.append('--force')
self(*submodule_update,
name='submodule update%s' % step_suffix,
can_fail_build=can_fail_build)
return retVal
def __iter__(self):
for value in chain(*(self.lookup[i] for i in self.indexes)):
yield value
# --------------------------------------------------------------------------
all_words = set([PAD_WORD])
wordchar2vector_path = os.path.join(data_folder,'wordchar2vector.dat')
print( 'Loading the wordchar2vector model {}'.format(wordchar2vector_path) )
wc2v = gensim.models.KeyedVectors.load_word2vec_format(wordchar2vector_path, binary=False)
wc2v_dims = len(wc2v.syn0[0])
print('wc2v_dims={0}'.format(wc2v_dims))
for word in wc2v.vocab:
all_words.add(word)
word2id = dict( [(c,i) for i,c in enumerate( itertools.chain([PAD_WORD], filter(lambda z:z!=PAD_WORD, all_words)))] )
nb_words = len(all_words)
print('nb_words={}'.format(nb_words))
# --------------------------------------------------------------------------
#w2v_path = '/home/eek/polygon/w2v/w2v.CBOW=0_WIN=5_DIM=32.txt'
w2v_path = '/home/eek/polygon/w2v/w2v.CBOW=0_WIN=5_DIM=48.txt'
#w2v_path = '/home/eek/polygon/WordSDR2/sdr.dat'
#w2v_path = '/home/eek/polygon/w2v/w2v.CBOW=0_WIN=5_DIM=128.txt'
#w2v_path = r'f:\Word2Vec\word_vectors_cbow=1_win=5_dim=32.txt'
print( 'Loading the w2v model {}'.format(w2v_path) )
w2v = gensim.models.KeyedVectors.load_word2vec_format(w2v_path, binary=False)
w2v_dims = len(w2v.syn0[0])
print('w2v_dims={0}'.format(w2v_dims))
def asdict(ghcn_source, antarc_source):
print "Load GHCN records"
return dict((record.uid, record)
for record in itertools.chain(ghcn_source, antarc_source))
def index():
if request.method == 'POST':
searchString = request.form['content'].replace(" ","")
try:
USR = "******"
PWD = "Test#12345"
# Connection URL
#CONNECTION_URL = f"mongodb+srv://{USR}:{PWD}@cluster0.zvs9q.mongodb.net/{DB_NAME}?retryWrites=true&w=majority"
CONNECTION_URL = "mongodb://localhost:27017/" # opening a connection to Mongo # opening a connection to Mongo
try:
# Establish a connection with mongoDB
dbConn = pymongo.MongoClient(CONNECTION_URL)
print("Connected successfully!!!")
except:
print("Could not connect to MongoDB")
# Enter into collection Name
dataBase = dbConn["Web_Scraping_Project"] # Specifiy a Database Name
collection = dataBase[searchString].find({})
# check collection count
if collection.count() > 500: # if there is a collection with searched keyword and it has records in it
return render_template('results1.html',reviews=collection) # show the results to user
else:
flipkart_url = "https://www.flipkart.com/search?q=" + searchString
uClient = uReq(flipkart_url)
flipkartPage = uClient.read()
uClient.close()
# Create a Collection Name
collection = dataBase[searchString]
review_soup = bs(flipkartPage, 'html.parser')
pagesize_content = review_soup.find("div", {"class": "_2MImiq"})
pagesize_content.prettify()
pagesize_info = pagesize_content.span.contents[0]
res = [int(i) for i in pagesize_info.split() if i.isdigit()]
page_size = res[-1]
page_size = 1
print(page_size)
for n in list(range(page_size)):
print("Entering into page:", n + 1)
product_scrap_data = [] # Create the list to store the final result
print("Pagesize****************:",n)
#Navigate the required page
page_url = "https://www.flipkart.com/search?q="+searchString+"+&%3Bas=on&%3Bas-show=on&%3Botracker=AS_Query_HistoryAutoSuggest_1_2_na_na_na&%3Botracker1=AS_Query_HistoryAutoSuggest_1_2_na_na_na&%3Bas-pos=1&%3Bas-type=HISTORY&%3BsuggestionId=samsung+&%3BrequestId=3deaa522-51bf-4a70-a7fc-d86c0a81128d&%3Bpage=4&page="+str(n)
eachpage = requests.get(page_url)
eachpages = bs(eachpage.content,'html.parser')
#Extratc the product size from the page
container = eachpages.find('span',attrs={"class":'_10Ermr'})
page_product_start_count=(int(container.text.split(' ')[1]))-1
page_product_end_count = (int(container.text.split(' ')[3]))
products_per_page = page_product_end_count-page_product_start_count
print("products_per_page:",products_per_page)
#Extratc the product list from that page
pre_product_name_list = ['None'] * products_per_page
product_name_list = []
try:
for container in eachpages.findAll('a',href=True,attrs={"class":'_1fQZEK'}):
product_name_list.append(container.get('href').split('/')[1])
except:
product_name_list.append("Product name is not available")
pre_product_name_list[:len(product_name_list)] = product_name_list[:min(len(pre_product_name_list),len(product_name_list))]
#print("product_name_list",len(pre_product_name_list))
product_name = pre_product_name_list
#print(product_name)
#Extarct the product rating
pre_product_rating_list = ['None'] * products_per_page
product_rating_list = []
try:
for container in eachpages.findAll('div',attrs={"class":'_3LWZlK'}):
product_rating_list.append(container.text)
except:
product_rating_list.append("Product rating is not available")
#Consider only poduct rating from the main list
pre_product_rating_list[:len(product_rating_list)] = product_rating_list[:min(len(pre_product_rating_list),len(product_rating_list))]
product_rating= pre_product_rating_list
#print("product_rating",len(product_rating))
#print(product_rating)
#Extarct the rating count & review count
pre_rating_counts_list = ['None'] * products_per_page
pre_review_counts_list = ['None'] * products_per_page
rating_counts_list = []
review_counts_list = []
try:
for container in eachpages.findAll('span',attrs={"class":'_2_R_DZ'}):
rating_counts_list.append((container.text.replace('\xa0','').split('&'))[0])
review_counts_list.append((container.text.replace('\xa0','').split('&'))[1])
except:
review_counts_list.append("Product review count is not available")
rating_counts_list.append("Product rating count is not available")
pre_rating_counts_list[:len(rating_counts_list)] = rating_counts_list[:min(len(pre_rating_counts_list),len(rating_counts_list))]
pre_review_counts_list[:len(review_counts_list)] = review_counts_list[:min(len(pre_review_counts_list),len(review_counts_list))]
#print("rating_counts_list",len(pre_rating_counts_list))
#print("review_counts_list",len(pre_review_counts_list))
rating_counts = pre_rating_counts_list
review_counts = pre_rating_counts_list
#print(rating_counts)
#print(review_counts)
#Extarct the product description
pre_product_description_list = ['None'] * products_per_page
product_description_list = []
try:
for container in eachpages.findAll('ul',attrs={"class":'_1xgFaf'}):
product_description_list.append(container.text)
except:
product_description_list.append("Product description is not available")
pre_product_description_list[:len(product_description_list)] = product_description_list[:min(len(pre_product_description_list),len(product_description_list))]
#print(len(pre_product_description_list))
product_description = pre_product_description_list
#print(product_description)
#Extarct the offer price
pre_offer_price_list = ['None'] * products_per_page
offer_price_list =[]
try:
for container in eachpages.findAll('div',attrs={"class":'_30jeq3 _1_WHN1'}):
offer_price_list.append(container.text)
except:
offer_price_list.appned("Product offer_price is not available")
pre_offer_price_list[:len(offer_price_list)] = offer_price_list[:min(len(pre_offer_price_list),len(offer_price_list))]
#print(len(pre_offer_price_list))
offer_price = pre_offer_price_list
#print(offer_price)
#Extarct the actual price
pre_actual_price_list = ['None'] * products_per_page
actual_price_list = []
try:
for container in eachpages.findAll('div',attrs={"class":'_3I9_wc _27UcVY'}):
actual_price_list.append(container.text)
actual_price_list = actual_price_list[0:page_product_count[0]]
except:
actual_price_list.append("Product actual_price is not available")
pre_actual_price_list[:len(actual_price_list)] = actual_price_list[:min(len(pre_actual_price_list),len(actual_price_list))]
#print("length of price list",len(pre_actual_price_list))
#print(actual_price_list)
actual_price = pre_actual_price_list
#print(actual_price)
#Extarct the offer percentage
pre_offer_percentage_list = ['None'] * products_per_page
offer_percentage_list = []
try:
for container in eachpages.findAll('div',attrs={"class":'_3Ay6Sb'}):
offer_percentage_list.append(container.text)
offer_percentage_list = offer_percentage_list[0:page_product_count[0]]
except:
offer_percentage_list.append("Product offer_percentage is not available")
pre_offer_percentage_list[:len(offer_percentage_list)] = offer_percentage_list[:min(len(pre_offer_percentage_list),len(offer_percentage_list))]
#print(len(pre_offer_percentage_list)
offer_percentage = pre_offer_percentage_list
#print(offer_percentage)
# Extarct the top level class content
container = eachpages.findAll("div", {"class": "_1YokD2 _3Mn1Gg"})
# print(len(raw_bigboxes))
# Navigate to product list section
raw_bigboxes = container[1].findAll("div", {"class": "_1AtVbE col-12-12"})
# print(len(bigboxes))
# Eliminate the last two uncessary product info link
bigboxes = raw_bigboxes[: len(raw_bigboxes) - 2]
# print(len(bigboxes))
review_list = []
for i in list(range(products_per_page)):
print("Entering into product", i + 1)
box = bigboxes[i]
productLink = "https://www.flipkart.com" + box.div.div.div.a['href']
prodlink_info = requests.get(productLink)
prod_detail_info_html = bs(prodlink_info.content,'html.parser')
# Function to extarct the product info
def product_info_finder(tag_name, class_name, required_info):
try:
for container in prod_detail_info_html.find(tag_name, attrs={"class": class_name}):
return container.text
except:
return required_info + " data is not available"
# Extarct the offer difference
offer_difference = (product_info_finder('div', '_1V_ZGU', 'offer_difference'))
# print(offer_difference)
# Extarct the avaliable offer
avaliable_offer = (product_info_finder('div', '_3TT44I', 'avaliable_offer'))
# Extarct the warrenty info
warranty_info = (product_info_finder('div', 'XcYV4g', 'warranty_info'))
# print(warranty_info)
# Extarct the highlights_info
highlights_info = (product_info_finder('div', '_2418kt', 'highlights_info'))
# print(highlights_info)
# Extarct the easy payment info
easy_payment_info = (product_info_finder('div', '_250Jnj', 'easy_payment_info'))
# print(easy_payment_info)
# Extarct the seller info
seller_info = (product_info_finder('div', '_1RLviY', 'seller_info'))
# print(seller_info)
# Extarct the description_info
description_info = (product_info_finder('div', '_2o-xpa', 'description_info'))
# print(description_info)
# Extarct the product specification
product_spec = []
try:
for containers in prod_detail_info_html.findAll('div', attrs={"class": '_3k-BhJ'}):
summary = containers.findAll('table', attrs={"class": '_14cfVK'})
for items in summary:
rows = items.findAll('tr')
product_spec.append(
[':'.join([td.findNext(text=True) for td in tr.findAll("td")]) for tr in rows])
except:
product_spec.append("product_specification is not available")
product_spec_jointed = itertools.chain(*product_spec)
# Join the list item into single string
product_specification = (' '.join(product_spec_jointed))
# print(product_specification)
# function to extract the review details
def review_info_finder(tag_name, class_name, required_info):
try:
commentbox = prod_detail_info_html.find('div', {'class': "_16PBlm"})
tag_result = commentbox.find(tag_name, attrs={"class": class_name})
return tag_result.text
except:
return required_info + " data is not available"
# Extarct the Reviewer name
reviewer_name = (review_info_finder('p', '_2sc7ZR _2V5EHH', 'reviewer_name'))
# print(reviewer_name)
# Extarct the reviewer_rating
reviewer_rating = (review_info_finder('div', '_3LWZlK _1BLPMq', 'reviewer_rating'))
# print(reviewer_rating)
# Extract the comment_heading
comment_heading = (review_info_finder('p', '_2-N8zT', 'comment_heading'))
# print(comment_heading)
# Extarct the review_comments
review_comments = (review_info_finder('div', 't-ZTKy', 'review_comments'))
# print(review_comments)
# Extract the thumps up
try:
commentbox = prod_detail_info_html.find('div', {'class': "_16PBlm"})
thumsup = (commentbox.div.div.find_all('span', {'_3c3Px5'})[0].text)
except:
thumsup = 'Thumsup data is not avaliable'
# print(thumsup)
# Extarct the thumpsdown
try:
commentbox = prod_detail_info_html.find('div', {'class': "_16PBlm"})
thumsdown = (commentbox.div.div.find_all('span', {'_3c3Px5'})[1].text)
except:
thumsdown = 'Thumsdown data is not avaliable'
# print(thumsdown)
mydict = {"Product_Name": product_name[n], "Product_Rating": product_rating[n],
"Rating_Counts": rating_counts[n],
"Review_Counts": review_counts[n],
"Product_Description": product_description[n],
"Offer_Price": offer_price[n],
"Actual_Price": actual_price[n], "Offer_Percentage": offer_percentage[n],
"Offer_Difference": offer_difference,
"Avaliable_Offer": avaliable_offer, "Warranty_Info": warranty_info,
"Highlights_Info": highlights_info,
"Easy_Payment_Info": easy_payment_info, "Seller_Info": seller_info,
"Description_Info": description_info,
"Product_Specification": product_specification, "Reviewer_Name": reviewer_name,
"Reviewer_Rating": reviewer_rating, "Comment_Heading": comment_heading,
"Review_Comments": review_comments, "Thums-Up": thumsup, "Thums_Down": thumsdown}
review_list.append(mydict)
table = dataBase[searchString]
# print(db.list_collection_names())
col_exists = searchString in dataBase.list_collection_names()
if col_exists == True:
table = dataBase[searchString]
table.drop # drop the existing collection
table = dataBase[searchString] # creating a collection with the same name as search string. Tables and Collections are analogous.
time.sleep(1)
table.insert_many(review_list) # insertig the dictionary containing the rview comments to the collection
time.sleep(1)
return render_template('results1.html', reviews=review_list)
except:
return 'something is wrong'
# return render_template('results.html')
else:
return render_template('index.html')
def rpad_wordseq(words, n):
return list( itertools.chain( words, itertools.repeat(PAD_WORD, n-len(words)) ) )
def build_vocabulary(cutoff, *texts):
combined_texts = chain(*texts)
return NgramModelVocabulary(cutoff, combined_texts)
def enrich_report_with_guidelines(scan_report: Report, guidelines: Dict[str, str]) -> None:
for record in itertools.chain(scan_report.failed_checks, scan_report.passed_checks, scan_report.skipped_checks):
if record.check_id in guidelines:
record.set_guideline(guidelines[record.check_id])
class Test_expand_operator:
# Conceptually, a lot of these tests are complete duplicates of
# `TestGateExpansion`, except that they explicitly target an underlying
# function in `qutip_qip.operations.gates` which (as of 2020-03-01) is not
# called in the majority of cases---it appears to be newer than the
# surrounding code, but the surrounding code wasn't updated.
@pytest.mark.parametrize(
'permutation',
tuple(itertools.chain(*[
itertools.permutations(range(k)) for k in [2, 3, 4]
])),
ids=_permutation_id)
def test_permutation_without_expansion(self, permutation):
base = qutip.tensor([qutip.rand_unitary(2) for _ in permutation])
test = gates.expand_operator(base,
N=len(permutation), targets=permutation)
expected = base.permute(_apply_permutation(permutation))
np.testing.assert_allclose(test.full(), expected.full(), atol=1e-15)
@pytest.mark.parametrize('n_targets', range(1, 5))
def test_general_qubit_expansion(self, n_targets):
# Test all permutations with the given number of targets.
n_qubits = 5
operation = qutip.rand_unitary(2**n_targets, dims=[[2]*n_targets]*2)
for targets in itertools.permutations(range(n_qubits), n_targets):
expected = _tensor_with_entanglement([qutip.qeye(2)] * n_qubits,
operation, targets)
test = gates.expand_operator(operation, n_qubits, targets)
np.testing.assert_allclose(test.full(), expected.full(),
atol=1e-15)
def test_cnot_explicit(self):
test = gates.expand_operator(gates.cnot(), 3, [2, 0]).full()
expected = np.array([[1, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 1, 0, 0],
[0, 0, 1, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 1, 0, 0, 0],
[0, 1, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 1, 0],
[0, 0, 0, 1, 0, 0, 0, 0]])
np.testing.assert_allclose(test, expected, atol=1e-15)
def test_cyclic_permutation(self):
operators = [qutip.sigmax(), qutip.sigmaz()]
test = gates.expand_operator(qutip.tensor(*operators), N=3,
targets=[0, 1], cyclic_permutation=True)
base_expected = qutip.tensor(*operators, qutip.qeye(2))
expected = [base_expected.permute(x)
for x in [[0, 1, 2], [1, 2, 0], [2, 0, 1]]]
assert len(expected) == len(test)
for element in expected:
assert element in test
@pytest.mark.parametrize('dimensions', [
pytest.param([3, 4, 5], id="standard"),
pytest.param([3, 3, 4, 4, 2], id="standard"),
pytest.param([1, 2, 3], id="1D space"),
])
def test_non_qubit_systems(self, dimensions):
n_qubits = len(dimensions)
for targets in itertools.permutations(range(n_qubits), 2):
operators = [qutip.rand_unitary(dimension) if n in targets
else qutip.qeye(dimension)
for n, dimension in enumerate(dimensions)]
expected = qutip.tensor(*operators)
base_test = qutip.tensor(*[operators[x] for x in targets])
test = gates.expand_operator(base_test, N=n_qubits,
targets=targets, dims=dimensions)
assert test.dims == expected.dims
np.testing.assert_allclose(test.full(), expected.full())
def _extract_drug_info(drug_xml: ElementTree.Element) -> Mapping[str, Any]:
"""Extract information from an XML element representing a drug."""
# assert drug_xml.tag == f'{ns}drug'
row = {
'type': drug_xml.get('type'),
'drugbank_id': drug_xml.findtext(f"{ns}drugbank-id[@primary='true']"),
'cas_number': drug_xml.findtext(f"{ns}cas-number"),
'name': drug_xml.findtext(f"{ns}name"),
'description': drug_xml.findtext(f"{ns}description"),
'groups': [
group.text
for group in drug_xml.findall(f"{ns}groups/{ns}group")
],
'atc_codes': [
code.get('code')
for code in drug_xml.findall(f"{ns}atc-codes/{ns}atc-code")
],
'categories': [
{
'name': x.findtext(f'{ns}category'),
'mesh_id': x.findtext(f'{ns}mesh-id'),
}
for x in drug_xml.findall(f"{ns}categories/{ns}category")
],
'patents': [
{
'patent_id': x.findtext(f'{ns}number'),
'country': x.findtext(f'{ns}country'),
'approved': datetime.datetime.strptime(x.findtext(f'{ns}approved'), '%Y-%m-%d'),
'expires': datetime.datetime.strptime(x.findtext(f'{ns}expires'), '%Y-%m-%d'),
'pediatric_extension': x.findtext(f'{ns}pediatric-extension') != 'false',
}
for x in drug_xml.findall(f"{ns}patents/{ns}patent")
],
'xrefs': [
{
'resource': x.findtext(f'{ns}resource'),
'identifier': x.findtext(f'{ns}identifier'),
}
for x in drug_xml.findall(f"{ns}external-identifiers/{ns}external-identifier")
],
'inchi': drug_xml.findtext(inchi_template),
'inchikey': drug_xml.findtext(inchikey_template),
'smiles': drug_xml.findtext(smiles_template),
}
# Add drug aliases
aliases = {
elem.text.strip() for elem in
itt.chain(
drug_xml.findall(f"{ns}international-brands/{ns}international-brand"),
drug_xml.findall(f"{ns}synonyms/{ns}synonym[@language='English']"),
drug_xml.findall(f"{ns}international-brands/{ns}international-brand"),
drug_xml.findall(f"{ns}products/{ns}product/{ns}name"),
)
if elem.text.strip()
}
aliases.add(row['name'])
row['aliases'] = aliases
row['protein_interactions'] = []
row['protein_group_interactions'] = []
for category, protein in _iterate_protein_stuff(drug_xml):
target_row = _extract_protein_info(category, protein)
if not target_row:
continue
row['protein_interactions'].append(target_row)
return row
