def merge_txt(raw_ann: AnnotationData, no_type: AnnType) -> AnnotationData:
"""Return a normalized (lines merged) version of ``raw_ann``:
'\n\n' in *.txt are the real lines, and are parsed as ''.
So to create real lines, we merge all the strings between ''.
"""
merged_ann: AnnotationData = AnnotationData(
annotation_difficulty=raw_ann.annotation_difficulty,
annotations=[])
types: Set = set()
lines: List[str] = []
i: int = 0
for virtual_line in raw_ann.annotations:
if virtual_line.text == '':
i += 1
# remove no_type from the virtual lines when part of the line was
# annotated
if len(types) > 1:
types.discard(no_type)
merged_ann.annotations.append(AnnotatedLine(line_num=i,
types=types,
text=' '.join(lines)))
types = set()
lines = []
else:
types |= virtual_line.types
lines.append(virtual_line.text)
return merged_ann
def __init__(self):
self.field_s: Set[FieldModel] = set()
# Related fields that shouldn't be targeted for randomization
self.dnr_field_s: Set[FieldModel] = set()
self.constraint_s: Set[ConstraintModel] = set()
self.field_l = None
self.all_field_l = None
def findAthletesToInsert(resultsFile, urlAthleteCheck):
table = []
with open(resultsFile, mode="r") as f:
for line in f:
table.append(line.split(";"))
firstNameId = table[0].index("FirstName")
lastNameId = table[0].index("LastName")
birthYearId = table[0].index("Yob")
clubCodeId = table[0].index("ClubCode")
clubNameId = table[0].index("ClubName")
genderId = table[0].index("Gender")
notExistingAthletes = set()
existingAthletes = set()
for l in table[1:10000]:
if l[clubCodeId] == tvuID or l[clubNameId] == tvuName:
fullName = l[firstNameId] + " " + l[lastNameId]
birthYear = l[birthYearId]
if not (fullName, birthYear) in notExistingAthletes and not (
fullName, birthYear) in existingAthletes:
if not checkAthleteExists(urlAthleteCheck, fullName,
birthYear):
notExistingAthletes.add((fullName, birthYear, l[genderId]))
else:
existingAthletes.add((fullName, birthYear))
exportCSV(
notExistingAthletes, "notExistingAthletes{}.csv".format(
datetime.datetime.now().strftime("%Y%m%d%H%M")))
if len(notExistingAthletes) == 0:
print("\n\nReady For Insertation. All Athletes are in the DB\n\n")
return notExistingAthletes
def __init__(self, *args, **kwargs):
fields = kwargs.pop('fields', None)
super(DynamicFieldsModelSerializer, self).__init__(*args, **kwargs)
if fields is not None:
allowed = set(fields)
existing = set(self.fields)
for field_name in existing - allowed:
self.fields.pop(field_name)
def post(self, request):
leaf_names = request.POST.getlist('leaf_name')
results = []
for leaf_name in leaf_names:
result = Plant.objects.filter(leafchars__chars=leaf_name)
if len(results):
results = list(set(result) & set(results))
else:
results = result
print(results)
return render(request, 'index.html', context={'results': results})
def load_all_fields(db, month: int, year: int, required_fields, **kwargs):
assert len(required_fields) > 0
db.market_quote_cache.create_index([('tag', pymongo.ASCENDING)], unique=True)
uber_df = None
impute_fields = kwargs.get('impute_fields', set())
market = kwargs.get('market', 'asx')
for field_name in required_fields:
print("Constructing matrix: {} {}-{}".format(field_name, month, year))
rows = load_field_dataframe(db, field_name, month, year)
if len(rows) == 0:
continue
df = pd.DataFrame.from_records(rows)
df = df.pivot(index='fetch_date', columns='asx_code', values='field_value')
if field_name in impute_fields: # some fields will break the viewer if too many NaN
before = df.isnull().sum().sum()
print(f"Imputing {field_name} - before {before} missing values")
df = df.fillna(method='pad').fillna(method='backfill')
after = df.isnull().sum().sum()
print(f"After imputation {field_name} - now {after} missing values")
rows = []
for asx_code, series in df.iteritems():
for fetch_date, field_value in series.iteritems():
rows.append({ 'asx_code': asx_code, 'fetch_date': fetch_date, 'field_value': field_value, 'field_name': field_name})
#print(rows)
if uber_df is None:
uber_df = pd.DataFrame.from_records(rows)
else:
uber_df = uber_df.append(rows)
if df.isnull().values.any():
dates_with_missing = set([df.loc[the_date].isnull().any() for the_date in df.index])
today = datetime.strftime(datetime.today(), "%Y-%m-%d")
if today in dates_with_missing:
print(f"WARNING: today's {field_name} matrix contains missing data! Continuing anyway.")
print(df.loc[today].isnull().values.any())
# FALLTHRU...
tag = "{}-{:02d}-{}-{}".format(field_name, month, year, market)
#save_dataframe(db, df, tag, field_name, status, market, scope, compression='gzip', n_days=len(df), n_stocks=len(df.columns))
all_stocks = set(uber_df['asx_code'])
print("Detected {} stocks during month ({} datapoints total)".format(len(all_stocks), len(uber_df)))
uber_tag = "uber-{:02d}-{}-{}".format(month, year, market)
all_fields = set(uber_df['field_name'])
# TODO FIXME: should we drop missing values from the uber parquet? might save space... since we will get them back on load
#print(all_fields)
#print(required_fields)
assert all_fields == set(required_fields)
print("% missing values: ", ((uber_df.isnull() | uber_df.isna()).sum() * 100 / uber_df.index.size).round(2))
n_stocks = uber_df['asx_code'].nunique()
n_days = uber_df['fetch_date'].nunique()
print(uber_df)
save_dataframe(db, uber_df, uber_tag, 'uber', n_days=n_days, n_stocks=n_stocks, **kwargs)
def run_game(self):
# run rounds until there is a winner
while (len(self.players) - len(self.inactive_players)) > 1:
self.run_round()
self.current_round += 1
# If there is only one player left, do some formatting
if (len(self.players) - len(self.inactive_players)) == 1:
winner = list(set(self.players) - set(self.inactive_players))
self.eliminated_players.insert(0, [winner[0].get_name()])
# sort list of disqualified players
self.disqualified_players.sort()
return self.eliminated_players, self.disqualified_players
def __init__(self, rng):
# TODO: need access to the random state
super().__init__()
self._pass = 0
self._field_s = set()
self._active_constraint = None
self._active_randset = None
self._randset_s: Set[RandSet] = set()
self._randset_field_m: Dict[FieldModel,
RandSet] = {} # map<field,randset>
self._constraint_s: List[ConstraintModel] = []
self._used_rand = True
self._in_generator = False
self.active_cp = None
self._rng = rng
def test_filtering_no_tags(self):
a = self.make_target('a', tags=['tag1'])
b = self.make_target('b', tags=['tag1', 'tag2'])
c = self.make_target('c', tags=['tag2'])
filtered_targets = TargetFiltering(set()).apply_tag_blacklist([a, b, c])
self.assertEqual([a, b, c], filtered_targets)
def handle_nick(data, match, client, channels):
"""
When a user changes their nick, tell everyone in all the channels they're in
about it.
'^NICK (?P<nick>.*)'
:type data: str
:type match: dict
:type client: Client
:type channels: list
"""
newnick = match['nick']
logger.info("Set new user's nick to '{newnick}'", newnick=newnick)
client.send(Protocol.Nick.response(client.nick, newnick))
announce = Protocol.Nick.announce(client, newnick)
for cl in set(chain.from_iterable(chan.clients for chan in client.channels)):
if cl is not client:
cl.send(announce)
client.nick = newnick
def __init__(self, rng):
# TODO: need access to the random state
super().__init__()
self._pass = 0
self._field_m = {}
self._field_l = []
self._active_constraint = None
self._active_randset = None
# Tracks the randsets added due to
# dependencies against ordered fields
self._active_order_randset_s = set()
self._randset_m: Dict[RandSet, int] = {}
self._randset_l = []
self._randset_field_m: Dict[FieldModel,
RandSet] = {} # map<field,randset>
self._constraint_s: List[ConstraintModel] = []
self._soft_priority = 0
self._used_rand = True
self._in_generator = False
self.active_cp = None
self._rng = rng
self._order_m = {}
self._expr2fm = Expr2FieldVisitor()
def FindWordWithMaxFrequency(self):
# convert all text to lowercase
doc_text = self.DataSourcetext.lower()
# remove numbers
doc_text = re.sub(r'\d+', '', doc_text)
# remove punctuation, characters and whitespaces
doc_text = re.sub('\W+', ' ', doc_text)
# remove stopwords and tokenize
stop_words = set(stopwords.words('english'))
tokens = word_tokenize(doc_text)
result = [i for i in tokens if not i in stop_words]
wordlist = []
for word in result:
if len(word) > 2:
wordlist.append(word)
wordfreq = []
for w in wordlist:
wordfreq.append(wordlist.count(w))
return max(list(zip(wordlist, wordfreq)), key=itemgetter(1))[0]
def set(arg, default=None):
val_list = list(arg, default=default)
if val_list is not None:
return builtins.set(val_list)
else:
return val_list
def ListMethods(dev):
"""List user-callable methods for the device.
Example:
>>> ListMethods(phi)
"""
dev = session.getDevice(dev, Device)
items = []
listed = builtins.set()
def _list(cls):
if cls in listed:
return
listed.add(cls)
for name, (args, doc, mcls, is_user) in sorted(cls.methods.items()):
if cls is mcls and is_user:
items.append((dev.name + '.' + name + args, cls.__name__, doc))
for base in cls.__bases__:
if issubclass(base, (Device, DeviceMixinBase)):
_list(base)
_list(dev.__class__)
dev.log.info('Device methods:')
printTable(('method', 'from class', 'description'), items,
session.log.info)
def __init__(self, global_context, raw_msg):
"""Parse the message, extracts and decode all headers and all
text parts.
"""
super(Message, self).__init__(global_context)
self.raw_msg = self.translate_line_breaks(raw_msg)
self.msg = email.message_from_string(self.raw_msg)
self.headers = _Headers()
self.raw_headers = _Headers()
self.addr_headers = _Headers()
self.name_headers = _Headers()
self.mime_headers = _Headers()
self.received_headers = _Headers()
self.raw_mime_headers = _Headers()
self.header_ips = _Headers()
self.text = ""
self.raw_text = ""
self.uri_list = set()
self.score = 0
self.rules_checked = dict()
self.interpolate_data = dict()
self.plugin_tags = dict()
# Data
self.sender_address = ""
self.hostname_with_ip = list()
self._parse_message()
self._hook_parsed_metadata()
def __init__(self, order=-1):
self.order = order
self.field_s: Set[FieldModel] = set()
self.field_rand_l = []
self.all_field_l = []
self.constraint_s: Set[ConstraintModel] = set()
self.constraint_l: List[ConstraintModel] = []
self.soft_constraint_s: Set[ConstraintModel] = set()
self.soft_constraint_l: List[ConstraintModel] = []
self.soft_priority = 0
self.dist_field_m = {}
# List of fields in each ordered set
# Only non-none if order constraints impact this randset
self.rand_order_l = None
def __init__(self, global_context, raw_msg):
"""Parse the message, extracts and decode all headers and all
text parts.
"""
super(Message, self).__init__(global_context)
self.raw_msg = self.translate_line_breaks(raw_msg)
self.msg = email.message_from_string(self.raw_msg)
self.headers = _Headers()
self.raw_headers = _Headers()
self.addr_headers = _Headers()
self.name_headers = _Headers()
self.mime_headers = _Headers()
self.received_headers = list()
self.raw_mime_headers = _Headers()
self.header_ips = _Headers()
self.text = ""
self.raw_text = ""
self.uri_list = set()
self.score = 0
self.rules_checked = dict()
self.interpolate_data = dict()
self.plugin_tags = dict()
# Data
self.sender_address = ""
self.hostname_with_ip = list()
self.internal_relays = []
self.external_relays = []
self.last_internal_relay_index = 0
self.last_trusted_relay_index = 0
self.trusted_relays = []
self.untrusted_relays = []
self._parse_message()
self._hook_parsed_metadata()
def maximumSum(a, m):
# logger.debug("a {} m {}".format(a, m))
suma_mod = partial(lambda m, x, y: (x % m + y % m) % m, m)
resta_mod = partial(lambda m, x, y: (x - y + m) % m, m)
ord_set = OrderedSet()
unord_set = set()
acum = 0
r = 0
for n in a:
acum = suma_mod(acum, n)
logger.debug("acm {}".format(acum))
sig = ord_set.find_gt(acum)
optim = acum
if sig:
optim = resta_mod(acum, sig)
logger.debug("optimus {}".format(optim))
if optim > r:
r = optim
if acum not in unord_set:
ord_set.add(acum)
unord_set.add(acum)
# todos = list(map(lambda n:n.key, ord_set))
# todos_s = sorted(todos)
# assert todos == todos_s, "esperado {} obtenido {}".format(todos_s, todos)
# logger.debug("r es {} ord set {}".format(r, list(map(lambda n:n.key, ord_set))))
# rt=fuerza_bruta(a, m)
# assert r==rt, "esperado {} real {}".format(r,rt)
return r
def _get_update_artifact_cache_work(self, vts_artifactfiles_pairs):
"""Create a Work instance to update an artifact cache, if we're configured to.
vts_artifactfiles_pairs - a list of pairs (vts, artifactfiles) where
- vts is single VersionedTargetSet.
- artifactfiles is a list of paths to artifacts for the VersionedTargetSet.
"""
cache = self._cache_factory.get_write_cache()
if cache:
if len(vts_artifactfiles_pairs) == 0:
return None
# Do some reporting.
targets = set()
for vts, _ in vts_artifactfiles_pairs:
targets.update(vts.targets)
self._report_targets(
'Caching artifacts for ',
list(targets),
'.',
logger=self.context.log.debug,
)
always_overwrite = self._cache_factory.overwrite()
# Cache the artifacts.
args_tuples = []
for vts, artifactfiles in vts_artifactfiles_pairs:
overwrite = always_overwrite or vts.cache_key in self._cache_key_errors
args_tuples.append((cache, vts.cache_key, artifactfiles, overwrite))
return Work(lambda x: self.context.subproc_map(call_insert, x), [(args_tuples,)], 'insert')
else:
return None
def handle_nick(data, match, client, channels):
"""
When a user changes their nick, tell everyone in all the channels they're in
about it.
'^NICK (?P<nick>.*)'
:type data: str
:type match: dict
:type client: Client
:type channels: list
"""
newnick = match['nick']
logger.info("Set new user's nick to '{newnick}'", newnick=newnick)
client.send(Protocol.Nick.response(client.nick, newnick))
announce = Protocol.Nick.announce(client, newnick)
for cl in set(chain.from_iterable(chan.clients
for chan in client.channels)):
if cl is not client:
cl.send(announce)
client.nick = newnick
def __resolve_single(self, from_idtype, to_idtype, ids):
from_mappings = self.mappers.get(from_idtype, {})
to_mappings = from_mappings.get(to_idtype, [])
if not to_mappings:
_log.warn("cannot find mapping from %s to %s", from_idtype,
to_idtype)
return [None for _ in ids]
def apply_mapping(mapper, ids: List[str]):
# Each mapper can define if it preserves the order of the incoming ids.
if hasattr(mapper, "preserves_order") and mapper.preserves_order:
return mapper(ids)
else:
# If this is not the case, we need to map every single id separately
return [mapper([id])[0] for id in ids]
if len(to_mappings) == 1:
# single mapping no need for merging
return apply_mapping(to_mappings[0], ids)
# two way to preserve the order of the results
r = [[] for _ in ids]
rset = [set() for _ in ids]
for mapper in to_mappings:
mapped_ids = apply_mapping(mapper, ids)
for mapped_id, rlist, rhash in zip(mapped_ids, r, rset):
for id in mapped_id:
if id not in rhash:
rlist.append(id)
rhash.add(id)
return r
def run_election(vp_file: open) -> {str}:
ballot_count = 1
vd = read_voter_preferences(vp_file)
print("Preferences: voter -> [candidates in order]")
print(dict_as_str(vd))
rc = {i for i in list(vd.values())[0]}
while len(rc) != 1:
if len(list(vd.values())[0]) <= ballot_count:
break
votes = evaluate_ballot(vd, rc)
print(
f'Vote count on ballot #{ballot_count}: candidates (sorted alphabetically) using only candidates in set {rc}'
)
for i, j in sorted(votes.items()):
print(f'{i} -> {j}')
print(
f'Vote count on ballot #{ballot_count}: candidates (sorted numerically) using only candidates in set {rc}'
)
for i, j in sorted(votes.items(), key=(lambda x: x[1]), reverse=True):
print(f'{i} -> {j}')
rc = remaining_candidates(votes)
ballot_count += 1
print(f'Election winner is {set(list(rc)[0])}' if len(rc) == 1 else
'Not any unique winner: all remaining candidates on the ballot tie')
return rc if len(rc) == 1 else set()
def __init__(self, datasets):
"""
Parameters
----------
datasets: List[dataset]
List of Datasets objects. Currently restricted to those generated by 'absorbance', 'fluorescence'
and 'luminescence' operations
"""
operation_set = set([ds.operation for ds in datasets])
if len(operation_set) > 1:
raise RuntimeError("Input Datasets must all be of the same type.")
self.operation = operation_set.pop()
if self.operation not in ["absorbance", "fluorescence", "luminescence"]:
raise RuntimeError("%s has to be of type absorbance, fluorescence or luminescence" % self.operation)
super(Spectrophotometry, self).__init__(datasets)
# Assume that well names are consistent across all runs
ref_dataset = datasets[0]
ref_container = ref_dataset.container
# Check if well_map is defined
if len(ref_container.well_map) != 0:
self.properties = pd.DataFrame.from_dict(ref_container.well_map, orient='index')
else:
self.properties = pd.DataFrame.from_dict({ref_container.container_type.robotize(x): x
for x in ref_dataset.data.columns
if x not in ["GAIN"]},
orient='index')
self.properties.columns = ['name']
self.properties.insert(1, "column", (self.properties.index % ref_container.container_type.col_count))
self.properties.insert(1, "row", (self.properties.index // ref_container.container_type.col_count))
self.properties.row = self.properties.row.apply(lambda x: "ABCDEFGHIJKLMNOPQRSTUVWXYZ"[x])
self.properties.index = [ref_container.container_type.humanize(int(x)) for x in list(self.properties.index)]
def hasCombat(self):
playerset = set()
for va in self.vaisseaux:
playerset.add(va.player)
if len(playerset) > 1:
return True
return False
def search(self, from_idtype, to_idtype, query, max_results=None):
"""
Searches for matches in the names of the given idtype.
This operation does not resolve transitive mappings.
:param query:
:param max_results
:return:
"""
from_mappings = self.mappers.get(from_idtype, {})
to_mappings = from_mappings.get(to_idtype, [])
to_mappings = [m for m in to_mappings if hasattr(m, "search")]
if not to_mappings:
_log.warn("cannot find mapping from %s to %s", from_idtype,
to_idtype)
return []
if len(to_mappings) == 1:
# single mapping no need for merging
return to_mappings[0].search(query, max_results)
rset = set()
for mapper in to_mappings:
results = mapper.search(query, max_results)
for r in results:
rset.add(r)
return list(rset)
def test_filtering_no_tags(self):
a = self.make_target('a', tags=['tag1'])
b = self.make_target('b', tags=['tag1', 'tag2'])
c = self.make_target('c', tags=['tag2'])
filtered_targets = TargetFiltering(set()).apply_tag_blacklist(
[a, b, c])
self.assertEqual([a, b, c], filtered_targets)
def set(
env_var: builtins.str,
default: Optional[Set[Any]] = None,
) -> Optional[Set[Any]]:
if env_var is None or env_var == "":
return default
return builtins.set(env_var.split(","))
def dirty(f, *args, **kwargs):
""" Dirty the cache key for the function invoked with the given parameters. """
# delete the key containing the cached value for this function
version = f.__version
shard = f.__shard
sorted_parameters = _sorted_parameters(f, *args, **kwargs)
keys_to_delete = builtins.set([_cache_key(f, sorted_parameters, version=version)])
# if we have keys to be excluded, then dirty all of those as well
if hasattr(f, '__wildcard'):
included_parameters = [e for e in sorted_parameters if e[0] not in f.__wildcard]
included_keys_key = _cache_key(f, included_parameters, INCLUDED_KEYS_PREFIX, version=version)
keys = get(included_keys_key, shard) or builtins.set()
keys_to_delete |= keys
return delete_multi(keys_to_delete, shard)
def merge_txt(raw_ann: List[Annotated_line], no_type: AnnType,
prelude_type: AnnType) -> List[Annotated_line]:
"""
Return a normalized (lines merged) version of ``raw_ann``
'\n\n' in *.txt are the real lines, and are parsed as ''.
So we merge all the strings between ''
"""
merged_ann: List[Annotated_line] = []
types: Set = set()
lines: List = []
i: int = 0
for virtual_line in raw_ann:
if virtual_line.text == '':
i += 1
# remove no_type from the virtual lines when part of the line was
# annotated
if len(types) > 1:
types.discard(no_type)
merged_ann += [
Annotated_line(line_num=i, types=types, text=' '.join(lines))
]
types = set()
lines = []
else:
types |= virtual_line.types
lines.append(virtual_line.text)
def has_no_type(ann_line: Annotated_line) -> bool:
return {no_type} == ann_line.types
# add the prelude_type to all the first untyped lines
for_prelude_ann: List[Annotated_line] = list(
itertools.takewhile(has_no_type, merged_ann))
with_prelude_ann: List[Annotated_line] = list(
map(
lambda x: Annotated_line(
line_num=x.line_num, types={prelude_type}, text=x.text),
for_prelude_ann))
tail_ann: List[Annotated_line] = list(
itertools.dropwhile(has_no_type, merged_ann))
merged_ann = list(with_prelude_ann) + (list(tail_ann))
return merged_ann
def set_toadd(i, n):
toAdd = set()
for idx in n.machine.nodes:
if idx != i:
toAdd.add(idx)
if not n.hidden:
for idx in conf_type.pubNodes:
if idx != i:
toAdd.add(idx)
return toAdd
def __init__(self, path, name):
self.path = path
self.name = name
self.folders = set()
self.toc = Toc()
self.last_updated = 0
self.ignore_updates = False
# add the addon name as a primary folder
self.folders.add(name)
def __init__(self, file):
with open(file , "r") as f:
config_json = json.load(f)
base_stations = config_json.get("base_stations")
if not base_stations:
raise "No base stations defined"
self._bssid_set = set(base_stations)
self._known_addrs = config_json.get("known_addresses")
if not self._known_addrs:
self._known_addrs = {}
def _filter_targets(self, targets):
included_targets = TargetFilter.scoped_instance(self).apply(targets)
excluded_targets = set(targets).difference(included_targets)
if excluded_targets:
self.context.log.info("{} target(s) excluded".format(len(excluded_targets)))
for target in excluded_targets:
self.context.log.debug("{} excluded".format(target.address.spec))
return included_targets
def if_row():
checkRow = splitList(gridsL, 9)
for i in range(9):
poslist = checkRow[i]
for j in range(9):
poslist[j] = abs(int(poslist[j]))
sl = list(set(poslist))
sl += [0] * (poslist.count(0) - 1)
if len(poslist) != len(sl):
return False
return True
def __init__(self, username, password, page_id, output):
if USE_VIRTUAL_DISPLAY:
self.display = Display(visible=0, size=(1366, 768))
self.display.start()
self.driver = get_driver(username, password)
self.page_id = page_id
self.output = output
self.post_count = 0
self.post_id_old = set()
self.current_year = datetime.datetime.now().year
self.scrap_page(FB_HOME + self.page_id)
def known_idtypes(self):
"""
returns a set of a all known id types in this mapping graph
:return:
"""
s = set()
for from_, v in self.mappers.items():
s.add(from_)
for to_ in list(v.keys()):
s.add(to_)
return s
def wrapper(*args, **kwargs):
if not hasattr(f, '__version'):
f.__version = version
if not hasattr(f, '__shard'):
f.__shard = shard
# check if value already exists in cache
sorted_parameters = _sorted_parameters(f, *args, **kwargs)
key = _cache_key(f, sorted_parameters, version=version)
cached = get(key, shard)
if cached is not None:
return cached
# value is not cached, so compute and store
result = f(*args, **kwargs)
set(key, result, shard, timeout=timeout)
# if this function has specified a list of keys to wildcard, then we need to store a separate
# list mapping a key without those parameters to a list of keys with those parameters
included_keys_key = None
if wildcard:
# keep track of the excluded parameters so we have it when we dirty later
wildcard_set = builtins.set(wildcard)
if not hasattr(f, '__wildcard'):
f.__wildcard = wildcard_set
# get a list of parameters without the keys designated excluded
included_parameters = [e for e in sorted_parameters if e[0] not in wildcard_set]
included_keys_key = _cache_key(f, included_parameters, INCLUDED_KEYS_PREFIX, version=version)
included_keys = get(included_keys_key, shard)
# add the current key to that set of keys
if not included_keys or not isinstance(included_keys, builtins.set):
included_keys = builtins.set()
included_keys.add(key)
set(included_keys_key, included_keys, shard)
return result
def _parse_message(self):
"""Parse the message."""
self._hook_check_start()
# Dump the message raw headers
for name, raw_value in self.msg._headers:
self.raw_headers[name].append(raw_value)
# XXX This is strange, but it's what SA does.
# The body starts with the Subject header(s)
body = list(self.get_decoded_header("Subject"))
raw_body = list()
for payload, part in self._iter_parts(self.msg):
# Extract any MIME headers
for name, raw_value in part._headers:
self.raw_mime_headers[name].append(raw_value)
text = None
if payload is not None:
# this must be a text part
self.uri_list.update(set(URL_RE.findall(payload)))
if part.get_content_subtype() == "html":
text = self.normalize_html_part(payload.replace("\n", " "))
text = " ".join(text)
body.append(text)
raw_body.append(payload)
else:
text = payload.replace("\n", " ")
body.append(text)
raw_body.append(payload)
self._hook_extract_metadata(payload, text, part)
self.text = " ".join(body)
self.raw_text = "\n".join(raw_body)
self._parse_sender()
received_headers = self.get_decoded_header("Received")
for header in self.ctxt.conf["originating_ip_headers"]:
headers = ["X-ORIGINATING-IP: %s" % x
for x in self.get_decoded_header(header)]
received_headers.extend(headers)
received_obj = ReceivedParser(received_headers)
self.received_headers = received_obj.received
self._parse_relays(self.received_headers)
try:
self._create_plugin_tags(self.received_headers[0])
except IndexError:
pass
for header in self.received_headers:
self.hostname_with_ip.append((header["rdns"], header["ip"]))
def disconnect(devices=None):
"""
"""
global targets
remove_devices = devices and devices or list(targets)
targets = targets - builtins.set(remove_devices)
# This means disconnect from all already established connections
if not devices:
global available_connections
for k, v in available_connections.items():
pass
# close socket in available_connections[k]["socket"]
available_connections.clear()
# print("disconnect {}".format(connections))
def modified(date=None, etag=None):
"""
Checks to see if the page has been modified since the version in the
requester's cache.
When you publish pages, you can include `Last-Modified` and `ETag`
with the date the page was last modified and an opaque token for
the particular version, respectively. When readers reload the page,
the browser sends along the modification date and etag value for
the version it has in its cache. If the page hasn't changed,
the server can just return `304 Not Modified` and not have to
send the whole page again.
This function takes the last-modified date `date` and the ETag `etag`
and checks the headers to see if they match. If they do, it returns
`True`, or otherwise it raises NotModified error. It also sets
`Last-Modified` and `ETag` output headers.
"""
try:
from builtins import set
except ImportError:
# for python 2.3
from sets import Set as set
n = set([x.strip('" ') for x in web.ctx.env.get('HTTP_IF_NONE_MATCH', '').split(',')])
m = net.parsehttpdate(web.ctx.env.get('HTTP_IF_MODIFIED_SINCE', '').split(';')[0])
validate = False
if etag:
if '*' in n or etag in n:
validate = True
if date and m:
# we subtract a second because
# HTTP dates don't have sub-second precision
if date-datetime.timedelta(seconds=1) <= m:
validate = True
if date: lastmodified(date)
if etag: web.header('ETag', '"' + etag + '"')
if validate:
raise web.notmodified()
else:
return True
def _parse_message(self):
"""Parse the message."""
self._hook_check_start()
# Dump the message raw headers
for name, raw_value in self.msg._headers:
self.raw_headers[name].append(raw_value)
# XXX This is strange, but it's what SA does.
# The body starts with the Subject header(s)
body = list(self.get_decoded_header("Subject"))
raw_body = list()
for payload, part in self._iter_parts(self.msg):
# Extract any MIME headers
for name, raw_value in part._headers:
self.raw_mime_headers[name].append(raw_value)
text = None
if payload is not None:
# this must be a text part
self.uri_list.update(set(URL_RE.findall(payload)))
if part.get_content_subtype() == "html":
text = self.normalize_html_part(payload.replace("\n", " "))
text = " ".join(text)
body.append(text)
raw_body.append(payload)
else:
text = payload.replace("\n", " ")
body.append(text)
raw_body.append(payload)
self._hook_extract_metadata(payload, text, part)
self.text = " ".join(body)
self.raw_text = "\n".join(raw_body)
self._parse_sender()
for value in self.get_received_headers("Received"):
if 'from' in value:
hostname = value.split(' ')[1]
ip = IPFRE.search(value).group()
clean_ip = ip.strip("[ ]();\n")
try:
self.hostname_with_ip.append((hostname, clean_ip))
except ValueError:
continue
def disconnect(devices=None):
"""[summary]
[description]
Keyword Arguments:
devices {[type]} -- [description] (default: {None})
"""
# Disconnect from selected devices or all targets
global targets
remove_devices = devices and devices or list(targets)
# Update target set for future operations
targets = targets - builtins.set(remove_devices)
# StreamWriters are closed only if no devices are specified (disconnect from all)
if not devices:
global async_event_loop
global available_connections
for gw in available_connections.keys():
s = available_connections[gw]["streams"]
try:
async_event_loop.run_until_complete(_close_stream(s[1]))
except Exception as e:
print("Exception while closing streams: {}".format(e))
available_connections.clear()
try:
async_event_loop.stop()
async_event_loop.close()
except Exception as e:
print("Disconnect: {}".format(e))
def __init__(self, context, workdir):
"""Subclass __init__ methods, if defined, *must* follow this idiom:
class MyTask(Task):
def __init__(self, *args, **kwargs):
super(MyTask, self).__init__(*args, **kwargs)
...
This allows us to change Task.__init__()'s arguments without
changing every subclass. If the subclass does not need its own
initialization, this method can (and should) be omitted entirely.
:API: public
"""
super(TaskBase, self).__init__()
self.context = context
self._workdir = workdir
self._task_name = type(self).__name__
self._cache_key_errors = set()
self._cache_factory = CacheSetup.create_cache_factory_for_task(self)
self._force_invalidated = False
def _parse_message(self):
"""Parse the message."""
self._hook_check_start()
# Dump the message raw headers
self.ctxt.log.debug("EMAIL %s", self.raw_msg)
for line in self.raw_msg:
self.ctxt.log.debug("LINE %s", line)
if not email.feedparser.headerRE.match(line):
# If we saw the RFC defined header/body separator
# (i.e. newline), just throw it away. Otherwise the line is
# part of the body so push it back.
if not email.feedparser.NLCRE.match(line):
self.missing_header_body_separator = True
break
for name, raw_value in self.msg._headers:
self.raw_headers[name].append(raw_value)
# XXX This is strange, but it's what SA does.
# The body starts with the Subject header(s)
body = list(self.get_decoded_header("Subject"))
raw_body = list()
for payload, part in self._iter_parts(self.msg):
if not part._headers:
self.missing_boundary_header = True
# Extract any MIME headers
for name, raw_value in part._headers:
self.raw_mime_headers[name].append(raw_value)
text = None
if payload is not None:
# this must be a text part
self.uri_list.update(set(URL_RE.findall(payload)))
if part.get_content_subtype() == "html":
text = self.normalize_html_part(payload.replace("\n", " "))
text = " ".join(text)
body.append(text)
raw_body.append(payload)
else:
text = payload.replace("\n", " ")
body.append(text)
raw_body.append(payload)
self._hook_extract_metadata(payload, text, part)
self.text = " ".join(body)
self.raw_text = "\n".join(raw_body)
received_headers = self.get_decoded_header("Received")
for header in self.ctxt.conf["originating_ip_headers"]:
headers = ["X-ORIGINATING-IP: %s" % x
for x in self.get_decoded_header(header)]
received_headers.extend(headers)
received_obj = ReceivedParser(received_headers)
self.received_headers = received_obj.received
self._parse_relays(self.received_headers)
self._parse_sender()
try:
self._create_plugin_tags(self.received_headers[0])
except IndexError:
pass
for header in self.received_headers:
self.hostname_with_ip.append((header["rdns"], header["ip"]))
import builtins import itertools from pyname import pyname try: targets except NameError: targets = builtins.set() try: available_connections except NameError: available_connections = dict() try: properties except NameError: properties = dict() def connect(devices, rbac_token, timeout_s): """ """ # Make a copy of the generator devices, devices_copy = itertools.tee(devices) global targets
def apply(self, targets): exclude_tags = set(self.get_options().exclude_tags) return TargetFiltering(exclude_tags).apply_tag_blacklist(targets)
def plot(self, wells="*", groupby=None, title=None, xlabel=None, ylabel=None, max_legend_len=20):
"""
This generates a plot of the kinetics curve. Note that this function is meant for use under a Jupyter notebook
environment
Example Usage:
.. code-block:: python
from transcriptic.analysis.kinetics import Spectrophotometry
growth_curve = Spectrophotometry(myRun.data.Datasets)
growth_curve.plot(wells=["A1", "A2", "B1", "B2"])
growth_curve.plot(wells=["A1", "A2", "B1", "B2"], groupby="row", title="Row Groups")
growth_curve.plot(wells=["A1", "A2", "B1", "B2"], groupby="name", ylabel="Absorbance Units")
growth_curve.plot(groupby="name", max_legend_len=40)
Parameters
----------
wells: Optional[list or str]
If not specified, this plots all the wells associated with the Datasets given. Otherwise, specifiy
a list of well indices (["A1", "B1"]) or a specific well ("A1")
groupby: Optional[str]
When specified, this groups the wells with the same property value together. On the plot, each group will
be represented by a single curve with the mean values and error bars of 1 std. dev. away from the mean
title: Optional[str]
Plot title. Default: "Kinectics Curve (`run-id`)"
xlabel: Optional[str]
Plot x-axis label. Default: "Time"
ylabel: Optional[str]
Plot y-axis label. Default: "`Operation` (`Wavelength`)"
max_legend_len
Maximum number of characters for the legend labels before truncating. Default: 20
Returns
-------
IPlot
Plotly iplot object. Will be rendered nicely in Jupyter notebook instance
"""
# TODO: Shift init_notebook_mode() to start of notebook instance
py.offline.init_notebook_mode()
if isinstance(wells, str):
if wells != "*":
wells = [wells]
else:
well_readings = self.readings
wells = list(self.properties.index)
if isinstance(wells, list):
well_readings = self.readings.loc[wells]
if not groupby:
traces = [go.Scatter(x=self.readings.columns, y=well_readings.loc[well],
name=self.properties["name"].loc[well]) for well in wells]
else:
if groupby not in self.properties.columns:
raise ValueError("\'%s\' not found in the properties table. Please specify a column which exists" %
groupby)
grouped = self.properties.groupby(groupby)
index_list = [grouped.get_group(group).index for group in grouped.groups]
reading_map = []
for indx in index_list:
common_set = set(well_readings.index).intersection(set(indx))
if len(common_set) != 0:
reading_map.append(well_readings.loc[common_set])
if len(reading_map) != 0:
traces = [go.Scatter(x=self.readings.columns,
y=reading.mean(),
name=self._truncate_name(self.properties[groupby].loc[reading.iloc[0].name],
max_legend_len),
error_y=dict(type='data', array=reading.std(), visible=True)
)
for reading in reading_map]
else:
raise ValueError("No common groups found for specified groupby: %s" % groupby)
# Assume all data is generated from the same run-id for now
if not title:
title = "Kinetics Curve (%s)" % self.datasets[0].attributes["instruction"]["run"]["id"]
if not xlabel:
xlabel = 'Time'
if not ylabel:
if self.operation == "absorbance":
ylabel = "RAU (%s)" % self.datasets[0].attributes["instruction"]["operation"]["wavelength"]
elif self.operation == "fluorescence":
ylabel = "RFU (%s/%s)" % (self.datasets[0].attributes["instruction"]["operation"]["excitation"],
self.datasets[0].attributes["instruction"]["operation"]["emission"])
elif self.operation == "luminescence":
ylabel = "Luminescence"
layout = go.Layout(
title=title,
xaxis=dict(
title=xlabel,
titlefont=dict(
family='Courier New, monospace',
size=18,
color='#7f7f7f'
)
),
yaxis=dict(
title=ylabel,
titlefont=dict(
family='Courier New, monospace',
size=18,
color='#7f7f7f'
)
),
legend=dict(
x=100,
y=1
)
)
fig = go.Figure(data=traces, layout=layout)
return py.offline.iplot(fig)
def _filter_fields(fields):
""" Filter out invalid field names. """
exclude = builtins.set(['__created__'])
return [e for e in fields if e not in exclude]
