def _do_discovery_for(
host_name: HostName,
ipaddress: Optional[HostAddress],
parsed_sections_broker: ParsedSectionsBroker,
run_plugin_names: Container[CheckPluginName],
only_new: bool,
discovery_parameters: DiscoveryParameters,
) -> None:
host_labels = analyse_host_labels(
host_name=host_name,
ipaddress=ipaddress,
parsed_sections_broker=parsed_sections_broker,
discovery_parameters=discovery_parameters,
)
service_result = analyse_discovered_services(
host_name=host_name,
ipaddress=ipaddress,
parsed_sections_broker=parsed_sections_broker,
discovery_parameters=discovery_parameters,
run_plugin_names=run_plugin_names,
only_new=only_new,
)
# TODO (mo): for the labels the corresponding code is in _host_labels.
# We should put the persisting and logging in one place.
autochecks.save_autochecks_file(host_name, service_result.present)
new_per_plugin = Counter(s.check_plugin_name for s in service_result.new)
for name, count in sorted(new_per_plugin.items()):
console.verbose("%s%3d%s %s\n" % (tty.green + tty.bold, count, tty.normal, name))
section.section_success("%s, %s" % (
f"Found {len(service_result.new)} services"
if service_result.new else "Found no%s services" % (" new" if only_new else ""),
f"{len(host_labels.new)} host labels" if host_labels.new else "no%s host labels" %
(" new" if only_new else ""),
))
async def search(self, ctx, *, command: str):
"""Search for command stats"""
await self.update_global()
data = await self.config.globaldata()
if not data:
return await ctx.send("No commands have been used yet.")
new_data = Counter({})
for cmd in data:
if command in cmd:
new_data[cmd] = data[cmd]
if not new_data:
return await ctx.send("No data can be found for your query.")
await GenericMenu(
source=EmbedFormat(self.build_data(new_data)),
title=f"Commands Statistics for {command}",
_type="Command",
ctx=ctx,
).start(
ctx=ctx,
wait=False,
)
def calc_weights(freqs: Counter) -> Counter:
"""Calculate weights for locust tasks.
Calculate locust task weights based on the frequency that each
document was requested, as given by `freqs`.
Parameters
----------
freqs : {Counter}
The number of requests for each document.
Returns
-------
Counter
The weight for each document when mapped as a locust task.
"""
weights = Counter(freqs)
_, least = freqs.most_common()[-1]
for key in weights:
weights[key] = round(weights[key] / least)
return weights
def set_vocab_size(self, *files: Path) -> None:
if self.vocab is not None:
log.warning("Warning: vocabulary already set!")
word_counts: Counter[str] = Counter() # count of each word
for file in files:
for token in get_tokens(file):
word_counts[token] += 1
self.show_progress()
vocab: Set[str] = set(w for w in word_counts
if word_counts[w] >= OOV_THRESHOLD)
vocab |= { # Union equals
OOV,
EOS,
} # add EOS to vocab (but not BOS, which is never a possible outcome but only a context)
self.vocab = vocab
print(
f"Vocabulary size is {self.vocab_size} types including OOV and EOS"
)
def deduplicates_names(departement_centers):
"""
Removes unique names by appending city name
in par_departement
see https://github.com/CovidTrackerFr/vitemadose/issues/173
"""
deduplicated_centers = []
departement_center_names_count = Counter(
[center["nom"] for center in departement_centers])
names_to_remove = {
departement
for departement in departement_center_names_count
if departement_center_names_count[departement] > 1
}
for center in departement_centers:
if center["nom"] in names_to_remove:
center[
"nom"] = f"{center['nom']} - {departementUtils.get_city(center['metadata']['address'])}"
deduplicated_centers.append(center)
return deduplicated_centers
def reconcile(self, overlapping_points):
transformations = []
for ref_points, points in overlapping_points:
ref_vec = ref_points[1] - ref_points[0]
vec = points[1] - points[0]
rotation_matrices = list(
filter(lambda m: np.array_equal(m @ vec, ref_vec),
ROTATIONS_MATRICES))
if len(rotation_matrices) == 1:
rotation_matrix = rotation_matrices[0]
translation = ref_points[0] - rotation_matrix @ points[0]
transformations.append((rotation_matrix, translation))
s, _ = Counter([str(t) for t in transformations]).most_common(1)[0]
self.rotation_matrix, self.translation = filter(
lambda t: str(t) == s, transformations).__next__()
self.points = [(self.rotation_matrix @ p + self.translation)
for p in self.points]
self.compute_distances()
return self
def test_debug_data_compile_gold_for_spans():
nlp = English()
spans_key = "sc"
pred = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."])
pred.spans[spans_key] = [Span(pred, 3, 6, "ORG"), Span(pred, 5, 6, "GPE")]
ref = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."])
ref.spans[spans_key] = [Span(ref, 3, 6, "ORG"), Span(ref, 5, 6, "GPE")]
eg = Example(pred, ref)
data = _compile_gold([eg], ["spancat"], nlp, True)
assert data["spancat"][spans_key] == Counter({"ORG": 1, "GPE": 1})
assert data["spans_length"][spans_key] == {"ORG": [3], "GPE": [1]}
assert data["spans_per_type"][spans_key] == {
"ORG": [Span(ref, 3, 6, "ORG")],
"GPE": [Span(ref, 5, 6, "GPE")],
}
assert data["sb_per_type"][spans_key] == {
"ORG": {"start": [ref[2:3]], "end": [ref[6:7]]},
"GPE": {"start": [ref[4:5]], "end": [ref[6:7]]},
}
def train(self, corpus: Path) -> List[str]:
"""Read the training corpus and collect any information that will be needed
by the prob function later on. Tokens are whitespace-delimited.
Note: In a real system, you wouldn't do this work every time you ran the
testing program. You'd do it only once and save the trained model to disk
in some format.
"""
log.info(f"Training from corpus {corpus}")
# The real work:
# accumulate the type and token counts into the global hash tables.
# If vocab size has not been set, build the vocabulary from training corpus
if self.vocab is None:
self.set_vocab_size(corpus)
# Clear out any previous training.
self.tokens = Counter()
# We save the corpus in memory to a list tokens_list. Notice that we
# prepended two BOS at the front of the list and appended an EOS at the end. You
# will need to add more BOS tokens if you want to use a longer context than
# trigram.
x, y = BOS, BOS # Previous two words. Initialized as "beginning of sequence"
# count the BOS context
self.tokens[(x, y)] = 1
self.tokens[(y,)] = 1 # The syntax for a 1-element tuple in Python
tokens_list = [x, y] # the corpus saved as a list
for z in get_tokens(corpus):
z = self.replace_missing(z)
self.count(x, y, z)
self.show_progress()
x, y = y, z # Shift over by 1 word.
tokens_list.append(z)
sys.stderr.write("\n") # done printing progress dots "...."
log.info(f"Finished counting {self.tokens[()]} tokens")
return tokens_list
def life(black_tiles: Set[Tuple[int, int]]) -> None:
"""Simulates hexagonal game of life, with tiles"""
tiles_to_flip: Set[Tuple[int, int]] = set()
white_tile_neighbour_counter: Counter[Tuple[int, int]] = Counter()
for black_tile in black_tiles:
neighbour_count = count_neighbours(black_tiles, black_tile)
if not 1 <= neighbour_count <= 2:
tiles_to_flip.add(black_tile)
# Also, mark all adjacent white tiles
for tile in get_neighbours(black_tile):
if tile in black_tiles:
continue
white_tile_neighbour_counter[tile] += 1
for black_tile, neighbour_count in white_tile_neighbour_counter.items():
if neighbour_count == 2:
tiles_to_flip.add(black_tile)
black_tiles ^= tiles_to_flip
def create_id_dict():
#Each number will have a unique frequency id, ie. number of occurences for each letter
numbers_key = {
'0': "abcefg",
'1': "cf",
'2': "acdeg",
'3': "acdfg",
'4': "bcdf",
'5': "abdfg",
'6': "abdefg",
'7': "acf",
'8': "abcdefg",
'9': "abcdfg"
}
frequency = Counter()
for val in numbers_key.values():
frequency.update(val)
id = {}
for val, string_id in numbers_key.items():
list_id = str(sorted([frequency[letter] for letter in string_id]))
id[list_id] = val
return id
def run_sim(inp, days):
# Create Counter to track number of fish with each counter value
# ([0..8])
c = Counter()
for fish in inp:
c[fish] += 1
for _ in range(days):
c0 = c[0]
# Move counts down by one place
for fc in range(1, 9):
c[fc - 1] = c[fc]
# Spawn fish with counter 8 for all counter 0 fish
c[8] = c0
# Move previous c[0] fish to c[6]
c[6] += c0
# Return count of all fish
return sum(c.values())
def part2(data: str):
"""Part 2 solution"""
grid = {
complex(row, col)
for row in range(128)
for col, val in enumerate(knot_hash(f"{data}-{row}")) if val == "1"
}
regions: dict[complex, int] = {}
current_region = 0
while grid:
todo = [next(iter(grid))]
current_region += 1
while todo:
pos = todo.pop()
if pos not in grid:
continue
regions[pos] = current_region
grid.remove(pos)
for n_pos in neighbors(pos):
if n_pos in grid:
todo.append(n_pos)
return len(Counter(regions.values()))
def find_next(
cards: List[Card],
index: Dict[str, int],
norm: Counter[str],
deck: List[str],
) -> str:
links: Counter[str] = Counter()
for name in deck:
card = cards[index[name.lower()]]
links.update(card["Forward"])
links.update(card["Reverse"])
links.update(card["Recommended"])
total = sum(links.values())
final_links: Dict[str, float] = dict(
(k, v / total) for k, v in links.items())
for name in list(links.keys()):
if name.lower() not in index:
del links[name]
continue
final_links[name] *= math.log(len(cards) / norm[name])
if not len(links):
return random.choice([c["Name"] for c in cards])
return random.choices(*zip(*links.items()))[0]
def myCode():
a = int(sys.stdin.readline())
num = [] * a
for _ in range(a):
num.append(int(sys.stdin.readline()))
sort_num = sorted(num)
num_cnt = Counter(sort_num).most_common()
print(round(sum(num) / a))
print(sort_num[(a // 2)])
if (len(num_cnt)) > 1:
if num_cnt[0][1] == num_cnt[1][1]:
print(num_cnt[1][0])
else:
print(num_cnt[0][0])
else:
print(num_cnt[0][0])
print(max(num) - min(num))
async def cleanup(self, ctx, search=100):
"""Cleans up the bot's messages from the channel."""
def check(m):
return m.author == ctx.me or m.content.startswith(ctx.prefix)
deleted = await ctx.channel.purge(limit=search,
check=check,
before=ctx.message)
spammers = Counter(m.author.display_name for m in deleted)
count = len(deleted)
messages = [
f'{count} message{" was" if count == 1 else "s were"} removed.'
]
if len(deleted) > 0:
messages.append("")
spammers = sorted(spammers.items(),
key=lambda t: t[1],
reverse=True)
messages.extend(f"– **{author}**: {count}"
for author, count in spammers)
await ctx.send("\n".join(messages), delete_after=5)
def leastBricks(self, wall: List[List[int]]) -> int:
def cumsum(lst):
for i in range(len(lst)):
if i == 0:
pass
else:
lst[i] = lst[i] + lst[i - 1]
return lst
from collections import Counter
from functools import reduce
if not wall: return 0
wall_num = len(wall)
wall_len = sum(wall[0])
c = reduce(lambda x, y: x + y, [Counter(cumsum(i)) for i in wall])
del c[wall_num]
del c[wall_len]
if c:
return wall_num - c.most_common(1)[0][1]
else:
return wall_num
def get_layout_document_with_text_or_graphic_replaced_by_graphics(
layout_document: LayoutDocument,
semantic_graphics: Iterable[SemanticGraphic],
is_replace_overlapping_text: bool) -> LayoutDocument:
page_by_page_number = {
page.meta.page_number: page
for page in layout_document.pages if page.meta
}
LOGGER.debug('page_by_page_number.keys: %r', page_by_page_number.keys())
has_changes = False
semantic_graphics_list = list(semantic_graphics)
semantic_graphic_count_by_page = Counter(
(semantic_graphic.layout_graphic.coordinates.page_number
for semantic_graphic in semantic_graphics_list
if (semantic_graphic.layout_graphic
and semantic_graphic.layout_graphic.coordinates)))
for semantic_graphic in semantic_graphics_list:
layout_graphic = semantic_graphic.layout_graphic
assert layout_graphic
if not layout_graphic.coordinates:
continue
page_number = layout_graphic.coordinates.page_number
page_by_page_number[page_number] = (
get_layout_page_with_text_or_graphic_replaced_by_graphic(
page_by_page_number[page_number],
semantic_graphic,
is_only_semantic_graphic_on_page=(
semantic_graphic_count_by_page[page_number] < 2),
is_replace_overlapping_text=is_replace_overlapping_text))
has_changes = True
if not has_changes:
return layout_document
pages = [
(page_by_page_number[page.meta.page_number] if page.meta else page)
for page in layout_document.pages
]
return layout_document.replace(pages=pages)
def _c(
self,
children_subterms: SubTerms,
) -> Terms:
"""
Dict representing the polynomial that multiply the polynomial of the
flipped child for length n.
"""
min_sizes = self._min_sizes[:self.idx] + self._min_sizes[self.idx + 1:]
max_sizes = self._max_sizes[:self.idx] + self._max_sizes[self.idx + 1:]
possible_sizes = utils.compositions(self._parent_shift,
self.number_of_children - 1,
min_sizes, max_sizes)
other_children_subterms = (children_subterms[:self.idx] +
children_subterms[self.idx + 1:])
res: Terms = Counter()
for sizes in possible_sizes:
for param_value_pairs in CartesianProduct.params_value_pairs_combinations(
sizes, other_children_subterms):
new_param = self._other_new_param(
*(p for p, _ in param_value_pairs))
res[new_param] += utils.prod((v for _, v in param_value_pairs))
assert res[new_param] >= 0
return res
def _a(
self,
n: int,
parent_subterm: Callable[[int], Terms],
children_subterms: SubTerms,
) -> Terms:
"""
A terms dict representing the polynomial for the rule parent terms minus the
contribution of the childs when the size of the flipped child in the composition
is smaller than n.
"""
max_sizes = (self._max_sizes[:self.idx] + (n - 1, ) +
self._max_sizes[self.idx + 1:])
possible_sizes = utils.compositions(n + self._parent_shift,
self.number_of_children,
self._min_sizes, max_sizes)
res: Terms = Counter(parent_subterm(n + self._parent_shift))
for sizes in possible_sizes:
for param_value_pairs in CartesianProduct.params_value_pairs_combinations(
sizes, children_subterms):
new_param = self._new_param(*(p for p, _ in param_value_pairs))
res[new_param] -= utils.prod((v for _, v in param_value_pairs))
assert res[new_param] >= 0
return res
def record(self, ctx, name):
guild = ctx.message.guild
if not ctx.message.author.bot:
if getattr(ctx, "assume_yes", False):
if name not in self.cache["automated"]:
self.cache["automated"][name] = 1
else:
self.cache["automated"][name] += 1
return
if guild is not None:
if str(guild.id) not in self.cache["guild"]:
self.cache["guild"][str(guild.id)] = Counter({})
if name not in self.cache["guild"][str(guild.id)]:
self.cache["guild"][str(guild.id)][name] = 1
else:
self.cache["guild"][str(guild.id)][name] += 1
if name not in self.cache["session"]:
self.cache["session"][name] = 1
else:
self.cache["session"][name] += 1
if name not in self.session:
self.session[name] = 1
else:
self.session[name] += 1
def balance_classes(training_features, training_labels):
# columns = ['age', 'degree-of-diffe', 'sex_2', 'histologic-type_2', 'histologic-type_3', 'bone_2', 'bone-marrow_2',
# 'lung_2', 'pleura_2', 'peritoneum_2', 'liver_2', 'brain_2', 'skin_2', 'neck_2', 'supraclavicular_2',
# 'axillar_2', 'mediastinum_2', 'abdominal_2']
# sm = SMOTENC(sampling_strategy='minority', categorical_features=columns) # You need to reduce k or increase the number of instances for the least represented class.
# sm = BorderlineSMOTE()
sm = RandomUnderSampler()
# sm = SMOTE(k_neighbors=3) # Supports multi-class resampling
# sm = SVMSMOTE() #BorderlineSMOTE, SMOTE, SVMSMOTE - supports multiclass by ovr scheme
# tomeks = TomekLinks(sampling_strategy='majority')
# # X_resampled, y_resampled = BorderlineSMOTE().fit_resample(X_resampled, y_resampled)
# steps = [('smote', sm), ('tomek', tomeks)]
# pipeline = Pipeline(steps=steps)
X_resampled, y_resampled = sm.fit_sample(training_features,
training_labels)
# X_resampled, y_resampled = tomeks.fit_sample(X_resampled, y_resampled)
print('SMOTE {}'.format(Counter(y_resampled)))
X_resampled.to_csv('../resources/datasets/X_resampled.csv', index=False)
y_resampled.to_csv('../resources/datasets/y_resampled.csv',
header=['binaryClass'],
index=False)
return X_resampled, y_resampled
def _check_service_lists(
*,
host_name: HostName,
services_by_transition: ServicesByTransition,
params: config.DiscoveryCheckParameters,
service_filters: _ServiceFilters,
discovery_mode: DiscoveryMode,
) -> Tuple[ActiveCheckResult, bool]:
status = 0
infotexts = []
long_infotexts = []
need_rediscovery = False
for transition, title, params_key, default_state, service_filter in [
("new", "unmonitored", "severity_unmonitored",
config.inventory_check_severity, service_filters.new),
("vanished", "vanished", "severity_vanished", 0,
service_filters.vanished),
]:
affected_check_plugin_names: Counter[CheckPluginName] = Counter()
unfiltered = False
for (discovered_service,
_found_on_nodes) in services_by_transition.get(transition, []):
affected_check_plugin_names[
discovered_service.check_plugin_name] += 1
if not unfiltered and service_filter(host_name,
discovered_service):
unfiltered = True
#TODO In service_filter:we use config.service_description(...)
# in order to finalize service_description (translation, etc.).
# Why do we use discovered_service.description here?
long_infotexts.append(u"%s: %s: %s" %
(title, discovered_service.check_plugin_name,
discovered_service.description))
if affected_check_plugin_names:
info = ", ".join(
["%s:%d" % e for e in affected_check_plugin_names.items()])
st = params.get(params_key, default_state)
status = worst_service_state(status, st, default=0)
infotexts.append(u"%d %s services (%s)%s" % (
sum(affected_check_plugin_names.values()),
title,
info,
state_markers[st],
))
if (unfiltered and ((transition == "new" and discovery_mode in
(DiscoveryMode.NEW, DiscoveryMode.FIXALL,
DiscoveryMode.REFRESH)) or
(transition == "vanished" and discovery_mode in
(DiscoveryMode.REMOVE, DiscoveryMode.FIXALL,
DiscoveryMode.REFRESH)))):
need_rediscovery = True
else:
infotexts.append(u"no %s services found" % title)
for (discovered_service,
_found_on_nodes) in services_by_transition.get("ignored", []):
long_infotexts.append(u"ignored: %s: %s" %
(discovered_service.check_plugin_name,
discovered_service.description))
return ActiveCheckResult(status, infotexts, long_infotexts,
[]), need_rediscovery
def __init__(self):
self.worst = 0
self.last_worst_line = ""
self._output_lines: List[str] = []
self._states_counter: Counter[str] = Counter()
def get_terms(self, n: int) -> Terms:
terms: Terms = Counter()
for obj in self.objects_of_size(n):
param = self.get_parameters(obj)
terms[param] += 1
return terms
def _poly_to_terms(poly: Union[sympy.Poly, int]) -> Terms:
if isinstance(poly, (int, sympy.core.numbers.Integer)):
if poly == 0:
return Counter()
return Counter({tuple(): int(poly)})
return Counter(poly.as_dict())
def extract_tables_and_properties(
props: List[Property]) -> Counter[PropertyIdentifier]:
return Counter(
(prop.key, prop.type, prop.group_type_index) for prop in props)
def isAnagram(s, t):
c_s = Counter(s)
c_t = Counter(t)
return c_t == c_s
def mostCommonWord(self, paragraph: str, banned: List[str]) -> str:
words = [
word for word in re.sub(r'[\W]', ' ', paragraph).lower().split()
if word not in banned
]
return Counter(words).most_common(1)[0][0]
def make_mixed_pos2i() -> Dict[str, int]:
print('Counting mixed POS tags ...')
tokens = Counter(tqdm.tqdm(iterate_mixed_pos_tags('train')))
most_common = (token for (token, __) in tokens.most_common())
return _make_any2i(most_common)
def __init__(self):
self.tokens: Counter[Ngram] = Counter() # the c(...) function.
self.vocab: Optional[Set[str]] = None
self.progress = 0 # To print progress.