def infer_model_type(checkpoint_weights: OrderedDict,
attention_mechanism: bool) -> (str, bool):
"""
Function to infer the model type using the weights matrix.
We first try to use the "model_type" key added by our retrain process.
If this fails, we infer it using our knowledge of the layers' names.
For example, BPEmb model uses an embedding network, thus, if `embedding_network.model.weight_ih_l0` is present,
we can say that it is such a type; otherwise, it is a FastText model.
Finally, to handle the attention model, we use a similar approach but using the
`decoder.linear_attention_mechanism_encoder_outputs.weight` layer name to deduct the presence of
attention mechanism.
Args:
checkpoint_weights (OrderedDict): The weights matrix.
attention_mechanism (bool): Either or not the model uses an attention mechanism or not.
Return:
A tuple where the first element is the model_type name and the second element is the attention_mechanism flag.
"""
inferred_model_type = checkpoint_weights.get("model_type")
if inferred_model_type is not None:
model_type = inferred_model_type
else:
if "embedding_network.model.weight_ih_l0" in checkpoint_weights.keys():
model_type = "bpemb"
else:
model_type = "fasttext"
if "decoder.linear_attention_mechanism_encoder_outputs.weight" in checkpoint_weights.keys(
):
attention_mechanism = True
return model_type, attention_mechanism
def assertOrderedDictEqual(od1: OrderedDict, od2: OrderedDict):
errors = []
if len(od1.keys()) != len(od2.keys()):
raise AssertionError("Number of items don't match: {} {}".format(od1, od2))
for i, j in zip(od1.items(), od2.items()):
if i[0] != j[0]:
errors.append("Keys in {} and {} don't match".format(i, j))
if i[1] != j[1]:
errors.append("Values in {} and {} don't match".format(i, j))
if errors:
raise AssertionError(", ".join(errors))
return True
def messages_per_actors_per_weekday(self, chats: List[Chat]) -> None:
"""
"""
dataframes: Dict[Chat, DataFrame] = {}
title = 'Participation Status (Messages per Actors per Weekday)'
bars = ['Qtd_messages']
types = {
'User Messages': 'messages',
'System Messages': 'system_messages'
}
msg = 'Choose the message type:'
result = select(msg, list(types.keys())).ask()
message_type = types[result]
for chat in chats:
data = OrderedDict({weekday: 0 for weekday in weekdays})
for message in getattr(chat, message_type):
data[message.created_at.strftime('%A')] += 1
index = list(data.keys())
rows = list(data.values())
dataframe = DataFrame(rows, index=index, columns=bars)
dataframes[chat] = dataframe
generate_chart(dataframes, bars=bars, lines=[], title=title)
def conditions_expr(dataset,
conditions,
query: Query,
parsing_context: ParsingContext,
depth=0):
"""
Return a boolean expression suitable for putting in the WHERE clause of the
query. The expression is constructed by ANDing groups of OR expressions.
Expansion of columns is handled, as is replacement of columns with aliases,
if the column has already been expanded and aliased elsewhere.
"""
from snuba.clickhouse.columns import Array
if not conditions:
return ''
if depth == 0:
# dedupe conditions at top level, but keep them in order
sub = OrderedDict(
(conditions_expr(dataset, cond, query, parsing_context, depth + 1),
None) for cond in conditions)
return u' AND '.join(s for s in sub.keys() if s)
elif is_condition(conditions):
lhs, op, lit = dataset.process_condition(conditions)
# facilitate deduping IN conditions by sorting them.
if op in ('IN', 'NOT IN') and isinstance(lit, tuple):
lit = tuple(sorted(lit))
# If the LHS is a simple column name that refers to an array column
# (and we are not arrayJoining on that column, which would make it
# scalar again) and the RHS is a scalar value, we assume that the user
# actually means to check if any (or all) items in the array match the
# predicate, so we return an `any(x == value for x in array_column)`
# type expression. We assume that operators looking for a specific value
# (IN, =, LIKE) are looking for rows where any array value matches, and
# exclusionary operators (NOT IN, NOT LIKE, !=) are looking for rows
# where all elements match (eg. all NOT LIKE 'foo').
columns = dataset.get_dataset_schemas().get_read_schema().get_columns()
if (isinstance(lhs, str) and lhs in columns
and isinstance(columns[lhs].type, Array)
and columns[lhs].base_name != query.get_arrayjoin()
and not isinstance(lit, (list, tuple))):
any_or_all = 'arrayExists' if op in POSITIVE_OPERATORS else 'arrayAll'
return u'{}(x -> assumeNotNull(x {} {}), {})'.format(
any_or_all, op, escape_literal(lit),
column_expr(dataset, lhs, query, parsing_context))
else:
return u'{} {} {}'.format(
column_expr(dataset, lhs, query, parsing_context), op,
escape_literal(lit))
elif depth == 1:
sub = (conditions_expr(dataset, cond, query, parsing_context,
depth + 1) for cond in conditions)
sub = [s for s in sub if s]
res = u' OR '.join(sub)
return u'({})'.format(res) if len(sub) > 1 else res
else:
raise InvalidConditionException(str(conditions))
def containsNearbyAlmostDuplicate(self, nums: List[int], k: int,
t: int) -> bool:
n = len(nums)
if n == 0 or k <= 0 or t < 0: return False
d = OrderedDict()
for i, e in enumerate(nums):
pool = [num[0] for num in d.keys()]
pool.sort()
print(f"Keys are: {' '.join(map(str, pool))}")
l_b = bisect.bisect_left(pool, e)
print(f"Foud lower bound {l_b} for e={e} at i={i}")
if l_b < len(pool) and pool[l_b] <= e + t: return True
u_b = bisect.bisect_right(pool, e - t - 1)
print(f"Foud upper bound {u_b} for e={e - t - 1} at i={i}")
if u_b < len(pool) and pool[u_b] >= e - t and pool[u_b] <= e:
return True
if len(d) == k:
d.popitem(last=False)
d[(e, i)] = i
return False
class record():
""" The record to return """
def __init__(self, source, target_source, id, title, link, abstract, details):
self.dct = OrderedDict()
self.dct['source'] = source
self.dct['target_source'] = target_source
self.dct['id'] = id
self.dct['title'] = title
self.dct['link'] = link
self.dct['abstract'] = abstract
self.dct['details'] = details
@staticmethod
def __clean_value(value):
return value.replace('\n', '').strip()
def to_dict(self):
_dict = {}
for key in self.dct.keys():
if isinstance(self.dct[key], str):
_dict[key] = self.__clean_value(self.dct[key])
elif isinstance(self.dct[key], Tag):
_dict[key] = self.__clean_value(self.dct[key].text)
elif isinstance(self.dct[key], ResultSet):
if key not in _dict:
_dict[key] = []
for elt in self.dct[key]:
_dict[key].append(self.__clean_value(elt.text))
else:
_dict[key] = '[' + \
str(type(self.dct[key])) + '] ' + str(self.dct[key])
return _dict
def main(args):
model_index_file = MMCLS_ROOT / 'model-index.yml'
model_index = Config.fromfile(model_index_file)
models = OrderedDict()
for file in model_index.Import:
metafile = Config.fromfile(MMCLS_ROOT / file)
models.update({model.Name: model for model in metafile.Models})
logger = get_root_logger(log_file='benchmark_test_image.log',
log_level=logging.INFO)
if args.models:
patterns = [re.compile(pattern) for pattern in args.models]
filter_models = {}
for k, v in models.items():
if any([re.match(pattern, k) for pattern in patterns]):
filter_models[k] = v
if len(filter_models) == 0:
print('No model found, please specify models in:')
print('\n'.join(models.keys()))
return
models = filter_models
summary_data = {}
for model_name, model_info in models.items():
config = Path(model_info.Config)
assert config.exists(), f'{model_name}: {config} not found.'
logger.info(f'Processing: {model_name}')
http_prefix = 'https://download.openmmlab.com/mmclassification/'
dataset = model_info.Results[0]['Dataset']
if args.checkpoint_root is not None:
root = Path(args.checkpoint_root)
checkpoint = root / model_info.Weights[len(http_prefix):]
checkpoint = str(checkpoint)
else:
checkpoint = None
try:
# build the model from a config file and a checkpoint file
result = inference(MMCLS_ROOT / config, checkpoint,
classes_map[dataset], args)
result['valid'] = 'PASS'
except Exception as e:
logger.error(f'"{config}" : {repr(e)}')
result = {'valid': 'FAIL'}
summary_data[model_name] = result
# show the results
if args.show:
imshow_infos(args.img, result, wait_time=args.wait_time)
show_summary(summary_data)
def print_test_list(test_list: typing.OrderedDict) -> None:
"""
Prints a list of all tests and demos in test_list.
"""
namelen = max(len(name) for name, _ in test_list.keys())
for current_type in ['test', 'demo', 'benchmark']:
for (name, type_), (_, lang, desc, _) in test_list.items():
if type_ == current_type:
print(f"[{type_} {lang:3}] {name:{namelen}} {desc}")
print("")
print("To see how to run them, add --help to your invocation!")
print("")
print("Remember: Testing is the future, and the future starts with: You.")
print("")
def filter_projects_by_name_and_progress(projects: OrderedDict,
filter_string: str,
progress_threshold: int) -> List[str]:
"""Filter projects by name (lowercase) and progress."""
selected_project_ids = []
for project_id in projects.keys():
name = projects[project_id]["name"]
progress = projects[project_id]["progress"]
if filter_string.lower() in name.lower(
) and progress >= progress_threshold:
selected_project_ids.append(project_id)
logger.info(
f"selected {len(selected_project_ids)} project(s) which contain(s) "
f"'{filter_string}' in the project name and progress >= {progress_threshold}%."
)
return selected_project_ids
def mostCommonWord(self, paragraph: str, banned: List[str]) -> str:
word_dic = {}
sorted_dic = {}
tmp = re.sub(pattern='\W', repl=' ', string=paragraph).lower().split()
for s in tmp:
if s not in banned:
if s in word_dic:
word_dic[s] += 1
else:
word_dic[s] = 1
# value기준으로 딕셔너리 정렬
sorted_dic = OrderedDict(sorted(word_dic.items(),
key=lambda x: x[1],
reverse=True))
for k in sorted_dic.keys():
return k
def combine_keys(dict_a: dict, dict_b: dict) -> Set[str]:
"""
Combines keys in one dict with keys in another
# NOTE: maybe get rid of this function as it is a one liner
Args:
dict_a: First dictionary to combine keys from
dict_b: Second dictionary to combine keys from
Returns:
Combined keys in both dictionaries
"""
keys = OrderedDict()
for key in dict_a:
keys[key] = None
for key in dict_b:
keys[key] = None
return keys.keys()
def main():
args = parser().parse_args()
results = OrderedDict()
max_results = 10
index = 0
home = Path(args.base) if args.base else Path().home()
ignore_case = False
print(args)
search = args.search.lower() if ignore_case else args.search
for path in home.glob('**/*'):
name = path.name.lower() if ignore_case else path.name
if path.is_dir() and name.endswith(search):
results[string.ascii_lowercase[index]] = path.as_posix()
index += 1
if len(results) >= max_results:
break
if len(results) < 1:
print(f'No such directory for <{args.search}>')
return -1
for index, elem in results.items():
print(index, '--', elem)
keys = list(results.keys())
wrong_key = True
while wrong_key:
choice = input(
f'Please enter a key between \'{keys[0]}\' and \'{keys[-1]}\'\n')
try:
print(f'Entering directory {results[choice]} ...')
wrong_key = False
except KeyError:
print(
f'Wrong key. Please chose acorrect one between {keys[0]} and {keys[-1]}'
)
wrong_key = True
async def shop(self, ctx):
shop = self.load_shop()
embeds = []
shop = OrderedDict(
sorted(shop.items(), key=lambda x: getitem(x[1], "raw_price")))
chunks = divide_chunks(list(shop.keys()), 5)
i = 0
for chunk in list(chunks):
i += 1
embed = discord.Embed(color=discord.Color.teal())
embed.title = "The Waifu Shop"
for item in chunk:
v = (f" / `{shop[item]['raw_price']} erin`"
if shop[item]['price']['item'] != "erin" else "")
embed.add_field(
name=item,
value=
f"{shop[item]['name']} {shop[item]['emoji']} | Costs `{shop[item]['price']['quantity']} {shop[item]['price']['item']}`"
+ v,
inline=False,
)
embed.set_footer(text=f"Page {i}/{len(chunks)}",
icon_url=ctx.author.avatar_url)
embeds.append(embed)
paginator = DiscordUtils.Pagination.CustomEmbedPaginator(ctx)
paginator.add_reaction(
"\N{Black Left-Pointing Double Triangle with Vertical Bar}",
"first")
paginator.add_reaction("\N{Black Left-Pointing Double Triangle}",
"back")
paginator.add_reaction("\N{CROSS MARK}", "lock")
paginator.add_reaction("\N{Black Right-Pointing Double Triangle}",
"next")
paginator.add_reaction(
"\N{Black Right-Pointing Double Triangle with Vertical Bar}",
"last")
await paginator.run(embeds)
def uniq_list(l):
od = OrderedDict()
for x in l:
od[x] = x
return list(od.keys())
def _check_data_with_labels(data: typing.OrderedDict, labels: pd.DataFrame):
data_block_ids = list(data.keys())
# remove labels for blocks not in data (this might happen if using only subset of data for debugging, e.g., HDFS1_100k)
label_block_ids = labels[labels["BlockId"].isin(data.keys())]["BlockId"]
assert all(data_block_ids == label_block_ids)
def forward_loss(
self,
speech_pre: torch.Tensor,
speech_lengths: torch.Tensor,
feature_mix: torch.Tensor,
feature_pre: torch.Tensor,
others: OrderedDict,
speech_ref: torch.Tensor,
noise_ref: torch.Tensor = None,
dereverb_speech_ref: torch.Tensor = None,
) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]:
# for calculating loss on estimated noise signals
if getattr(self.separator, "predict_noise", False):
assert "noise1" in others, others.keys()
if noise_ref is not None and "noise1" in others:
for n in range(self.num_noise_type):
key = "noise{}".format(n + 1)
others[key] = self.decoder(others[key], speech_lengths)[0]
# for calculating loss on dereverberated signals
if getattr(self.separator, "predict_dereverb", False):
assert "dereverb1" in others, others.keys()
if dereverb_speech_ref is not None and "dereverb1" in others:
for spk in range(self.num_spk):
key = "dereverb{}".format(spk + 1)
if key in others:
others[key] = self.decoder(others[key], speech_lengths)[0]
loss = 0.0
stats = {}
o = {}
for loss_wrapper in self.loss_wrappers:
criterion = loss_wrapper.criterion
if getattr(criterion, "only_for_test", False) and self.training:
continue
if getattr(criterion, "is_noise_loss", False):
if noise_ref is None:
raise ValueError(
"No noise reference for training!\n"
'Please specify "--use_noise_ref true" in run.sh')
signal_ref = noise_ref
signal_pre = [
others["noise{}".format(n + 1)]
for n in range(self.num_noise_type)
]
elif getattr(criterion, "is_dereverb_loss", False):
if dereverb_speech_ref is None:
raise ValueError(
"No dereverberated reference for training!\n"
'Please specify "--use_dereverb_ref true" in run.sh')
signal_ref = dereverb_speech_ref
signal_pre = [
others["dereverb{}".format(n + 1)]
for n in range(self.num_noise_type)
if "dereverb{}".format(n + 1) in others
]
if len(signal_pre) == 0:
signal_pre = None
else:
signal_ref = speech_ref
signal_pre = speech_pre
if isinstance(criterion, TimeDomainLoss):
assert signal_pre is not None
sref, spre = self._align_ref_pre_channels(signal_ref,
signal_pre,
ch_dim=2,
force_1ch=True)
# for the time domain criterions
l, s, o = loss_wrapper(sref, spre, {**others, **o})
elif isinstance(criterion, FrequencyDomainLoss):
sref, spre = self._align_ref_pre_channels(signal_ref,
signal_pre,
ch_dim=2,
force_1ch=False)
# for the time-frequency domain criterions
if criterion.compute_on_mask:
# compute loss on masks
if getattr(criterion, "is_noise_loss", False):
tf_ref, tf_pre = self._get_noise_masks(
criterion,
feature_mix,
speech_ref,
signal_ref,
signal_pre,
speech_lengths,
others,
)
elif getattr(criterion, "is_dereverb_loss", False):
tf_ref, tf_pre = self._get_dereverb_masks(
criterion,
feature_mix,
noise_ref,
signal_ref,
signal_pre,
speech_lengths,
others,
)
else:
tf_ref, tf_pre = self._get_speech_masks(
criterion,
feature_mix,
noise_ref,
signal_ref,
signal_pre,
speech_lengths,
others,
)
else:
# compute on spectrum
tf_ref = [
self.encoder(sr, speech_lengths)[0] for sr in sref
]
tf_pre = [
self.encoder(sp, speech_lengths)[0] for sp in spre
]
l, s, o = loss_wrapper(tf_ref, tf_pre, {**others, **o})
else:
raise NotImplementedError("Unsupported loss type: %s" %
str(criterion))
loss += l * loss_wrapper.weight
stats.update(s)
if self.training and isinstance(loss, float):
raise AttributeError(
"At least one criterion must satisfy: only_for_test=False")
stats["loss"] = loss.detach()
# force_gatherable: to-device and to-tensor if scalar for DataParallel
batch_size = speech_ref[0].shape[0]
loss, stats, weight = force_gatherable((loss, stats, batch_size),
loss.device)
return loss, stats, weight
def generateDiff(phab, repo, username, publickey):
path = PuppetRoot
if os.path.isdir(path):
path = os.path.join(path, WireguardManifestFilePath)
f = open(path, 'r')
fileContents = f.read()
code = round_trip_load(fileContents, preserve_quotes=True)
# make the modifications
peers = code['wireguard_server::wireguard_peers']
# we know the keys should be sorted, but make no assumption
# on the manifest that they indeed are. Let's just do a
# best-effort search for the appropriate insertion index
orderedPeers = OrderedDict(peers.copy())
peersList = list(orderedPeers.keys())
ipAddress = getNextValidIpAddress(orderedPeers)
index = 0
for key in peersList:
if key > username:
break
index += 1
peers.insert(index, username, {
'PublicKey': publickey,
'IPAddress': ipAddress
})
fileContents = round_trip_dump(code, block_seq_indent=2,
explicit_start=True)
# surround the values with single quotes to match the rest of the
# YAML file, as library we used doesn't supply the new values with quotes.
fileContents = re.sub(r'PublicKey: ([a-zA-Z0-9+=/]+)',
f'PublicKey: \'{publickey}\'', fileContents)
fileContents = re.sub(r'IPAddress: ([0-9.]+)',
f'IPAddress: \'{ipAddress}\'', fileContents)
if username in peersList and not click.confirm(f'The username \'{username}\' already exists in \
the Wireguard peers registry, continue?', default=False):
click.echo('Operation aborted.')
return
click.echo(f'The next valid IP Address in the subrange 10.3.128.1/17 \
appears to be: {ipAddress}')
path = PuppetRoot
if os.path.isdir(path):
path = os.path.join(path, WireguardManifestFilePath)
else:
click.echo(
'Not a directory! Please correctly specify the puppet repo root.')
return
if not os.path.isfile(path):
click.echo(f'Unable to open {path}, is it a file?')
return
wireguardManifest = open(path, "w")
wireguardManifest.write(fileContents)
wireguardManifest.close()
click.echo(
f'Updated file {WireguardManifestFilePath}.')
ticketId = None
if click.confirm(f'Would you like to create a ticket?', default=True):
ticketId = createTicket(phab, username, publickey, ipAddress)
commitChanges(repo, username=username, ticketId=ticketId)
if not click.confirm(f'Proceed with creating a diff?', default=True):
return
# execute arc diff to prepare a diff for a syseng reviewer.
os.chdir(PuppetRoot)
command = 'arc diff --browse --create --draft --nolint \
--skip-staging --nounit --verbatim'
click.echo(f'Running shell command: {command}')
os.system(command)
class ContentCache(CacheABC):
# pylint: disable=too-many-instance-attributes
def __init__(self,
cache_folder: str,
temporary_dir: str,
max_cache_size_bytes: int = 1024 * 1024 * 1024,
max_workers: int = 10,
contents_load: bool = True,
contents_save_interval_secs: float = 5.0,
url_resolver: URLResolverABC = URLResolver()):
print(
f'ContentCache.__init__: cache_folder={cache_folder}, temporary_dir={temporary_dir}'
)
self.cache_folder: str = cache_folder
self.max_cache_size_bytes: int = max_cache_size_bytes
self.temporary_dir = temporary_dir
self.contents_save_interval_secs = contents_save_interval_secs
self.url_resolver = url_resolver
self._lock = threading.RLock()
self._executor = ThreadPoolExecutor(max_workers)
self._tasks: Dict[URL, Task] = {}
self._contents: OrderedDict[str, Content] = OrderedDict()
self._contents_size: int = 0
self._contents_save_timer = None
if contents_load:
self._load_contents()
def __del__(self):
self._save_contents()
self._executor.shutdown()
def _load_contents(self):
contents_json_path = os.path.join(self.cache_folder, 'contents.json')
if not os.path.exists(contents_json_path):
return
with self._lock:
with open(contents_json_path, 'r') as file:
content_json = json.load(file, object_pairs_hook=OrderedDict)
self._contents = OrderedDict({
key: Content(id=value['id'],
state=Content.State[value['state']],
filepath=os.path.join(self.cache_folder,
value['filepath']),
type=value['type'],
length=value['length'])
for key, value in content_json.items()
})
self._contents_size = sum(
[c.length for c in self._contents.values()])
print(
f'_load_contents: {len(self._contents)} from {contents_json_path}'
)
def _save_contents(self):
contents_json_path = os.path.join(self.cache_folder, 'contents.json')
with self._lock:
print(
f'_save_contents: {len(self._contents)} to {contents_json_path}'
)
content_json = {
key: {
'id':
value.id,
'state':
value.state.name,
'filepath':
os.path.basename(value.filepath) if value.filepath else '',
'type':
value.type,
'length':
value.length
}
for key, value in self._contents.items()
}
with open(contents_json_path, 'w') as file:
json.dump(content_json, file)
def _schedule_save_contents(self):
if self._contents_save_timer is not None:
self._contents_save_timer.cancel()
self._contents_save_timer = threading.Timer(
self.contents_save_interval_secs, self._save_contents)
self._contents_save_timer.start()
def _to_content_filepath(self, content_id: str) -> str:
return os.path.join(self.cache_folder, content_id)
def _fetch(self, task: Task):
# pylint: disable=too-many-statements
with self._lock:
if task.state is not Task.State.QUEUING:
raise ValueError(
f'task (={task.url}) is invalid state (={task.state})')
task.state = Task.State.RUNNING
content_id = task.content_id
content_filepath = self._to_content_filepath(content_id)
content = Content(content_id, Content.State.FETCHING)
try:
temp_fd, temp_path = tempfile.mkstemp(dir=self.temporary_dir)
with os.fdopen(temp_fd, 'bw') as temp_file:
response = self.url_resolver.resolve(task.url)
response.raise_for_status()
content_type = response.headers.get('Content-Type')
content_length_text = response.headers.get('Content-Length')
content_length = int(
content_length_text) if content_length_text else 0
fetch_size = 0
with self._lock:
task.content_length = content_length
task.fetched_size = fetch_size
for chunk in response.iter_content(chunk_size=65536):
fetch_size += len(chunk)
with self._lock:
task.fetched_size = fetch_size
if task.state is not Task.State.RUNNING:
raise InterruptedError(
f'task (={task.url}) fetch was interrupted')
temp_file.write(chunk)
os.rename(temp_path, content_filepath)
content_length = os.path.getsize(content_filepath)
with self._lock:
self._contents_size += content_length
task.state = Task.State.SUCCESS
content.state = Content.State.CACHED
content.filepath = content_filepath
content.length = content_length
content.type = content_type
except: # pylint: disable=bare-except
traceback.print_exc()
with self._lock:
content.state = Content.State.FAILED
if task.state is Task.State.RUNNING:
task.state = Task.State.FAILURE
else:
pass # keep state
if temp_path is not None:
os.remove(temp_path)
finally:
with self._lock:
self._contents[content_id] = content
del self._tasks[task.url]
self._invoke_callbacks(task)
self._schedule_save_contents()
return task
@staticmethod
def _invoke_callback(callback, content):
try:
callback(content)
except: # pylint: disable=bare-except
traceback.print_exc()
def _invoke_callbacks(self, task: Task):
with self._lock:
task_callbacks_copy = list(task.callbacks)
task.callbacks.clear()
content = self._contents[task.content_id]
for callback in task_callbacks_copy:
self._invoke_callback(callback, content)
return task
def cancel_fetch(self, url: URL):
with self._lock:
task = self.try_get_task(url)
if task is None:
return
if task.state != Task.State.RUNNING:
return
task.state = Task.State.CANCELED
def remove_content(self, url: URL) -> bool:
with self._lock:
content = self.try_get_content(url)
if content is None:
return False
del self._contents[content.id]
self._schedule_save_contents()
return True
def try_get_content(self, url: URL) -> Union[Content, None]:
content_id = Content.to_content_id(url)
with self._lock:
if content_id not in self._contents:
return None
content = self._contents[content_id]
# LRU implementation
self._contents.move_to_end(content_id)
excess_cache_size = max(
0, self._contents_size - self.max_cache_size_bytes)
if excess_cache_size > 0:
for content_id in self._contents.keys():
content = self._contents[content_id]
if content.length == 0:
continue
del self._contents[content_id]
self._contents_size -= content.length
if os.path.exists(content.file_path):
try:
os.remove(content.file_path)
except: # pylint: disable=bare-except
traceback.print_exc()
excess_cache_size -= content.length
if excess_cache_size <= 0:
break
self._schedule_save_contents()
return content
def try_get_task(self, url: URL) -> Union[Task, None]:
with self._lock:
return self._tasks[url] if url in self._tasks else None
def async_get_content(self, url: URL, callback: Callback) -> Future:
with self._lock:
content = self.try_get_content(url)
if content is not None:
if content.state in {
Content.State.CACHED, Content.State.FETCHING
}:
return self._executor.submit(self._invoke_callback,
callback, content)
self.remove_content(url)
elif url in self._tasks:
task = self._tasks[url]
if task.state in {Task.State.QUEUING, Task.State.RUNNING}:
task.callbacks.append(callback)
return task.future
task = Task(url, Task.State.QUEUING, [callback])
task.future = self._executor.submit(self._fetch, task)
self._tasks[url] = task
return task.future
class NetworkBlock:
"""
Base block object that will be used for each block in the network.
Attributes
----------
name : str
Name of the block which will be used to write the BNGL text
comment : (str, str)
comment at the begin {block} or end {block} statements, tuple
items : OrderedDict
all the model objects in the block
Methods
-------
add_item((name,value))
sets self.item[name] = value to add a particular model object
into a block
add_items(item_list)
loops over every element in the list and uses add_item on it
gen_string()
for every block this method generates the BNGL string of the
block. it has to be overwritten for each block.
"""
def __init__(self) -> None:
self.name = "NetworkBlock"
self.comment = (None, None)
self.items = OrderedDict()
def __str__(self) -> str:
return self.gen_string()
def __len__(self) -> int:
return len(self.items)
def __repr__(self) -> str:
# overwrites what the class representation
# shows the items in the model block in
# say ipython
repr_str = "{} block with {} item(s): {}".format(
self.name, len(self.items),
list([i.name for i in self.items.values()]))
return repr_str
def __getitem__(self, key):
if isinstance(key, int):
# get the item in order
return list(self.items.keys())[key]
return self.items[key]
def __setitem__(self, key, value) -> None:
self.items[key] = value
def __delitem__(self, key) -> None:
if key in self.items:
self.items.pop(key)
else:
print("Item {} not found".format(key))
def __iter__(self):
return self.items.keys().__iter__()
def __contains__(self, key) -> bool:
return key in self.items
# TODO: Think extensively how this is going to work
def __setattr__(self, name, value) -> None:
changed = False
if hasattr(self, "items"):
if name in self.items.keys():
try:
new_value = float(value)
changed = True
self.items[name] = new_value
except:
self.items[name] = value
if changed:
self._changes[name] = new_value
self.__dict__[name] = new_value
else:
self.__dict__[name] = value
def gen_string(self) -> str:
# each block can have a comment at the start
if self.comment[0] is not None:
block_lines = ["\nbegin {} #{}".format(self.name, self.comment[0])]
else:
block_lines = ["\nbegin {}".format(self.name)]
# now we just loop over lines
for item in self.items.keys():
block_lines.append(self.items[item].print_line())
# each block can have a comment at the start
if self.comment[1] is not None:
block_lines.append("end {} #{}\n".format(self.name,
self.comment[1]))
else:
block_lines.append("end {}\n".format(self.name))
# join everything with new lines
return "\n".join(block_lines)
def add_item(self, item_tpl) -> None:
# TODO: try adding evaluation of the parameter here
# for the future, in case we want people to be able
# to adjust the math
# TODO: Error handling, some names will definitely break this
name, value = item_tpl
# allow for empty addition, uses index
if name is None:
name = len(self.items)
# set the line
self.items[name] = value
# if the name is a string, try adding as an attribute
if isinstance(name, str):
try:
setattr(self, name, value)
except:
# print("can't set {} to {}".format(name, value))
pass
# we just added an item to a block, let's assume we need
# to recompile if we have a compiled simulator
self._recompile = True
def add_items(self, item_list) -> None:
for item in item_list:
self.add_item(item)
available_matches = np.zeros((len(tiles), len(tiles)))
i = 0
for id, im in tiles.items():
edges = get_edges(im)
j = 0
for id2, im2 in tiles.items():
if id != id2:
edges2 = get_edges(im2)
available_matches[i, j] = count_matches(edges, edges2)
j += 1
i += 1
indices_with_two_matches = np.where((available_matches > 0).sum(
axis=0) == 2)[0]
prod = 1
for index, id in enumerate(tiles.keys()):
if index in indices_with_two_matches:
prod *= id
print(f'a) {prod}')
sidelen = int(np.sqrt(len(tiles)))
tile_ims = list(tiles.values())
starting_index = indices_with_two_matches[0]
def apply_flip_rot(im, fliprot):
if fliprot[0] == 1:
im = np.fliplr(im)
if fliprot[1] == 1:
im = np.flipud(im)
class LRUCacheStrategy(MemoryCacheStrategy[K, V]):
"""strategy which enforces a size limit with LRU"""
__slots__ = ("storage", "lock", "max_entries")
storage: OrderedDict[K, V]
lock: Lock # OrderedDict is not thread safe
max_entries: int
def __init__(self, max_entries: int) -> None:
self.storage = OrderedDict()
self.lock = Lock()
self.max_entries = max_entries
def __eq__(self, other: object) -> bool:
if isinstance(other, LRUCacheStrategy):
return self.storage == other.storage \
and self.max_entries == other.max_entries
return NotImplemented
def __getitem__(self, key: K) -> V:
"""get a value, setting it as the most recently used one"""
with self.lock:
self.storage.move_to_end(
key, last=False) # higher index = longer time since last use
return self.storage[key]
def __setitem__(self, key: K, value: V) -> None:
"""set a value, removing old ones if necessary"""
with self.lock:
if key not in self.storage and len(
self.storage) == self.max_entries:
self.storage.popitem(
) # make space for new entry by removing the last element
self.storage[key] = value
def __delitem__(self, key: K) -> None:
"""remove a value"""
with self.lock:
del self.storage[key]
def __iter__(self) -> Iterator[K]:
return iter(self.storage)
def __len__(self) -> int:
return len(self.storage)
def __contains__(self, key: object) -> bool:
return key in self.storage
def keys(self) -> KeysView[K]:
return self.storage.keys()
def values(self) -> ValuesView[V]:
return self.storage.values()
def items(self) -> ItemsView[K, V]:
return self.storage.items()
def peek(self, key: K) -> V:
"""get the value of key without triggering side effects like changing its priority"""
with self.lock:
return self.storage[key]
@overload
def pop(self, key: K) -> V:
...
@overload
def pop(self, key: K, default: Union[V, T] = ...) -> Union[V, T]:
...
def pop(self,
key: K,
default: Union[V,
T] = POP_SENTINEL) -> Union[V, T]: # type: ignore
"""remove a value and return it"""
with self.lock:
if default is POP_SENTINEL:
return self.storage.pop(key)
return self.storage.pop(key, default)
def popitem(self) -> Tuple[K, V]:
"""remove the least recently used key-value pair and return it"""
with self.lock:
return self.storage.popitem()
def clear(self) -> None:
"""remove all values"""
with self.lock:
self.storage.clear()
