def _conf_intervals_to_df(cis: OrderedDict) -> DataFrame:
"""Convert lmfit confidence intervals to pandas.DataFrame."""
ncis = len(list(cis.values())[0]) // 2 # e.g. [μ - σ, μ, μ + σ]
return DataFrame(
((name, *[climit for _, climit in climits])
for name, climits in cis.items()),
columns=(
"name",
*[
f"{sign}{clevel:.3f}"
for sign, (clevel, _) in zip(("-", ) * ncis + ("", ) +
("+", ) * ncis,
list(cis.values())[0])
],
),
)
def messages_per_actors_per_weekday(self, chats: List[Chat]) -> None:
"""
"""
dataframes: Dict[Chat, DataFrame] = {}
title = 'Participation Status (Messages per Actors per Weekday)'
bars = ['Qtd_messages']
types = {
'User Messages': 'messages',
'System Messages': 'system_messages'
}
msg = 'Choose the message type:'
result = select(msg, list(types.keys())).ask()
message_type = types[result]
for chat in chats:
data = OrderedDict({weekday: 0 for weekday in weekdays})
for message in getattr(chat, message_type):
data[message.created_at.strftime('%A')] += 1
index = list(data.keys())
rows = list(data.values())
dataframe = DataFrame(rows, index=index, columns=bars)
dataframes[chat] = dataframe
generate_chart(dataframes, bars=bars, lines=[], title=title)
def _training_step(self, kwargs: OrderedDict) -> ClosureResult:
"""Performs the actual train step with the tied hooks.
Args:
kwargs: the kwargs passed down to the hooks.
Returns:
A ``ClosureResult`` containing the training step output.
"""
# manually capture logged metrics
training_step_output = self.trainer._call_strategy_hook("training_step", *kwargs.values())
self.trainer.strategy.post_training_step()
model_output = self.trainer._call_lightning_module_hook("training_step_end", training_step_output)
strategy_output = self.trainer._call_strategy_hook("training_step_end", training_step_output)
training_step_output = strategy_output if model_output is None else model_output
self._hiddens = _extract_hiddens(training_step_output, self.trainer.lightning_module.truncated_bptt_steps)
result = self.output_result_cls.from_training_step_output(
training_step_output, self.trainer.accumulate_grad_batches
)
if self.trainer.move_metrics_to_cpu:
# hiddens and the training step output are not moved as they are not considered "metrics"
assert self.trainer._results is not None
self.trainer._results.cpu()
return result
def verticalTraversal(self, root: TreeNode) -> List[List[int]]:
node_list = []
def BFS(root):
queue = deque()
queue.append((root, 0, 0))
while queue:
node, row, column = queue.popleft()
if node:
node_list.append((column, row, node.val))
queue.append((node.left, row + 1, column - 1))
queue.append((node.right, row + 1, column + 1))
# step 1 - construct the node list with coordinates
BFS(root)
# step 2 - sort the node list according to coordinates
node_list.sort()
# step 3 - retrive sorted results partitioned by columns
# use an ordered dictionary for keys
output = OrderedDict()
for column, row, value in node_list:
if not column in output:
output[column] = [value]
else:
output[column] = output[column] + [value]
return output.values()
def get_info(row: OrderedDict):
result = ",".join(row.values())
try:
content = requestHTML(row["url"])
matched_meta_info = match_meta_info(content)
get_image(matched_meta_info["alt_img_url"], row)
except Exception as i:
print(i)
print("Parsed Faild: " + row["url"])
return result
row.update(matched_meta_info)
# Add local image path
row["img_path"] = "/assets/img/works/posts/" + \
"{id}.jpeg".format(id=row["url"][-6:])
print("解析成功：#" + row["id"] + " " + row["title"])
return ",".join(row.values())
def test(cls, cfg, model, evaluators=None):
"""
Args:
cfg (CfgNode):
model (nn.Module):
evaluators (list[DatasetEvaluator] or None): if None, will call
:meth:`build_evaluator`. Otherwise, must have the same length as
``cfg.DATASETS.TEST``.
Returns:
dict: a dict of result metrics
"""
logger = logging.getLogger(__name__)
if isinstance(evaluators, DatasetEvaluator):
evaluators = [evaluators]
if evaluators is not None:
assert len(
cfg.DATASETS.TEST) == len(evaluators), "{} != {}".format(
len(cfg.DATASETS.TEST), len(evaluators))
results = OrderedDict()
for idx, dataset_name in enumerate(cfg.DATASETS.TEST):
data_loader = cls.build_test_loader(cfg, dataset_name)
# When evaluators are passed in as arguments,
# implicitly assume that evaluators can be created before data_loader.
if evaluators is not None:
evaluator = evaluators[idx]
else:
try:
evaluator = cls.build_evaluator(cfg, dataset_name)
except NotImplementedError:
logger.warn(
"No evaluator found. Use `DefaultTrainer.test(evaluators=)`, "
"or implement its `build_evaluator` method.")
results[dataset_name] = {}
continue
results_i = inference_on_dataset(model, data_loader, evaluator)
results[dataset_name] = results_i
if comm.is_main_process():
assert isinstance(
results_i, dict
), "Evaluator must return a dict on the main process. Got {} instead.".format(
results_i)
logger.info("Evaluation results for {} in csv format:".format(
dataset_name))
#print_csv_format(results_i)
print(results_i)
if len(results) == 1:
results = list(results.values())[0]
return results
def get_tail_frequencies(self,
date: datetime) -> Tuple[int, Dict[int, float]]:
# TODO: 其实没必要在这里保持有序
lower_bound, upper_bound = self._get_boundaries(date)
self.cur.execute(r'''SELECT * FROM get_tail_count(%s, %s)''',
(lower_bound, upper_bound))
rows = self.cur.fetchall()
counts = OrderedDict({r[0]: r[1] for r in rows})
sum_count = sum(counts.values())
frequencies = OrderedDict(
(tail, float(count) / sum_count) for tail, count in counts.items())
return (sum_count, frequencies)
edges2 = get_edges(im2)
available_matches[i, j] = count_matches(edges, edges2)
j += 1
i += 1
indices_with_two_matches = np.where((available_matches > 0).sum(
axis=0) == 2)[0]
prod = 1
for index, id in enumerate(tiles.keys()):
if index in indices_with_two_matches:
prod *= id
print(f'a) {prod}')
sidelen = int(np.sqrt(len(tiles)))
tile_ims = list(tiles.values())
starting_index = indices_with_two_matches[0]
def apply_flip_rot(im, fliprot):
if fliprot[0] == 1:
im = np.fliplr(im)
if fliprot[1] == 1:
im = np.flipud(im)
for _ in range(fliprot[2]):
im = np.rot90(im)
return im
def place_tile(puzzle, puzzle_as_inds, pos, tile_ind):
top_to_match = None
def build_model(config: Mapping, cardinalities: Mapping[str,
int]) -> keras.Model:
"""Construct model specified in the configuration.
Also create optimizer and set the loss function.
Args:
config: Dictionary representing configuration file.
cardinalities: Cardinalities of categorical features (needed to
construct their embeddings).
Return:
Compiled model.
"""
model_config = config['model']
if isinstance(model_config, str):
model = keras.models.load_model(
model_config,
custom_objects={'loss_fn': _create_loss(config['loss'])})
return model
features = Features(config['features'])
inputs_all = []
# Constituents of different types
constituent_types = [
key for key in sorted(model_config.keys()) # Ensure order
if key not in {'head', 'load_weights'}
]
outputs_constituents = []
for constituent_type in constituent_types:
inputs_numerical = keras.Input(
shape=(None, len(features.numerical(constituent_type))),
ragged=True,
name=f'{constituent_type}_numerical')
inputs_categorical = OrderedDict()
for feature in features.categorical(constituent_type):
inputs_categorical[feature] = keras.Input(shape=(None, ),
ragged=True,
name=feature)
inputs_all.append(inputs_numerical)
inputs_all.extend(inputs_categorical.values())
outputs = _apply_deep_set(inputs_numerical, inputs_categorical,
model_config[constituent_type],
cardinalities, constituent_type)
outputs_constituents.append(outputs)
# Head
inputs_global_numerical = keras.Input(shape=(len(
features.numerical('global')), ),
name='global_numerical')
inputs_global_categorical = OrderedDict()
for feature in features.categorical('global'):
inputs_global_categorical[feature] = keras.Input(shape=(None, ),
name=feature)
embeddings_global = {
feature: Embedding(cardinalities[feature],
model_config['head']['embeddings'][feature],
name=feature + '_embeddings')(inputs)
for feature, inputs in inputs_global_categorical.items()
}
inputs_all.append(inputs_global_numerical)
inputs_all.extend(inputs_global_categorical.values())
inputs_head = Concatenate(
name='head_concatenate')([inputs_global_numerical] + [
embeddings_global[feature]
for feature in inputs_global_categorical.values()
] + outputs_constituents)
outputs = _apply_dense_from_config(inputs_head,
model_config['head'],
name_prefix='head_')
outputs = Dense(1, name='head_dense_output')(outputs) # Output unit
model = keras.Model(inputs=inputs_all, outputs=outputs, name='full')
model.compile(optimizer=_create_optimizer(config.get('optimizer', None)),
loss=_create_loss(config['loss']))
if 'load_weights' in model_config:
# Normally, a saved model should be loaded
# keras.models.load_model at the beginning of thsi function.
# However, this is currently not supported for models that use
# ragged tensors [1]. As a workaround, construct the model anew
# and then load saved weights. The path to weights would
# usually be "{model_directory}/variables/variables", with the
# ".index" file extension stripped off. This doesn't restore
# the state of the optimizer.
# [1] https://github.com/tensorflow/tensorflow/issues/41034
model.load_weights(model_config['load_weights'])
return model
class ContentCache(CacheABC):
# pylint: disable=too-many-instance-attributes
def __init__(self,
cache_folder: str,
temporary_dir: str,
max_cache_size_bytes: int = 1024 * 1024 * 1024,
max_workers: int = 10,
contents_load: bool = True,
contents_save_interval_secs: float = 5.0,
url_resolver: URLResolverABC = URLResolver()):
print(
f'ContentCache.__init__: cache_folder={cache_folder}, temporary_dir={temporary_dir}'
)
self.cache_folder: str = cache_folder
self.max_cache_size_bytes: int = max_cache_size_bytes
self.temporary_dir = temporary_dir
self.contents_save_interval_secs = contents_save_interval_secs
self.url_resolver = url_resolver
self._lock = threading.RLock()
self._executor = ThreadPoolExecutor(max_workers)
self._tasks: Dict[URL, Task] = {}
self._contents: OrderedDict[str, Content] = OrderedDict()
self._contents_size: int = 0
self._contents_save_timer = None
if contents_load:
self._load_contents()
def __del__(self):
self._save_contents()
self._executor.shutdown()
def _load_contents(self):
contents_json_path = os.path.join(self.cache_folder, 'contents.json')
if not os.path.exists(contents_json_path):
return
with self._lock:
with open(contents_json_path, 'r') as file:
content_json = json.load(file, object_pairs_hook=OrderedDict)
self._contents = OrderedDict({
key: Content(id=value['id'],
state=Content.State[value['state']],
filepath=os.path.join(self.cache_folder,
value['filepath']),
type=value['type'],
length=value['length'])
for key, value in content_json.items()
})
self._contents_size = sum(
[c.length for c in self._contents.values()])
print(
f'_load_contents: {len(self._contents)} from {contents_json_path}'
)
def _save_contents(self):
contents_json_path = os.path.join(self.cache_folder, 'contents.json')
with self._lock:
print(
f'_save_contents: {len(self._contents)} to {contents_json_path}'
)
content_json = {
key: {
'id':
value.id,
'state':
value.state.name,
'filepath':
os.path.basename(value.filepath) if value.filepath else '',
'type':
value.type,
'length':
value.length
}
for key, value in self._contents.items()
}
with open(contents_json_path, 'w') as file:
json.dump(content_json, file)
def _schedule_save_contents(self):
if self._contents_save_timer is not None:
self._contents_save_timer.cancel()
self._contents_save_timer = threading.Timer(
self.contents_save_interval_secs, self._save_contents)
self._contents_save_timer.start()
def _to_content_filepath(self, content_id: str) -> str:
return os.path.join(self.cache_folder, content_id)
def _fetch(self, task: Task):
# pylint: disable=too-many-statements
with self._lock:
if task.state is not Task.State.QUEUING:
raise ValueError(
f'task (={task.url}) is invalid state (={task.state})')
task.state = Task.State.RUNNING
content_id = task.content_id
content_filepath = self._to_content_filepath(content_id)
content = Content(content_id, Content.State.FETCHING)
try:
temp_fd, temp_path = tempfile.mkstemp(dir=self.temporary_dir)
with os.fdopen(temp_fd, 'bw') as temp_file:
response = self.url_resolver.resolve(task.url)
response.raise_for_status()
content_type = response.headers.get('Content-Type')
content_length_text = response.headers.get('Content-Length')
content_length = int(
content_length_text) if content_length_text else 0
fetch_size = 0
with self._lock:
task.content_length = content_length
task.fetched_size = fetch_size
for chunk in response.iter_content(chunk_size=65536):
fetch_size += len(chunk)
with self._lock:
task.fetched_size = fetch_size
if task.state is not Task.State.RUNNING:
raise InterruptedError(
f'task (={task.url}) fetch was interrupted')
temp_file.write(chunk)
os.rename(temp_path, content_filepath)
content_length = os.path.getsize(content_filepath)
with self._lock:
self._contents_size += content_length
task.state = Task.State.SUCCESS
content.state = Content.State.CACHED
content.filepath = content_filepath
content.length = content_length
content.type = content_type
except: # pylint: disable=bare-except
traceback.print_exc()
with self._lock:
content.state = Content.State.FAILED
if task.state is Task.State.RUNNING:
task.state = Task.State.FAILURE
else:
pass # keep state
if temp_path is not None:
os.remove(temp_path)
finally:
with self._lock:
self._contents[content_id] = content
del self._tasks[task.url]
self._invoke_callbacks(task)
self._schedule_save_contents()
return task
@staticmethod
def _invoke_callback(callback, content):
try:
callback(content)
except: # pylint: disable=bare-except
traceback.print_exc()
def _invoke_callbacks(self, task: Task):
with self._lock:
task_callbacks_copy = list(task.callbacks)
task.callbacks.clear()
content = self._contents[task.content_id]
for callback in task_callbacks_copy:
self._invoke_callback(callback, content)
return task
def cancel_fetch(self, url: URL):
with self._lock:
task = self.try_get_task(url)
if task is None:
return
if task.state != Task.State.RUNNING:
return
task.state = Task.State.CANCELED
def remove_content(self, url: URL) -> bool:
with self._lock:
content = self.try_get_content(url)
if content is None:
return False
del self._contents[content.id]
self._schedule_save_contents()
return True
def try_get_content(self, url: URL) -> Union[Content, None]:
content_id = Content.to_content_id(url)
with self._lock:
if content_id not in self._contents:
return None
content = self._contents[content_id]
# LRU implementation
self._contents.move_to_end(content_id)
excess_cache_size = max(
0, self._contents_size - self.max_cache_size_bytes)
if excess_cache_size > 0:
for content_id in self._contents.keys():
content = self._contents[content_id]
if content.length == 0:
continue
del self._contents[content_id]
self._contents_size -= content.length
if os.path.exists(content.file_path):
try:
os.remove(content.file_path)
except: # pylint: disable=bare-except
traceback.print_exc()
excess_cache_size -= content.length
if excess_cache_size <= 0:
break
self._schedule_save_contents()
return content
def try_get_task(self, url: URL) -> Union[Task, None]:
with self._lock:
return self._tasks[url] if url in self._tasks else None
def async_get_content(self, url: URL, callback: Callback) -> Future:
with self._lock:
content = self.try_get_content(url)
if content is not None:
if content.state in {
Content.State.CACHED, Content.State.FETCHING
}:
return self._executor.submit(self._invoke_callback,
callback, content)
self.remove_content(url)
elif url in self._tasks:
task = self._tasks[url]
if task.state in {Task.State.QUEUING, Task.State.RUNNING}:
task.callbacks.append(callback)
return task.future
task = Task(url, Task.State.QUEUING, [callback])
task.future = self._executor.submit(self._fetch, task)
self._tasks[url] = task
return task.future
class NetworkBlock:
"""
Base block object that will be used for each block in the network.
Attributes
----------
name : str
Name of the block which will be used to write the BNGL text
comment : (str, str)
comment at the begin {block} or end {block} statements, tuple
items : OrderedDict
all the model objects in the block
Methods
-------
add_item((name,value))
sets self.item[name] = value to add a particular model object
into a block
add_items(item_list)
loops over every element in the list and uses add_item on it
gen_string()
for every block this method generates the BNGL string of the
block. it has to be overwritten for each block.
"""
def __init__(self) -> None:
self.name = "NetworkBlock"
self.comment = (None, None)
self.items = OrderedDict()
def __str__(self) -> str:
return self.gen_string()
def __len__(self) -> int:
return len(self.items)
def __repr__(self) -> str:
# overwrites what the class representation
# shows the items in the model block in
# say ipython
repr_str = "{} block with {} item(s): {}".format(
self.name, len(self.items),
list([i.name for i in self.items.values()]))
return repr_str
def __getitem__(self, key):
if isinstance(key, int):
# get the item in order
return list(self.items.keys())[key]
return self.items[key]
def __setitem__(self, key, value) -> None:
self.items[key] = value
def __delitem__(self, key) -> None:
if key in self.items:
self.items.pop(key)
else:
print("Item {} not found".format(key))
def __iter__(self):
return self.items.keys().__iter__()
def __contains__(self, key) -> bool:
return key in self.items
# TODO: Think extensively how this is going to work
def __setattr__(self, name, value) -> None:
changed = False
if hasattr(self, "items"):
if name in self.items.keys():
try:
new_value = float(value)
changed = True
self.items[name] = new_value
except:
self.items[name] = value
if changed:
self._changes[name] = new_value
self.__dict__[name] = new_value
else:
self.__dict__[name] = value
def gen_string(self) -> str:
# each block can have a comment at the start
if self.comment[0] is not None:
block_lines = ["\nbegin {} #{}".format(self.name, self.comment[0])]
else:
block_lines = ["\nbegin {}".format(self.name)]
# now we just loop over lines
for item in self.items.keys():
block_lines.append(self.items[item].print_line())
# each block can have a comment at the start
if self.comment[1] is not None:
block_lines.append("end {} #{}\n".format(self.name,
self.comment[1]))
else:
block_lines.append("end {}\n".format(self.name))
# join everything with new lines
return "\n".join(block_lines)
def add_item(self, item_tpl) -> None:
# TODO: try adding evaluation of the parameter here
# for the future, in case we want people to be able
# to adjust the math
# TODO: Error handling, some names will definitely break this
name, value = item_tpl
# allow for empty addition, uses index
if name is None:
name = len(self.items)
# set the line
self.items[name] = value
# if the name is a string, try adding as an attribute
if isinstance(name, str):
try:
setattr(self, name, value)
except:
# print("can't set {} to {}".format(name, value))
pass
# we just added an item to a block, let's assume we need
# to recompile if we have a compiled simulator
self._recompile = True
def add_items(self, item_list) -> None:
for item in item_list:
self.add_item(item)
class LRUCacheStrategy(MemoryCacheStrategy[K, V]):
"""strategy which enforces a size limit with LRU"""
__slots__ = ("storage", "lock", "max_entries")
storage: OrderedDict[K, V]
lock: Lock # OrderedDict is not thread safe
max_entries: int
def __init__(self, max_entries: int) -> None:
self.storage = OrderedDict()
self.lock = Lock()
self.max_entries = max_entries
def __eq__(self, other: object) -> bool:
if isinstance(other, LRUCacheStrategy):
return self.storage == other.storage \
and self.max_entries == other.max_entries
return NotImplemented
def __getitem__(self, key: K) -> V:
"""get a value, setting it as the most recently used one"""
with self.lock:
self.storage.move_to_end(
key, last=False) # higher index = longer time since last use
return self.storage[key]
def __setitem__(self, key: K, value: V) -> None:
"""set a value, removing old ones if necessary"""
with self.lock:
if key not in self.storage and len(
self.storage) == self.max_entries:
self.storage.popitem(
) # make space for new entry by removing the last element
self.storage[key] = value
def __delitem__(self, key: K) -> None:
"""remove a value"""
with self.lock:
del self.storage[key]
def __iter__(self) -> Iterator[K]:
return iter(self.storage)
def __len__(self) -> int:
return len(self.storage)
def __contains__(self, key: object) -> bool:
return key in self.storage
def keys(self) -> KeysView[K]:
return self.storage.keys()
def values(self) -> ValuesView[V]:
return self.storage.values()
def items(self) -> ItemsView[K, V]:
return self.storage.items()
def peek(self, key: K) -> V:
"""get the value of key without triggering side effects like changing its priority"""
with self.lock:
return self.storage[key]
@overload
def pop(self, key: K) -> V:
...
@overload
def pop(self, key: K, default: Union[V, T] = ...) -> Union[V, T]:
...
def pop(self,
key: K,
default: Union[V,
T] = POP_SENTINEL) -> Union[V, T]: # type: ignore
"""remove a value and return it"""
with self.lock:
if default is POP_SENTINEL:
return self.storage.pop(key)
return self.storage.pop(key, default)
def popitem(self) -> Tuple[K, V]:
"""remove the least recently used key-value pair and return it"""
with self.lock:
return self.storage.popitem()
def clear(self) -> None:
"""remove all values"""
with self.lock:
self.storage.clear()
