def _conf_intervals_to_df(cis: OrderedDict) -> DataFrame: """Convert lmfit confidence intervals to pandas.DataFrame.""" ncis = len(list(cis.values())[0]) // 2 # e.g. [μ - σ, μ, μ + σ] return DataFrame( ((name, *[climit for _, climit in climits]) for name, climits in cis.items()), columns=( "name", *[ f"{sign}{clevel:.3f}" for sign, (clevel, _) in zip(("-", ) * ncis + ("", ) + ("+", ) * ncis, list(cis.values())[0]) ], ), )
def messages_per_actors_per_weekday(self, chats: List[Chat]) -> None: """ """ dataframes: Dict[Chat, DataFrame] = {} title = 'Participation Status (Messages per Actors per Weekday)' bars = ['Qtd_messages'] types = { 'User Messages': 'messages', 'System Messages': 'system_messages' } msg = 'Choose the message type:' result = select(msg, list(types.keys())).ask() message_type = types[result] for chat in chats: data = OrderedDict({weekday: 0 for weekday in weekdays}) for message in getattr(chat, message_type): data[message.created_at.strftime('%A')] += 1 index = list(data.keys()) rows = list(data.values()) dataframe = DataFrame(rows, index=index, columns=bars) dataframes[chat] = dataframe generate_chart(dataframes, bars=bars, lines=[], title=title)
def _training_step(self, kwargs: OrderedDict) -> ClosureResult: """Performs the actual train step with the tied hooks. Args: kwargs: the kwargs passed down to the hooks. Returns: A ``ClosureResult`` containing the training step output. """ # manually capture logged metrics training_step_output = self.trainer._call_strategy_hook("training_step", *kwargs.values()) self.trainer.strategy.post_training_step() model_output = self.trainer._call_lightning_module_hook("training_step_end", training_step_output) strategy_output = self.trainer._call_strategy_hook("training_step_end", training_step_output) training_step_output = strategy_output if model_output is None else model_output self._hiddens = _extract_hiddens(training_step_output, self.trainer.lightning_module.truncated_bptt_steps) result = self.output_result_cls.from_training_step_output( training_step_output, self.trainer.accumulate_grad_batches ) if self.trainer.move_metrics_to_cpu: # hiddens and the training step output are not moved as they are not considered "metrics" assert self.trainer._results is not None self.trainer._results.cpu() return result
def verticalTraversal(self, root: TreeNode) -> List[List[int]]: node_list = [] def BFS(root): queue = deque() queue.append((root, 0, 0)) while queue: node, row, column = queue.popleft() if node: node_list.append((column, row, node.val)) queue.append((node.left, row + 1, column - 1)) queue.append((node.right, row + 1, column + 1)) # step 1 - construct the node list with coordinates BFS(root) # step 2 - sort the node list according to coordinates node_list.sort() # step 3 - retrive sorted results partitioned by columns # use an ordered dictionary for keys output = OrderedDict() for column, row, value in node_list: if not column in output: output[column] = [value] else: output[column] = output[column] + [value] return output.values()
def get_info(row: OrderedDict): result = ",".join(row.values()) try: content = requestHTML(row["url"]) matched_meta_info = match_meta_info(content) get_image(matched_meta_info["alt_img_url"], row) except Exception as i: print(i) print("Parsed Faild: " + row["url"]) return result row.update(matched_meta_info) # Add local image path row["img_path"] = "/assets/img/works/posts/" + \ "{id}.jpeg".format(id=row["url"][-6:]) print("解析成功:#" + row["id"] + " " + row["title"]) return ",".join(row.values())
def test(cls, cfg, model, evaluators=None): """ Args: cfg (CfgNode): model (nn.Module): evaluators (list[DatasetEvaluator] or None): if None, will call :meth:`build_evaluator`. Otherwise, must have the same length as ``cfg.DATASETS.TEST``. Returns: dict: a dict of result metrics """ logger = logging.getLogger(__name__) if isinstance(evaluators, DatasetEvaluator): evaluators = [evaluators] if evaluators is not None: assert len( cfg.DATASETS.TEST) == len(evaluators), "{} != {}".format( len(cfg.DATASETS.TEST), len(evaluators)) results = OrderedDict() for idx, dataset_name in enumerate(cfg.DATASETS.TEST): data_loader = cls.build_test_loader(cfg, dataset_name) # When evaluators are passed in as arguments, # implicitly assume that evaluators can be created before data_loader. if evaluators is not None: evaluator = evaluators[idx] else: try: evaluator = cls.build_evaluator(cfg, dataset_name) except NotImplementedError: logger.warn( "No evaluator found. Use `DefaultTrainer.test(evaluators=)`, " "or implement its `build_evaluator` method.") results[dataset_name] = {} continue results_i = inference_on_dataset(model, data_loader, evaluator) results[dataset_name] = results_i if comm.is_main_process(): assert isinstance( results_i, dict ), "Evaluator must return a dict on the main process. Got {} instead.".format( results_i) logger.info("Evaluation results for {} in csv format:".format( dataset_name)) #print_csv_format(results_i) print(results_i) if len(results) == 1: results = list(results.values())[0] return results
def get_tail_frequencies(self, date: datetime) -> Tuple[int, Dict[int, float]]: # TODO: 其实没必要在这里保持有序 lower_bound, upper_bound = self._get_boundaries(date) self.cur.execute(r'''SELECT * FROM get_tail_count(%s, %s)''', (lower_bound, upper_bound)) rows = self.cur.fetchall() counts = OrderedDict({r[0]: r[1] for r in rows}) sum_count = sum(counts.values()) frequencies = OrderedDict( (tail, float(count) / sum_count) for tail, count in counts.items()) return (sum_count, frequencies)
edges2 = get_edges(im2) available_matches[i, j] = count_matches(edges, edges2) j += 1 i += 1 indices_with_two_matches = np.where((available_matches > 0).sum( axis=0) == 2)[0] prod = 1 for index, id in enumerate(tiles.keys()): if index in indices_with_two_matches: prod *= id print(f'a) {prod}') sidelen = int(np.sqrt(len(tiles))) tile_ims = list(tiles.values()) starting_index = indices_with_two_matches[0] def apply_flip_rot(im, fliprot): if fliprot[0] == 1: im = np.fliplr(im) if fliprot[1] == 1: im = np.flipud(im) for _ in range(fliprot[2]): im = np.rot90(im) return im def place_tile(puzzle, puzzle_as_inds, pos, tile_ind): top_to_match = None
def build_model(config: Mapping, cardinalities: Mapping[str, int]) -> keras.Model: """Construct model specified in the configuration. Also create optimizer and set the loss function. Args: config: Dictionary representing configuration file. cardinalities: Cardinalities of categorical features (needed to construct their embeddings). Return: Compiled model. """ model_config = config['model'] if isinstance(model_config, str): model = keras.models.load_model( model_config, custom_objects={'loss_fn': _create_loss(config['loss'])}) return model features = Features(config['features']) inputs_all = [] # Constituents of different types constituent_types = [ key for key in sorted(model_config.keys()) # Ensure order if key not in {'head', 'load_weights'} ] outputs_constituents = [] for constituent_type in constituent_types: inputs_numerical = keras.Input( shape=(None, len(features.numerical(constituent_type))), ragged=True, name=f'{constituent_type}_numerical') inputs_categorical = OrderedDict() for feature in features.categorical(constituent_type): inputs_categorical[feature] = keras.Input(shape=(None, ), ragged=True, name=feature) inputs_all.append(inputs_numerical) inputs_all.extend(inputs_categorical.values()) outputs = _apply_deep_set(inputs_numerical, inputs_categorical, model_config[constituent_type], cardinalities, constituent_type) outputs_constituents.append(outputs) # Head inputs_global_numerical = keras.Input(shape=(len( features.numerical('global')), ), name='global_numerical') inputs_global_categorical = OrderedDict() for feature in features.categorical('global'): inputs_global_categorical[feature] = keras.Input(shape=(None, ), name=feature) embeddings_global = { feature: Embedding(cardinalities[feature], model_config['head']['embeddings'][feature], name=feature + '_embeddings')(inputs) for feature, inputs in inputs_global_categorical.items() } inputs_all.append(inputs_global_numerical) inputs_all.extend(inputs_global_categorical.values()) inputs_head = Concatenate( name='head_concatenate')([inputs_global_numerical] + [ embeddings_global[feature] for feature in inputs_global_categorical.values() ] + outputs_constituents) outputs = _apply_dense_from_config(inputs_head, model_config['head'], name_prefix='head_') outputs = Dense(1, name='head_dense_output')(outputs) # Output unit model = keras.Model(inputs=inputs_all, outputs=outputs, name='full') model.compile(optimizer=_create_optimizer(config.get('optimizer', None)), loss=_create_loss(config['loss'])) if 'load_weights' in model_config: # Normally, a saved model should be loaded # keras.models.load_model at the beginning of thsi function. # However, this is currently not supported for models that use # ragged tensors [1]. As a workaround, construct the model anew # and then load saved weights. The path to weights would # usually be "{model_directory}/variables/variables", with the # ".index" file extension stripped off. This doesn't restore # the state of the optimizer. # [1] https://github.com/tensorflow/tensorflow/issues/41034 model.load_weights(model_config['load_weights']) return model
class ContentCache(CacheABC): # pylint: disable=too-many-instance-attributes def __init__(self, cache_folder: str, temporary_dir: str, max_cache_size_bytes: int = 1024 * 1024 * 1024, max_workers: int = 10, contents_load: bool = True, contents_save_interval_secs: float = 5.0, url_resolver: URLResolverABC = URLResolver()): print( f'ContentCache.__init__: cache_folder={cache_folder}, temporary_dir={temporary_dir}' ) self.cache_folder: str = cache_folder self.max_cache_size_bytes: int = max_cache_size_bytes self.temporary_dir = temporary_dir self.contents_save_interval_secs = contents_save_interval_secs self.url_resolver = url_resolver self._lock = threading.RLock() self._executor = ThreadPoolExecutor(max_workers) self._tasks: Dict[URL, Task] = {} self._contents: OrderedDict[str, Content] = OrderedDict() self._contents_size: int = 0 self._contents_save_timer = None if contents_load: self._load_contents() def __del__(self): self._save_contents() self._executor.shutdown() def _load_contents(self): contents_json_path = os.path.join(self.cache_folder, 'contents.json') if not os.path.exists(contents_json_path): return with self._lock: with open(contents_json_path, 'r') as file: content_json = json.load(file, object_pairs_hook=OrderedDict) self._contents = OrderedDict({ key: Content(id=value['id'], state=Content.State[value['state']], filepath=os.path.join(self.cache_folder, value['filepath']), type=value['type'], length=value['length']) for key, value in content_json.items() }) self._contents_size = sum( [c.length for c in self._contents.values()]) print( f'_load_contents: {len(self._contents)} from {contents_json_path}' ) def _save_contents(self): contents_json_path = os.path.join(self.cache_folder, 'contents.json') with self._lock: print( f'_save_contents: {len(self._contents)} to {contents_json_path}' ) content_json = { key: { 'id': value.id, 'state': value.state.name, 'filepath': os.path.basename(value.filepath) if value.filepath else '', 'type': value.type, 'length': value.length } for key, value in self._contents.items() } with open(contents_json_path, 'w') as file: json.dump(content_json, file) def _schedule_save_contents(self): if self._contents_save_timer is not None: self._contents_save_timer.cancel() self._contents_save_timer = threading.Timer( self.contents_save_interval_secs, self._save_contents) self._contents_save_timer.start() def _to_content_filepath(self, content_id: str) -> str: return os.path.join(self.cache_folder, content_id) def _fetch(self, task: Task): # pylint: disable=too-many-statements with self._lock: if task.state is not Task.State.QUEUING: raise ValueError( f'task (={task.url}) is invalid state (={task.state})') task.state = Task.State.RUNNING content_id = task.content_id content_filepath = self._to_content_filepath(content_id) content = Content(content_id, Content.State.FETCHING) try: temp_fd, temp_path = tempfile.mkstemp(dir=self.temporary_dir) with os.fdopen(temp_fd, 'bw') as temp_file: response = self.url_resolver.resolve(task.url) response.raise_for_status() content_type = response.headers.get('Content-Type') content_length_text = response.headers.get('Content-Length') content_length = int( content_length_text) if content_length_text else 0 fetch_size = 0 with self._lock: task.content_length = content_length task.fetched_size = fetch_size for chunk in response.iter_content(chunk_size=65536): fetch_size += len(chunk) with self._lock: task.fetched_size = fetch_size if task.state is not Task.State.RUNNING: raise InterruptedError( f'task (={task.url}) fetch was interrupted') temp_file.write(chunk) os.rename(temp_path, content_filepath) content_length = os.path.getsize(content_filepath) with self._lock: self._contents_size += content_length task.state = Task.State.SUCCESS content.state = Content.State.CACHED content.filepath = content_filepath content.length = content_length content.type = content_type except: # pylint: disable=bare-except traceback.print_exc() with self._lock: content.state = Content.State.FAILED if task.state is Task.State.RUNNING: task.state = Task.State.FAILURE else: pass # keep state if temp_path is not None: os.remove(temp_path) finally: with self._lock: self._contents[content_id] = content del self._tasks[task.url] self._invoke_callbacks(task) self._schedule_save_contents() return task @staticmethod def _invoke_callback(callback, content): try: callback(content) except: # pylint: disable=bare-except traceback.print_exc() def _invoke_callbacks(self, task: Task): with self._lock: task_callbacks_copy = list(task.callbacks) task.callbacks.clear() content = self._contents[task.content_id] for callback in task_callbacks_copy: self._invoke_callback(callback, content) return task def cancel_fetch(self, url: URL): with self._lock: task = self.try_get_task(url) if task is None: return if task.state != Task.State.RUNNING: return task.state = Task.State.CANCELED def remove_content(self, url: URL) -> bool: with self._lock: content = self.try_get_content(url) if content is None: return False del self._contents[content.id] self._schedule_save_contents() return True def try_get_content(self, url: URL) -> Union[Content, None]: content_id = Content.to_content_id(url) with self._lock: if content_id not in self._contents: return None content = self._contents[content_id] # LRU implementation self._contents.move_to_end(content_id) excess_cache_size = max( 0, self._contents_size - self.max_cache_size_bytes) if excess_cache_size > 0: for content_id in self._contents.keys(): content = self._contents[content_id] if content.length == 0: continue del self._contents[content_id] self._contents_size -= content.length if os.path.exists(content.file_path): try: os.remove(content.file_path) except: # pylint: disable=bare-except traceback.print_exc() excess_cache_size -= content.length if excess_cache_size <= 0: break self._schedule_save_contents() return content def try_get_task(self, url: URL) -> Union[Task, None]: with self._lock: return self._tasks[url] if url in self._tasks else None def async_get_content(self, url: URL, callback: Callback) -> Future: with self._lock: content = self.try_get_content(url) if content is not None: if content.state in { Content.State.CACHED, Content.State.FETCHING }: return self._executor.submit(self._invoke_callback, callback, content) self.remove_content(url) elif url in self._tasks: task = self._tasks[url] if task.state in {Task.State.QUEUING, Task.State.RUNNING}: task.callbacks.append(callback) return task.future task = Task(url, Task.State.QUEUING, [callback]) task.future = self._executor.submit(self._fetch, task) self._tasks[url] = task return task.future
class NetworkBlock: """ Base block object that will be used for each block in the network. Attributes ---------- name : str Name of the block which will be used to write the BNGL text comment : (str, str) comment at the begin {block} or end {block} statements, tuple items : OrderedDict all the model objects in the block Methods ------- add_item((name,value)) sets self.item[name] = value to add a particular model object into a block add_items(item_list) loops over every element in the list and uses add_item on it gen_string() for every block this method generates the BNGL string of the block. it has to be overwritten for each block. """ def __init__(self) -> None: self.name = "NetworkBlock" self.comment = (None, None) self.items = OrderedDict() def __str__(self) -> str: return self.gen_string() def __len__(self) -> int: return len(self.items) def __repr__(self) -> str: # overwrites what the class representation # shows the items in the model block in # say ipython repr_str = "{} block with {} item(s): {}".format( self.name, len(self.items), list([i.name for i in self.items.values()])) return repr_str def __getitem__(self, key): if isinstance(key, int): # get the item in order return list(self.items.keys())[key] return self.items[key] def __setitem__(self, key, value) -> None: self.items[key] = value def __delitem__(self, key) -> None: if key in self.items: self.items.pop(key) else: print("Item {} not found".format(key)) def __iter__(self): return self.items.keys().__iter__() def __contains__(self, key) -> bool: return key in self.items # TODO: Think extensively how this is going to work def __setattr__(self, name, value) -> None: changed = False if hasattr(self, "items"): if name in self.items.keys(): try: new_value = float(value) changed = True self.items[name] = new_value except: self.items[name] = value if changed: self._changes[name] = new_value self.__dict__[name] = new_value else: self.__dict__[name] = value def gen_string(self) -> str: # each block can have a comment at the start if self.comment[0] is not None: block_lines = ["\nbegin {} #{}".format(self.name, self.comment[0])] else: block_lines = ["\nbegin {}".format(self.name)] # now we just loop over lines for item in self.items.keys(): block_lines.append(self.items[item].print_line()) # each block can have a comment at the start if self.comment[1] is not None: block_lines.append("end {} #{}\n".format(self.name, self.comment[1])) else: block_lines.append("end {}\n".format(self.name)) # join everything with new lines return "\n".join(block_lines) def add_item(self, item_tpl) -> None: # TODO: try adding evaluation of the parameter here # for the future, in case we want people to be able # to adjust the math # TODO: Error handling, some names will definitely break this name, value = item_tpl # allow for empty addition, uses index if name is None: name = len(self.items) # set the line self.items[name] = value # if the name is a string, try adding as an attribute if isinstance(name, str): try: setattr(self, name, value) except: # print("can't set {} to {}".format(name, value)) pass # we just added an item to a block, let's assume we need # to recompile if we have a compiled simulator self._recompile = True def add_items(self, item_list) -> None: for item in item_list: self.add_item(item)
class LRUCacheStrategy(MemoryCacheStrategy[K, V]): """strategy which enforces a size limit with LRU""" __slots__ = ("storage", "lock", "max_entries") storage: OrderedDict[K, V] lock: Lock # OrderedDict is not thread safe max_entries: int def __init__(self, max_entries: int) -> None: self.storage = OrderedDict() self.lock = Lock() self.max_entries = max_entries def __eq__(self, other: object) -> bool: if isinstance(other, LRUCacheStrategy): return self.storage == other.storage \ and self.max_entries == other.max_entries return NotImplemented def __getitem__(self, key: K) -> V: """get a value, setting it as the most recently used one""" with self.lock: self.storage.move_to_end( key, last=False) # higher index = longer time since last use return self.storage[key] def __setitem__(self, key: K, value: V) -> None: """set a value, removing old ones if necessary""" with self.lock: if key not in self.storage and len( self.storage) == self.max_entries: self.storage.popitem( ) # make space for new entry by removing the last element self.storage[key] = value def __delitem__(self, key: K) -> None: """remove a value""" with self.lock: del self.storage[key] def __iter__(self) -> Iterator[K]: return iter(self.storage) def __len__(self) -> int: return len(self.storage) def __contains__(self, key: object) -> bool: return key in self.storage def keys(self) -> KeysView[K]: return self.storage.keys() def values(self) -> ValuesView[V]: return self.storage.values() def items(self) -> ItemsView[K, V]: return self.storage.items() def peek(self, key: K) -> V: """get the value of key without triggering side effects like changing its priority""" with self.lock: return self.storage[key] @overload def pop(self, key: K) -> V: ... @overload def pop(self, key: K, default: Union[V, T] = ...) -> Union[V, T]: ... def pop(self, key: K, default: Union[V, T] = POP_SENTINEL) -> Union[V, T]: # type: ignore """remove a value and return it""" with self.lock: if default is POP_SENTINEL: return self.storage.pop(key) return self.storage.pop(key, default) def popitem(self) -> Tuple[K, V]: """remove the least recently used key-value pair and return it""" with self.lock: return self.storage.popitem() def clear(self) -> None: """remove all values""" with self.lock: self.storage.clear()