def validate(self, data: OrderedDict) -> OrderedDict: # Take out fields that cannot be handled by standard serializer logic self.code = data.pop('code') self.name = data.pop('name') self.course_url = data.pop('course_url') self.teachers = data.pop('teachers') data = super().validate(data) return data
def _process_function_base(self, raw: T.OrderedDict, obj: T.Optional[Object] = None) -> Function: # Handle arguments posargs = raw.pop('posargs', {}) optargs = raw.pop('optargs', {}) varargs = raw.pop('varargs', None) kwargs = raw.pop('kwargs', {}) # Fix kwargs_inherit if isinstance(raw['kwargs_inherit'], str): raw['kwargs_inherit'] = [raw['kwargs_inherit']] # Parse args posargs_mapped: T.List[PosArg] = [] optargs_mapped: T.List[PosArg] = [] varargs_mapped: T.Optional[VarArgs] = None kwargs_mapped: T.Dict[str, Kwarg] = {} for k, v in posargs.items(): v['type'] = Type(v['type']) posargs_mapped += [PosArg(name=k, **v)] for k, v in optargs.items(): v['type'] = Type(v['type']) optargs_mapped += [PosArg(name=k, **v)] for k, v in kwargs.items(): v['type'] = Type(v['type']) kwargs_mapped[k] = Kwarg(name=k, **v) if varargs is not None: varargs['type'] = Type(varargs['type']) varargs_mapped = VarArgs(**varargs) raw['returns'] = Type(raw['returns']) # Build function object if obj is not None: return Method( posargs=posargs_mapped, optargs=optargs_mapped, varargs=varargs_mapped, kwargs=kwargs_mapped, obj=obj, **raw, ) return Function( posargs=posargs_mapped, optargs=optargs_mapped, varargs=varargs_mapped, kwargs=kwargs_mapped, **raw, )
def format_image_link(line: OrderedDict): image_pattern = re.compile(r'.*"(?P<link>.*)".*') image: str = line.pop('image') res = image_pattern.search(image) link = res.group('link') line.update({'image': link}) return line
def load_states_from_checkpoint(model_file: str) -> CheckpointState: print("Reading saved model from %s", model_file) state_dict = torch.load( model_file, map_location=lambda s, l: default_restore_location(s, "cpu")) for k in [("model_dict", "model_state_dict"), ("optimizer_dict", "optimizer_state_dict")]: state_dict[k[0]] = state_dict.pop(k[-1], None) new_model_dict = OrderedDict() for key, value in state_dict["model_dict"].items(): if key.startswith("bert."): key = "bert_model." + key[5:] # key = key.replace("bert_model.encoder", "bert_model") new_model_dict[key] = value state_dict["model_dict"] = new_model_dict for key in ["linear.weight"]: new_model_dict.pop(key) return CheckpointState(**state_dict)
def main(args: List[str]): logging.debug(f"args: {args}") args = parse_args(prog=args[0], args=args[1:]) if args.dump_folder_path.exists(): shutil.rmtree(args.dump_folder_path) args.dump_folder_path.mkdir(parents=True) (args.dump_folder_path / "pages").mkdir(parents=True) data_folder_path = args.luwei_downloaded_thread_folder_path / "data" max_page_number = get_max_page_number(data_folder_path) for page_number in range(1, max_page_number + 1): data_file_path = data_folder_path / f"{page_number}.data" with open(data_file_path) as data_file: data_raw = data_file.read()[11:-2] thread_page = json.loads(data_raw, object_pairs_hook=OrderedDict) last_dumped_post_id = int(thread_page["replys"][-1]["id"]) if page_number == max_page_number: thread_body = OrderedDict(thread_page) thread_body.pop("replys") thread_body.pop("replyCount") with open(args.dump_folder_path / "thread.json", "w+") as thread_file: json.dump(thread_body, thread_file, indent=2, ensure_ascii=False) thread_file.write("\n") replies = list( filter(lambda post: post["userid"] != "芦苇", thread_page["replys"])) with open(args.dump_folder_path / "pages" / f"{page_number}.json", "w+") as page_file: json.dump(replies, page_file, indent=2, ensure_ascii=False) page_file.write("\n")
def create_advert(data: t.OrderedDict) -> Advert: input_stack = data.pop("stack") new_advert = Advert.objects.create(**data) for s in input_stack: stack, created = Stack.objects.get_or_create(name=s) # TODO: change to unpack list new_advert.add(*[list_with_stack]) if created: new_advert.stack.add(stack.pk) else: new_advert.stack.add(stack.pk) new_advert.save() calculate_similarity_coefficient.delay(new_advert.pk) return new_advert
class ISBNCache: def __init__(self, size): self.cache = OrderedDict() self.size = size # O(1) def lookup(self, isbn): if isbn in self.cache: self.cache.move_to_end(isbn) return self.cache[isbn] return -1 # O(1) def insert(self, isbn, price): if isbn in self.cache: del self.cache[isbn] self.cache[isbn] = price if len(self.cache) > self.size: self.cache.popitem(last=False) # Remove least recent isbn # O(1) def remove(self, isbn): return self.cache.pop(isbn, None) is not None
def remove_web_scrapper_order(line: OrderedDict): line.pop('\ufeffweb-scraper-order') return line
def remove_coctail_preview_href(line: OrderedDict): line.pop('coctail-preview-href') return line
def remove_coctail_preview(line: OrderedDict): line.pop('coctail-preview') return line
def remove_web_scraper_start_url(line: OrderedDict): line.pop('web-scraper-start-url') return line
class NetworkBlock: """ Base block object that will be used for each block in the network. Attributes ---------- name : str Name of the block which will be used to write the BNGL text comment : (str, str) comment at the begin {block} or end {block} statements, tuple items : OrderedDict all the model objects in the block Methods ------- add_item((name,value)) sets self.item[name] = value to add a particular model object into a block add_items(item_list) loops over every element in the list and uses add_item on it gen_string() for every block this method generates the BNGL string of the block. it has to be overwritten for each block. """ def __init__(self) -> None: self.name = "NetworkBlock" self.comment = (None, None) self.items = OrderedDict() def __str__(self) -> str: return self.gen_string() def __len__(self) -> int: return len(self.items) def __repr__(self) -> str: # overwrites what the class representation # shows the items in the model block in # say ipython repr_str = "{} block with {} item(s): {}".format( self.name, len(self.items), list([i.name for i in self.items.values()])) return repr_str def __getitem__(self, key): if isinstance(key, int): # get the item in order return list(self.items.keys())[key] return self.items[key] def __setitem__(self, key, value) -> None: self.items[key] = value def __delitem__(self, key) -> None: if key in self.items: self.items.pop(key) else: print("Item {} not found".format(key)) def __iter__(self): return self.items.keys().__iter__() def __contains__(self, key) -> bool: return key in self.items # TODO: Think extensively how this is going to work def __setattr__(self, name, value) -> None: changed = False if hasattr(self, "items"): if name in self.items.keys(): try: new_value = float(value) changed = True self.items[name] = new_value except: self.items[name] = value if changed: self._changes[name] = new_value self.__dict__[name] = new_value else: self.__dict__[name] = value def gen_string(self) -> str: # each block can have a comment at the start if self.comment[0] is not None: block_lines = ["\nbegin {} #{}".format(self.name, self.comment[0])] else: block_lines = ["\nbegin {}".format(self.name)] # now we just loop over lines for item in self.items.keys(): block_lines.append(self.items[item].print_line()) # each block can have a comment at the start if self.comment[1] is not None: block_lines.append("end {} #{}\n".format(self.name, self.comment[1])) else: block_lines.append("end {}\n".format(self.name)) # join everything with new lines return "\n".join(block_lines) def add_item(self, item_tpl) -> None: # TODO: try adding evaluation of the parameter here # for the future, in case we want people to be able # to adjust the math # TODO: Error handling, some names will definitely break this name, value = item_tpl # allow for empty addition, uses index if name is None: name = len(self.items) # set the line self.items[name] = value # if the name is a string, try adding as an attribute if isinstance(name, str): try: setattr(self, name, value) except: # print("can't set {} to {}".format(name, value)) pass # we just added an item to a block, let's assume we need # to recompile if we have a compiled simulator self._recompile = True def add_items(self, item_list) -> None: for item in item_list: self.add_item(item)
class LRUCacheStrategy(MemoryCacheStrategy[K, V]): """strategy which enforces a size limit with LRU""" __slots__ = ("storage", "lock", "max_entries") storage: OrderedDict[K, V] lock: Lock # OrderedDict is not thread safe max_entries: int def __init__(self, max_entries: int) -> None: self.storage = OrderedDict() self.lock = Lock() self.max_entries = max_entries def __eq__(self, other: object) -> bool: if isinstance(other, LRUCacheStrategy): return self.storage == other.storage \ and self.max_entries == other.max_entries return NotImplemented def __getitem__(self, key: K) -> V: """get a value, setting it as the most recently used one""" with self.lock: self.storage.move_to_end( key, last=False) # higher index = longer time since last use return self.storage[key] def __setitem__(self, key: K, value: V) -> None: """set a value, removing old ones if necessary""" with self.lock: if key not in self.storage and len( self.storage) == self.max_entries: self.storage.popitem( ) # make space for new entry by removing the last element self.storage[key] = value def __delitem__(self, key: K) -> None: """remove a value""" with self.lock: del self.storage[key] def __iter__(self) -> Iterator[K]: return iter(self.storage) def __len__(self) -> int: return len(self.storage) def __contains__(self, key: object) -> bool: return key in self.storage def keys(self) -> KeysView[K]: return self.storage.keys() def values(self) -> ValuesView[V]: return self.storage.values() def items(self) -> ItemsView[K, V]: return self.storage.items() def peek(self, key: K) -> V: """get the value of key without triggering side effects like changing its priority""" with self.lock: return self.storage[key] @overload def pop(self, key: K) -> V: ... @overload def pop(self, key: K, default: Union[V, T] = ...) -> Union[V, T]: ... def pop(self, key: K, default: Union[V, T] = POP_SENTINEL) -> Union[V, T]: # type: ignore """remove a value and return it""" with self.lock: if default is POP_SENTINEL: return self.storage.pop(key) return self.storage.pop(key, default) def popitem(self) -> Tuple[K, V]: """remove the least recently used key-value pair and return it""" with self.lock: return self.storage.popitem() def clear(self) -> None: """remove all values""" with self.lock: self.storage.clear()