def validate(self, data: OrderedDict) -> OrderedDict:
# Take out fields that cannot be handled by standard serializer logic
self.code = data.pop('code')
self.name = data.pop('name')
self.course_url = data.pop('course_url')
self.teachers = data.pop('teachers')
data = super().validate(data)
return data
def _process_function_base(self, raw: T.OrderedDict, obj: T.Optional[Object] = None) -> Function:
# Handle arguments
posargs = raw.pop('posargs', {})
optargs = raw.pop('optargs', {})
varargs = raw.pop('varargs', None)
kwargs = raw.pop('kwargs', {})
# Fix kwargs_inherit
if isinstance(raw['kwargs_inherit'], str):
raw['kwargs_inherit'] = [raw['kwargs_inherit']]
# Parse args
posargs_mapped: T.List[PosArg] = []
optargs_mapped: T.List[PosArg] = []
varargs_mapped: T.Optional[VarArgs] = None
kwargs_mapped: T.Dict[str, Kwarg] = {}
for k, v in posargs.items():
v['type'] = Type(v['type'])
posargs_mapped += [PosArg(name=k, **v)]
for k, v in optargs.items():
v['type'] = Type(v['type'])
optargs_mapped += [PosArg(name=k, **v)]
for k, v in kwargs.items():
v['type'] = Type(v['type'])
kwargs_mapped[k] = Kwarg(name=k, **v)
if varargs is not None:
varargs['type'] = Type(varargs['type'])
varargs_mapped = VarArgs(**varargs)
raw['returns'] = Type(raw['returns'])
# Build function object
if obj is not None:
return Method(
posargs=posargs_mapped,
optargs=optargs_mapped,
varargs=varargs_mapped,
kwargs=kwargs_mapped,
obj=obj,
**raw,
)
return Function(
posargs=posargs_mapped,
optargs=optargs_mapped,
varargs=varargs_mapped,
kwargs=kwargs_mapped,
**raw,
)
def format_image_link(line: OrderedDict):
image_pattern = re.compile(r'.*"(?P<link>.*)".*')
image: str = line.pop('image')
res = image_pattern.search(image)
link = res.group('link')
line.update({'image': link})
return line
def load_states_from_checkpoint(model_file: str) -> CheckpointState:
print("Reading saved model from %s", model_file)
state_dict = torch.load(
model_file,
map_location=lambda s, l: default_restore_location(s, "cpu"))
for k in [("model_dict", "model_state_dict"),
("optimizer_dict", "optimizer_state_dict")]:
state_dict[k[0]] = state_dict.pop(k[-1], None)
new_model_dict = OrderedDict()
for key, value in state_dict["model_dict"].items():
if key.startswith("bert."):
key = "bert_model." + key[5:]
# key = key.replace("bert_model.encoder", "bert_model")
new_model_dict[key] = value
state_dict["model_dict"] = new_model_dict
for key in ["linear.weight"]:
new_model_dict.pop(key)
return CheckpointState(**state_dict)
def main(args: List[str]):
logging.debug(f"args: {args}")
args = parse_args(prog=args[0], args=args[1:])
if args.dump_folder_path.exists():
shutil.rmtree(args.dump_folder_path)
args.dump_folder_path.mkdir(parents=True)
(args.dump_folder_path / "pages").mkdir(parents=True)
data_folder_path = args.luwei_downloaded_thread_folder_path / "data"
max_page_number = get_max_page_number(data_folder_path)
for page_number in range(1, max_page_number + 1):
data_file_path = data_folder_path / f"{page_number}.data"
with open(data_file_path) as data_file:
data_raw = data_file.read()[11:-2]
thread_page = json.loads(data_raw, object_pairs_hook=OrderedDict)
last_dumped_post_id = int(thread_page["replys"][-1]["id"])
if page_number == max_page_number:
thread_body = OrderedDict(thread_page)
thread_body.pop("replys")
thread_body.pop("replyCount")
with open(args.dump_folder_path / "thread.json",
"w+") as thread_file:
json.dump(thread_body,
thread_file,
indent=2,
ensure_ascii=False)
thread_file.write("\n")
replies = list(
filter(lambda post: post["userid"] != "芦苇",
thread_page["replys"]))
with open(args.dump_folder_path / "pages" / f"{page_number}.json",
"w+") as page_file:
json.dump(replies, page_file, indent=2, ensure_ascii=False)
page_file.write("\n")
def create_advert(data: t.OrderedDict) -> Advert:
input_stack = data.pop("stack")
new_advert = Advert.objects.create(**data)
for s in input_stack:
stack, created = Stack.objects.get_or_create(name=s)
# TODO: change to unpack list new_advert.add(*[list_with_stack])
if created:
new_advert.stack.add(stack.pk)
else:
new_advert.stack.add(stack.pk)
new_advert.save()
calculate_similarity_coefficient.delay(new_advert.pk)
return new_advert
class ISBNCache:
def __init__(self, size):
self.cache = OrderedDict()
self.size = size
# O(1)
def lookup(self, isbn):
if isbn in self.cache:
self.cache.move_to_end(isbn)
return self.cache[isbn]
return -1
# O(1)
def insert(self, isbn, price):
if isbn in self.cache:
del self.cache[isbn]
self.cache[isbn] = price
if len(self.cache) > self.size:
self.cache.popitem(last=False) # Remove least recent isbn
# O(1)
def remove(self, isbn):
return self.cache.pop(isbn, None) is not None
def remove_web_scrapper_order(line: OrderedDict):
line.pop('\ufeffweb-scraper-order')
return line
def remove_coctail_preview_href(line: OrderedDict):
line.pop('coctail-preview-href')
return line
def remove_coctail_preview(line: OrderedDict):
line.pop('coctail-preview')
return line
def remove_web_scraper_start_url(line: OrderedDict):
line.pop('web-scraper-start-url')
return line
class NetworkBlock:
"""
Base block object that will be used for each block in the network.
Attributes
----------
name : str
Name of the block which will be used to write the BNGL text
comment : (str, str)
comment at the begin {block} or end {block} statements, tuple
items : OrderedDict
all the model objects in the block
Methods
-------
add_item((name,value))
sets self.item[name] = value to add a particular model object
into a block
add_items(item_list)
loops over every element in the list and uses add_item on it
gen_string()
for every block this method generates the BNGL string of the
block. it has to be overwritten for each block.
"""
def __init__(self) -> None:
self.name = "NetworkBlock"
self.comment = (None, None)
self.items = OrderedDict()
def __str__(self) -> str:
return self.gen_string()
def __len__(self) -> int:
return len(self.items)
def __repr__(self) -> str:
# overwrites what the class representation
# shows the items in the model block in
# say ipython
repr_str = "{} block with {} item(s): {}".format(
self.name, len(self.items),
list([i.name for i in self.items.values()]))
return repr_str
def __getitem__(self, key):
if isinstance(key, int):
# get the item in order
return list(self.items.keys())[key]
return self.items[key]
def __setitem__(self, key, value) -> None:
self.items[key] = value
def __delitem__(self, key) -> None:
if key in self.items:
self.items.pop(key)
else:
print("Item {} not found".format(key))
def __iter__(self):
return self.items.keys().__iter__()
def __contains__(self, key) -> bool:
return key in self.items
# TODO: Think extensively how this is going to work
def __setattr__(self, name, value) -> None:
changed = False
if hasattr(self, "items"):
if name in self.items.keys():
try:
new_value = float(value)
changed = True
self.items[name] = new_value
except:
self.items[name] = value
if changed:
self._changes[name] = new_value
self.__dict__[name] = new_value
else:
self.__dict__[name] = value
def gen_string(self) -> str:
# each block can have a comment at the start
if self.comment[0] is not None:
block_lines = ["\nbegin {} #{}".format(self.name, self.comment[0])]
else:
block_lines = ["\nbegin {}".format(self.name)]
# now we just loop over lines
for item in self.items.keys():
block_lines.append(self.items[item].print_line())
# each block can have a comment at the start
if self.comment[1] is not None:
block_lines.append("end {} #{}\n".format(self.name,
self.comment[1]))
else:
block_lines.append("end {}\n".format(self.name))
# join everything with new lines
return "\n".join(block_lines)
def add_item(self, item_tpl) -> None:
# TODO: try adding evaluation of the parameter here
# for the future, in case we want people to be able
# to adjust the math
# TODO: Error handling, some names will definitely break this
name, value = item_tpl
# allow for empty addition, uses index
if name is None:
name = len(self.items)
# set the line
self.items[name] = value
# if the name is a string, try adding as an attribute
if isinstance(name, str):
try:
setattr(self, name, value)
except:
# print("can't set {} to {}".format(name, value))
pass
# we just added an item to a block, let's assume we need
# to recompile if we have a compiled simulator
self._recompile = True
def add_items(self, item_list) -> None:
for item in item_list:
self.add_item(item)
class LRUCacheStrategy(MemoryCacheStrategy[K, V]):
"""strategy which enforces a size limit with LRU"""
__slots__ = ("storage", "lock", "max_entries")
storage: OrderedDict[K, V]
lock: Lock # OrderedDict is not thread safe
max_entries: int
def __init__(self, max_entries: int) -> None:
self.storage = OrderedDict()
self.lock = Lock()
self.max_entries = max_entries
def __eq__(self, other: object) -> bool:
if isinstance(other, LRUCacheStrategy):
return self.storage == other.storage \
and self.max_entries == other.max_entries
return NotImplemented
def __getitem__(self, key: K) -> V:
"""get a value, setting it as the most recently used one"""
with self.lock:
self.storage.move_to_end(
key, last=False) # higher index = longer time since last use
return self.storage[key]
def __setitem__(self, key: K, value: V) -> None:
"""set a value, removing old ones if necessary"""
with self.lock:
if key not in self.storage and len(
self.storage) == self.max_entries:
self.storage.popitem(
) # make space for new entry by removing the last element
self.storage[key] = value
def __delitem__(self, key: K) -> None:
"""remove a value"""
with self.lock:
del self.storage[key]
def __iter__(self) -> Iterator[K]:
return iter(self.storage)
def __len__(self) -> int:
return len(self.storage)
def __contains__(self, key: object) -> bool:
return key in self.storage
def keys(self) -> KeysView[K]:
return self.storage.keys()
def values(self) -> ValuesView[V]:
return self.storage.values()
def items(self) -> ItemsView[K, V]:
return self.storage.items()
def peek(self, key: K) -> V:
"""get the value of key without triggering side effects like changing its priority"""
with self.lock:
return self.storage[key]
@overload
def pop(self, key: K) -> V:
...
@overload
def pop(self, key: K, default: Union[V, T] = ...) -> Union[V, T]:
...
def pop(self,
key: K,
default: Union[V,
T] = POP_SENTINEL) -> Union[V, T]: # type: ignore
"""remove a value and return it"""
with self.lock:
if default is POP_SENTINEL:
return self.storage.pop(key)
return self.storage.pop(key, default)
def popitem(self) -> Tuple[K, V]:
"""remove the least recently used key-value pair and return it"""
with self.lock:
return self.storage.popitem()
def clear(self) -> None:
"""remove all values"""
with self.lock:
self.storage.clear()
