def _parse_losses(self, losses):
"""Parse the raw outputs (losses) of the network.
Args:
losses (dict): Raw output of the network, which usually contain
losses and other necessary information.
Returns:
tuple[Tensor, dict]: (loss, log_vars), loss is the loss tensor \
which may be a weighted sum of all losses, log_vars contains \
all the variables to be sent to the logger.
"""
log_vars = OrderedDict()
for loss_name, loss_value in losses.items():
if isinstance(loss_value, torch.Tensor):
log_vars[loss_name] = loss_value.mean()
elif isinstance(loss_value, list):
log_vars[loss_name] = sum(_loss.mean() for _loss in loss_value)
else:
raise TypeError(
f'{loss_name} is not a tensor or list of tensor')
loss = sum(_value for _key, _value in log_vars.items()
if 'loss' in _key)
log_vars['loss'] = loss
for loss_name, loss_value in log_vars.items():
if dist.is_available() and dist.is_initialized():
loss_value = loss_value.data.clone()
dist.all_reduce(loss_value.div_(dist.get_world_size()))
log_vars[loss_name] = loss_value.item()
return loss, log_vars
def main(args):
model_index_file = MMCLS_ROOT / 'model-index.yml'
model_index = Config.fromfile(model_index_file)
models = OrderedDict()
for file in model_index.Import:
metafile = Config.fromfile(MMCLS_ROOT / file)
models.update({model.Name: model for model in metafile.Models})
logger = get_root_logger(log_file='benchmark_test_image.log',
log_level=logging.INFO)
if args.models:
patterns = [re.compile(pattern) for pattern in args.models]
filter_models = {}
for k, v in models.items():
if any([re.match(pattern, k) for pattern in patterns]):
filter_models[k] = v
if len(filter_models) == 0:
print('No model found, please specify models in:')
print('\n'.join(models.keys()))
return
models = filter_models
summary_data = {}
for model_name, model_info in models.items():
config = Path(model_info.Config)
assert config.exists(), f'{model_name}: {config} not found.'
logger.info(f'Processing: {model_name}')
http_prefix = 'https://download.openmmlab.com/mmclassification/'
dataset = model_info.Results[0]['Dataset']
if args.checkpoint_root is not None:
root = Path(args.checkpoint_root)
checkpoint = root / model_info.Weights[len(http_prefix):]
checkpoint = str(checkpoint)
else:
checkpoint = None
try:
# build the model from a config file and a checkpoint file
result = inference(MMCLS_ROOT / config, checkpoint,
classes_map[dataset], args)
result['valid'] = 'PASS'
except Exception as e:
logger.error(f'"{config}" : {repr(e)}')
result = {'valid': 'FAIL'}
summary_data[model_name] = result
# show the results
if args.show:
imshow_infos(args.img, result, wait_time=args.wait_time)
show_summary(summary_data)
def assertOrderedDictEqual(od1: OrderedDict, od2: OrderedDict):
errors = []
if len(od1.keys()) != len(od2.keys()):
raise AssertionError("Number of items don't match: {} {}".format(od1, od2))
for i, j in zip(od1.items(), od2.items()):
if i[0] != j[0]:
errors.append("Keys in {} and {} don't match".format(i, j))
if i[1] != j[1]:
errors.append("Values in {} and {} don't match".format(i, j))
if errors:
raise AssertionError(", ".join(errors))
return True
class GetSetInstruction(InstructionBase):
"""Aggregates a number instructions that allow a CPU to get and set values.
The instruction uses funct7 to identify a "Xetter" to which to delegate.
"""
def __init__(self, xetters):
"""Constructor
Parameters
----------
xetters: mapping of register number (integer 0-127) to a subclass of Xetter.
"""
super().__init__()
self.xetters = OrderedDict(xetters)
def elab(self, m):
# By default, wire done to start to prevent hangs
m.d.comb += self.done.eq(self.start)
# Based on funct7, route start, done and output signals
m.submodules['f7buf'] = f7buf = ValueBuffer(self.funct7, self.start)
for reg, x in self.xetters.items():
m.d.comb += [
x.in0.eq(self.in0),
x.in1.eq(self.in1),
]
with m.If(f7buf.output == reg):
m.d.comb += [
x.start.eq(self.start),
self.output.eq(x.output),
self.done.eq(x.done),
]
def _walk_validator(
instance: OrderedDict,
validator_dict: OrderedDict,
validator_function: Callable[[Mapping, Any, str],
Iterator[ValidationError]],
position=None,
allow_missing_keys: bool = False,
) -> Iterator[ValidationError]:
"""Walk instance and validation dict entries in parallel and apply a validator func.
This function can be used to recursively walk both the instance dictionary and the
custom validation dictionary in parallel. Once a leaf dictionary entry is reached,
the validation function is applied to the selected items.
Parameters
----------
instance:
Tree serialization (with default dtypes) of the instance
validator_dict:
OrderedDict representation of the validation structure.
validator_function:
Custom python validator function to apply along the (nested) dictionary
position:
String representation of the current nested schema position
allow_missing_keys:
If True will skip validation if the requested key to validate does not exist.
Yields
------
asdf.ValidationError
"""
if position is None: # pragma: no cover
position = []
if isinstance(validator_dict, dict):
for key, item in validator_dict.items():
if isinstance(item, Mapping):
yield from _walk_validator(
instance[key],
validator_dict[key],
validator_function,
position=position + [key],
allow_missing_keys=allow_missing_keys,
)
else:
if key in instance:
yield from validator_function(instance[key], item,
position + [key])
elif allow_missing_keys: # pragma: no cover
pass
else: # pragma: no cover
pass
# TODO: if a property is not required the key might be missing
# yield ValidationError(f"Missing key {key}")
else:
yield from validator_function(instance, validator_dict, position)
def process(self, data: OrderedDict) -> OrderedDict:
"""
Processing PyTorch model parameter.
The data is a state_dict of a PyTorch model.
"""
new_data = OrderedDict()
for layer_name, layer_params in data.items():
new_data[layer_name] = self._process_layer(layer_params)
return new_data
def parse_launch_arguments(
launch_arguments: List[Text]) -> List[Tuple[Text, Text]]:
"""Parse the given launch arguments from the command line, into list of tuples for launch."""
parsed_launch_arguments = OrderedDict() # type: ignore
for argument in launch_arguments:
count = argument.count(':=')
if count == 0 or argument.startswith(':=') or (
count == 1 and argument.endswith(':=')):
raise RuntimeError(
"malformed launch argument '{}', expected format '<name>:=<value>'"
.format(argument))
name, value = argument.split(':=', maxsplit=1)
parsed_launch_arguments[name] = value # last one wins is intentional
return parsed_launch_arguments.items()
def print_test_list(test_list: typing.OrderedDict) -> None:
"""
Prints a list of all tests and demos in test_list.
"""
namelen = max(len(name) for name, _ in test_list.keys())
for current_type in ['test', 'demo', 'benchmark']:
for (name, type_), (_, lang, desc, _) in test_list.items():
if type_ == current_type:
print(f"[{type_} {lang:3}] {name:{namelen}} {desc}")
print("")
print("To see how to run them, add --help to your invocation!")
print("")
print("Remember: Testing is the future, and the future starts with: You.")
print("")
def _conf_intervals_to_df(cis: OrderedDict) -> DataFrame:
"""Convert lmfit confidence intervals to pandas.DataFrame."""
ncis = len(list(cis.values())[0]) // 2 # e.g. [μ - σ, μ, μ + σ]
return DataFrame(
((name, *[climit for _, climit in climits])
for name, climits in cis.items()),
columns=(
"name",
*[
f"{sign}{clevel:.3f}"
for sign, (clevel, _) in zip(("-", ) * ncis + ("", ) +
("+", ) * ncis,
list(cis.values())[0])
],
),
)
def get_tail_frequencies(self,
date: datetime) -> Tuple[int, Dict[int, float]]:
# TODO: 其实没必要在这里保持有序
lower_bound, upper_bound = self._get_boundaries(date)
self.cur.execute(r'''SELECT * FROM get_tail_count(%s, %s)''',
(lower_bound, upper_bound))
rows = self.cur.fetchall()
counts = OrderedDict({r[0]: r[1] for r in rows})
sum_count = sum(counts.values())
frequencies = OrderedDict(
(tail, float(count) / sum_count) for tail, count in counts.items())
return (sum_count, frequencies)
def process_data(data):
prerequisites = OrderedDict()
list_data = []
for line in data:
sentence = line.split()
first = sentence[1]
second = sentence[7]
list_data.append((first, second))
if second in prerequisites and first not in prerequisites[second]:
prerequisites[second].append(first)
else:
prerequisites[second] = [first]
prerequisites[second].sort()
sorted = list(prerequisites.items())
sorted.sort(key=lambda x: x[0])
prerequisites = {x[0]: x[1] for x in sorted}
return prerequisites, list_data
def reduce_duplicated_timestamps(ys, ts, verbose=False):
''' Reduce duplicated timestamps in a timeseries by averaging the values
Parameters
----------
ys : array
Dependent variable
ts : array
Independent variable
verbose : bool
If True, will print a warning message
Returns
-------
ys : array
Dependent variable
ts : array
Independent variable, with duplicated timestamps reduced by averaging the values
'''
ys = np.asarray(ys, dtype=np.float)
ts = np.asarray(ts, dtype=np.float)
assert ys.size == ts.size, 'The size of time axis and data value should be equal!'
if len(ts) != len(set(ts)):
value = OrderedDict()
for t, y in zip(ts, ys):
if t not in value:
value[t] = [y]
else:
value[t].append(y)
ts = []
ys = []
for k, v in value.items():
ts.append(k)
ys.append(np.mean(v))
ts = np.array(ts)
ys = np.array(ys)
if verbose:
print(
'Duplicate timestamps have been combined by averaging values.')
return ys, ts
class ScopedSymbolTable:
def __init__(self, scope_name, scope_level, enclosing_scope=None):
self._symbols = OrderedDict()
self.scope_name = scope_name
self.scope_level = scope_level
self.enclosing_scope = enclosing_scope
def _init_builtins(self):
self.insert(BuiltinTypeSymbol('INTEGER'))
self.insert(BuiltinTypeSymbol('REAL'))
def __str__(self):
h1 = 'SCOPE (SCOPED SYMBOL TABLE)'
lines = ['\n', h1, '=' * len(h1)]
for header_name, header_value in (('Scope name', self.scope_name),
('Scope level', self.scope_level),
('Enclosing scope',
self.enclosing_scope.scope_name
if self.enclosing_scope else None)):
lines.append('%-15s: %s' % (header_name, header_value))
h2 = 'Scope (Scoped symbol table) contents'
lines.extend([h2, '-' * len(h2)])
lines.extend(
('%7s: %r' % (key, value)) for key, value in self._symbols.items())
lines.append('\n')
s = '\n'.join(lines)
return s
__repr__ = __str__
def insert(self, symbol):
print('Insert: %s' % symbol.name)
self._symbols[symbol.name] = symbol
def lookup(self, name):
print('Lookup: %s. (Scope name: %s)' % (name, self.scope_name))
# 'symbol' is either an instance of the Symbol class or None
symbol = self._symbols.get(name)
if symbol is not None:
return symbol
# recursivamente va hacia arriba en la cadena buscando el nombre.
if self.enclosing_scope is not None:
return self.enclosing_scope.lookup(name)
def ordered_dict_insert(dct: Field,
new_key: str,
new_value: Union[str, bool],
before_key: Optional[str] = None,
after_key: Optional[str] = None) -> None:
output = OrderedDict()
inserted: bool = False
for key, value in dct.items():
if not inserted and before_key is not None and key == before_key:
output[new_key] = new_value
inserted = True
output[key] = value
if not inserted and after_key is not None and key == after_key:
output[new_key] = new_value
inserted = True
if not inserted:
output[new_key] = new_value
dct.clear()
for key, value in output.items():
dct[key] = value
def run_update_project_status(filter_string: str) -> None:
"""Run the workflow to update project status for all filtered projects."""
logger.info("### Start update project status workflow ###")
active_projects = get_projects(status="active")
finished_projects = filter_projects_by_name_and_progress(
active_projects,
filter_string,
progress_threshold=100,
)
inactive_projects = get_projects(status="inactive")
# We sort projects by their attribute "projectNumber" to ensure that
# always the lowest one will be set to "status=active" next.
inactive_projects = OrderedDict(
sorted(inactive_projects.items(),
key=lambda x: int(x[1]["projectNumber"])))
new_active_projects = filter_projects_by_name_and_progress(
inactive_projects,
filter_string,
progress_threshold=0,
)[0:len(finished_projects)]
# Here we check that there is at least one inactive project
# which can be activated in the app.
# We do this to avoid that there is no project left in the app.
if len(new_active_projects) > 0:
for project_id in finished_projects:
project_name = active_projects[project_id]["name"]
set_status_in_firebase(project_id,
project_name,
new_status="finished")
for project_id in new_active_projects:
project_name = inactive_projects[project_id]["name"]
set_status_in_firebase(project_id,
project_name,
new_status="active")
logger.info("### Finished update project status workflow ###")
async def shop(self, ctx):
shop = self.load_shop()
embeds = []
shop = OrderedDict(
sorted(shop.items(), key=lambda x: getitem(x[1], "raw_price")))
chunks = divide_chunks(list(shop.keys()), 5)
i = 0
for chunk in list(chunks):
i += 1
embed = discord.Embed(color=discord.Color.teal())
embed.title = "The Waifu Shop"
for item in chunk:
v = (f" / `{shop[item]['raw_price']} erin`"
if shop[item]['price']['item'] != "erin" else "")
embed.add_field(
name=item,
value=
f"{shop[item]['name']} {shop[item]['emoji']} | Costs `{shop[item]['price']['quantity']} {shop[item]['price']['item']}`"
+ v,
inline=False,
)
embed.set_footer(text=f"Page {i}/{len(chunks)}",
icon_url=ctx.author.avatar_url)
embeds.append(embed)
paginator = DiscordUtils.Pagination.CustomEmbedPaginator(ctx)
paginator.add_reaction(
"\N{Black Left-Pointing Double Triangle with Vertical Bar}",
"first")
paginator.add_reaction("\N{Black Left-Pointing Double Triangle}",
"back")
paginator.add_reaction("\N{CROSS MARK}", "lock")
paginator.add_reaction("\N{Black Right-Pointing Double Triangle}",
"next")
paginator.add_reaction(
"\N{Black Right-Pointing Double Triangle with Vertical Bar}",
"last")
await paginator.run(embeds)
def main():
args = parser().parse_args()
results = OrderedDict()
max_results = 10
index = 0
home = Path(args.base) if args.base else Path().home()
ignore_case = False
print(args)
search = args.search.lower() if ignore_case else args.search
for path in home.glob('**/*'):
name = path.name.lower() if ignore_case else path.name
if path.is_dir() and name.endswith(search):
results[string.ascii_lowercase[index]] = path.as_posix()
index += 1
if len(results) >= max_results:
break
if len(results) < 1:
print(f'No such directory for <{args.search}>')
return -1
for index, elem in results.items():
print(index, '--', elem)
keys = list(results.keys())
wrong_key = True
while wrong_key:
choice = input(
f'Please enter a key between \'{keys[0]}\' and \'{keys[-1]}\'\n')
try:
print(f'Entering directory {results[choice]} ...')
wrong_key = False
except KeyError:
print(
f'Wrong key. Please chose acorrect one between {keys[0]} and {keys[-1]}'
)
wrong_key = True
def select_name_v3(voiceList: OrderedDict) -> int:
print(*(f'{x["vVoiceName"]}: {y}' for (x, y) in voiceList.items()),
sep='\n')
return int(input('?'))
class ContentCache(CacheABC):
# pylint: disable=too-many-instance-attributes
def __init__(self,
cache_folder: str,
temporary_dir: str,
max_cache_size_bytes: int = 1024 * 1024 * 1024,
max_workers: int = 10,
contents_load: bool = True,
contents_save_interval_secs: float = 5.0,
url_resolver: URLResolverABC = URLResolver()):
print(
f'ContentCache.__init__: cache_folder={cache_folder}, temporary_dir={temporary_dir}'
)
self.cache_folder: str = cache_folder
self.max_cache_size_bytes: int = max_cache_size_bytes
self.temporary_dir = temporary_dir
self.contents_save_interval_secs = contents_save_interval_secs
self.url_resolver = url_resolver
self._lock = threading.RLock()
self._executor = ThreadPoolExecutor(max_workers)
self._tasks: Dict[URL, Task] = {}
self._contents: OrderedDict[str, Content] = OrderedDict()
self._contents_size: int = 0
self._contents_save_timer = None
if contents_load:
self._load_contents()
def __del__(self):
self._save_contents()
self._executor.shutdown()
def _load_contents(self):
contents_json_path = os.path.join(self.cache_folder, 'contents.json')
if not os.path.exists(contents_json_path):
return
with self._lock:
with open(contents_json_path, 'r') as file:
content_json = json.load(file, object_pairs_hook=OrderedDict)
self._contents = OrderedDict({
key: Content(id=value['id'],
state=Content.State[value['state']],
filepath=os.path.join(self.cache_folder,
value['filepath']),
type=value['type'],
length=value['length'])
for key, value in content_json.items()
})
self._contents_size = sum(
[c.length for c in self._contents.values()])
print(
f'_load_contents: {len(self._contents)} from {contents_json_path}'
)
def _save_contents(self):
contents_json_path = os.path.join(self.cache_folder, 'contents.json')
with self._lock:
print(
f'_save_contents: {len(self._contents)} to {contents_json_path}'
)
content_json = {
key: {
'id':
value.id,
'state':
value.state.name,
'filepath':
os.path.basename(value.filepath) if value.filepath else '',
'type':
value.type,
'length':
value.length
}
for key, value in self._contents.items()
}
with open(contents_json_path, 'w') as file:
json.dump(content_json, file)
def _schedule_save_contents(self):
if self._contents_save_timer is not None:
self._contents_save_timer.cancel()
self._contents_save_timer = threading.Timer(
self.contents_save_interval_secs, self._save_contents)
self._contents_save_timer.start()
def _to_content_filepath(self, content_id: str) -> str:
return os.path.join(self.cache_folder, content_id)
def _fetch(self, task: Task):
# pylint: disable=too-many-statements
with self._lock:
if task.state is not Task.State.QUEUING:
raise ValueError(
f'task (={task.url}) is invalid state (={task.state})')
task.state = Task.State.RUNNING
content_id = task.content_id
content_filepath = self._to_content_filepath(content_id)
content = Content(content_id, Content.State.FETCHING)
try:
temp_fd, temp_path = tempfile.mkstemp(dir=self.temporary_dir)
with os.fdopen(temp_fd, 'bw') as temp_file:
response = self.url_resolver.resolve(task.url)
response.raise_for_status()
content_type = response.headers.get('Content-Type')
content_length_text = response.headers.get('Content-Length')
content_length = int(
content_length_text) if content_length_text else 0
fetch_size = 0
with self._lock:
task.content_length = content_length
task.fetched_size = fetch_size
for chunk in response.iter_content(chunk_size=65536):
fetch_size += len(chunk)
with self._lock:
task.fetched_size = fetch_size
if task.state is not Task.State.RUNNING:
raise InterruptedError(
f'task (={task.url}) fetch was interrupted')
temp_file.write(chunk)
os.rename(temp_path, content_filepath)
content_length = os.path.getsize(content_filepath)
with self._lock:
self._contents_size += content_length
task.state = Task.State.SUCCESS
content.state = Content.State.CACHED
content.filepath = content_filepath
content.length = content_length
content.type = content_type
except: # pylint: disable=bare-except
traceback.print_exc()
with self._lock:
content.state = Content.State.FAILED
if task.state is Task.State.RUNNING:
task.state = Task.State.FAILURE
else:
pass # keep state
if temp_path is not None:
os.remove(temp_path)
finally:
with self._lock:
self._contents[content_id] = content
del self._tasks[task.url]
self._invoke_callbacks(task)
self._schedule_save_contents()
return task
@staticmethod
def _invoke_callback(callback, content):
try:
callback(content)
except: # pylint: disable=bare-except
traceback.print_exc()
def _invoke_callbacks(self, task: Task):
with self._lock:
task_callbacks_copy = list(task.callbacks)
task.callbacks.clear()
content = self._contents[task.content_id]
for callback in task_callbacks_copy:
self._invoke_callback(callback, content)
return task
def cancel_fetch(self, url: URL):
with self._lock:
task = self.try_get_task(url)
if task is None:
return
if task.state != Task.State.RUNNING:
return
task.state = Task.State.CANCELED
def remove_content(self, url: URL) -> bool:
with self._lock:
content = self.try_get_content(url)
if content is None:
return False
del self._contents[content.id]
self._schedule_save_contents()
return True
def try_get_content(self, url: URL) -> Union[Content, None]:
content_id = Content.to_content_id(url)
with self._lock:
if content_id not in self._contents:
return None
content = self._contents[content_id]
# LRU implementation
self._contents.move_to_end(content_id)
excess_cache_size = max(
0, self._contents_size - self.max_cache_size_bytes)
if excess_cache_size > 0:
for content_id in self._contents.keys():
content = self._contents[content_id]
if content.length == 0:
continue
del self._contents[content_id]
self._contents_size -= content.length
if os.path.exists(content.file_path):
try:
os.remove(content.file_path)
except: # pylint: disable=bare-except
traceback.print_exc()
excess_cache_size -= content.length
if excess_cache_size <= 0:
break
self._schedule_save_contents()
return content
def try_get_task(self, url: URL) -> Union[Task, None]:
with self._lock:
return self._tasks[url] if url in self._tasks else None
def async_get_content(self, url: URL, callback: Callback) -> Future:
with self._lock:
content = self.try_get_content(url)
if content is not None:
if content.state in {
Content.State.CACHED, Content.State.FETCHING
}:
return self._executor.submit(self._invoke_callback,
callback, content)
self.remove_content(url)
elif url in self._tasks:
task = self._tasks[url]
if task.state in {Task.State.QUEUING, Task.State.RUNNING}:
task.callbacks.append(callback)
return task.future
task = Task(url, Task.State.QUEUING, [callback])
task.future = self._executor.submit(self._fetch, task)
self._tasks[url] = task
return task.future
def make_initial_simplex_table(simplex_table: OrderedDict, constraints: list,
f_o: dict, big_m, m1, urs):
"""
<summary>
<args>
simplex_table: type OrderedDict, se le ingresa vacío para utilizarlo para hacer la matriz de f_o y constraints.
constraints: type lista de diccionarios, trae la información de los constraints.
f_o: type dict, trae la información de la función objetivo.
</args>
Crea la simplex table inicial preparando para tener big m.
Retorna la tabla inicial preparada.
</summary>
"""
if (m1):
big_m = -big_m
# Agarrar las variables pertinentes.
variables = set()
for i in constraints:
for k in i.keys():
variables.add(k)
for k in f_o.keys():
variables.add(k)
f_o_var = set(variables) - set([x for x in f_o.keys()])
for i in f_o_var:
f_o.update({i: 0})
index = 0
for i in constraints:
constraints_i = variables - set(i.keys())
for j in constraints_i:
constraints[index].update({j: 0})
index += 1
temp = {}
temp.update({k: 0 for k, v in f_o.items()})
for i in constraints:
temp.update({k: 0 for k, v in i.items()})
del temp['symbol']
#
# print('\n\n\n\n\n\n\n')
# print(constraints)
# print(temp)
print('\n\n\n')
# Add the actual variables.
actual_vars = []
for k, v in temp.items():
if (is_actual_variable(k)):
actual_vars.append(k)
actual_vars.sort(key=lambda x: x[1:])
for i in actual_vars:
simplex_table.update({i: []})
# Add the slack/excess variables.
s_e_vars = []
for k, v in temp.items():
if (is_slack_excess_variable(k)):
s_e_vars.append(k)
s_e_vars.sort(key=lambda x: x[1:])
for i in s_e_vars:
simplex_table.update({i: []})
# Add the artifitial variables.
a_vars = []
for k, v in temp.items():
if (is_artifitial_variable(k)):
a_vars.append(k)
a_vars.sort(key=lambda x: x[1:])
for i in a_vars:
simplex_table.update({i: []})
# Add the z.
simplex_table.update({'z': []})
# Add the c.
simplex_table.update({'c': []})
# Copying the information in f_p and constraints to the simplex table:
for k, v in simplex_table.items():
simplex_table[k].append(f_o[k])
for k, v in simplex_table.items():
for i in range(len(constraints)):
simplex_table[k].append(constraints[i][k])
# {'X1': [-2, 0.5, 1, 1], 'X2': [-3, 0.25, 3, 1], 's1': [0, 1, 0, 0], 'e2': [0, 0, -1, 0], 'a2': [-10000, 0, 1, 0], 'a3': [-10000, 0, 0, 1], 'z': [1, 0, 0, 0], 'c': [0, 4, 20, 10], 'pivot': [0, 0, 0, 0], 'VB': [0, 0, 0, 0], 'index': [0, 1, 2, 3]}
# Transpose.
simplex_table_T = [{} for x in range(len(simplex_table['z']))]
for k, v in simplex_table.items():
for i in range(len(simplex_table_T)):
simplex_table_T[i].update({k: v[i]})
# Add URS vars.
urs_variables = list(zip(actual_vars, urs))
for i in range(len(simplex_table_T)):
for j in urs_variables:
if (j[1] == 1):
simplex_table_T[i].update({
(str(j[0]) + "''"):
-simplex_table_T[i][j[0]]
})
# for i in simplex_table_T: print(i)
# exit()
# new_row:
# print("simplex_table_T: ", simplex_table_T)
new_row_0 = []
for i in simplex_table_T:
contributes_to_new_row = False
for k, v in i.items():
if ((k[0] == 'a') and (v != 0)):
contributes_to_new_row = True
if contributes_to_new_row:
new_row_0.append(i.copy())
if (len(new_row_0) == 0):
new_row_0 = [simplex_table_T[0].copy()]
# print("new_row_0", new_row_0)
new_row = []
for i in new_row_0:
is_first_row = False
for k, v in i.items():
if ((k == 'z') and (v == 1)):
is_first_row = True
break
if is_first_row:
new_row.append(i)
else:
new_row.append(mult_row(i, big_m))
# Sum all the interesting rows.
# print(new_row)
new_row_0 = {k: 0 for k, v in new_row[0].items()}
for i in new_row:
for k, v in i.items():
new_row_0[k] += v
# Make artificial letters 0.
for k, v in new_row_0.items():
if (k[0] == 'a'):
new_row_0[k] = 0
# Making the new_row_0 the new zeroth row.
simplex_table_T[0] = new_row_0
# Adding pivot column and VB column. Adding an index.
index = 0
for i in range(len(simplex_table_T)):
simplex_table_T[i].update({'pivot': 0})
# simplex_table_T[i].update( {'VB':0} )
simplex_table_T[i].update({'index': index})
index += 1
# Setting up VB.
n = 1
index = 0
for i in simplex_table_T[1:]:
if (i.get(f"s{n}") != None):
simplex_table_T[index]['VB'] = simplex_table_T[index][
f"s{n}"] * simplex_table_T[index]['c']
elif (i.get(f"e{n}") != None):
simplex_table_T[index]['VB'] = simplex_table_T[index][
f"e{n}"] * simplex_table_T[index]['c']
index += 1
n += 1
return simplex_table_T
def _generate_daily_qst_reference(self) -> Optional[str]:
stuff = []
# 跑团日报
if self.daily_qst_thread_id is not None:
daily_qsts = self.db.get_responses_match(self.date,
self.daily_qst_thread_id,
r'^\[头条\]\s*?<br />\r?$')
if len(daily_qsts) > 0:
daily_qst = daily_qsts[-1]
line1 = daily_qst[1].splitlines()[0]
m = re.search(r'(day .+?)\s*?<br />', line1)
if m is None:
issue_text = ''
else:
issue = m.group(1)
if len(issue) > 10:
issue = issue[:10] + "…"
issue_text = f"〔{issue}〕"
stuff.append(
f"跑团日报{issue_text}：>>No.{daily_qst[0]} (位于原串第{(daily_qst[2]-1)//19+1}页)"
)
# 每日鸽报
if True:
# 由于有「11.5期」这样的实例，要考虑一天发多期的情况
daily_dovess = self.db.get_responses_match(
self.date, 36939614, r'^Daily Dove 每日鸽报.*?<br />\r?$')
daily_dove_dict = OrderedDict()
for daily_dove in daily_dovess:
lines = daily_dove[1].splitlines()
if len(lines) < 2:
continue
line2 = lines[1]
m = re.search(r'第(\S*?)期\s*?<br />', line2)
if m is None:
issue = None
else:
issue = m.group(1)
daily_dove_dict[issue] = daily_dove
for issue, daily_dove in daily_dove_dict.items():
if issue is None:
issue_text = ''
else:
issue_text = f"〔第{issue}期〕"
stuff.append(
f"每日鸽报{issue_text}：>>No.{daily_dove[0]} (位于原串第{(daily_dove[2]-1)//19+1}页)"
)
# 有趣团推荐报
if True:
third_newspapers = self.db.get_responses_match(
self.date, 37777146, r'^『.*?报.*?』.*?<br />\r?$')
third_newspaper_name = None
if len(third_newspapers) > 0:
# 假设每日最多一期
third_newspaper = third_newspapers[-1]
line1 = third_newspaper[1].splitlines()[0]
m = re.search(r'『(.*?报)(.*?)』：?(.*?)<br />', line1)
if m is not None: # always true
third_newspaper_name = m.group(1)
issue_text = ''
subhead = m.group(2)
if len(subhead) <= 5:
issue_text = subhead
else:
issue_text = subhead[:5] + "…"
issue_date = m.group(3)
if len(issue_date) > 0:
if issue_text != "":
issue_text += " "
if len(issue) <= 5:
issue_text += issue_date
else:
issue_text += issue_date[:5] + "…"
if issue_text != "":
issue_text = f"〔{issue_text}〕"
stuff.append(
f"{third_newspaper_name}{issue_text}：>>No.{third_newspaper[0]} (位于原串第{(third_newspaper[2]-1)//19+1}页)"
)
if len(stuff) == 0:
return None
return '\n'.join(["当日刊物："] + stuff) + '\n'
# Having built with `yarn build`, run `python3 src/tools/count-build.py extension/background.js`
import sys
from typing import OrderedDict
prefix =" !*** "
suffix = " ***!"
line_counts = OrderedDict()
active_file = None
active_line_count = None
def store_line_count():
if active_file and active_line_count:
line_counts[active_file] = active_line_count
with open(sys.argv[1], 'r') as f:
for line in f.readlines():
if line.startswith(prefix):
store_line_count()
active_file = line[len(prefix):-len(suffix)]
active_line_count = 0
elif active_file:
active_line_count += 1
store_line_count()
print("\n".join(( " " + str(line_count).ljust(10) + file_name) for file_name, line_count in line_counts.items()))
def main():
print("\n Paddlepaddle version: {}\n".format(paddle.__version__))
args = parse_args()
# 初始化并行环境
# dist.init_parallel_env()
# 加载数据集
train_dataset = paddle.vision.datasets.MNIST(mode='train',
transform=ToTensor())
val_dataset = paddle.vision.datasets.MNIST(mode='test',
transform=ToTensor())
train_loader = paddle.io.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True)
test_loader = paddle.io.DataLoader(val_dataset, batch_size=args.batch_size)
# 模型搭建
mnist = Mnist()
paddle.summary(net=mnist, input_size=(-1, 1, 28, 28))
# 增加paddle.DataParallel封装
# mnist = paddle.DataParallel(mnist)
optim = paddle.optimizer.Adam(parameters=mnist.parameters())
loss_fn = paddle.nn.CrossEntropyLoss()
start_epoch = 0
epochs = args.epochs
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
info = np.load('./weights/info.npy', allow_pickle=True).item()
start_epoch = info['epoch'] + 1
val_loss = info['loss']
val_acc = info['acc']
print('Epoch {}, validation loss is: {loss:.4f}, validation accuracy is {acc:.4f}\n'\
.format(start_epoch,loss=val_loss,acc=val_acc))
mnist_state_dict = paddle.load('./weights/mnist.pdparams')
mnist.set_state_dict(mnist_state_dict)
optim_state_dict = paddle.load('./weights/optim.pdopt')
optim.set_state_dict(optim_state_dict)
best_acc = 0.0
for epoch in range(start_epoch, epochs):
# 训练
mnist.train()
loader = tqdm.tqdm(train_loader)
for batch_id, (image, label) in enumerate(loader):
predicts = mnist(image)
loss = loss_fn(predicts, label)
acc = paddle.metric.accuracy(predicts, label)
loss.backward()
optim.step()
optim.clear_grad()
description = (
'Epoch {} (loss: {loss:.4f}, acc: {acc:.4f})'.format(
epoch, loss=loss.numpy().item(), acc=acc.numpy().item()))
loader.set_description(description)
# 测试
mnist.eval()
losses = 0.0
accuracy = 0.0
count = 0
for batch_id, (image, label) in enumerate(test_loader):
predicts = mnist(image)
loss = loss_fn(predicts, label)
acc = paddle.metric.accuracy(predicts, label)
count += 1
losses += loss.numpy().item()
accuracy += acc.numpy().item()
val_loss = losses / count
val_acc = accuracy / count
print("Testing: loss:{loss:.4f}, acc: {acc:.4f}".format(loss=val_loss,
acc=val_acc))
# 保存测试过程结果
result = OrderedDict()
result['timestamp'] = datetime.now()
result['epoch'] = epoch
result['loss'] = val_loss
result['accuracy'] = val_acc
result_dir = './result/'
if not os.path.exists(result_dir) and result_dir != '':
os.makedirs(result_dir)
result_file = os.path.join(result_dir, 'valid_results.csv')
write_heading = not os.path.exists(result_file)
with open(result_file, mode='a') as out:
if write_heading:
out.write(",".join([str(k) for k, v in result.items()]) + '\n')
out.write(",".join([str(v) for k, v in result.items()]) + '\n')
# 保存参数
print('Saving checkpoint..')
state = {'epoch': epoch, 'loss': val_loss, 'acc': val_acc}
# 目前仅支持存储 Layer 或者 Optimizer 的 state_dict 。
np.save('./weights/info.npy', state, allow_pickle=True) # 保存相关参数
paddle.save(mnist.state_dict(), './weights/mnist.pdparams')
paddle.save(optim.state_dict(), './weights/optim.pdopt')
# 保存用于部署的模型和参数
if val_acc > best_acc:
best_acc = val_acc
paddle.jit.save(
mnist,
'./deploy/mnist',
input_spec=[InputSpec(shape=[1, 1, 28, 28], dtype='float32')])
def count_matches(ed1, ed2):
m = 0
for e1 in ed1:
hasmatch = False
for e2 in ed2:
if (e1 == e2).all():
hasmatch = True
break
if hasmatch:
m += 1
return m
available_matches = np.zeros((len(tiles), len(tiles)))
i = 0
for id, im in tiles.items():
edges = get_edges(im)
j = 0
for id2, im2 in tiles.items():
if id != id2:
edges2 = get_edges(im2)
available_matches[i, j] = count_matches(edges, edges2)
j += 1
i += 1
indices_with_two_matches = np.where((available_matches > 0).sum(
axis=0) == 2)[0]
prod = 1
for index, id in enumerate(tiles.keys()):
if index in indices_with_two_matches:
prod *= id
def build_model(config: Mapping, cardinalities: Mapping[str,
int]) -> keras.Model:
"""Construct model specified in the configuration.
Also create optimizer and set the loss function.
Args:
config: Dictionary representing configuration file.
cardinalities: Cardinalities of categorical features (needed to
construct their embeddings).
Return:
Compiled model.
"""
model_config = config['model']
if isinstance(model_config, str):
model = keras.models.load_model(
model_config,
custom_objects={'loss_fn': _create_loss(config['loss'])})
return model
features = Features(config['features'])
inputs_all = []
# Constituents of different types
constituent_types = [
key for key in sorted(model_config.keys()) # Ensure order
if key not in {'head', 'load_weights'}
]
outputs_constituents = []
for constituent_type in constituent_types:
inputs_numerical = keras.Input(
shape=(None, len(features.numerical(constituent_type))),
ragged=True,
name=f'{constituent_type}_numerical')
inputs_categorical = OrderedDict()
for feature in features.categorical(constituent_type):
inputs_categorical[feature] = keras.Input(shape=(None, ),
ragged=True,
name=feature)
inputs_all.append(inputs_numerical)
inputs_all.extend(inputs_categorical.values())
outputs = _apply_deep_set(inputs_numerical, inputs_categorical,
model_config[constituent_type],
cardinalities, constituent_type)
outputs_constituents.append(outputs)
# Head
inputs_global_numerical = keras.Input(shape=(len(
features.numerical('global')), ),
name='global_numerical')
inputs_global_categorical = OrderedDict()
for feature in features.categorical('global'):
inputs_global_categorical[feature] = keras.Input(shape=(None, ),
name=feature)
embeddings_global = {
feature: Embedding(cardinalities[feature],
model_config['head']['embeddings'][feature],
name=feature + '_embeddings')(inputs)
for feature, inputs in inputs_global_categorical.items()
}
inputs_all.append(inputs_global_numerical)
inputs_all.extend(inputs_global_categorical.values())
inputs_head = Concatenate(
name='head_concatenate')([inputs_global_numerical] + [
embeddings_global[feature]
for feature in inputs_global_categorical.values()
] + outputs_constituents)
outputs = _apply_dense_from_config(inputs_head,
model_config['head'],
name_prefix='head_')
outputs = Dense(1, name='head_dense_output')(outputs) # Output unit
model = keras.Model(inputs=inputs_all, outputs=outputs, name='full')
model.compile(optimizer=_create_optimizer(config.get('optimizer', None)),
loss=_create_loss(config['loss']))
if 'load_weights' in model_config:
# Normally, a saved model should be loaded
# keras.models.load_model at the beginning of thsi function.
# However, this is currently not supported for models that use
# ragged tensors [1]. As a workaround, construct the model anew
# and then load saved weights. The path to weights would
# usually be "{model_directory}/variables/variables", with the
# ".index" file extension stripped off. This doesn't restore
# the state of the optimizer.
# [1] https://github.com/tensorflow/tensorflow/issues/41034
model.load_weights(model_config['load_weights'])
return model
def switch_keys_with_values(dictionary: OrderedDictType) -> OrderedDictType:
result = OrderedDict([(v, k) for k, v in dictionary.items()])
return result
class InventoryAllocator:
def __init__(self, inventory):
"""
function that initializes the warehouse.
params:
inventory: dictionary mapping warehouse to it's inventory
"""
self.warehouse_stock = OrderedDict()
self.set_warehouse(inventory)
def set_warehouse(self, inventory):
"""
function that sets and updates inventories .
params:
inventory : dict
dictionary containing information about single/multiple warehouses and their inventory
"""
for warehouse in inventory:
if warehouse["name"] not in self.warehouse_stock:
self.warehouse_stock[
warehouse["name"]] = warehouse["inventory"]
else:
for item, quantity in warehouse["inventory"].items():
if item not in self.warehouse_stock[warehouse["name"]]:
self.warehouse_stock[warehouse["name"]][
item] = warehouse["inventory"][item]
else:
self.warehouse_stock[warehouse["name"]][item] = (
quantity +
self.warehouse_stock[warehouse["name"]][item])
def update_stock(self, warehouse_name, item_name, item_stock):
"""
function that updates stock in a specific warehouse.
params:
warehouse_name : str
Name of warehouse to update.
item_name : str
Name of item to update.
item_stock : int
Amount of item to update.
"""
self.warehouse_stock[warehouse_name][item_name] = item_stock
def create_shipment_detail(self, order):
"""
function used to find cheapest way to fullfill order
params:
order: A dictionary that maps item needed to amount of item needed
"""
warehouse_item_distribution_amounts = defaultdict(lambda: {})
for item_name, amount_required in order.items():
item_distribution_amounts = {}
for warehouse_name, warehouse_inventory in self.warehouse_stock.items(
):
if item_name not in warehouse_inventory or warehouse_inventory[
item_name] <= 0:
continue
item_stock = warehouse_inventory[item_name]
if amount_required <= item_stock:
item_distribution_amounts[warehouse_name] = amount_required
item_stock -= amount_required
amount_required = 0
self.update_stock(warehouse_name, item_name, item_stock)
break
elif amount_required > item_stock:
item_distribution_amounts[warehouse_name] = item_stock
amount_required -= item_stock
item_stock = 0
self.update_stock(warehouse_name, item_name, item_stock)
if amount_required > 0:
return []
for warehouse_name, amount in item_distribution_amounts.items():
warehouse_item_distribution_amounts[warehouse_name][
item_name] = amount
final_allocation = []
for name, items in warehouse_item_distribution_amounts.items():
final_allocation.append({name: items})
return final_allocation
def make_initial_simplex_table(simplex_table: OrderedDict, constraints: list,
f_o: dict, big_m):
"""
<summary>
</summary>
"""
# Agarrar las variables pertinentes.
variables = set()
for i in constraints:
for k in i.keys():
variables.add(k)
for k in f_o.keys():
variables.add(k)
f_o_var = set(variables) - set([x for x in f_o.keys()])
for i in f_o_var:
f_o.update({i: 0})
index = 0
for i in constraints:
constraints_i = variables - set(i.keys())
for j in constraints_i:
constraints[index].update({j: 0})
index += 1
temp = {}
temp.update({k: 0 for k, v in f_o.items()})
for i in constraints:
temp.update({k: 0 for k, v in i.items()})
del temp['symbol']
# Add the actual variables.
actual_vars = []
for k, v in temp.items():
if (is_actual_variable(k)):
actual_vars.append(k)
actual_vars.sort(key=lambda x: x[1:])
for i in actual_vars:
simplex_table.update({i: []})
# Add the slack/excess variables.
s_e_vars = []
for k, v in temp.items():
if (is_slack_excess_variable(k)):
s_e_vars.append(k)
s_e_vars.sort(key=lambda x: x[1:])
for i in s_e_vars:
simplex_table.update({i: []})
# Add the artifitial variables.
a_vars = []
for k, v in temp.items():
if (is_artifitial_variable(k)):
a_vars.append(k)
a_vars.sort(key=lambda x: x[1:])
for i in a_vars:
simplex_table.update({i: []})
# Add the z.
simplex_table.update({'z': []})
# Add the c.
simplex_table.update({'c': []})
# Copying the information in f_p and constraints to the simplex table:
for k, v in simplex_table.items():
simplex_table[k].append(f_o[k])
for k, v in simplex_table.items():
for i in range(len(constraints)):
simplex_table[k].append(constraints[i][k])
# {'X1': [-2, 0.5, 1, 1], 'X2': [-3, 0.25, 3, 1], 's1': [0, 1, 0, 0], 'e2': [0, 0, -1, 0], 'a2': [-10000, 0, 1, 0], 'a3': [-10000, 0, 0, 1], 'z': [1, 0, 0, 0], 'c': [0, 4, 20, 10], 'pivot': [0, 0, 0, 0], 'VB': [0, 0, 0, 0], 'index': [0, 1, 2, 3]}
# Transpose.
simplex_table_T = [{} for x in range(len(simplex_table['z']))]
for k, v in simplex_table.items():
for i in range(len(simplex_table_T)):
simplex_table_T[i].update({k: v[i]})
# new_row:
new_row_0 = []
for i in simplex_table_T:
contributes_to_new_row = False
for k, v in i.items():
if ((k[0] == 'a') and (v != 0)):
contributes_to_new_row = True
if contributes_to_new_row:
new_row_0.append(i.copy())
new_row = []
for i in new_row_0:
is_first_row = False
for k, v in i.items():
if ((k == 'z') and (v == 1)):
is_first_row = True
break
if is_first_row:
new_row.append(i)
else:
new_row.append(mult_row(i, big_m))
# Sum all the interesting rows.
new_row_0 = {k: 0 for k, v in new_row[0].items()}
for i in new_row:
for k, v in i.items():
new_row_0[k] += v
# Make artificial letters 0.
for k, v in new_row_0.items():
if (k[0] == 'a'):
new_row_0[k] = 0
# Making the new_row_0 the new zeroth row.
simplex_table_T[0] = new_row_0
# Adding pivot column and VB column. Adding an index.
index = 0
for i in range(len(simplex_table_T)):
simplex_table_T[i].update({'pivot': 0})
simplex_table_T[i].update({'VB': 0})
simplex_table_T[i].update({'index': index})
index += 1
for i in simplex_table_T:
print(i)
return simplex_table_T
class LRUCacheStrategy(MemoryCacheStrategy[K, V]):
"""strategy which enforces a size limit with LRU"""
__slots__ = ("storage", "lock", "max_entries")
storage: OrderedDict[K, V]
lock: Lock # OrderedDict is not thread safe
max_entries: int
def __init__(self, max_entries: int) -> None:
self.storage = OrderedDict()
self.lock = Lock()
self.max_entries = max_entries
def __eq__(self, other: object) -> bool:
if isinstance(other, LRUCacheStrategy):
return self.storage == other.storage \
and self.max_entries == other.max_entries
return NotImplemented
def __getitem__(self, key: K) -> V:
"""get a value, setting it as the most recently used one"""
with self.lock:
self.storage.move_to_end(
key, last=False) # higher index = longer time since last use
return self.storage[key]
def __setitem__(self, key: K, value: V) -> None:
"""set a value, removing old ones if necessary"""
with self.lock:
if key not in self.storage and len(
self.storage) == self.max_entries:
self.storage.popitem(
) # make space for new entry by removing the last element
self.storage[key] = value
def __delitem__(self, key: K) -> None:
"""remove a value"""
with self.lock:
del self.storage[key]
def __iter__(self) -> Iterator[K]:
return iter(self.storage)
def __len__(self) -> int:
return len(self.storage)
def __contains__(self, key: object) -> bool:
return key in self.storage
def keys(self) -> KeysView[K]:
return self.storage.keys()
def values(self) -> ValuesView[V]:
return self.storage.values()
def items(self) -> ItemsView[K, V]:
return self.storage.items()
def peek(self, key: K) -> V:
"""get the value of key without triggering side effects like changing its priority"""
with self.lock:
return self.storage[key]
@overload
def pop(self, key: K) -> V:
...
@overload
def pop(self, key: K, default: Union[V, T] = ...) -> Union[V, T]:
...
def pop(self,
key: K,
default: Union[V,
T] = POP_SENTINEL) -> Union[V, T]: # type: ignore
"""remove a value and return it"""
with self.lock:
if default is POP_SENTINEL:
return self.storage.pop(key)
return self.storage.pop(key, default)
def popitem(self) -> Tuple[K, V]:
"""remove the least recently used key-value pair and return it"""
with self.lock:
return self.storage.popitem()
def clear(self) -> None:
"""remove all values"""
with self.lock:
self.storage.clear()
