def __iter__(self):
for i in range(self._n_data):
with open(join(self._path, '{}.json'.format(i))) as f:
data = json.loads(f.read())
for s in concatv(data['article'], data['abstract']):
if s != '<E>':
yield ['<s>'] + s.lower().split() + [r'<\s>']
def collate_by_padding(batch: List[Tuple[Tensor, Tensor]],
padding_value: int = 0):
"""Collates a batch of sequences.
Args:
batch (List[Tuple[LongTensor, LongTensor]])
padding_value (int)
Returns:
articles, articles_len
prev_input, prev_inputs_len
gold_summaries
"""
batch.sort(key=lambda x: len(x[0]),
reverse=True) # sort by decreasing article
# length
articles, prev_inputs, gold_summaries = zip(*batch)
n_agents = articles[0].shape[1]
articles_len = torch.tensor(list(
concatv(*(n_agents * [len(article)] for article in articles))),
dtype=torch.long)
articles = pad_sequence(articles, padding_value=padding_value)
prev_inputs_len = torch.tensor(
[len(prev_input) for prev_input in prev_inputs], dtype=torch.long)
prev_inputs = pad_sequence(prev_inputs, padding_value=padding_value)
gold_summaries = pad_sequence(gold_summaries, padding_value=padding_value)
return (articles, articles_len), (prev_inputs,
prev_inputs_len), gold_summaries
def _put_registers_last(x):
# no need to do anything unless parameter (i.e. float) is found last
if not isinstance(x[-1], float):
return x
# swap this last group of floats with the penultimate group of integers
parts = tuple(cytoolz.partitionby(type, x))
return tuple(cytoolz.concatv(*parts[:-2], parts[-1], parts[-2]))
def __iter__(self):
for i in range(self._n_data):
if not os.path.isfile(os.path.join(self._path,
'{}.json'.format(i))):
continue
with open(os.path.join(self._path, '{}.json'.format(i))) as f:
data = json.loads(f.read())
for s in concatv(data['article'], data['abstract']):
yield ['<s>'] + s.lower().split() + [r'<\s>']
def __iter__(self):
for i in range(self._n_data):
with open(join(self._path, '{}.json'.format(i))) as f:
content = f.read()
if content:
data = json.loads(content)
for s in concatv(data['article'], data['abstract']):
yield ['<s>'] + s.lower().split() + [r'<\s>']
else:
continue
def get_balance(self, address: str) -> List[BalanceItem]:
native_balances = self.fcd.get_native_balances(address)
staking_balances = self.fcd.get_staking_balances(address)
cw20_balances = self.mantle.get_cw20_balances(address)
return list(concatv(native_balances, staking_balances, cw20_balances))
def __iter__(self):
for i in tqdm(concatv(self.text_char, self.summary_char)):
yield i
def get_compiled_contracts(self, source_file_paths, import_remappings):
self.logger.debug("Import remappings: %s", import_remappings)
self.logger.debug("Compiler Settings PRE: %s",
pprint.pformat(self.compiler_settings))
# DEBUG
self.compiler_settings['output_values'] = []
self.logger.debug("Compiler Settings POST: %s",
pprint.pformat(self.compiler_settings))
if 'remappings' in self.compiler_settings and import_remappings is not None:
self.logger.warn(
"Import remappings setting will by overridden by backend settings"
)
sources = build_standard_input_sources(source_file_paths)
std_input = {
'language': 'Solidity',
'sources': sources,
'settings': {
'remappings': import_remappings,
'outputSelection': {
'*': {
'*': REQUIRED_OUTPUT_SELECTION
}
}
}
}
# solc command line options as passed to solc_wrapper()
# https://github.com/ethereum/py-solc/blob/3a6de359dc31375df46418e6ffd7f45ab9567287/solc/wrapper.py#L20
command_line_options = self.compiler_settings.get(
"command_line_options", {})
# Get Solidity Input Description settings section
# http://solidity.readthedocs.io/en/develop/using-the-compiler.html#input-description
std_input_settings = self.compiler_settings.get("stdin", {})
std_input['settings'].update(std_input_settings)
# Make sure the output selection has all of the required output values.
if has_nested_key(std_input, OUTPUT_SELECTION_KEY):
current_selection = get_nested_key(std_input, OUTPUT_SELECTION_KEY)
output_selection = list(
set(concatv(current_selection, REQUIRED_OUTPUT_SELECTION)))
else:
output_selection = REQUIRED_OUTPUT_SELECTION
set_nested_key(std_input, OUTPUT_SELECTION_KEY, output_selection)
self.logger.debug("std_input sections: %s", std_input.keys())
self.logger.debug("Input Description JSON settings are: %s",
std_input["settings"])
self.logger.debug("Command line options are: %s", command_line_options)
try:
compilation_result = compile_standard(std_input,
**command_line_options)
except ContractsNotFound:
return {}
compiled_contracts = normalize_compilation_result(compilation_result)
return compiled_contracts
def __iter__(self):
for i in range(self._n_data):
with open(join(self._path, '{}.json'.format(i))) as f:
data = json.loads(f.read())
for s in concatv(data['article'], data['abstract']):
yield ['<s>'] + s.lower().split() + [r'<\s>']
def __iter__(self):
for i in range(self._n_data):
with open(join(self._path, '{}.json'.format(i))) as f:
data = json.loads(f.read())
for s in concatv(data['reviewText'], data['summary']):
yield ['<s>'] + s.lower().split() + [r'<\s>']