def __iter__(self): for i in range(self._n_data): with open(join(self._path, '{}.json'.format(i))) as f: data = json.loads(f.read()) for s in concatv(data['article'], data['abstract']): if s != '<E>': yield ['<s>'] + s.lower().split() + [r'<\s>']
def collate_by_padding(batch: List[Tuple[Tensor, Tensor]], padding_value: int = 0): """Collates a batch of sequences. Args: batch (List[Tuple[LongTensor, LongTensor]]) padding_value (int) Returns: articles, articles_len prev_input, prev_inputs_len gold_summaries """ batch.sort(key=lambda x: len(x[0]), reverse=True) # sort by decreasing article # length articles, prev_inputs, gold_summaries = zip(*batch) n_agents = articles[0].shape[1] articles_len = torch.tensor(list( concatv(*(n_agents * [len(article)] for article in articles))), dtype=torch.long) articles = pad_sequence(articles, padding_value=padding_value) prev_inputs_len = torch.tensor( [len(prev_input) for prev_input in prev_inputs], dtype=torch.long) prev_inputs = pad_sequence(prev_inputs, padding_value=padding_value) gold_summaries = pad_sequence(gold_summaries, padding_value=padding_value) return (articles, articles_len), (prev_inputs, prev_inputs_len), gold_summaries
def _put_registers_last(x): # no need to do anything unless parameter (i.e. float) is found last if not isinstance(x[-1], float): return x # swap this last group of floats with the penultimate group of integers parts = tuple(cytoolz.partitionby(type, x)) return tuple(cytoolz.concatv(*parts[:-2], parts[-1], parts[-2]))
def __iter__(self): for i in range(self._n_data): if not os.path.isfile(os.path.join(self._path, '{}.json'.format(i))): continue with open(os.path.join(self._path, '{}.json'.format(i))) as f: data = json.loads(f.read()) for s in concatv(data['article'], data['abstract']): yield ['<s>'] + s.lower().split() + [r'<\s>']
def __iter__(self): for i in range(self._n_data): with open(join(self._path, '{}.json'.format(i))) as f: content = f.read() if content: data = json.loads(content) for s in concatv(data['article'], data['abstract']): yield ['<s>'] + s.lower().split() + [r'<\s>'] else: continue
def get_balance(self, address: str) -> List[BalanceItem]: native_balances = self.fcd.get_native_balances(address) staking_balances = self.fcd.get_staking_balances(address) cw20_balances = self.mantle.get_cw20_balances(address) return list(concatv(native_balances, staking_balances, cw20_balances))
def __iter__(self): for i in tqdm(concatv(self.text_char, self.summary_char)): yield i
def get_compiled_contracts(self, source_file_paths, import_remappings): self.logger.debug("Import remappings: %s", import_remappings) self.logger.debug("Compiler Settings PRE: %s", pprint.pformat(self.compiler_settings)) # DEBUG self.compiler_settings['output_values'] = [] self.logger.debug("Compiler Settings POST: %s", pprint.pformat(self.compiler_settings)) if 'remappings' in self.compiler_settings and import_remappings is not None: self.logger.warn( "Import remappings setting will by overridden by backend settings" ) sources = build_standard_input_sources(source_file_paths) std_input = { 'language': 'Solidity', 'sources': sources, 'settings': { 'remappings': import_remappings, 'outputSelection': { '*': { '*': REQUIRED_OUTPUT_SELECTION } } } } # solc command line options as passed to solc_wrapper() # https://github.com/ethereum/py-solc/blob/3a6de359dc31375df46418e6ffd7f45ab9567287/solc/wrapper.py#L20 command_line_options = self.compiler_settings.get( "command_line_options", {}) # Get Solidity Input Description settings section # http://solidity.readthedocs.io/en/develop/using-the-compiler.html#input-description std_input_settings = self.compiler_settings.get("stdin", {}) std_input['settings'].update(std_input_settings) # Make sure the output selection has all of the required output values. if has_nested_key(std_input, OUTPUT_SELECTION_KEY): current_selection = get_nested_key(std_input, OUTPUT_SELECTION_KEY) output_selection = list( set(concatv(current_selection, REQUIRED_OUTPUT_SELECTION))) else: output_selection = REQUIRED_OUTPUT_SELECTION set_nested_key(std_input, OUTPUT_SELECTION_KEY, output_selection) self.logger.debug("std_input sections: %s", std_input.keys()) self.logger.debug("Input Description JSON settings are: %s", std_input["settings"]) self.logger.debug("Command line options are: %s", command_line_options) try: compilation_result = compile_standard(std_input, **command_line_options) except ContractsNotFound: return {} compiled_contracts = normalize_compilation_result(compilation_result) return compiled_contracts
def __iter__(self): for i in range(self._n_data): with open(join(self._path, '{}.json'.format(i))) as f: data = json.loads(f.read()) for s in concatv(data['article'], data['abstract']): yield ['<s>'] + s.lower().split() + [r'<\s>']
def __iter__(self): for i in range(self._n_data): with open(join(self._path, '{}.json'.format(i))) as f: data = json.loads(f.read()) for s in concatv(data['reviewText'], data['summary']): yield ['<s>'] + s.lower().split() + [r'<\s>']