def filter_empty_income(incomes): employer_incomes = incomes.get('employer_incomes') other_incomes = incomes.get('other_incomes') employer_incomes = _.filter(employer_incomes, lambda x: x.get('amount', None)) other_incomes = _.filter(other_incomes, lambda x: x.get('amount', None)) return employer_incomes + other_incomes
def get_dataset(overlap=5, window_size=10, time_steps=20, language=None, max_len=80): if language: dataset = _.filter(processed, lambda x: x['language'] == language) else: dataset = processed num_features = 7 # number of features on each time step # x,y,meta = [],[],[] genuine = forged = None if not max_len: pass else: for val in dataset: update = False g = val['values'][:20] f = val['values'][20:] g = np.array( _.map( _.filter(g, lambda i: len(i) >= max_len), lambda x: window( trim(x, max_len).reshape(-1), window_size * num_features, overlap * num_features, True))) f = np.array( _.map( _.filter(f, lambda i: len(i) >= max_len), lambda x: window( trim(x, max_len).reshape(-1), window_size * num_features, overlap * num_features, True))) if genuine is None and g.size > 0: genuine = g update = True if forged is None and f.size > 0: forged = f update = True continue if (g.size > 0 or f.size > 0) and not update: genuine = genuine if g.size == 0 else np.append( genuine, g, axis=0) forged = forged if f.size == 0 else np.append( forged, f, axis=0) print(genuine.shape ) # (num_samples, num_time_steps, window_size*num_features) print( f'Options\nWindow size --> {window_size}\nOverlap (in samples) --> {overlap}\nMax length {max_len}\n{"— "*20}' ) print( f'Number of time steps --> {genuine.shape[1]}\nNumber of features at each step --> {genuine.shape[2]}' ) print(f'Number of genuine samples --> {genuine.shape[0]}') print(f'Number of forged samples --> {forged.shape[0]}') x = {"genuine": genuine, 'forged': forged} return x
def _to_release_note_lines( self, nodes: typing.List[TitleNode], level: int, rls_note_objs: typing.List[ReleaseNote], ) -> typing.List[str]: lines = list() for node in nodes: filtered_rls_note_objects = _.filter( rls_note_objs, lambda rls_note_obj: get_or_call( rls_note_obj, node.matches_rls_note_field_path) in node. identifiers) if not filtered_rls_note_objects: continue if node.nodes: release_note_lines = self._to_release_note_lines( nodes=node.nodes, level=level + 1, rls_note_objs=filtered_rls_note_objects) lines.append(self._title(node, level)) lines.extend(release_note_lines) else: bullet_points = self._to_bullet_points( tag=node.title, rls_note_objs=filtered_rls_note_objects) # title is used as bullet point tag -> no need for additional title lines.extend(bullet_points) return lines
def __search_deep(self, signal: S): value = self.__search_current(signal) if value is not None: return value # gather all value from children if it cannot be found in current store values = _.filter( _.map(self.children, lambda store: store.__search_deep(signal)), lambda x: x != None) if values: return _.flatten(values)
def __match_list(self, template_list, data_list, path, current_store): # To handle xmltodict, if the template request a list, but the data isn't one, # make the data the only element in the list if not isinstance(data_list, self.list_like): data_list = [data_list] for idx, template in enumerate(template_list): for data in data_list: self.match(template, data, self.__next_path(path, str(idx)), self.__new_store(current_store)) # When no processing happen (e.g: static matching), the store will be empty # clean empty store created. current_store.children = _.filter(current_store.children, lambda x: bool(x.values))
def reachable_release_tags_from_commit( self, repo: git.Repo, commit: git.objects.Commit) -> typing.List[str]: '''Returns a list of release-tags whose tagged commits are ancestors of the given commit. The returned list is sorted in descending order, putting the greatest reachable tag first. ''' tags = self.release_tags() visited = set() queue = list() queue.append(commit) visited.add(commit.hexsha) reachable_tags = list() while queue: commit = queue.pop(0) if commit.hexsha in tags: reachable_tags.append(tags[commit.hexsha]) not_visited_parents = _.filter( commit.parents, lambda parent_commit: parent_commit.hexsha not in visited) if not_visited_parents: queue.extend(not_visited_parents) visited |= set( _.map(not_visited_parents, lambda commit: commit.hexsha)) reachable_tags.sort(key=lambda t: version.parse_to_semver(t), reverse=True) if not reachable_tags: logger.warning('no release tag found, falling back to root commit') root_commits = repo.iter_commits(rev=commit, max_parents=0) root_commit = next(root_commits, None) if not root_commit: fail( f'could not determine root commit from rev {commit.hexsha}' ) if next(root_commits, None): fail( 'cannot determine range for release notes. Repository has multiple root ' 'commits. Specify range via commit_range parameter.') reachable_tags.append(root_commit.hexsha) return reachable_tags
def reachable_release_tags_from_commit(github_helper: GitHubRepositoryHelper, repo: git.Repo, commit: git.objects.Commit) -> [str]: tags = release_tags(github_helper, repo) visited = set() queue = list() queue.append(commit) visited.add(commit.hexsha) reachable_tags = list() while queue: commit = queue.pop(0) if commit.hexsha in tags: reachable_tags.append(tags[commit.hexsha]) not_visited_parents = _.filter( commit.parents, lambda parent_commit: parent_commit.hexsha not in visited) if not_visited_parents: queue.extend(not_visited_parents) visited |= set( _.map(not_visited_parents, lambda commit: commit.hexsha)) reachable_tags.sort(key=lambda t: parse_version_info(t), reverse=True) if not reachable_tags: warning('no release tag found, falling back to root commit') root_commits = repo.iter_commits(rev=commit, max_parents=0) root_commit = next(root_commits, None) if not root_commit: fail('could not determine root commit from rev {rev}'.format( rev=commit.hexsha)) if next(root_commits, None): fail( 'cannot determine range for release notes. Repository has multiple root commits. ' 'Specify range via commit_range parameter.') reachable_tags.append(root_commit.hexsha) return reachable_tags
processed.append({ 'user': i, 'values': [ np.loadtxt(f'{DATASET_PATH}/U{i}S{x}.TXT', skiprows=1) for x in range(1, 41) ], # 'values': [np.loadtxt('{}/U{}S{}.TXT'.format(DATASET_PATH,i,x), skiprows=1) for x in range(1,41)], 'language': 'english' if i in ENGLISH else 'chinese' }) min_length = defaultdict(lambda: float('inf')) max_length = defaultdict(lambda: float('-inf')) for lang in langs: for val in _.filter(processed, lambda x: x['language'] == lang): curr_min = min(len(x) for x in val['values']) curr_max = max(len(x) for x in val['values']) if curr_min < min_length[lang]: min_length[lang] = curr_min if curr_max > min_length[lang]: max_length[lang] = curr_max print(f'{len(processed)} signatures loaded\n\n') print( f'English min length: {min_length["english"]}\nChinese min length: {min_length["chinese"]}\n\n' ) print( f'English max length: {max_length["english"]}\nChinese max length: {max_length["chinese"]}' )
def get_public_properties(cls): """Public properties are ones begining with 'auth_' and ending with '_id'""" return _.filter(set(dir(cls)), lambda x: x.startswith('auth_') and ( x.endswith('_id') or x.endswith('_icon')))
def get_public_properties(cls): """Public properties are ones begining with 'auth_' and ending with '_id'""" return _.filter( set(dir(cls)), lambda x: x.startswith('auth_') and (x.endswith('_id') or x.endswith('_icon')))
def build_keyring(self): c = self.app.conf log = self.app.log s = self.app.services spinner = self.app.spinner if path.isdir(c.paths.keyring): shutil.rmtree(c.paths.keyring) os.makedirs(c.paths.keyring) s.util.chown(c.paths.keyring) os.chdir(c.paths.keyring) try: command = 'apt-get source ubuntu-keyring' log.debug('command: ' + command) stdout = check_output(command, stderr=STDOUT, shell=True).decode('utf-8') log.debug(stdout) s.util.chown(c.paths.keyring) except CalledProcessError as err: matches = re.findall(r'Temporary\sfailure\sresolving', err.output.decode('utf-8')) if len(matches) > 0: spinner.fail('no internet connection') if c.debug: if err.output: log.error(err.output.decode('utf-8')) raise err exit(1) ubuntu_keyring_path = _.filter(os.listdir(c.paths.keyring), os.path.isdir) if len(ubuntu_keyring_path) > 0: ubuntu_keyring_path = path.join(c.paths.keyring, ubuntu_keyring_path[0]) else: raise Exception('failed to source ubuntu-keyring') keyrings_path = path.join(ubuntu_keyring_path, 'keyrings') os.chdir(keyrings_path) s.util.subproc('gpg --import < ' + path.join(keyrings_path, 'ubuntu-archive-keyring.gpg')) gpg_key = self.get_key() s.util.subproc('gpg --export ' + ' '.join(self.ubuntu_keys) + ' ' + gpg_key.pub.key.short + ' > ' + path.join(keyrings_path, 'ubuntu-archive-keyring.gpg')) os.chdir(ubuntu_keyring_path) stdout = s.util.subproc( 'cat ' + path.join(keyrings_path, 'ubuntu-archive-keyring.gpg') + ' | sha512sum') sha512sum = stdout.split(' ')[0] lines = [] with open(path.join(ubuntu_keyring_path, 'SHA512SUMS.txt.asc'), 'r') as f: for line in f.readlines(): line = re.sub( r'(\w+)(?=\s+keyrings/ubuntu-archive-keyring.gpg)', sha512sum, line) lines.append(line) with open(path.join(ubuntu_keyring_path, 'SHA512SUMS.txt.asc'), 'w') as f: f.writelines(lines) self.app.spinner.stop() s.util.subproc('dpkg-buildpackage -rfakeroot -m"' + gpg_key.name + ' <' + gpg_key.email + '>" -k' + gpg_key.pub.key.short) self.app.spinner.start() keyring_deb_path = _.filter(os.listdir(c.paths.keyring), lambda x: x[len(x) - 4:] == '.deb') if len(keyring_deb_path) > 0: keyring_deb_path = path.join(c.paths.keyring, keyring_deb_path[0]) else: raise Exception('failed to build ubuntu-keyring') for deb_path in glob.glob( path.join(c.paths.keyring, 'ubuntu-keyring*deb')): shutil.copy(deb_path, path.join(c.paths.mount, 'pool/main/u/ubuntu-keyring')) if not path.isdir(path.join(c.paths.filesystem, 'var/lib/apt/keyrings')): os.makedirs(path.join(c.paths.filesystem, 'var/lib/apt/keyrings')) if not path.isdir(path.join(c.paths.filesystem, 'etc/apt')): os.makedirs(path.join(c.paths.filesystem, 'etc/apt')) if not path.isdir(path.join(c.paths.filesystem, 'var/lib/apt/keyrings')): os.makedirs(path.join(c.paths.filesystem, 'var/lib/apt/keyrings')) shutil.copyfile( keyring_deb_path, path.join(c.paths.filesystem, 'usr/share/keyrings/ubuntu-archive-keyring.gpg')) shutil.copyfile(keyring_deb_path, path.join(c.paths.filesystem, 'etc/apt/trusted.gpg')) shutil.copyfile( keyring_deb_path, path.join(c.paths.filesystem, 'var/lib/apt/keyrings/ubuntu-archive-keyring.gpg')) os.chdir(c.paths.cwd)