def only_changed( safelist: List[str], flags: int = glob.BRACE | glob.DOTGLOB | glob.GLOBSTAR | glob.NEGATE ) -> bool: """ Return True if we're in a Pull Request and the only files changed by this PR are in the `safelist`. This function can be used to skip tests if only files named in the `safelist` have been modified. """ github_type, files_changed = _change_detector.get_changed_files() if github_type != CI_PULL_REQUEST: # return False for all other types, including trains, to force all tests. return False assert files_changed is not None matches = glob.globfilter(files_changed, safelist, flags=flags) not_safe = set(files_changed) - set(matches) # When used in a `skipif`, this function is called during collection, so is # logged away from the test. Add the file and lineno to identify the test. caller = inspect.stack()[1] caller_id = '{}:{}'.format(trailing_path(caller.filename, 2), caller.lineno) if not_safe: logging.info('%s: Changed files not in safelist: %s', caller_id, tuple(not_safe)) return False else: logging.info('%s: All changed files in safelist', caller_id) return True
def run(self): """Run command.""" self.syntaxes = [ os.path.splitext(x)[0].replace('Packages/', '', 1) for x in glob.globfilter(sublime.find_resources('*'), '**/*.@(sublime-syntax|tmLanguage)', flags=GLOB_FLAGS | glob.I) ] if self.syntaxes: self.window.show_quick_panel(self.syntaxes, self.check_selection)
def filter_excludes(arr: FrozenSet[Path], excludes: List[str]) -> FrozenSet[Path]: """ Returns all elements in arr that do not match any excludes pattern If excludes is empty, returns arr unchanged """ if not excludes: return arr excludes = TargetManager.preprocess_path_patterns(excludes) return arr - frozenset( wcglob.globfilter( arr, excludes, flags=wcglob.GLOBSTAR | wcglob.DOTGLOB))
def filter_includes(arr: FrozenSet[Path], includes: Sequence[str]) -> FrozenSet[Path]: """ Returns all elements in arr that match any includes pattern If includes is empty, returns arr unchanged """ if not includes: return arr includes = TargetManager.preprocess_path_patterns(includes) return frozenset( wcglob.globfilter(arr, includes, flags=wcglob.GLOBSTAR | wcglob.DOTGLOB))
def glob(self, pattern: str) -> List["DPPath"]: """Search path using the glob pattern. Parameters ---------- pattern : str glob pattern Returns ------- List[DPPath] list of paths """ # got paths starts with current path first, which is faster subpaths = [ii for ii in self._keys if ii.startswith(self.name)] return list([ type(self)("%s#%s" % (self.root_path, pp)) for pp in globfilter(subpaths, self._connect_path(pattern)) ])
def _list_agave(uri, globstr, agave): """ List contents of agave URI. Args: uri: parsed URI to list. agave: dict that contains: agave_wrapper: Agave wrapper object. Returns: On success: a list of filenames (basenames only). On failure: False. """ # recurse to depth based on glob depth = -1 if '**' in globstr else 1 if depth == 1: depth = globstr.count('/')+1 file_list = agave['agave_wrapper'].files_list( uri['authority'], uri['chopped_path'], depth=depth ) if file_list is False: Log.an().error( 'cannot get file list for uri: %s', uri['chopped_uri'] ) return False # apply glob pattern to filter file list path_len = len(uri['chopped_path'])+1 globbed_file_list = glob.globfilter( [str(f['path']+'/'+f['name'])[path_len:] for f in file_list], globstr, flags=glob.EXTGLOB|glob.GLOBSTAR ) return globbed_file_list
def only_changed( safelist: List[str], flags: int = glob.BRACE | glob.DOTGLOB | glob.GLOBSTAR | glob.NEGATE ) -> bool: """ Return True if we're in a Pull Request and the only files changed by this PR are in the `safelist`. This function can be used to skip tests if only files named in the `safelist` have been modified. """ # Ensure flags enable globstar required for default safelist flags |= glob.GLOBSTAR github_type, files_changed = _change_detector.get_changed_files() if github_type == CI_PULL_REQUEST: # use supplied safelist pass elif github_type == CI_TRAIN: # For trains we want to run if the build has been affected in any way, # but can skip for changes that do not affect the build or installation # of DC/OS. To do this, skip e2e tests only when files in the default # safelist are changed. safelist = E2E_SAFE_DEFAULT else: # return False for all other types to force all tests. return False assert files_changed is not None matches = glob.globfilter(files_changed, safelist, flags=flags) not_safe = set(files_changed) - set(matches) # When used in a `skipif`, this function is called during collection, so is # logged away from the test. Add the file and lineno to identify the test. caller = inspect.stack()[1] caller_id = '{}:{}'.format(trailing_path(caller.filename, 2), caller.lineno) if not_safe: logging.info('%s: Changed files not in safelist: %s', caller_id, tuple(not_safe)) return False else: logging.info('%s: All changed files in safelist', caller_id) return True
def _get_map_uri_list(self): """ Get the contents of the map URI (agave URI). Args: self: class instance. Returns: Array of base file names in the map URI. Returns False on exception. """ combined_file_list = [] for uri in self._parsed_map_uris: # make sure map URI is compatible scheme (agave) if uri['scheme'] != 'agave': msg = 'invalid map uri scheme for this step: {}'.format( uri['scheme'] ) Log.an().error(msg) return self._fatal(msg) # get file list from URI file_list = DataManager.list( parsed_uri=uri, globstr=self._step['map']['glob'], agave=self._agave ) if file_list is False: msg = 'cannot get contents of map uri: {}'\ .format(uri['chopped_uri']) Log.an().error(msg) return self._fatal(msg) if self._step['map']['inclusive']: # filter with glob if glob.globfilter( [uri['name']], self._step['map']['glob'], flags=glob.EXTGLOB|glob.GLOBSTAR ): combined_file_list.append({ 'chopped_uri': '{}://{}{}'.format( uri['scheme'], uri['authority'], uri['folder'] ), 'filename': uri['name'] }) for f in file_list: if '/' in f: # reparse uri to correctly represent recursive elements new_uri = URIParser.parse('{}/{}'.format(uri['chopped_uri'], f)) combined_file_list.append({ 'chopped_uri': '{}://{}{}'.format( new_uri['scheme'], new_uri['authority'], new_uri['folder'] ), 'filename': new_uri['name'] }) else: combined_file_list.append({ 'chopped_uri': uri['chopped_uri'], 'filename': f }) return combined_file_list
print('Flags?') print(case) print(flags, cls.flags) flags = cls.flags ^ flags pat = case[0] if isinstance(case[0], list) else [case[0]] if split and cls.skip_split: return if split: new_pat = [] for x in pat: new_pat.extend(list(glob.globsplit(x, flags=flags))) pat = new_pat print("PATTERN: ", case[0]) print("FILES: ", files) print("FLAGS: ", bin(flags)) result = sorted(glob.globfilter(files, pat, flags=flags)) source = sorted(cls.norm_files(case[1], flags)) print("TEST: ", result, '<==>', source, '\n') cls.assert_equal(result, source) @pytest.mark.parametrize("case", cases) def test_glob_filter(self, case): """Test wildcard parsing.""" _wcparse._compile.cache_clear() self._filter(case) @pytest.mark.parametrize("case", cases) def test_glob_split_filter(self, case): """Test wildcard parsing by first splitting on `|`."""
def to_system_data(f: h5py.File, folder: str, type_map: list = None, labels: bool = True): """Load a HDF5 file. Parameters ---------- f : h5py.File HDF5 file object folder : str path in the HDF5 file type_map : list type map labels : bool labels """ g = f[folder] if folder else f data = {} data['atom_types'] = g['type.raw'][:] ntypes = np.max(data['atom_types']) + 1 natoms = data['atom_types'].size data['atom_numbs'] = [] for ii in range(ntypes): data['atom_numbs'].append(np.count_nonzero(data['atom_types'] == ii)) data['atom_names'] = [] # if find type_map.raw, use it if 'type_map.raw' in g.keys(): my_type_map = list(np.char.decode(g['type_map.raw'][:])) # else try to use arg type_map elif type_map is not None: my_type_map = type_map # in the last case, make artificial atom names else: my_type_map = [] for ii in range(ntypes): my_type_map.append('Type_%d' % ii) assert (len(my_type_map) >= len(data['atom_numbs'])) for ii in range(len(data['atom_numbs'])): data['atom_names'].append(my_type_map[ii]) data['orig'] = np.zeros([3]) if 'nopbc' in g.keys(): data['nopbc'] = True sets = globfilter(g.keys(), 'set.*') data_types = { 'cells': { 'fn': 'box', 'labeled': False, 'shape': (3, 3), 'required': 'nopbc' not in data }, 'coords': { 'fn': 'coord', 'labeled': False, 'shape': (natoms, 3), 'required': True }, 'energies': { 'fn': 'energy', 'labeled': True, 'shape': tuple(), 'required': False }, 'forces': { 'fn': 'force', 'labeled': True, 'shape': (natoms, 3), 'required': False }, 'virials': { 'fn': 'virial', 'labeled': True, 'shape': (3, 3), 'required': False }, } for dt, prop in data_types.items(): all_data = [] for ii in sets: set = g[ii] fn = '%s.npy' % prop['fn'] if fn in set.keys(): dd = set[fn][:] nframes = dd.shape[0] all_data.append(np.reshape(dd, (nframes, *prop['shape']))) elif prop['required']: raise RuntimeError("%s/%s/%s not found" % (folder, ii, fn)) if len(all_data) > 0: data[dt] = np.concatenate(all_data, axis=0) return data
flags = cls.flags ^ flags pat = case[0] if isinstance(case[0], list) else [case[0]] if split and cls.skip_split: return if split: new_pat = [] for x in pat: new_pat.extend(list(glob.globsplit(x, flags=flags))) pat = new_pat print("PATTERN: ", case[0]) print("FILES: ", files) print("FLAGS: ", bin(flags)) result = sorted( glob.globfilter( files, pat, flags=flags ) ) source = sorted(cls.norm_files(case[1], flags)) print("TEST: ", result, '<==>', source, '\n') cls.assert_equal(result, source) @pytest.mark.parametrize("case", cases) def test_glob_filter(self, case): """Test wildcard parsing.""" _wcparse._compile.cache_clear() self._filter(case)
result = [] joinResult = [] for p1 in pairs1: for p2 in pairs2: if p1[0]['id'] == p2[2]['id'] and p1[1]['id'] == p2[0]['id'] and p1[2][ 'id'] == p2[1]['id']: # agent compare result = [ p1[0]['properties']['name'], p1[0]['properties']['affiliation'], #Agent p1[2]['properties']['name'], p1[2]['properties']['date'], p1[1]['properties']['name'], #Entity p1[1]['properties']['type'], p1[1]['properties']['price'], p1[1]['properties']['device'] ] #Activity delimiter = '-' joinResult.append(delimiter.join(result)) joinResult = list(set(joinResult)) #리스트 중복제거 searchTerm = ['*Emily*'] re = glob.globfilter(joinResult, searchTerm) print(re) #search term '*Emily*2018' 이런 식으로 넣으면 검색 가능 #js에서 검색 시 * 붙일 수 있도록 구현 #python에서 함수 호출 #json으로 export하는 cypher : CALL apoc.export.json.query("query 적기","filename.json",{})
raise SystemExit( "This file is intended to be executed as an executable program. You cannot use it as a module." f"To run this script, run the ./{__file__} command" ) CONFIG_KEY = 'labelPRBasedOnFilePath' current_files = subprocess.check_output(["git", "ls-files"]).decode().splitlines() git_root = Path(subprocess.check_output(['git', 'rev-parse', '--show-toplevel']).decode().strip()) cyborg_config_path = git_root / ".github" / "boring-cyborg.yml" cyborg_config = yaml.safe_load(cyborg_config_path.read_text()) if CONFIG_KEY not in cyborg_config: raise SystemExit(f"Missing section {CONFIG_KEY}") errors = [] for label, patterns in cyborg_config[CONFIG_KEY].items(): for pattern in patterns: if glob.globfilter(current_files, pattern, flags=glob.G | glob.E): continue yaml_path = f'{CONFIG_KEY}.{label}' errors.append( f"Unused pattern [{colored(pattern, 'cyan')}] in [{colored(yaml_path, 'cyan')}] section." ) if errors: print(f"Found {colored(str(len(errors)), 'red')} problems:") print("\n".join(errors)) sys.exit(1) else: print("No found problems. Have a good day!")