def fnmatch(filename, pattern):
"""fnmatch replacement for accept pattern list"""
if isinstance(pattern,list):
for p in pattern:
if _fnmatch(filename,p):
return True
return False
else:
return _fnmatch(filename, pattern)
def fnmatch(path, pattern):
name = split(path)[1]
if isinstance(pattern, basestring):
if _fnmatch(name, pattern) or\
_fnmatch(path, pattern):
return True
elif hasattr(pattern, 'match'): # A regex-like object
if pattern.match(name):
return True
elif isinstance(pattern, tuple) or isinstance(pattern, list):
for subpattern in strlist(pattern):
if fnmatch(path, subpattern):
return True
return False
def _ls_AA_in_df(AA_patt, df):
r""" Same as find_AA but using dataframe syntax...between 10 and 100 times slower (200mus to 20ms)"""
from fnmatch import fnmatch as _fnmatch
_AA = str(AA_patt)
match = lambda val: _fnmatch(val, _AA)
idxs = _np.flatnonzero(
df.applymap(lambda val: str(val)).applymap(match).values.any(
1)).tolist()
return idxs
def fnmatch(path, pattern):
"""helper function to test multiple patterns
"""
patterns = pattern
if isinstance(patterns, basestring):
patterns = [patterns]
for pattern in patterns:
if _fnmatch(path, pattern):
return True
return False
def get_debian_package(self, node_module):
result = {}
result['info'] = None
result['name'] = None
result['version'] = None
result['suite'] = None
result['repr'] = None
db_package = None
if node_module in self.json:
db_package = self.json[node_module]
else:
for pattern in self.json.keys():
if _fnmatch(node_module, pattern):
db_package = self.json[pattern]
break
if db_package:
if 'replace' in db_package:
result['name'] = db_package['replace']
if 'info' in db_package:
result['info'] = ('info', db_package['info'])
self.append_warning('info', node_module, db_package['info'])
elif 'warning' in db_package:
result['info'] = ('warning', db_package['warning'])
self.append_warning('warning', node_module,
db_package['warning'])
elif 'error' in db_package:
result['info'] = ('error', db_package['error'])
self.append_warning('error', node_module, db_package['error'])
else:
result['name'] = 'node-%s' % _debianize_name(node_module)
if not result['name']:
return result
data = _urlopen(MADISON_URL % result['name']).read().decode('utf-8')
packages = _parseJSON(data)
if len(packages) < 1:
result['name'] = None
return result
result['version'] = '0:0'
for suite, versions in packages[0][result['name']].items():
for version, source in versions.items():
if apt_pkg.version_compare(version, result['version']) > 0:
result['version'] = version
result['suite'] = suite
result['name'] = source['source']
result['repr'] = '%s (%s)' % (result['name'],
result['version'])
result['version'] = result['version']
if result['version'] == '0:0':
result['version'] = None
return result
def fnmatch_ex(patterns_as_csv, list_of_keys):
r"""
Match the keys in :obj:`list_of_keys` against some naming patterns
using Unix filename pattern matching
TODO include link: https://docs.python.org/3/library/fnmatch.html
This method also allows for exclusions (grep -e)
TODO: find out if regular expression re.findall() is better
Uses fnmatch under the hood
Parameters
----------
patterns_as_csv : str
Patterns to include or exclude, separated by commas, e.g.
* "H*,-H8" will include all TMs but not H8
* "G.S*" will include all beta-sheets
list_of_keys : list
Keys against which to match the patterns, e.g.
* ["H1","ICL1", "H2"..."ICL3","H6", "H7", "H8"]
Returns
-------
matching_keys : list
"""
include_patterns = [pattern for pattern in patterns_as_csv.split(",") if not pattern.startswith("-")]
exclude_patterns = [pattern[1:] for pattern in patterns_as_csv.split(",") if pattern.startswith("-")]
#print(include_patterns)
#print(exclude_patterns)
# Define the match using a lambda
matches_include = lambda key : any([_fnmatch(str(key), patt) for patt in include_patterns])
matches_exclude = lambda key : any([_fnmatch(str(key), patt) for patt in exclude_patterns])
passes_filter = lambda key : matches_include(key) and not matches_exclude(key)
outgroup = []
for key in list_of_keys:
#print(key, matches_include(key),matches_exclude(key),include_patterns, exclude_patterns)
if passes_filter(key):
outgroup.append(key)
return outgroup
def get_debian_package(self, node_module):
result = {}
result['info'] = None
result['name'] = None
result['version'] = None
result['suite'] = None
result['repr'] = None
db_package = None
if node_module in self.json:
db_package = self.json[node_module]
else:
for pattern in self.json.keys():
if _fnmatch(node_module, pattern):
db_package = self.json[pattern]
break
if db_package:
if 'replace' in db_package:
result['name'] = db_package['replace']
if 'info' in db_package:
result['info'] = ('info', db_package['info'])
self.append_warning('info', node_module, db_package['info'])
elif 'warning' in db_package:
result['info'] = ('warning', db_package['warning'])
self.append_warning('warning', node_module,
db_package['warning'])
elif 'error' in db_package:
result['info'] = ('error', db_package['error'])
self.append_warning('error', node_module, db_package['error'])
else:
result['name'] = 'node-%s' % _debianize_name(node_module)
if not result['name']:
return result
madison = _getstatusoutput('rmadison -u debian "%s" | grep source' %
result['name'])
if madison[0] != 0:
result['name'] = None
return result
tmp = madison[1].split('|')
if len(tmp) >= 2:
result['name'] = tmp[0].strip()
result['version'] = tmp[1].strip()
result['suite'] = tmp[2].strip()
result['repr'] = '%s (%s)' % (result['name'], result['version'])
return result
def search(filename_pattern,
repository='http://ftp.imp.fu-berlin.de/pub/cmb-data/'):
r""" Returns a list of available files matching the :obj:filename_pattern
:param filname_pattern: String with the filename pattern, allows for Unix shell-style wildcards, eg. di-Ala*xtc
:param repository: ftp repository where to look for files
:return: list with matching files in the :obj:repository
"""
avail_files = _avail_files_dict(repository.lstrip('http://').split('/'))
return [
key for key in sorted(avail_files.keys())
if _fnmatch(key, filename_pattern)
]
def find(pattern, path):
"""
Will search for files we some patter in certain path. Use:
find('*.txt', '/path/to/dir')
:param pattern:String with the pattern to search for
:param path:Where to search for (absolute or relative path)
:return:Results of the search
"""
result = []
for root, dirs, files in os.walk(path):
for name in files:
if _fnmatch(name, pattern):
result.append(os.path.join(root, name))
return result
def __on_received_search(self, message):
pattern = message.pattern
if pattern == '':
pattern = '*'
self.__log_event(_tr('Searching for \'{pattern}\'').format(pattern=pattern))
if self.__username is None:
self.send(_messages.Error(_tr('Search failed: {reason}').format(reason=_tr('Not logged in'))))
self.__log_status(_tr('Error: {reason}').format(reason=_tr('Not logged in')))
return
files = []
for parent, _, directory_files in _os.walk(_config['root-directory']):
for file in directory_files:
file = _os.path.relpath(_os.path.join(parent, file), _config['root-directory'])
if _fnmatch(file, pattern):
files.append(file)
self.send(_messages.SetAll(files))
self.__log_status(_tr('Success'))
def load_audio(path,
with_path=True,
recursive=True,
ignore_failure=True,
random_order=False):
"""
Loads WAV file(s) from a path.
Parameters
----------
path : str
Path to WAV files to be loaded.
with_path : bool, optional
Indicates whether a path column is added to the returned SFrame.
recursive : bool, optional
Indicates whether ``load_audio`` should do a recursive directory traversal,
or only load audio files directly under ``path``.
ignore_failure : bool, optional
If True, only print warnings for failed files and keep loading the remaining
audio files.
random_order : bool, optional
Load audio files in random order.
Returns
-------
out : SFrame
Returns an SFrame with either an 'audio' column or both an 'audio' and
a 'path' column. The 'audio' column is a column of dictionaries.
Each dictionary contains two items. One item is the sample rate, in
samples per second (int type). The other item will be the data in a numpy
array. If the wav file has a single channel, the array will have a single
dimension. If there are multiple channels, the array will have shape
(L,C) where L is the number of samples and C is the number of channels.
Examples
--------
>>> audio_path = "~/Documents/myAudioFiles/"
>>> audio_sframe = tc.audio_analysis.load_audio(audio_path, recursive=True)
"""
from scipy.io import wavfile as _wavfile
all_wav_files = []
if _fnmatch(path, '*.wav'): # single file
all_wav_files.append(path)
elif recursive:
for (dir_path, _, file_names) in _os.walk(path):
for cur_file in file_names:
if _fnmatch(cur_file, '*.wav'):
all_wav_files.append(dir_path + '/' + cur_file)
else:
all_wav_files = _glob(path + '/*.wav')
if random_order:
_shuffle(all_wav_files)
result_builder = _tc.SFrameBuilder(column_types=[dict, str],
column_names=['audio', 'path'])
for cur_file_path in all_wav_files:
try:
sample_rate, data = _wavfile.read(cur_file_path)
except Exception as e:
error_string = "Could not read {}: {}".format(cur_file_path, e)
if not ignore_failure:
raise _ToolkitError(error_string)
else:
print(error_string)
continue
result_builder.append([{
'sample_rate': sample_rate,
'data': data
}, cur_file_path])
result = result_builder.close()
if not with_path:
del result['path']
return result
def _match(self, key, value):
for pattern, callback in self._callback_registry.iteritems():
if _fnmatch(key, pattern):
callback(key, value)
def fnmatch(pattern, filename) -> bool:
"""
Just swapping *arg order in fnmatch.fnmatch
"""
return _fnmatch(filename, pattern)