def gnomekeyring_get_all_entries():
"""Returns a dict of all keyring containers with a list of all their
entries.
"""
out = {}
tried_to_unlock = False
for c in gnomekeyring.list_keyring_names_sync():
out[c] = []
for i in gnomekeyring.list_item_ids_sync(c):
item = None
try:
item = gnomekeyring.item_get_info_sync(c, i)
except gnomekeyring.IOError as e:
logging.info('%s', e)
if not tried_to_unlock:
tried_to_unlock = True
if gnomekeyring_unlock():
# Try again.
try:
item = gnomekeyring.item_get_info_sync(c, i)
except gnomekeyring.IOError as e:
logging.info('%s', e)
if item:
out[c].append(GnomeKeyringItem(c, i, item))
else:
logging.error('Failed to access %s-%-2d: %s', c, i, e)
natsort.natsort(out[c], key=str)
return out
def gnomekeyring_get_all_entries():
"""Returns a dict of all keyring containers with a list of all their
entries.
"""
out = {}
tried_to_unlock = False
for c in gnomekeyring.list_keyring_names_sync():
out[c] = []
for i in gnomekeyring.list_item_ids_sync(c):
item = None
try:
item = gnomekeyring.item_get_info_sync(c, i)
except gnomekeyring.IOError as e:
logging.info('%s', e)
if not tried_to_unlock:
tried_to_unlock = True
if gnomekeyring_unlock():
# Try again.
try:
item = gnomekeyring.item_get_info_sync(c, i)
except gnomekeyring.IOError as e:
logging.info('%s', e)
if item:
out[c].append(GnomeKeyringItem(c, i, item))
else:
logging.error('Failed to access %s-%-2d: %s', c, i, e)
natsort.natsort(out[c], key=str)
return out
def runTest(self):
# This is plain old sorting (what we don't want).
assert sorted(["1", "5", "10", "50"]) == ["1", "10", "5", "50"]
# This is version sorting (what we're after).
assert natsort(["1", "5", "10", "50"]) == ["1", "5", "10", "50"]
# This is version sorting reversed.
assert natsort(["1", "5", "10", "50"], reverse=True) == ["50", "10", "5", "1"]
# This covers a previously fixed bug. I've purposefully shuffled the
# order on the left side to avoid false positives caused by stable
# sorting.
assert natsort(["1.5", "1.0"]) == ["1.0", "1.5"]
def __init__(self, directoryName, fileSuffix, revertOrder=False):
self.directoryName = directoryName
self.fileSuffix = fileSuffix
self.fileNames = os.listdir(directoryName)
self.fileNames = natsort(self.fileNames)
if revertOrder:
self.fileNames = self.fileNames[::-1]
def collect_backups(self, bucketname, prefix):
"""
Collect the backups in the given s3 bucket.
:param bucket: s3 backup bucket (a string).
:returns: A sorted :class:`list` of :class:`Backup` objects (the
backups are sorted by their date).
"""
backups = []
bucket = self.conn.get_bucket(bucketname)
logger.info("Scanning for backups: s3://%s/%s", bucketname, prefix)
for entry in natsort([key.name for key in bucket.list(prefix)]):
# Check for a time stamp in the directory entry's name.
match = TIMESTAMP_PATTERN.search(entry)
if match:
# Make sure the entry matches the given include/exclude patterns.
if self.exclude_list and any(fnmatch.fnmatch(entry, p) for p in self.exclude_list):
logger.debug("Excluded %r (it matched the exclude list).", entry)
elif self.include_list and not any(fnmatch.fnmatch(entry, p) for p in self.include_list):
logger.debug("Excluded %r (it didn't match the include list).", entry)
else:
backups.append(S3Backup(
pathname=entry,
timestamp=datetime.datetime(*(int(group, 10) for group in match.groups('0'))),
))
else:
logger.debug("Failed to match time stamp in filename: %s", entry)
if backups:
logger.info("Found %i timestamped backups in %s.", len(backups), bucket)
return sorted(backups)
def calc_feat_df(cough_dir_list,feat_col_Names,N_SPLITS): df = pd.DataFrame(columns = feat_col_Names) k = 0.0 # Init progress bar printProgress(k,len(cough_dir_list),prefix = "Progress:",suffix = "Complete",barLength = 50) for dir in cough_dir_list: coughs_list = glob(dir + "/*.wav") # first numerically sort the coughs list coughs_list = natsort(coughs_list) # extract spectral features for all coughs in dir feat_matrix = extract_features(dir,coughs_list, meta_data,N_SPLITS) temp_df = pd.DataFrame(feat_matrix,columns = feat_col_Names) df = df.append(temp_df) k += 1 printProgress(k,len(cough_dir_list),prefix = "Progress:",suffix = "Complete",barLength = 50) return df
def test_python_3_compatibility(self):
"""
Test the Python 3 incompatibility reported in `issue 2`_.
.. _issue 2: https://github.com/xolox/python-naturalsort/issues/2
"""
assert natsort(['1', 'a']) == ['1', 'a']
def readStreamVid(dataDir='data'):
"""
Reads the streamline time sequence.
call signature:
readStreamVid(dataDir = 'data')
Keyword arguments:
*dataDir*:
Data directory.
"""
files = listdir(dataDir)
files = natsort.natsort(files)
nFiles = 0
for i in range(len(files)):
if ((str.find(files[i], 'stream') == 0) and (files[i] != 'stream.vtk')
and (str.find(files[i], 'Init') == -1)):
nFiles += 1
# initialize the array of streamline objects
s = []
for f in files:
if ((str.find(f, 'stream') == 0) and (f != 'stream.vtk')
and (str.find(f, 'Init') == -1)):
s.append(gm.readStream(dataDir=dataDir, streamFile=f))
return s
def collect_backups(self, location):
"""
Collect the backups at the given location.
:param location: Any value accepted by :func:`coerce_location()`.
:returns: A sorted :class:`list` of :class:`Backup` objects (the
backups are sorted by their date).
:raises: :exc:`~exceptions.ValueError` when the given directory doesn't
exist or isn't readable.
"""
backups = []
location = coerce_location(location)
logger.info("Scanning %s for backups ..", location)
location.ensure_readable()
pattern=re.compile(self.timestamp, re.VERBOSE)
for entry in natsort(location.context.list_entries(location.directory)):
match = pattern.search( entry)
if match:
if self.exclude_list and any(fnmatch.fnmatch(entry, p) for p in self.exclude_list):
logger.verbose("Excluded %s (it matched the exclude list).", entry)
elif self.include_list and not any(fnmatch.fnmatch(entry, p) for p in self.include_list):
logger.verbose("Excluded %s (it didn't match the include list).", entry)
else:
try:
backups.append(Backup(
pathname=os.path.join(location.directory, entry),
timestamp=datetime.datetime(*(int(group, 10) for group in match.groups('0'))),
))
except ValueError as e:
logger.notice("Ignoring %s due to invalid date (%s).", entry, e)
else:
logger.debug("Failed to match time stamp in filename: %s", entry)
if backups:
logger.info("Found %i timestamped backups in %s.", len(backups), location)
return sorted(backups)
def update_file(self, force=None):
"""
Update the file with the contents of the files in the ``.d`` directory.
:param force: Override the value of :attr:`force` (a boolean or
:data:`None`).
:raises: :exc:`RefuseToOverwrite` when :attr:`force` is :data:`False`
and the contents of :attr:`filename` were modified.
"""
if force is None:
force = self.force
if not self.context.is_directory(self.directory):
# Create the .d directory.
logger.info("Creating directory %s ..", format_path(self.directory))
self.context.execute('mkdir', '-p', self.directory, tty=False)
# Move the original file into the .d directory.
local_file = os.path.join(self.directory, 'local')
logger.info("Moving %s to %s ..", format_path(self.filename), format_path(local_file))
self.context.execute('mv', self.filename, local_file, tty=False)
# Read the modular configuration file(s).
blocks = []
for entry in natsort(self.context.list_entries(self.directory)):
if not entry.startswith('.'):
filename = os.path.join(self.directory, entry)
if self.context.is_executable(filename):
blocks.append(self.execute_file(filename))
else:
blocks.append(self.read_file(filename))
contents = b"\n\n".join(blocks)
# Make sure the generated file was not modified? We skip this on the
# first run, when the original file was just moved into the newly
# created directory (see above).
if all(map(self.context.is_file, (self.filename, self.checksum_file))):
logger.info("Checking for local changes to %s ..", format_path(self.filename))
if self.new_checksum != self.old_checksum:
if force:
logger.warning(compact(
"""
The contents of the file to generate ({filename})
were modified but --force was used so overwriting
anyway!
""",
filename=format_path(self.filename),
))
else:
raise RefuseToOverwrite(compact(
"""
The contents of the file to generate ({filename})
were modified and I'm refusing to overwrite it! If
you're sure you want to proceed, use the --force
option or delete the file {checksum_file} and
retry.
""",
filename=format_path(self.filename),
checksum_file=format_path(self.checksum_file),
))
# Update the generated configuration file.
self.write_file(self.filename, contents)
# Update the checksum file.
self.context.write_file(self.checksum_file, self.new_checksum)
def calc_feat_df(cough_dir_list, WIN_SIZE):
feat_col_Names = get_colNames()
df = pd.DataFrame(columns=feat_col_Names)
k = 0.0
for dir in cough_dir_list:
coughs_list = glob(dir + "/*.wav")
coughs_list = natsort(coughs_list)
feat_matrix = extract_features(dir, coughs_list, meta_data, WIN_SIZE)
temp_df = pd.DataFrame(feat_matrix, columns=feat_col_Names)
df = df.append(temp_df)
k += 1
printProgress(k, len(cough_dir_list), prefix="Progress:", suffix="Complete", barLength=50)
return df
def rename(file, folder):
file_content = pd.read_excel(file)
files = natsort(os.listdir(folder))
i = 0
for file in files:
file_format = PurePosixPath(file).suffix
file_name = file.split(str(file_format))[0]
old_name = file_content['nome_atual'].tolist()[i]
new_name = file_content['novo_nome'].tolist()[i] + str(file_format)
if not os.path.exists(folder + '/Renomeados'):
os.mkdir(folder + '/Renomeados')
destino = folder + '/Renomeados'
if str(file) == str(old_name):
old_name = os.path.join(folder, old_name)
new_name = os.path.join(destino, new_name)
os.rename(old_name, new_name)
i += 1
else:
sg.PopupError(
"A lista de arquivos da planilha nao coincide com a pasta indicada!"
)
break
def readPoincareVid(dataDir='data'):
"""
Reads the Poncare map time sequence.
call signature:
readPoincareVid(dataDir = 'data')
Keyword arguments:
*dataDir*:
Data directory.
"""
# find the number of dump files and do some sorting
files = listdir(dataDir)
files = natsort.natsort(files)
nFiles = 0
for i in range(len(files)):
if ((str.find(files[i], 'poincare') == 0)
and (files[i] != 'poincare.vtk')
and (str.find(files[i], 'Init') == -1)):
nFiles += 1
# initialize the array of Poncare objects
po = []
for f in files:
if ((str.find(f, 'poincare') == 0) and (f != 'poincare.vtk')
and (str.find(f, 'Init') == -1)):
print(f)
po.append(gm.readPoincare(dataDir=dataDir, poincare=f))
return po
def collect_backups(self, location):
"""
Collect the backups at the given location.
:param location: Any value accepted by :func:`coerce_location()`.
:returns: A sorted :class:`list` of :class:`Backup` objects (the
backups are sorted by their date).
:raises: :exc:`~exceptions.ValueError` when the given directory doesn't
exist or isn't readable.
"""
backups = []
location = coerce_location(location)
logger.info("Scanning %s for backups ..", location)
location.ensure_readable()
for entry in natsort(location.context.list_entries(location.directory)):
match = TIMESTAMP_PATTERN.search(entry)
if match:
if self.exclude_list and any(fnmatch.fnmatch(entry, p) for p in self.exclude_list):
logger.verbose("Excluded %s (it matched the exclude list).", entry)
elif self.include_list and not any(fnmatch.fnmatch(entry, p) for p in self.include_list):
logger.verbose("Excluded %s (it didn't match the include list).", entry)
else:
try:
backups.append(Backup(
pathname=os.path.join(location.directory, entry),
timestamp=datetime.datetime(*(int(group, 10) for group in match.groups('0'))),
))
except ValueError as e:
logger.notice("Ignoring %s due to invalid date (%s).", entry, e)
else:
logger.debug("Failed to match time stamp in filename: %s", entry)
if backups:
logger.info("Found %i timestamped backups in %s.", len(backups), location)
return sorted(backups)
def readMapVid(dataDir = 'data'):
"""
Reads the integration map time sequence.
call signature:
readMapVid(dataDir = 'data')
Keyword arguments:
*dataDir*:
Data directory.
"""
files = listdir(dataDir)
files = natsort.natsort(files)
nFiles = 0
for i in range(len(files)):
if ((str.find(files[i], 'map') == 0) and (files[i] != 'map.vtk') and (str.find(files[i], 'Init') == -1)):
nFiles += 1
# initialize the array of streamline objects
m = []
for f in files:
if ((str.find(f, 'map') == 0) and (f != 'map.vtk') and (str.find(f, 'Init') == -1)):
m.append(gm.readMap(dataDir = dataDir, mapFile = f))
return m
def collect_backups(self, directory, rotate_type):
"""
Collect the backups in the given directory. on local or google drive
:param directory: The pathname of an existing directory (a string).
:param rotate_type: The rotate type if local o remote on Google Drive
:returns: A sorted :class:`list` of :class:`Backup` objects (the
backups are sorted by their date).
"""
backups = []
# directory = os.path.abspath(directory)
directory = os.path.abspath(directory) if not rotate_type == 'remote' else directory
logger.info("Scanning %s directory for backups: %s", rotate_type, self.custom_format_path(directory))
# get files from local if rotate_type is local else get files from GoogleDrive
files = os.listdir(directory) if not rotate_type == 'remote' else self.gdrivecm.get_files(directory)
for entry in natsort(files):
# Check for a time stamp in the directory entry's name.
match = TIMESTAMP_PATTERN.search(entry)
if match:
# Make sure the entry matches the given include/exclude patterns.
if self.exclude_list and any(fnmatch.fnmatch(entry, p) for p in self.exclude_list):
logger.debug("Excluded %r (it matched the exclude list).", entry)
elif self.include_list and not any(fnmatch.fnmatch(entry, p) for p in self.include_list):
logger.debug("Excluded %r (it didn't match the include list).", entry)
else:
backups.append(Backup(
pathname=os.path.join(directory, entry) if not rotate_type == 'remote' else entry,
datetime=datetime.datetime(*(int(group, 10) for group in match.groups('0'))),
))
else:
logger.debug("Failed to match time stamp in filename: %s", entry)
if backups:
logger.info("Found %i timestamped backups in %s.", len(backups), self.custom_format_path(directory))
return sorted(backups)
def test_dotted_sorting(self):
"""
Test a previously fixed bug to prevent regressions.
I've purposefully shuffled the order on the left side to avoid false
positives caused by stable sorting.
"""
assert natsort(['1.5', '1.0']) == ['1.0', '1.5']
def update_file(self, force=None):
"""
Update the file with the contents of the files in the ``.d`` directory.
:param force: Override the value of :attr:`force` (a boolean or
:data:`None`).
:raises: :exc:`RefuseToOverwrite` when :attr:`force` is :data:`False`
and the contents of :attr:`filename` were modified.
"""
if force is None:
force = self.force
if not self.context.is_directory(self.directory):
# Create the .d directory.
logger.info("Creating directory %s", format_path(self.directory))
self.context.execute('mkdir', '-p', self.directory, tty=False)
# Move the original file into the .d directory.
local_file = os.path.join(self.directory, 'local')
logger.info("Moving %s to %s", format_path(self.filename), format_path(local_file))
self.context.execute('mv', self.filename, local_file, tty=False)
# Read the modularized configuration file(s).
blocks = []
for filename in natsort(self.context.list_entries(self.directory)):
if not filename.startswith('.'):
blocks.append(self.read_file(os.path.join(self.directory, filename)))
contents = b"\n\n".join(blocks)
# Make sure the generated file was not modified? We skip this on the
# first run, when the original file was just moved into the newly
# created directory (see above).
if all(map(self.context.is_file, (self.filename, self.checksum_file))):
logger.info("Checking for local changes to %s ..", format_path(self.filename))
if self.hash_contents() != self.context.read_file(self.checksum_file):
if force:
logger.warning(compact(
"""
The contents of the file to generate ({filename})
were modified but --force was used so overwriting
anyway!
""",
filename=format_path(self.filename),
))
else:
raise RefuseToOverwrite(compact(
"""
The contents of the file to generate ({filename})
were modified and I'm refusing to overwrite it! If
you're sure you want to proceed, use the --force
option or delete the file {checksum_file} and
retry.
""",
filename=format_path(self.filename),
checksum_file=format_path(self.checksum_file),
))
# Update the generated configuration file.
self.write_file(self.filename, contents)
# Update the checksum file.
self.context.write_file(self.checksum_file, self.hash_contents())
def test_more_complex_versions(self):
"""
Test the implementation of the ``NaturalOrderKey`` class.
This test uses some more complex version strings that were sorted
incorrectly by the initial (way too naive) implementation in 1.4.
"""
sorted_versions = ['1532-44349', '1534-44658', '1536-44582', '1536-44935', '1538-44874', '1538-44920']
random_versions = ['1534-44658', '1536-44935', '1532-44349', '1538-44920', '1536-44582', '1538-44874']
assert sorted_versions == natsort(random_versions)
def test_input_order_irrelevant(self):
"""
Test that order of input does not adversely affect order of output.
Works by shuffling the input and checking that all 10.000 iterations
result in the same output.
"""
sorted_strings = ['1532-44349', '1534-44658', '1536-44582', '1536-44935', '1538-44874', '1538-44920']
mutable_copy = list(sorted_strings)
for i in range(10000):
random.shuffle(mutable_copy)
assert natsort(mutable_copy) == sorted_strings
def lines_for_timestamp(request):
try:
try:
d = dateutil.parser.parse(request.params['datetime'])
except:
d = DBSession.query(BusDelay).order_by(BusDelay.time.desc()).first().time
lines = DBSession.query(BusDelay.line).filter(BusDelay.time == d).group_by(BusDelay.line).all()
output=[]
for l in lines:
output.append(l[0])
except DBAPIError:
return Response("a problem occured", content_type='text/plain', status_int=500)
return {'lines': natsort(output)}
def entries(self):
"""A list of :class:`PasswordEntry` objects."""
timer = Timer()
passwords = []
logger.info("Scanning %s ..", format_path(self.directory))
listing = self.context.capture("find", "-type", "f", "-name", "*.gpg", "-print0")
for filename in split(listing, "\0"):
basename, extension = os.path.splitext(filename)
if extension == ".gpg":
# We use os.path.normpath() to remove the leading `./' prefixes
# that `find' adds because it searches the working directory.
passwords.append(PasswordEntry(name=os.path.normpath(basename), store=self))
logger.verbose("Found %s in %s.", pluralize(len(passwords), "password"), timer)
return natsort(passwords, key=lambda e: e.name)
def sorted(self, objects):
"""
Sort the given objects in a human friendly way.
:param objects: The objects to sort (an iterable).
:returns: The sorted objects (a list).
If all of the objects are strings they are sorted using natural
order sorting, otherwise the :func:`sorted()` function is used.
"""
if all(isinstance(o, string_types) for o in objects):
return natsort(objects)
else:
return sorted(objects)
def load_training_dict(self, folder_name='Brats17TrainingData'):
"""Creating a dictionary of training HGG and LGG scans."""
"""
Arguments:
folder_name: folder where the training data is stored
"""
grades = ['HGG', 'LGG']
for g in grades:
scans = os.listdir(os.path.join(self.db_path, folder_name, g))
scans = ns.natsort(scans)
for s in scans:
if s not in self.train_dict:
s_relative_path = os.path.join(folder_name, g, s)
self.train_dict[s] = ScanBRATS(s, s_relative_path, 'train')
def available_files(self):
"""
The filenames of the available configuration files (a list of strings).
The value of :attr:`available_files` is computed the first time its
needed by searching for available configuration files that match
:attr:`filename_patterns` using :func:`~glob.glob()`. If you set
:attr:`available_files` this effectively disables searching for
configuration files.
"""
matches = []
for pattern in self.filename_patterns:
logger.debug("Matching filename pattern: %s", pattern)
matches.extend(natsort(glob.glob(parse_path(pattern))))
return matches
def poincareVid(dataDir='data',
poincareInit='poincareInit.vtk',
interpolation='weighted'):
"""
Creates a Poincare map time sequence.
call signature:
poincareVid(dataDir = 'data', poincareInit = 'poincareInit.vtk', interpolation = 'weighted')
Keyword arguments:
*dataDir*:
Data directory.
*poincareInit*:
Read the initial Poincare maps from this file.
*interpolation*:
Interpolation of the vector field.
'mean': takes the mean of the adjacent grid point.
'weighted': weights the adjacent grid points according to their distance.
"""
# find the number of dump files and do some sorting
files = listdir(dataDir)
files = natsort.natsort(files)
nFiles = 0
for i in range(len(files)):
if (str.find(files[i], 'dump') == 0):
nFiles += 1
# initialize the array of Poncare objects
po = []
for f in files:
if (str.find(f, 'dump') == 0):
poincareFile = f.replace('dump', 'poincare')
print(f)
po.append(
gm.mapPoincare(dataDir=dataDir,
poincareInit=poincareInit,
poincare=poincareFile,
dumpFile=f,
interpolation=interpolation))
print(po[-1].p.t)
return po
def streamVid(dataDir='data',
streamFileInit='streamInit.vtk',
interpolation='weighted'):
"""
Creates a streamline time sequence.
call signature:
streamVid(dataDir = 'data', streamFileInit = 'streamInit.vtk', interpolation = 'weighted')
Keyword arguments:
*dataDir*:
Data directory.
*streamFileInit*:
Read the initial streamline in this file.
*interpolation*:
Interpolation of the vector field.
'mean': takes the mean of the adjacent grid point.
'weighted': weights the adjacent grid points according to their distance.
"""
# find the number of dump files
files = listdir(dataDir)
files = natsort.natsort(files)
nFiles = 0
for i in range(len(files)):
if (str.find(files[i], 'dump') == 0):
nFiles += 1
# initialize the array of streamline objects
s = []
for f in files:
if (str.find(f, 'dump') == 0):
streamFile = f.replace('dump', 'stream')
print(f)
s.append(
gm.mapStream(dataDir=dataDir,
streamFileInit=streamFileInit,
streamFile=streamFile,
dumpFile=f,
interpolation=interpolation))
print(s[-1].p.t)
return s
def lines_for_timestamp(request):
try:
try:
d = dateutil.parser.parse(request.params['datetime'])
except:
d = DBSession.query(BusDelay).order_by(
BusDelay.time.desc()).first().time
lines = DBSession.query(BusDelay.line).filter(
BusDelay.time == d).group_by(BusDelay.line).all()
output = []
for l in lines:
output.append(l[0])
except DBAPIError:
return Response("a problem occured",
content_type='text/plain',
status_int=500)
return {'lines': natsort(output)}
def TsToMp4(Folder, OutPutFileName=""):
Files = os.listdir(Folder)
Files = natsort(Files)
fullPathFiles = []
for i in Files:
path = os.path.join(Folder, i)
if os.path.isfile(path):
if i.endswith(".ts"):
fullPathFiles.append(path)
if fullPathFiles == []:
print "Thu muc %s khong co file .ts" % Folder
return
DirComplete = os.path.join(Folder[0:Folder.rfind("\\")], 'complete')
if not os.path.exists(DirComplete): os.mkdir(DirComplete)
DirLog = os.path.join(DirComplete, 'log')
if not os.path.exists(DirLog): os.mkdir(DirLog)
concatFile = os.path.join(DirComplete, 'concat.txt')
with open(concatFile, 'w') as f:
for i in fullPathFiles:
if os.path.isfile(i):
f.write("file '%s'\n" % i.replace("'", "'\\''"))
if OutPutFileName:
outputFile = os.path.join(DirComplete, OutPutFileName)
FileLog = os.path.join(DirLog, OutPutFileName + ".log")
else:
outputFile = os.path.join(DirComplete, 'output.mp4')
FileLog = os.path.join(DirLog, 'output.log')
args = [
FFMPEG_PATH, '-f', 'concat', '-i', concatFile, '-c', 'copy', '-bsf:a',
'aac_adtstoasc', outputFile
]
with open(FileLog, 'w') as f:
process = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
for line in iter(lambda: process.stdout.read(1), ''):
sys.stdout.write(line)
f.write(line.rstrip('\n'))
os.remove(concatFile)
def train_valid_split(self, folder_name='ISLES2017_Training'):
"""Splitting training data into train and valid subsets."""
"""
Arguments:
folder_name: name of the folder where training data is stored
"""
np.random.seed(123456)
scan_list = ns.natsort(self.train_dict.keys())
n_scans = len(self.train_dict)
select_valid = np.random.choice(n_scans,
int(np.round(n_scans *
self.valid_p)),
replace=False)
for s_idx, s in enumerate(scan_list):
if s_idx in select_valid:
self.valid_scans.append(s)
else:
self.train_scans.append(s)
def _summarize_digit_list(list_or_set):
# In some cases there are hundreds of "missing" boxes. To make the output more human-readable, the code below
# contracts sequential series of missing numbers into a range, e.g. [1, 2, 3, 4] into "1-4"
# First extract each series of sequential numbers
# (code from the python docs, see https://docs.python.org/2.6/library/itertools.html#examples)
range_lists = []
for k, g in groupby(enumerate(list_or_set), lambda ix: ix[0] - ix[1]):
range_lists.append(map(itemgetter(1), g))
# make each sequence human-readable, by grabbing the first and last values of each sequence list
ranges = []
for range in range_lists:
if len(range) > 1:
ranges.append(u"{0}-{1}".format(range[0], range[-1]))
elif len(range) == 1:
ranges.append(unicode(range[0]))
# Since this is a list of strings that sometimes contain non-numeric characters, we use the naturalsort library to
# return the results in the expected order. Otherwise ["2-10", "11-19", "20"] would be sorted ["11-19", "2-10", "20"]
return natsort(ranges)
def _summarize_digit_list(list_or_set):
# In some cases there are hundreds of "missing" boxes. To make the output more human-readable, the code below
# contracts sequential series of missing numbers into a range, e.g. [1, 2, 3, 4] into "1-4"
# First extract each series of sequential numbers
# (code from the python docs, see https://docs.python.org/2.6/library/itertools.html#examples)
range_lists = []
for k, g in groupby(enumerate(list_or_set), lambda ix: ix[0] - ix[1]):
range_lists.append(map(itemgetter(1), g))
# make each sequence human-readable, by grabbing the first and last values of each sequence list
ranges = []
for range in range_lists:
if len(range) > 1:
ranges.append(u"{0}-{1}".format(range[0], range[-1]))
elif len(range) == 1:
ranges.append(unicode(range[0]))
# Since this is a list of strings that sometimes contain non-numeric characters, we use the naturalsort library to
# return the results in the expected order. Otherwise ["2-10", "11-19", "20"] would be sorted ["11-19", "2-10", "20"]
return natsort(ranges)
def create_matriz(folder):
try:
linhas_planilha = []
files = natsort(os.listdir(folder))
for file in files:
linha = {}
linha['nome_atual'] = file
linha['nome_atual 2'] = str(file.split("-")[0])
linha['novo_nome'] = '[INSIRA O NOME NOVO]'
linhas_planilha.append(linha)
try:
data = pd.DataFrame(linhas_planilha)
data.to_excel('Matriz.xlsx', index=False)
except ValueError as err:
error_matriz = "Erro ao salvar Matriz: " + str(err)
sg.PopupError(error_matriz)
except ValueError as err:
error_matriz = "Erro ao Criar Matriz: " + str(err)
sg.PopupError(error_matriz)
def get_missing_numbers(list_or_set):
# set the maximum and minimum box numbers found in the given box-number list
min_number = min(list_or_set)
max_number = max(list_or_set)
missing_numbers = []
# for each integer between the minimum and maximum value, check if that integer is in the box list.
# If it does not, that box is missing.
i = min_number
while i < max_number:
if i not in list_or_set:
missing_numbers.append(i)
i += 1
# sort the list of missing numbers
missing_numbers.sort()
# In some cases there are hundreds of "missing" boxes. To make the output more human-readable, the code below
# contracts sequential series of missing numbers into a range, e.g. [1, 2, 3, 4] into "1-4"
# First extract each series of sequential numbers
# (code from the python docs, see https://docs.python.org/2.6/library/itertools.html#examples)
range_lists = []
for k, g in groupby(enumerate(missing_numbers), lambda ix: ix[0]-ix[1]):
range_lists.append(map(itemgetter(1), g))
# make each sequence human-readable, by grabbing the first and last values of each sequence list
ranges = []
for range in range_lists:
if len(range) > 1:
ranges.append("{0}-{1}".format(range[0], range[-1]))
elif len(range) == 1:
ranges.append(str(range[0]))
# Since this is a list of strings that sometimes contain non-numeric characters, we use the naturalsort library to
# return the results in the expected order. Otherwise ["2-10", "11-19", "20"] would be sorted ["11-19", "2-10", "20"]
return natsort(ranges)
def get_missing_numbers(list_or_set):
# set the maximum and minimum box numbers found in the given box-number list
min_number = min(list_or_set)
max_number = max(list_or_set)
missing_numbers = []
# for each integer between the minimum and maximum value, check if that integer is in the box list.
# If it does not, that box is missing.
i = min_number
while i < max_number:
if i not in list_or_set:
missing_numbers.append(i)
i += 1
# sort the list of missing numbers
missing_numbers.sort()
# In some cases there are hundreds of "missing" boxes. To make the output more human-readable, the code below
# contracts sequential series of missing numbers into a range, e.g. [1, 2, 3, 4] into "1-4"
# First extract each series of sequential numbers
# (code from the python docs, see https://docs.python.org/2.6/library/itertools.html#examples)
range_lists = []
for k, g in groupby(enumerate(missing_numbers), lambda ix: ix[0] - ix[1]):
range_lists.append(map(itemgetter(1), g))
# make each sequence human-readable, by grabbing the first and last values of each sequence list
ranges = []
for range in range_lists:
if len(range) > 1:
ranges.append("{0}-{1}".format(range[0], range[-1]))
elif len(range) == 1:
ranges.append(str(range[0]))
# Since this is a list of strings that sometimes contain non-numeric characters, we use the naturalsort library to
# return the results in the expected order. Otherwise ["2-10", "11-19", "20"] would be sorted ["11-19", "2-10", "20"]
return natsort(ranges)
def test_version_sorting(self):
"""Test version sorting (what we're after)."""
assert natsort(['1', '5', '10', '50']) == ['1', '5', '10', '50']
try:
os.makedirs(dbdir)
except:
print("ERROR: Unable to create directory: {}".format(dbdir),
file=sys.stderr)
exit(1)
# Output data to country spesific files
dbdir = config.get('DBDIR')
for cc in ccdata:
alldata = []
typedata = ccdata[cc]
types = sorted(typedata.keys())
for iptype in types:
filepath = os.path.join(dbdir, cc + '_' + iptype)
if iptype == "IPV6":
data = sorted(typedata[iptype])
else:
data = natsort.natsort(typedata[iptype])
alldata += data
with open(filepath, 'w+') as f:
for item in data:
print(item, file=f)
# Create a combined file as well
filepath = os.path.join(dbdir, cc + '_ALL')
with open(filepath, 'w+') as f:
for item in alldata:
print(item, file=f)
try:
os.makedirs(dbdir)
except:
print("ERROR: Unable to create directory: {}".format(dbdir),
file=sys.stderr)
exit(1)
# Output data to country spesific files
dbdir = config.get('DBDIR')
for cc in ccdata:
alldata = []
typedata = ccdata[cc]
types = sorted(typedata.keys())
for iptype in types:
filepath = os.path.join(dbdir, cc + '_' + iptype)
if iptype == "IPV6":
data = sorted(typedata[iptype])
else:
data = natsort.natsort(typedata[iptype])
alldata += data
with open(filepath, 'w+') as f:
for item in data:
print(item, file=f)
# Create a combined file as well
filepath = os.path.join(dbdir, cc + '_ALL')
with open(filepath, 'w+') as f:
for item in alldata:
print(item, file=f)
def run(root_dir, list_file_path, n):
# check list file for selection
list_file = open(list_file_path, "r")
list_content = list_file.readlines()
list_file.close()
filter_dict = {}
for i in range(0, len(list_content)):
filter_dict[list_content[i].split(".avi")[0]] = True
# print list_content
# read the content of the root directory and filter all directories
directory_names = map(lambda f: os.path.join(root_dir, f), os.listdir(root_dir))
# directory_names = filter(lambda f: os.path.basename(f).startswith('M'), directory_names)
directories = filter(os.path.isdir, directory_names)
# assign each 'top-level' directory to a topic id
topics = { dir : topicId for (topicId, dir) in enumerate(directories) }
# open ouput files
train_name = os.path.join(root_dir, OUTPUT_FILE) % os.path.basename(list_file_path)
train_file = open(train_name, "w")
videos = []
# for every topic read all its image files
for topic_dir in directories:
for parent_dir, sub_dirs, files in os.walk(topic_dir):
# only take folder if it is in list
parent_dir_split = parent_dir.split("/")
identifier = parent_dir_split[-2] + "/" + parent_dir_split[-1]
if (not identifier in filter_dict):
continue
if len(files) == 0:
continue
def writeFiles(sub_files):
sub_file_count = len(sub_files)
# all frames
if (n == -1):
count = 1
start = 0
# every 'count' frames
else:
count = sub_file_count / n
start = int(ceil((sub_file_count - count * n) / 2.0))
current_sequence = []
for i in range(start, len(sub_files), count):
if len(current_sequence) != n:
absolute_file = os.path.join(parent_dir, sub_files[i])
if (sub_files[i].endswith(("jpeg", "jpg", "png"))):
# write absolute file path and the corresponding topic Id to the output file
line = '{} {}\n'.format(absolute_file, topics[topic_dir])
current_sequence.append(line)
if n >= 0 and len(current_sequence) != n:
print("sub_file_count")
print(sub_file_count)
print("count")
print(count)
print("n")
print(n)
print("len(current_sequence)")
print(len(current_sequence))
assert(n == -1 or len(current_sequence) == n)
videos.append(current_sequence)
# end writeFiles()
# sort files
files = natsort(files)
file_count = len(files)
sub_video_count = 10
assert(file_count % sub_video_count == 0)
nrFramesPerCrop = file_count / sub_video_count
writeFiles(files[0 * nrFramesPerCrop:1 * nrFramesPerCrop])
writeFiles(files[1 * nrFramesPerCrop:2 * nrFramesPerCrop])
writeFiles(files[2 * nrFramesPerCrop:3 * nrFramesPerCrop])
writeFiles(files[3 * nrFramesPerCrop:4 * nrFramesPerCrop])
writeFiles(files[4 * nrFramesPerCrop:5 * nrFramesPerCrop])
writeFiles(files[5 * nrFramesPerCrop:6 * nrFramesPerCrop])
writeFiles(files[6 * nrFramesPerCrop:7 * nrFramesPerCrop])
writeFiles(files[7 * nrFramesPerCrop:8 * nrFramesPerCrop])
writeFiles(files[8 * nrFramesPerCrop:9 * nrFramesPerCrop])
writeFiles(files[9 * nrFramesPerCrop:10 * nrFramesPerCrop])
if (len(videos) % 5000 == 0):
print "%d videos processed..." % (len(videos) / 10)
# shuffle the video lists, not the frames inside one video
shuffle(videos)
video_number = len(videos)
split_point = int(video_number * TRAIN_PERCENTAGE)
for sequence in videos:
for line in sequence:
train_file.write(line)
# close output file
train_file.close()
# just some logging
sys.stdout.write("Done. Exported %s/%s \n\n" % (root_dir, OUTPUT_FILE))
sys.stdout.write("Please, call Caffee's 'convert_image' tool now:\n")
sys.stdout.write("$CAFFE_ROOT/build/tools/convert_image \"\" %s UCF101\n" % OUTPUT_FILE)
def run(root_dir, list_file_path, n):
# check list file for selection
list_file = open(list_file_path, "r")
list_content = list_file.readlines()
list_file.close()
filter_dict = {}
for i in range(0, len(list_content)):
filter_dict[list_content[i].split(".avi")[0]] = True
# print list_content
# read the content of the root directory and filter all directories
directory_names = map(lambda f: os.path.join(root_dir, f),
os.listdir(root_dir))
# directory_names = filter(lambda f: os.path.basename(f).startswith('M'), directory_names)
directories = filter(os.path.isdir, directory_names)
# assign each 'top-level' directory to a topic id
topics = {dir: topicId for (topicId, dir) in enumerate(directories)}
# open ouput files
train_name = os.path.join(root_dir,
OUTPUT_FILE) % os.path.basename(list_file_path)
train_file = open(train_name, "w")
videos = []
# for every topic read all its image files
for topic_dir in directories:
for parent_dir, sub_dirs, files in os.walk(topic_dir):
# only take folder if it is in list
parent_dir_split = parent_dir.split("/")
identifier = parent_dir_split[-2] + "/" + parent_dir_split[-1]
if (not identifier in filter_dict):
continue
if len(files) == 0:
continue
def writeFiles(sub_files):
sub_file_count = len(sub_files)
# all frames
if (n == -1):
count = 1
start = 0
# every 'count' frames
else:
count = sub_file_count / n
start = int(ceil((sub_file_count - count * n) / 2.0))
current_sequence = []
for i in range(start, len(sub_files), count):
if len(current_sequence) != n:
absolute_file = os.path.join(parent_dir, sub_files[i])
if (sub_files[i].endswith(("jpeg", "jpg", "png"))):
# write absolute file path and the corresponding topic Id to the output file
line = '{} {}\n'.format(absolute_file,
topics[topic_dir])
current_sequence.append(line)
if n >= 0 and len(current_sequence) != n:
print("sub_file_count")
print(sub_file_count)
print("count")
print(count)
print("n")
print(n)
print("len(current_sequence)")
print(len(current_sequence))
assert (n == -1 or len(current_sequence) == n)
videos.append(current_sequence)
# end writeFiles()
# sort files
files = natsort(files)
file_count = len(files)
sub_video_count = 10
assert (file_count % sub_video_count == 0)
nrFramesPerCrop = file_count / sub_video_count
writeFiles(files[0 * nrFramesPerCrop:1 * nrFramesPerCrop])
writeFiles(files[1 * nrFramesPerCrop:2 * nrFramesPerCrop])
writeFiles(files[2 * nrFramesPerCrop:3 * nrFramesPerCrop])
writeFiles(files[3 * nrFramesPerCrop:4 * nrFramesPerCrop])
writeFiles(files[4 * nrFramesPerCrop:5 * nrFramesPerCrop])
writeFiles(files[5 * nrFramesPerCrop:6 * nrFramesPerCrop])
writeFiles(files[6 * nrFramesPerCrop:7 * nrFramesPerCrop])
writeFiles(files[7 * nrFramesPerCrop:8 * nrFramesPerCrop])
writeFiles(files[8 * nrFramesPerCrop:9 * nrFramesPerCrop])
writeFiles(files[9 * nrFramesPerCrop:10 * nrFramesPerCrop])
if (len(videos) % 5000 == 0):
print "%d videos processed..." % (len(videos) / 10)
# shuffle the video lists, not the frames inside one video
shuffle(videos)
video_number = len(videos)
split_point = int(video_number * TRAIN_PERCENTAGE)
for sequence in videos:
for line in sequence:
train_file.write(line)
# close output file
train_file.close()
# just some logging
sys.stdout.write("Done. Exported %s/%s \n\n" % (root_dir, OUTPUT_FILE))
sys.stdout.write("Please, call Caffee's 'convert_image' tool now:\n")
sys.stdout.write("$CAFFE_ROOT/build/tools/convert_image \"\" %s UCF101\n" %
OUTPUT_FILE)
'--array',
metavar='NS',
type=int,
default=-1,
help='Submit the jobs as an array, with <=NS running simultaneous.'
'Good for avoiding using all the Abaqus licenses. NS=0 runs all array jobs simultaneously.'
)
args = parser.parse_args()
tC = 0
tA = 0
# Handle inputs.
if args.sort:
try:
from natsort import natsort
args.inp = natsort(args.inp)
except:
print 'WARNING: no natsort module found, sorting not available.'
if args.cluster: cp = cluster_properties(args.cluster)
if not args.procs:
if args.cluster: np = cp['np']
else: np = 8
else: np = args.procs
# Print a summary.
print 'Tasks to run:', len(args.inp)
for inp in args.inp:
print '\t', inp
# Run tasks.
jobs = list()
def test_zero_padding(self):
"""Test that zero padding semantics are respected."""
assert natsort(['1.5.1', '1.5']) == ['1.5', '1.5.1']
if args.labels and len(args.labels)<len(args.data):
utility.print_error('Number of given labels must be the same as the number of given files.',True)
# Separate the data files into FEM and EXP.
femdata = []
expdata = []
for df in args.data:
if df.endswith('.rpt'): femdata.append(df)
else: expdata.append(df)
# Try to sort the data.
if args.sort or args.sort_reverse:
try:
from natsort import natsort
femdata = natsort(femdata)
expdata = natsort(expdata)
if args.sort_reverse:
femdata = list(reversed(femdata))
expdata = list(reversed(expdata))
except:
print 'WARNING: no natsort module found, sorting not available.'
# Prepare the plots.
fig,ax = pyplot.subplots()
cm = pyplot.get_cmap('gist_rainbow')
# cm = pyplot.get_cmap('jet')
if len(femdata)==len(expdata):
colors = [cm(float(i)/len(femdata)) for i in range(len(femdata))]
else:
colors = [cm(float(i)/len(args.data)) for i in range(len(args.data))]
somecounter+=1
cigartup= read.cigartuples
cigarstr = read.cigarstring
unfiltreadidx = read.get_reference_positions(full_length=True)
unfiltread = list(read.query_sequence)
readqual = read.query_qualities
cigarseq = cigarTranslate(cigartup)
wee = analyzeRead(cigartup,cigarstr,unfiltreadidx,unfiltread,readqual,cigarseq)
bigconsensus = []
keylist = []
for boop in CONSENSUSDICT:
keylist.append(boop)
a = natsort(keylist)
coveragechecker = 0
for i in a:
# print i,CONSENSUSDICT[i]
if 'I' in i: #and sum(CONSENSUSDICT[i].values()) < 200: #Include insertion!?!?!?
pass
else:
x = CONSENSUSDICT[i]
maxkey = max(x.iteritems(), key=operator.itemgetter(1))[0]
# for key, value in sorted(x.iteritems(), key=lambda (k,v): (v,k)):
# maxkey = key
# break
# print>>readreportfile,i,maxkey,refdict[SEGMENT][int(i)],CONSENSUSDICT[i] #use me for snpcheck
if maxkey != refdict[SEGMENT][int(i)]:
if __name__ == '__main__': meta_data = np.loadtxt(labels_csv,delimiter=";",dtype = str,skiprows = 1) coughs_dir_list = glob(coughs_dir + "C*") recordings_list = [os.path.basename(f) for f in coughs_dir_list ] for dir in coughs_dir_list: if EXTRACT: print "Extracting features for coughs in:",dir,"...", # List of cough wav files sorted numerically cough_wavs = natsort(list(glob(dir + "/*.wav"))) extract_features(cough_wavs,dir,meta_data) print "done\n" # Copy all generated feature files to # the all folder if COPY_ALL: print "Copying features to all folder...", copy_to_all() print "done!\n" print "\ndone\n" if CONVERT_SUPERVECTORS: print "Converting data into supervectors..."
pyplot.rc('mathtext', default='regular') # Don't use italics for mathmode.
# pyplot.rc('font',size=16)
pyplot.rc('axes', grid=True)
pyplot.rc('figure', dpi=300)
pyplot.rc('savefig', dpi=300)
# Remove any datafiles that have already been processed.
datafiles = []
for dfile in args.datafiles:
if not path.splitext(dfile)[0].endswith('--CLEAN'):
datafiles.append(dfile)
# Try to sort the data.
try:
from natsort import natsort
datafiles = natsort(datafiles)
except:
print 'WARNING: no natsort module found, sorting not available.'
# Prepare the multiplots.
if len(datafiles) > 1:
cm = pyplot.get_cmap('gist_rainbow')
colors = [cm(float(i) / len(datafiles)) for i in range(len(datafiles))]
fig1, ax1 = pyplot.subplots(3, 1, figsize=(15, 11))
fig2, ax2 = pyplot.subplots(1, 3, figsize=(15, 6))
if args.title:
fig1.suptitle(args.title, fontweight="bold")
fig2.suptitle(args.title, fontweight="bold")
ax1[2].axis('off')
ax2[2].axis('off')
ax1[0].set_color_cycle(colors)
def create_window():
file_list_column = [
[
sg.Text("Selecione o Diretório de Origem: "),
sg.In(size=(25, 1), enable_events=True, key="-FOLDER-"),
sg.FolderBrowse(),
],
[
sg.Listbox(values=[],
enable_events=True,
size=(57, 20),
key="-FILE LIST-")
],
[sg.Button("Baixar Planilha Matriz")],
[
sg.Text("Selecione o Arquivo Matriz: "),
sg.In(size=(25, 1), enable_events=True, key="-FILE-"),
sg.FileBrowse(),
],
[sg.Button("Rename")],
[sg.Button(" Cancel ")],
]
layout = [[sg.Column(file_list_column)]]
window = sg.Window("Rename Files", layout)
# Run the Event Loop
while True:
event, values = window.read()
if event == " Cancel " or event == sg.WIN_CLOSED:
break
# Folder name was filled in, make a list of files in the folder
if event == "-FOLDER-":
folder = values["-FOLDER-"]
try:
# Get list of files in folder
file_list = natsort(os.listdir(folder))
except:
file_list = []
fnames = [
f for f in file_list
if os.path.isfile(os.path.join(folder, f)) and
f.lower().endswith((".rar", ".pdf", ".xls", ".xlsx", ".doc",
".docx", ".txt", ".png", ".jpg", ".bmp"))
]
window["-FILE LIST-"].update(fnames)
elif event == "-FILE LIST-": # A file was chosen from the listbox
try:
filename = os.path.join(values["-FOLDER-"],
values["-FILE LIST-"][0])
except:
pass
if event == "-FILE-":
file = values["-FILE-"]
if event == "Baixar Planilha Matriz":
create_matriz(folder)
sg.PopupOK("Arquivo Matriz Criado.")
if event == "Rename":
try:
backup(folder)
try:
rename(file, folder)
sg.PopupOK(
'Processo concluido: Verifique a pasta indicada!')
except ValueError as err:
error_rename = "Erro ao Renomear: " + str(err)
sg.PopupError(error_rename)
except ValueError as err:
error_backup = "Erro ao criar backup: " + str(err)
sg.PopupError(error_backup)
window.close()
def parse_xml(filename, hostname):
mini_graph_path, root = reconcile_mini_graph_locations(filename, hostname)
u_neighbors = None
u_devices = None
hwsku = None
bgp_sessions = None
bgp_asn = None
intfs = None
vlan_intfs = None
pc_intfs = None
mgmt_intf = None
lo_intf = None
neighbors = None
devices = None
hwsku_qn = QName(ns, "HwSku")
for child in root:
if child.tag == str(hwsku_qn):
hwsku = child.text
# port_alias_map maps ngs port name to sonic port name
if hwsku == "Force10-S6000":
for i in range(0, 128, 4):
port_alias_map["fortyGigE0/%d" % i] = "Ethernet%d" % i
elif hwsku == "Force10-S6100":
for i in range(0, 4):
for j in range(0, 16):
port_alias_map["fortyGigE1/%d/%d" % (i+1, j+1)] = "Ethernet%d" % (i * 16 + j)
elif hwsku == "Arista-7050-QX32":
for i in range(1, 25):
port_alias_map["Ethernet%d/1" % i] = "Ethernet%d" % ((i - 1) * 4)
for i in range(25, 33):
port_alias_map["Ethernet%d" % i] = "Ethernet%d" % ((i - 1) * 4)
else:
for i in range(0, 128, 4):
port_alias_map["Ethernet%d" % i] = "Ethernet%d" % i
for child in root:
if child.tag == str(QName(ns, "DpgDec")):
(intfs, lo_intfs, mgmt_intf, vlan_intfs, pc_intfs) = parse_dpg(child, hostname)
elif child.tag == str(QName(ns, "CpgDec")):
(bgp_sessions, bgp_asn) = parse_cpg(child, hostname)
elif child.tag == str(QName(ns, "PngDec")):
(neighbors, devices, console_dev, console_port, mgmt_dev, mgmt_port) = parse_png(child, hostname)
elif child.tag == str(QName(ns, "UngDec")):
(u_neighbors, u_devices, _, _, _, _) = parse_png(child, hostname)
# Replace port with alias in Vlan interfaces members
if vlan_intfs is not None:
for vlan in vlan_intfs:
for i,member in enumerate(vlan['members']):
vlan['members'][i] = port_alias_map[member]
# Convert vlan members into a space-delimited string
vlan['members'] = " ".join(vlan['members'])
# Replace port with alias in port channel interfaces members
if pc_intfs is not None:
for pc in pc_intfs:
for i,member in enumerate(pc['members']):
pc['members'][i] = port_alias_map[member]
# Create port index map. Since we currently output a mix of NGS names
# and SONiC mapped names, we include both in this map.
# SONiC aliases, when sorted in natural sort order, match the phyical port
# index order, so we sort by SONiC port alias, and map
# back to NGS names after sorting using this inverted map
#
# TODO: Move all alias-related code out of minigraph_facts.py and into
# its own module to be used as another layer after parsing the minigraph.
inverted_port_alias_map = {v: k for k, v in port_alias_map.iteritems()}
# Start by creating a list of all port aliases
port_alias_list = []
for k, v in port_alias_map.iteritems():
port_alias_list.append(v)
# Sort the list in natural order
port_alias_list_sorted = natsort(port_alias_list)
# Create map from SONiC alias to physical index and NGS name to physical index
port_index_map = {}
for idx, val in enumerate(port_alias_list_sorted):
port_index_map[val] = idx
port_index_map[inverted_port_alias_map[val]] = idx
# Generate results
Tree = lambda: defaultdict(Tree)
results = Tree()
results['minigraph_hwsku'] = hwsku
# sorting by lambdas are not easily done without custom filters.
# TODO: add jinja2 filter to accept a lambda to sort a list of dictionaries by attribute.
# TODO: alternatively (preferred), implement class containers for multiple-attribute entries, enabling sort by attr
results['minigraph_bgp'] = sorted(bgp_sessions, key=lambda x: x['addr'])
results['minigraph_bgp_asn'] = bgp_asn
# TODO: sort does not work properly on all interfaces of varying lengths. Need to sort by integer group(s).
results['minigraph_interfaces'] = sorted(intfs, key=lambda x: x['name'])
results['minigraph_vlan_interfaces'] = vlan_intfs
results['minigraph_portchannel_interfaces'] = pc_intfs
results['minigraph_mgmt_interface'] = mgmt_intf
results['minigraph_lo_interfaces'] = lo_intfs
results['minigraph_neighbors'] = neighbors
results['minigraph_devices'] = devices
results['minigraph_underlay_neighbors'] = u_neighbors
results['minigraph_underlay_devices'] = u_devices
# note - this may include files under acs/ansible/minigraph, or those under the default cache folder.
# (see ANSIBLE_USER_MINIGRAPH_PATH at the top of the module)
results['minigraph_as_xml'] = mini_graph_path
results['minigraph_console'] = get_console_info(devices, console_dev, console_port)
results['minigraph_mgmt'] = get_mgmt_info(devices, mgmt_dev, mgmt_port)
results['minigraph_port_indices'] = port_index_map
return results
def entries(self):
"""A list of :class:`PasswordEntry` objects."""
passwords = []
for store in self.stores:
passwords.extend(store.entries)
return natsort(passwords, key=lambda e: e.name)
def test_reversed_version_sorting(self):
"""Test reversed version sorting."""
assert natsort(['1', '5', '10', '50'], reverse=True) == ['50', '10', '5', '1']
consensus = []
for read in samplebamdict[SAMPLENAME].fetch(SEGMENT):
cigartup= read.cigartuples
cigarstr = read.cigarstring
unfiltreadidx = read.get_reference_positions(full_length=True)
unfiltread = list(read.query_sequence)
readqual = read.query_qualities
cigarseq = cigarTranslate(cigartup)
analyzeRead(cigartup,cigarstr,unfiltreadidx,unfiltread,readqual,cigarseq,read.is_reverse)
bigconsensus = []
keylist = []
for boop in CONSENSUSDICT:
keylist.append(boop)
a = natsort(keylist) #sorting our nucleotide
coveragechecker = 0
tempconsensus = []
for i in a: #move consensus earlier, we're going to need to use it later.
if 'I' in i: #and sum(CONSENSUSDICT[i].values()) < 200: #Include insertion!?!?!?
pass
else:
if sum(CONSENSUSDICT[i].values()) < 200:
coveragechecker+=1
if sum(CONSENSUSDICT[i].values()) < args.covercutoff: #Do we want to filter?!?!?!
bigconsensus.append('N')
tempconsensus.append('N')
else:
help='Optional min and max for y-axis.')
parser.add_argument('--title', help='Optional title for plot.')
parser.add_argument(
"--paper",
action='store_true',
help="Create plots designed for the paper, rather than for general use."
)
args = parser.parse_args()
pyplot.rc('mathtext', default='regular') # Don't use italics for mathmode.
# Try to sort the data.
expdata = args.data
if args.sort:
try:
from natsort import natsort
expdata = natsort(expdata)
except:
print 'WARNING: no natsort module found, sorting not available.'
# Prepare the plots.
fig, ax = pyplot.subplots()
# TODO - remove longest common prefix from label strings.
# Plot the EXP data.
lines = ['-', '--', '-.', ':']
if args.test == 'linU':
xlabel = 'Time (s)'
ylabel = 'Displacement (mm)'
title = 'Displacements vs Time'
elif args.test == 'linF':
xlabel = 'Time (s)'
