def nestedRenameFiles(dir_path, old_filenames, new_filenames, silent_level=0):
"""
Rename all files in "old_filenames" under "dir_path" to
"new_filenames". The first file in "old_filenames" will be
renamed to the first file in "new_filenames", and similar for
the rest. If "leaf_only" is specified as "True" then only
files in leaf subdirectories are modified. "silent_level"=0:
no output on screen; 1: short output; 2: full output
"""
old_filenames = listR.toList(old_filenames)
new_filenames = listR.toList(new_filenames)
for old_name, new_name in zip(old_filenames, new_filenames):
dirL = dirR.listNestedDirContainsFiles(dir_path, old_name)
for aDir in dirL:
if path.exists(path.join(aDir, new_name)):
print("File "+path.join(aDir, new_name)+" already exists! skipped.")
else:
os.rename(path.join(aDir, old_name), path.join(aDir, new_name))
if silent_level==0:
pass
elif silent_level==1:
print("File "+_relativePath(dir_path, full_path)
+" renamed to "+_relativePath(dir_path,new_full_path))
else:
print("File "+full_path+" renamed to "+new_full_path)
def combineFilesWithParas(dir_path, filename, targetDir=None, connector="="):
""" Combine all files with name filename. Also, parameters will be added as seperated
columns in the combined file. The order of parameters of corresponding to the inserted
columns is indicated by the file name. """
filenameL = listR.toList(filename)
for filename in filenameL: # do the following for each file:
# get list of common parameters:
allParas = []
for aDir in dirR.listNestedDirContainsFiles(dir_path, filename):
allParas.append(listR.readCSESD(path.join(aDir, filename)).keys())
allParas = listR.removeDuplicates(listR.intersect(allParas))
# generate modified file name:
if targetDir == None: targetDir=getcwd()
new_filename = path.join(targetDir,path.splitext(filename)[0]+"("+",".join(listR.stringizeL(allParas))+")"+path.splitext(filename)[1])
# generate the combined data file:
outFile = open(new_filename, "w")
for aDir in dirR.listNestedDirContainsFiles(dir_path, filename):
to_read = path.join(aDir, filename) # target file: path+name
parasD = listR.readCSESD(to_read) # get a dictionary of parameter values
para_str = " ".join(listR.stringizeL(listR.getValueListFromDict(allParas, parasD))) # get the associated parameter list in string format
inFile = open(to_read, "r")
buffer = inFile.readlines() # read target file
for aLine in buffer: # concarnate target file with parameter list:
outFile.write(para_str+" "+aLine)
inFile.close()
outFile.close()
def copyFiles(fileNames, sourceDir, targetDir, silent=True):
""" Copy files whose names are given in the list filenames
from sourceDir to targetDir.
"""
fileNames = listR.toList(fileNames)
for filename in fileNames:
copyFile(filename, sourceDir, targetDir, None, silent)
def preExePost(argValueList,
preFncs=[],
exeFile="",
postFncs=[],
order=[],
sleepTime=0): # exeFile="" to do nothing
"""
Execute exeFile with a choice of arguments specified in
argNameList. argValueList is a nested list, elements from
each sublist consist a "choice" of arguments. order is applied
before this choice of arguments is used. For each possible
combination of values of arguments, preFncs are executed first,
then exeFile is executed, then postFncs are executed.
preFnc is used to do some possible clean work between each
run, it has one parameter: a list of choice of parameters.
postFun is similar, only executed after each run.
Note that exeFile is a string that contains the executable
file, and the string can contain other arguments, as long as
these arguments are positioned in front of those automatically
generated arguments list of the form argNameList. The list
argValueList is very free, for example, the list
[["1 2","3 4"],...] will use "1 2" or "3 4" as the first
generated parameter. This is especially useful if a program
reads argument for a loop to generate one file.
"""
for valueCmb in listR.outer(
map(listR.toList, listR.toList(argValueList)
)): # take a perticular list of possible combination of values
valueCmb = list(listR.FL(valueCmb))
# execute preFnc
if preFncs != []:
for pre_func in listR.toList(preFncs):
pre_func(valueCmb)
# execute exeFile with given choice of arguments
if exeFile != "":
runCommand(
exeFile + " " +
listR.listToStr(listR.applyOrderList(order, valueCmb)),
os.path.dirname(exeFile), sleepTime)
# execute postFnc
if postFncs != []:
for post_func in listR.toList(postFncs):
post_func(valueCmb)
def copyinPreExeMkdirCopyoutPost(exeFile,
toBeCopiedDir,
copyToBaseDir,
toBeCopiedFileNames,
argNameList,
argValueList,
callOrderList=None,
sleepTime=0,
preFncs=[],
postFncs=[],
connectionSymbol="=",
seperationSymbol=","):
"""
Generate data files using exeFile. argNameList is a list of
variable names. argValueList is a nested list, whose sublist
at position i is a list of possible values of variable given at
position i in argNameList. For each possible combination of
values of arguments, exeFile is executed, then toBeCopiedFileNames
in toBeCopiedDir are copied to generated directory under
copyToBaseDir.The structure of the directories under
copyToBaseDir are like (when connectionSymbol is "="):
copyToBaseDir/arg1=xx/arg2=xx/arg3=xx/...
the data file corresponding to a certain combination of
values of argument is copied to the bottom directory with
which the name of the path indicates the choice of arguments.
callOrderList specifies the order of argument when being
called by exeFile. For example, if argNameList is like
["a1","a2","a3",...], callOrderList can be like
["a3","a1","a2",...]
preFnc is used to do some possible clean work between each
run, it has two parameters: a list of choice of parameters
and a path to the generated directory corresponding to this
choice of paramters.
Note that exeFile is a string that contains the executable
file, and the string can contain other arguments, as long as
these arguments are positioned in front of those automatically
generated arguments list of the form argNameList. The list
argValueList is very free, for example, the list
[["1 2","3 4"],...] will use "1 2" or "3 4" as the first
generated parameter. This is especially useful if a program
reads argument for a loop to generate one file.
See help for preExeMkdirPost for additional information.
"""
def copyinHook(valueCmb):
# make a list of type [arg1=xx, arg2=xx,...], join them with os.path.join
copyToFullDir = _argListToDir(argNameList, valueCmb, copyToBaseDir)
copyFiles(toBeCopiedFileNames, toBeCopiedDir, copyToFullDir)
# add copyinHook to preFncs:
preFncs = listR.toList(preFncs)
preFncs.insert(0, copyinHook)
preExeMkdirPost(argNameList, argValueList, copyToBaseDir, preFncs, exeFile,
callOrderList, postFncs, sleepTime, connectionSymbol,
seperationSymbol)
def readCSED(dir_name, connectionSymbol="=", seperationSymbol=","): #CSE: Comma Seperated Equations
""" Return a dic of the form {arg1:value1, ...} if with
connectionSymbol="=" and seperationSymbol=",", dir_name is
like arg1=value1,arg2=value2,...
Values are in string form.
"""
if connectionSymbol not in dir_name: return {}
return dict(map(lambda x:listR.split(x, connectionSymbol), listR.toList(dir_name.split(seperationSymbol))))
def nestedDeleteFiles(dir_path, filenames, silence_level=0, leaf_only=False):
"""
Delete all files in the list "filenames" under "dir_path".
"""
filenames = listR.toList(filenames)
for name in filenames:
dirL = dirR.listNestedDirContainsFiles(dir_path, name)
for aDir in dirL:
remove(path.join(aDir, name))
if silence_level>0: print("File "+full_path+" deleted.")
def collectFile(pathDir, filenames, targetDir=None):
""" Collect files of name filenames under pathDir to the targetDir,
then rename them according to the parameters.
"""
filenames = listR.toList(filenames)
if targetDir == None: targetDir=getcwd()
for aDir in dirR.listNestedDirContainsFiles(pathDir, filenames):
for aFile in filenames:
paraList = readCSEFullpathD(aDir).items() # read directory structure
toAdd="-" + ",".join(map(lambda x:"=".join(x), paraList)) # transform the directory strunction into a string
changedName = path.splitext(aFile)[0] + toAdd + path.splitext(aFile)[1] # add the string to the name of the file
copyFile(aFile, aDir, targetDir, changedName) # copy file
def descendDirTree(baseDir, mustBeDefined, connectionSymbol="=", seperationSymbol=","):
"""
Return a list of directories and a list of the corresponding var_name:var_value
dictionaries under baseDir. The valuse are given using string, like "1" instead of 1.
The subdirectaries are so chosen that all variables in mustBeDefined
must be defined.
"""
mustBeDefined = listR.toList(mustBeDefined)
tmp_result = dirR.listDir(baseDir)
def qf(var): # used in the filter, any result that does not fulled define vars in mustBeDefined will be filterd
return listR.containedIn(mustBeDefined, readCSEFullpathD(var, connectionSymbol="=", seperationSymbol=",").keys())
return filter(qf, tmp_result)
def preExeMkdirPost(argNameList, argValueList, baseDir, preFncs=[], exeFile="", callOrderList=None, postFncs=[], sleepTime=0, connectionSymbol="=", seperationSymbol=","):
"""
Execute exeFile with a choice of arguments specified in
argNameList. argValueList is a nested list, elements from
each sublist consist a "choice" of arguments. order is applied
before this choice of arguments is used. For each possible
combination of values of arguments, preFncs are executed first,
then exeFile is executed, then postFncs are executed.
Note that exeFile is a string that contains the executable
file, and the string can contain other arguments, as long as
these arguments are positioned in front of those automatically
generated arguments list of the form argNameList. The list
argValueList is very free, for example, the list
[["1 2","3 4"],...] will use "1 2" or "3 4" as the first
generated parameter. This is especially useful if a program
reads argument for a loop to generate one file.
A directory tree is constructed, with type (with connectionSymbol="=")
arg1=xx/arg2=xx/...
If argNameList and argValueList are nested (once), such as
argNameList=[[arg11,arg12],arg2,...] with argValueList as
[[["a","e"],["b","f"]],["c","d"]...], then directories are
like arg11=a, arg12=e/arg2=c/...
argNameList and argValueList must be consistent, i.e. one
name corresponds to one list of values; a group of names
correspond to one list of group of values.
preFnc is used to do some possible clean work between each
run, it has two parameters: a list of choice of parameters, and
a path of the associated directory. postFun is similar, only
executed after each run.
callOrderList specifies the order of parameters when the external
program is called.
"""
if len(argNameList) != len(argValueList):
print("The lists argNameList and argValueList have different length!\n")
return False
def mkdirHook(valueCmb):
# make a directory corresponding to a value combination:
makeDir(_argListToDir(argNameList, valueCmb, baseDir, connectionSymbol, seperationSymbol))
# add mkdirHook to preFncs:
preFncs=listR.toList(preFncs)
preFncs.insert(0,mkdirHook)
if callOrderList == None: callOrderList = argNameList
order = listR.createOrderList(listR.FLL(callOrderList), listR.FLL(argNameList))
preExePost(argValueList, preFncs, exeFile, postFncs, order, sleepTime)
def takeRatioWithFirstLine(filename, columns=[2], newfilename=""):
""" For all specified columns, convert all elements to the ratio of it over
the 1st one in this column. Note that columns are indexed from 1. """
# initialize
if newfilename == "": newfilename = filename;
columns = listR.toList(columns);
buffer = readData(filename); # read data from disk
first = list(buffer[0]); # a list of first elements in each column
for i in range(len(buffer)): # loop through rows
for j in columns: # loop through columns
buffer[i][j-1] = buffer[i][j-1] / float(first[j-1]); # take ratio
writeData(newfilename, buffer); # write data back to disk
def preExePost(argValueList, preFncs=[], exeFile="", postFncs=[], order=[], sleepTime=0): # exeFile="" to do nothing
"""
Execute exeFile with a choice of arguments specified in
argNameList. argValueList is a nested list, elements from
each sublist consist a "choice" of arguments. order is applied
before this choice of arguments is used. For each possible
combination of values of arguments, preFncs are executed first,
then exeFile is executed, then postFncs are executed.
preFnc is used to do some possible clean work between each
run, it has one parameter: a list of choice of parameters.
postFun is similar, only executed after each run.
Note that exeFile is a string that contains the executable
file, and the string can contain other arguments, as long as
these arguments are positioned in front of those automatically
generated arguments list of the form argNameList. The list
argValueList is very free, for example, the list
[["1 2","3 4"],...] will use "1 2" or "3 4" as the first
generated parameter. This is especially useful if a program
reads argument for a loop to generate one file.
"""
for valueCmb in listR.outer(map(listR.toList, listR.toList(argValueList))): # take a perticular list of possible combination of values
valueCmb = list(listR.FL(valueCmb))
# execute preFnc
if preFncs!=[]:
for pre_func in listR.toList(preFncs):
pre_func(valueCmb)
# execute exeFile with given choice of arguments
if exeFile!="":
runCommand(exeFile + " " + listR.listToStr(listR.applyOrderList(order,valueCmb)), path.dirname(exeFile), sleepTime)
# execute postFnc
if postFncs!=[]:
for post_func in listR.toList(postFncs):
post_func(valueCmb)
def copyinPreExeMkdirCopyoutPost(exeFile, toBeCopiedDir, copyToBaseDir, toBeCopiedFileNames,
argNameList, argValueList, callOrderList=None, sleepTime=0, preFncs=[], postFncs=[],
connectionSymbol="=",seperationSymbol=","):
"""
Generate data files using exeFile. argNameList is a list of
variable names. argValueList is a nested list, whose sublist
at position i is a list of possible values of variable given at
position i in argNameList. For each possible combination of
values of arguments, exeFile is executed, then toBeCopiedFileNames
in toBeCopiedDir are copied to generated directory under
copyToBaseDir.The structure of the directories under
copyToBaseDir are like (when connectionSymbol is "="):
copyToBaseDir/arg1=xx/arg2=xx/arg3=xx/...
the data file corresponding to a certain combination of
values of argument is copied to the bottom directory with
which the name of the path indicates the choice of arguments.
callOrderList specifies the order of argument when being
called by exeFile. For example, if argNameList is like
["a1","a2","a3",...], callOrderList can be like
["a3","a1","a2",...]
preFnc is used to do some possible clean work between each
run, it has two parameters: a list of choice of parameters
and a path to the generated directory corresponding to this
choice of paramters.
Note that exeFile is a string that contains the executable
file, and the string can contain other arguments, as long as
these arguments are positioned in front of those automatically
generated arguments list of the form argNameList. The list
argValueList is very free, for example, the list
[["1 2","3 4"],...] will use "1 2" or "3 4" as the first
generated parameter. This is especially useful if a program
reads argument for a loop to generate one file.
See help for preExeMkdirPost for additional information.
"""
def copyinHook(valueCmb):
# make a list of type [arg1=xx, arg2=xx,...], join them with path.join
copyToFullDir = _argListToDir(argNameList, valueCmb, copyToBaseDir)
copyFiles(toBeCopiedFileNames, toBeCopiedDir, copyToFullDir)
# add copyinHook to preFncs:
preFncs=listR.toList(preFncs)
preFncs.insert(0,copyinHook)
preExeMkdirPost(argNameList, argValueList, copyToBaseDir, preFncs,
exeFile, callOrderList, postFncs, sleepTime, connectionSymbol, seperationSymbol)
def nestedRenameFilesAdd(dir_path, filenames, str_add, add_to_front=True):
"""
Rename all files in "filenames" under "dir_path" by adding
the string "str_add" to the front. If "add_to_front" is
specified as "False", changes will be made to the end of the
file name, otherwise (by default) it will be add to the front
of the file name.
"""
filenames = listR.toList(filenames)
for name in filenames:
if add_to_front == True:
str_mode = str_add+"%s%s"
else:
str_mode = "%s" + str_add + "%s"
new_filename = str_mode % (path.splitext(name))
nestedRenameFiles(dir_path, name, new_filename)
def _argListToDir(argNameList, valueCmb, baseDir, connectionSymbol="=", seperationSymbol=","):
""" Return a associated directory (path+name) to
argNameList and the corresponding valueCmb.
argNameList can be one level nested, e.g:
[[a,b,c],[d,e],f,...]
The corresponding valueCmb is flattened:
[1,2,3,4,5,6,...]
And the generated dir name is (w/ default "=" & ",")
a=1,b=2,c=3/d=4,e=5/f=6/...
"""
# make a list of type [[arg1=xx, arg2=xx],arg3=xx,...]:
tmp = listR.mimic(argNameList, listR.strZip(listR.FL(argNameList), listR.FL(valueCmb), connectionSymbol))
# join sublist using seperationSymbol:
tmp = map(lambda x:seperationSymbol.join(listR.toList(x)), tmp)
# join them with path.join
return path.join(baseDir, path.sep.join(tmp))
def addColumnsToFile(filename, columns, add_before_original=True):
""" Add columns of data into a file with filename, before or after the original data
in each line. The variable "columns" should be a list of strings. Each string will be
inserted accordingly into the file. This list will be re-used if it is shorter than the
length of the file.
"""
tempFile = "TEMP.tmp"
outFile = open(tempFile, "w")
inFile = open(filename, "r")
columns = listR.toList(columns)
index = 0
aLine = inFile.readline()
while aLine:
if add_before_original:
outFile.write(columns[index] + aLine)
else:
outFile.write(aLine + columns[index])
index = listR.next(columns, index)
aLine = inFile.readline()
inFile.close
outFile.close()
copy(tempFile, filename)