Esempio n. 1
0
def nestedRenameFiles(dir_path, old_filenames, new_filenames, silent_level=0):
    """
        Rename all files in "old_filenames" under "dir_path" to
        "new_filenames". The first file in "old_filenames" will be
        renamed to the first file in "new_filenames", and similar for
        the rest. If "leaf_only" is specified as "True" then only
        files in leaf subdirectories are modified. "silent_level"=0:
        no output on screen; 1: short output; 2: full output
    """
    old_filenames = listR.toList(old_filenames)
    new_filenames = listR.toList(new_filenames)
    for old_name, new_name in zip(old_filenames, new_filenames):
        dirL = dirR.listNestedDirContainsFiles(dir_path, old_name)
        for aDir in dirL:
            if path.exists(path.join(aDir, new_name)):
                print("File "+path.join(aDir, new_name)+" already exists! skipped.")
            else:
                os.rename(path.join(aDir, old_name), path.join(aDir, new_name))
                if silent_level==0:
                    pass
                elif silent_level==1:
                    print("File "+_relativePath(dir_path, full_path)
                          +" renamed to "+_relativePath(dir_path,new_full_path))
                else:
                    print("File "+full_path+" renamed to "+new_full_path)
Esempio n. 2
0
def combineFilesWithParas(dir_path, filename, targetDir=None, connector="="):
    """ Combine all files with name filename. Also, parameters will be added as seperated
    columns in the combined file. The order of parameters of corresponding to the inserted
    columns is indicated by the file name. """
    filenameL = listR.toList(filename)

    for filename in filenameL: # do the following for each file:
        # get list of common parameters:
        allParas = []
        for aDir in dirR.listNestedDirContainsFiles(dir_path, filename):
            allParas.append(listR.readCSESD(path.join(aDir, filename)).keys())
        allParas = listR.removeDuplicates(listR.intersect(allParas))

        # generate modified file name:
        if targetDir == None: targetDir=getcwd()
        new_filename = path.join(targetDir,path.splitext(filename)[0]+"("+",".join(listR.stringizeL(allParas))+")"+path.splitext(filename)[1])

        # generate the combined data file:
        outFile = open(new_filename, "w")
        for aDir in dirR.listNestedDirContainsFiles(dir_path, filename):
            to_read = path.join(aDir, filename) # target file: path+name
            parasD = listR.readCSESD(to_read) # get a dictionary of parameter values
            para_str = " ".join(listR.stringizeL(listR.getValueListFromDict(allParas, parasD))) # get the associated parameter list in string format
            inFile = open(to_read, "r")
            buffer = inFile.readlines() # read target file
            for aLine in buffer: # concarnate target file with parameter list:
                outFile.write(para_str+" "+aLine)
            inFile.close()
        outFile.close()
Esempio n. 3
0
def copyFiles(fileNames, sourceDir, targetDir, silent=True):
  """ Copy files whose names are given in the list filenames
  from sourceDir to targetDir.
  """
  fileNames = listR.toList(fileNames)
  for filename in fileNames:
    copyFile(filename, sourceDir, targetDir, None, silent)
Esempio n. 4
0
def preExePost(argValueList,
               preFncs=[],
               exeFile="",
               postFncs=[],
               order=[],
               sleepTime=0):  # exeFile="" to do nothing
    """
		Execute exeFile with a choice of arguments specified in
		argNameList. argValueList is a nested list, elements from
		each sublist consist a "choice" of arguments. order is applied
		before this choice of arguments is used. For each possible
		combination of values of arguments, preFncs are executed first,
		then exeFile is executed, then postFncs are executed.

		preFnc is used to do some possible clean work between each
		run, it has one parameter: a list of choice of parameters.
		postFun is similar, only executed after each run.

		Note that exeFile is a string that contains the executable
		file, and the string can contain other arguments, as long as
		these arguments are positioned in front of those automatically
		generated arguments list of the form argNameList. The list
		argValueList is very free, for example, the list
		[["1 2","3 4"],...] will use "1 2" or "3 4" as the first
		generated parameter. This is especially useful if a program
		reads argument for a loop to generate one file.
	"""
    for valueCmb in listR.outer(
            map(listR.toList, listR.toList(argValueList)
                )):  # take a perticular list of possible combination of values
        valueCmb = list(listR.FL(valueCmb))
        # execute preFnc
        if preFncs != []:
            for pre_func in listR.toList(preFncs):
                pre_func(valueCmb)

        # execute exeFile with given choice of arguments
        if exeFile != "":
            runCommand(
                exeFile + " " +
                listR.listToStr(listR.applyOrderList(order, valueCmb)),
                os.path.dirname(exeFile), sleepTime)

        # execute postFnc
        if postFncs != []:
            for post_func in listR.toList(postFncs):
                post_func(valueCmb)
Esempio n. 5
0
def copyinPreExeMkdirCopyoutPost(exeFile,
                                 toBeCopiedDir,
                                 copyToBaseDir,
                                 toBeCopiedFileNames,
                                 argNameList,
                                 argValueList,
                                 callOrderList=None,
                                 sleepTime=0,
                                 preFncs=[],
                                 postFncs=[],
                                 connectionSymbol="=",
                                 seperationSymbol=","):
    """
		Generate data files using exeFile. argNameList is a list of
		variable names. argValueList is a nested list, whose sublist
		at position i is a list of possible values of variable given at
		position i in argNameList. For each possible combination of
		values of arguments, exeFile is executed, then toBeCopiedFileNames
		in toBeCopiedDir are copied to generated directory under
		copyToBaseDir.The structure of the directories under
		copyToBaseDir are like (when connectionSymbol is "="):
		copyToBaseDir/arg1=xx/arg2=xx/arg3=xx/...
		the data file corresponding to a certain combination of
		values of argument is copied to the bottom directory with
		which the name of the path indicates the choice of arguments.

		callOrderList specifies the order of argument when being
		called by exeFile. For example, if argNameList is like
		["a1","a2","a3",...], callOrderList can be like
		["a3","a1","a2",...]

		preFnc is used to do some possible clean work between each
		run, it has two parameters: a list of choice of parameters
		and a path to the generated directory corresponding to this
		choice of paramters.

		Note that exeFile is a string that contains the executable
		file, and the string can contain other arguments, as long as
		these arguments are positioned in front of those automatically
		generated arguments list of the form argNameList. The list
		argValueList is very free, for example, the list
		[["1 2","3 4"],...] will use "1 2" or "3 4" as the first
		generated parameter. This is especially useful if a program
		reads argument for a loop to generate one file.

		See help for preExeMkdirPost for additional information.
	"""
    def copyinHook(valueCmb):
        # make a list of type [arg1=xx, arg2=xx,...], join them with os.path.join
        copyToFullDir = _argListToDir(argNameList, valueCmb, copyToBaseDir)
        copyFiles(toBeCopiedFileNames, toBeCopiedDir, copyToFullDir)

    # add copyinHook to preFncs:
    preFncs = listR.toList(preFncs)
    preFncs.insert(0, copyinHook)

    preExeMkdirPost(argNameList, argValueList, copyToBaseDir, preFncs, exeFile,
                    callOrderList, postFncs, sleepTime, connectionSymbol,
                    seperationSymbol)
Esempio n. 6
0
def readCSED(dir_name, connectionSymbol="=", seperationSymbol=","): #CSE: Comma Seperated Equations
  """ Return a dic of the form {arg1:value1, ...} if with
  connectionSymbol="=" and seperationSymbol=",", dir_name is
  like arg1=value1,arg2=value2,...
  Values are in string form.
  """
  if connectionSymbol not in dir_name: return {}
  return dict(map(lambda x:listR.split(x, connectionSymbol), listR.toList(dir_name.split(seperationSymbol))))
Esempio n. 7
0
def nestedDeleteFiles(dir_path, filenames, silence_level=0, leaf_only=False):
    """
        Delete all files in the list "filenames" under "dir_path".
    """
    filenames = listR.toList(filenames)
    for name in filenames:
        dirL = dirR.listNestedDirContainsFiles(dir_path, name)
        for aDir in dirL:
            remove(path.join(aDir, name))
            if silence_level>0: print("File "+full_path+" deleted.")
Esempio n. 8
0
def collectFile(pathDir, filenames, targetDir=None):
  """ Collect files of name filenames under pathDir to the targetDir,
  then rename them according to the parameters.
  """
  filenames = listR.toList(filenames)
  if targetDir == None: targetDir=getcwd()
  for aDir in dirR.listNestedDirContainsFiles(pathDir, filenames):
    for aFile in filenames:
      paraList = readCSEFullpathD(aDir).items() # read directory structure
      toAdd="-" + ",".join(map(lambda x:"=".join(x), paraList)) # transform the directory strunction into a string
      changedName = path.splitext(aFile)[0] + toAdd + path.splitext(aFile)[1] # add the string to the name of the file
      copyFile(aFile, aDir, targetDir, changedName) # copy file
Esempio n. 9
0
def descendDirTree(baseDir, mustBeDefined, connectionSymbol="=", seperationSymbol=","):
  """
    Return a list of directories and a list of the corresponding var_name:var_value
    dictionaries under baseDir. The valuse are given using string, like "1" instead of 1.
    The subdirectaries are so chosen that all variables in mustBeDefined
    must be defined.
  """
  mustBeDefined = listR.toList(mustBeDefined)
  tmp_result = dirR.listDir(baseDir)
  def qf(var): # used in the filter, any result that does not fulled define vars in mustBeDefined will be filterd
    return listR.containedIn(mustBeDefined, readCSEFullpathD(var, connectionSymbol="=", seperationSymbol=",").keys())
  return filter(qf, tmp_result)
Esempio n. 10
0
def preExeMkdirPost(argNameList, argValueList, baseDir, preFncs=[], exeFile="", callOrderList=None, postFncs=[], sleepTime=0, connectionSymbol="=", seperationSymbol=","):
  """
    Execute exeFile with a choice of arguments specified in
    argNameList. argValueList is a nested list, elements from
    each sublist consist a "choice" of arguments. order is applied
    before this choice of arguments is used. For each possible
    combination of values of arguments, preFncs are executed first,
    then exeFile is executed, then postFncs are executed.

    Note that exeFile is a string that contains the executable
    file, and the string can contain other arguments, as long as
    these arguments are positioned in front of those automatically
    generated arguments list of the form argNameList. The list
    argValueList is very free, for example, the list
    [["1 2","3 4"],...] will use "1 2" or "3 4" as the first
    generated parameter. This is especially useful if a program
    reads argument for a loop to generate one file.

    A directory tree is constructed, with type (with connectionSymbol="=")
    arg1=xx/arg2=xx/...
    If argNameList and argValueList are nested (once), such as
    argNameList=[[arg11,arg12],arg2,...] with argValueList as
    [[["a","e"],["b","f"]],["c","d"]...], then directories are
    like arg11=a, arg12=e/arg2=c/...
    argNameList and argValueList must be consistent, i.e. one
    name corresponds to one list of values; a group of names
    correspond to one list of group of values.

    preFnc is used to do some possible clean work between each
    run, it has two parameters: a list of choice of parameters, and
    a path of the associated directory. postFun is similar, only
    executed after each run.

    callOrderList specifies the order of parameters when the external
    program is called.
  """
  if len(argNameList) != len(argValueList):
    print("The lists argNameList and argValueList have different length!\n")
    return False

  def mkdirHook(valueCmb):
    # make a directory corresponding to a value combination:
    makeDir(_argListToDir(argNameList, valueCmb, baseDir, connectionSymbol, seperationSymbol))

  # add mkdirHook to preFncs:
  preFncs=listR.toList(preFncs)
  preFncs.insert(0,mkdirHook)

  if callOrderList == None: callOrderList = argNameList
  order = listR.createOrderList(listR.FLL(callOrderList), listR.FLL(argNameList))

  preExePost(argValueList, preFncs, exeFile, postFncs, order, sleepTime)
Esempio n. 11
0
def takeRatioWithFirstLine(filename, columns=[2], newfilename=""):
    """ For all specified columns, convert all elements to the ratio of it over
    the 1st one in this column. Note that columns are indexed from 1. """
    # initialize
    if newfilename == "": newfilename = filename;
    columns = listR.toList(columns);

    buffer = readData(filename); # read data from disk
    first = list(buffer[0]); # a list of first elements in each column

    for i in range(len(buffer)): # loop through rows
        for j in columns: # loop through columns
            buffer[i][j-1] = buffer[i][j-1] / float(first[j-1]); # take ratio
    writeData(newfilename, buffer); # write data back to disk
Esempio n. 12
0
def preExePost(argValueList, preFncs=[], exeFile="", postFncs=[], order=[], sleepTime=0): # exeFile="" to do nothing
  """
    Execute exeFile with a choice of arguments specified in
    argNameList. argValueList is a nested list, elements from
    each sublist consist a "choice" of arguments. order is applied
    before this choice of arguments is used. For each possible
    combination of values of arguments, preFncs are executed first,
    then exeFile is executed, then postFncs are executed.

    preFnc is used to do some possible clean work between each
    run, it has one parameter: a list of choice of parameters.
    postFun is similar, only executed after each run.

    Note that exeFile is a string that contains the executable
    file, and the string can contain other arguments, as long as
    these arguments are positioned in front of those automatically
    generated arguments list of the form argNameList. The list
    argValueList is very free, for example, the list
    [["1 2","3 4"],...] will use "1 2" or "3 4" as the first
    generated parameter. This is especially useful if a program
    reads argument for a loop to generate one file.
  """
  for valueCmb in listR.outer(map(listR.toList, listR.toList(argValueList))): # take a perticular list of possible combination of values
    valueCmb = list(listR.FL(valueCmb))
    # execute preFnc
    if preFncs!=[]:
      for pre_func in listR.toList(preFncs):
        pre_func(valueCmb)

    # execute exeFile with given choice of arguments
    if exeFile!="":
      runCommand(exeFile + " " + listR.listToStr(listR.applyOrderList(order,valueCmb)), path.dirname(exeFile), sleepTime)

    # execute postFnc
    if postFncs!=[]:
      for post_func in listR.toList(postFncs):
        post_func(valueCmb)
Esempio n. 13
0
def copyinPreExeMkdirCopyoutPost(exeFile, toBeCopiedDir, copyToBaseDir, toBeCopiedFileNames,
  argNameList, argValueList, callOrderList=None, sleepTime=0, preFncs=[], postFncs=[],
  connectionSymbol="=",seperationSymbol=","):
  """
    Generate data files using exeFile. argNameList is a list of
    variable names. argValueList is a nested list, whose sublist
    at position i is a list of possible values of variable given at
    position i in argNameList. For each possible combination of
    values of arguments, exeFile is executed, then toBeCopiedFileNames
    in toBeCopiedDir are copied to generated directory under
    copyToBaseDir.The structure of the directories under
    copyToBaseDir are like (when connectionSymbol is "="):
    copyToBaseDir/arg1=xx/arg2=xx/arg3=xx/...
    the data file corresponding to a certain combination of
    values of argument is copied to the bottom directory with
    which the name of the path indicates the choice of arguments.

    callOrderList specifies the order of argument when being
    called by exeFile. For example, if argNameList is like
    ["a1","a2","a3",...], callOrderList can be like
    ["a3","a1","a2",...]

    preFnc is used to do some possible clean work between each
    run, it has two parameters: a list of choice of parameters
    and a path to the generated directory corresponding to this
    choice of paramters.

    Note that exeFile is a string that contains the executable
    file, and the string can contain other arguments, as long as
    these arguments are positioned in front of those automatically
    generated arguments list of the form argNameList. The list
    argValueList is very free, for example, the list
    [["1 2","3 4"],...] will use "1 2" or "3 4" as the first
    generated parameter. This is especially useful if a program
    reads argument for a loop to generate one file.

    See help for preExeMkdirPost for additional information.
  """
  def copyinHook(valueCmb):
    # make a list of type [arg1=xx, arg2=xx,...], join them with path.join
    copyToFullDir = _argListToDir(argNameList, valueCmb, copyToBaseDir)
    copyFiles(toBeCopiedFileNames, toBeCopiedDir, copyToFullDir)
  # add copyinHook to preFncs:
  preFncs=listR.toList(preFncs)
  preFncs.insert(0,copyinHook)

  preExeMkdirPost(argNameList, argValueList, copyToBaseDir, preFncs,
    exeFile, callOrderList, postFncs, sleepTime, connectionSymbol, seperationSymbol)
Esempio n. 14
0
def nestedRenameFilesAdd(dir_path, filenames, str_add, add_to_front=True):
  """
    Rename all files in "filenames" under "dir_path" by adding
    the string "str_add" to the front. If "add_to_front" is
    specified as "False", changes will be made to the end of the
    file name, otherwise (by default) it will be add to the front
    of the file name.
  """
  filenames = listR.toList(filenames)
  for name in filenames:
    if add_to_front == True:
      str_mode = str_add+"%s%s"
    else:
      str_mode = "%s" + str_add + "%s"
    new_filename = str_mode % (path.splitext(name))
    nestedRenameFiles(dir_path, name, new_filename)
Esempio n. 15
0
def _argListToDir(argNameList, valueCmb, baseDir, connectionSymbol="=", seperationSymbol=","):
  """ Return a associated directory (path+name) to
  argNameList and the corresponding valueCmb.
    argNameList can be one level nested, e.g:
    [[a,b,c],[d,e],f,...]
    The corresponding valueCmb is flattened:
    [1,2,3,4,5,6,...]
    And the generated dir name is (w/ default "=" & ",")
    a=1,b=2,c=3/d=4,e=5/f=6/...
  """
  # make a list of type [[arg1=xx, arg2=xx],arg3=xx,...]:
  tmp = listR.mimic(argNameList, listR.strZip(listR.FL(argNameList), listR.FL(valueCmb), connectionSymbol))
  # join sublist using seperationSymbol:
  tmp = map(lambda x:seperationSymbol.join(listR.toList(x)), tmp)
  # join them with path.join
  return path.join(baseDir, path.sep.join(tmp))
Esempio n. 16
0
def addColumnsToFile(filename, columns, add_before_original=True):
  """ Add columns of data into a file with filename, before or after the original data
  in each line. The variable "columns" should be a list of strings. Each string will be
  inserted accordingly into the file. This list will be re-used if it is shorter than the
  length of the file.
  """
  tempFile = "TEMP.tmp"
  outFile = open(tempFile, "w")
  inFile = open(filename, "r")
  columns = listR.toList(columns)
  index = 0
  aLine = inFile.readline()
  while aLine:
    if add_before_original:
      outFile.write(columns[index] + aLine)
    else:
      outFile.write(aLine + columns[index])
    index = listR.next(columns, index)
    aLine = inFile.readline()
  inFile.close
  outFile.close()
  copy(tempFile, filename)