def fetchFiles(self, symLink):
"""
Fn for testFetchJobStoreFiles() and testFetchJobStoreFilesWSymlinks().
Runs a workflow that imports 'B.txt' and 'mkFile.py' into the
jobStore. 'A.txt', 'C.txt', 'ABC.txt' are then created. This test then
attempts to get a list of these files and copy them over into our
output diectory from the jobStore, confirm that they are present, and
then delete them.
"""
contents = ['A.txt', 'B.txt', 'C.txt', 'ABC.txt', 'mkFile.py']
cmd = [python, os.path.abspath('src/toil/utils/toilDebugFile.py'),
self.jobStoreDir,
'--fetch', '*A.txt', '*B.txt', '*C.txt', '*ABC.txt', '*mkFile.py',
'--localFilePath=' + self.outputDir,
'--useSymlinks=' + str(symLink)]
print(cmd)
subprocess.check_call(cmd)
for xfile in contents:
matchingFilesFound = glob(glob_pattern='*' + xfile, directoryname=self.outputDir)
self.assertGreaterEqual(len(matchingFilesFound), 1)
for fileFound in matchingFilesFound:
assert fileFound.endswith(xfile) and os.path.exists(fileFound)
if fileFound.endswith('-' + xfile):
os.remove(fileFound)
def printContentsOfJobStore(jobStorePath: str,
nameOfJob: Optional[str] = None) -> None:
"""
Fetch a list of all files contained in the jobStore directory input if
nameOfJob is not declared, otherwise it only prints out the names of files
for that specific job for which it can find a match. Also creates a logFile
containing this same record of job files in the working directory.
:param jobStorePath: Directory path to recursively look for files.
:param nameOfJob: Default is None, which prints out all files in the jobStore.
If specified, it will print all jobStore files that have been written to the
jobStore by that job.
"""
if nameOfJob:
glob_pattern = "*" + nameOfJob + "*"
logFile = nameOfJob + "_fileset.txt"
else:
glob_pattern = "*"
logFile = "jobstore_files.txt"
nameOfJob = ""
list_of_files = glob(directoryname=jobStorePath, glob_pattern=glob_pattern)
if os.path.exists(logFile):
os.remove(logFile)
for gfile in sorted(list_of_files):
if not gfile.endswith('.new'):
logger.debug(f"{nameOfJob} File: {os.path.basename(gfile)}")
with open(logFile, "a+") as f:
f.write(os.path.basename(gfile))
f.write("\n")
def fetchJobStoreFiles(jobStore, options):
"""
Takes a list of file names as glob patterns, searches for these within a
given directory, and attempts to take all of the files found and copy them
into options.localFilePath.
:param jobStore: A fileJobStore object.
:param options.fetch: List of file glob patterns to search
for in the jobStore and copy into options.localFilePath.
:param options.localFilePath: Local directory to copy files into.
:param options.jobStore: The path to the jobStore directory.
"""
for jobStoreFile in options.fetch:
jobStoreHits = glob(directoryname=options.jobStore,
glob_pattern=jobStoreFile)
for jobStoreFileID in jobStoreHits:
logger.debug(
f"Copying job store file: {jobStoreFileID} to {options.localFilePath[0]}"
)
jobStore.readFile(jobStoreFileID,
os.path.join(options.localFilePath[0],
os.path.basename(jobStoreFileID)),
symlink=options.useSymlinks)