def _importFile(self, otherCls, url, sharedFileName=None, hardlink=False):
"""
Import the file at the given URL using the given job store class to retrieve that file.
See also :meth:`.importFile`. This method applies a generic approach to importing: it
asks the other job store class for a stream and writes that stream as either a regular or
a shared file.
:param AbstractJobStore otherCls: The concrete subclass of AbstractJobStore that supports
reading from the given URL and getting the file size from the URL.
:param urlparse.ParseResult url: The location of the file to import.
:param str sharedFileName: Optional name to assign to the imported file within the job store
:return The jobStoreFileId of imported file or None if sharedFileName was given
:rtype: toil.fileStore.FileID or None
"""
if sharedFileName is None:
with self.writeFileStream() as (writable, jobStoreFileID):
otherCls._readFromUrl(url, writable)
return FileID(jobStoreFileID, otherCls.getSize(url))
else:
self._requireValidSharedFileName(sharedFileName)
with self.writeSharedFileStream(sharedFileName) as writable:
otherCls._readFromUrl(url, writable)
return None
def _importFile(self, otherCls, url, sharedFileName=None):
if issubclass(otherCls, FileJobStore):
if sharedFileName is None:
fd, absPath = self._getTempFile(
) # use this to get a valid path to write to in job store
os.close(fd)
os.unlink(absPath)
try:
os.link(os.path.realpath(self._extractPathFromUrl(url)),
absPath)
except OSError:
shutil.copyfile(self._extractPathFromUrl(url), absPath)
return FileID(self._getRelativePath(absPath),
os.stat(absPath).st_size)
else:
self._requireValidSharedFileName(sharedFileName)
path = self._getSharedFilePath(sharedFileName)
try:
os.link(os.path.realpath(self._extractPathFromUrl(url)),
path)
except:
shutil.copyfile(self._extractPathFromUrl(url), path)
return None
else:
return super(FileJobStore,
self)._importFile(otherCls,
url,
sharedFileName=sharedFileName)
def _importFile(self, otherCls, url, sharedFileName=None):
if issubclass(otherCls, FileJobStore):
if sharedFileName is None:
absPath = self._getUniqueName(url.path) # use this to get a valid path to write to in job store
self._copyOrLink(url, absPath)
return FileID(self._getRelativePath(absPath), os.stat(absPath).st_size)
else:
self._requireValidSharedFileName(sharedFileName)
path = self._getSharedFilePath(sharedFileName)
self._copyOrLink(url, path)
return None
else:
return super(FileJobStore, self)._importFile(otherCls, url,
sharedFileName=sharedFileName)
def _importFile(self, otherCls, url, sharedFileName=None):
if issubclass(otherCls, FileJobStore):
if sharedFileName is None:
fd, absPath = self._getTempFile()
shutil.copyfile(self._extractPathFromUrl(url), absPath)
os.close(fd)
return FileID(self._getRelativePath(absPath), os.stat(absPath).st_size)
else:
self._requireValidSharedFileName(sharedFileName)
with self.writeSharedFileStream(sharedFileName) as writable:
with open(self._extractPathFromUrl(url), 'r') as readable:
shutil.copyfileobj(readable, writable)
return None
else:
return super(FileJobStore, self)._importFile(otherCls, url,
sharedFileName=sharedFileName)
def _importFile(self, otherCls, url, sharedFileName=None, hardlink=False):
if issubclass(otherCls, FileJobStore):
if sharedFileName is None:
absPath = self._getUniqueName(url.path) # use this to get a valid path to write to in job store
with self.optionalHardCopy(hardlink):
self._copyOrLink(url, absPath)
# TODO: os.stat(absPath).st_size consistently gives values lower than
# getDirSizeRecursively()
return FileID(self._getRelativePath(absPath), os.stat(absPath).st_size)
else:
self._requireValidSharedFileName(sharedFileName)
path = self._getSharedFilePath(sharedFileName)
with self.optionalHardCopy(hardlink):
self._copyOrLink(url, path)
return None
else:
return super(FileJobStore, self)._importFile(otherCls, url,
sharedFileName=sharedFileName)