def _copyOrLink(self, srcURL, destPath):
# linking is not done be default because of issue #1755
srcPath = self._extractPathFromUrl(srcURL)
if self.linkImports:
os.symlink(os.path.realpath(srcPath), destPath)
else:
atomic_copy(srcPath, destPath)
def updateFile(self, jobStoreFileID, localFilePath):
self._checkJobStoreFileID(jobStoreFileID)
jobStoreFilePath = self._getFilePathFromId(jobStoreFileID)
if os.path.samefile(jobStoreFilePath, localFilePath):
# The files are already the same file. We can't copy on eover the other.
return
atomic_copy(localFilePath, jobStoreFilePath)
def _exportFile(self, otherCls, jobStoreFileID, url):
if issubclass(otherCls, FileJobStore):
srcPath = self._getFilePathFromId(jobStoreFileID)
destPath = self._extractPathFromUrl(url)
if self.moveExports:
self._move_and_linkback(srcPath, destPath)
else:
atomic_copy(srcPath, destPath)
else:
super(FileJobStore, self)._defaultExportFile(otherCls, jobStoreFileID, url)
def _exportFile(self, otherCls, jobStoreFileID, url):
if issubclass(otherCls, FileJobStore):
srcPath = self._getFilePathFromId(jobStoreFileID)
destPath = self._extractPathFromUrl(url)
if self.moveExports:
self._move_and_linkback(srcPath, destPath)
else:
executable = False
if getattr(jobStoreFileID, 'executable', False):
executable = jobStoreFileID.executable
atomic_copy(srcPath, destPath, executable=executable)
else:
super(FileJobStore,
self)._defaultExportFile(otherCls, jobStoreFileID, url)
def readFile(self, jobStoreFileID, localFilePath, symlink=False):
self._checkJobStoreFileID(jobStoreFileID)
jobStoreFilePath = self._getFilePathFromId(jobStoreFileID)
localDirPath = os.path.dirname(localFilePath)
if not symlink and os.path.islink(localFilePath):
# We had a symlink and want to clobber it with a hardlink or copy.
os.unlink(localFilePath)
if os.path.exists(localFilePath) and os.path.samefile(
jobStoreFilePath, localFilePath):
# The files are already the same: same name, hardlinked, or
# symlinked. There is nothing to do, and trying to shutil.copyfile
# one over the other will fail.
return
if symlink:
# If the reader will accept a symlink, so always give them one.
# There's less that can go wrong.
try:
os.symlink(jobStoreFilePath, localFilePath)
# It worked!
return
except OSError as e:
if e.errno == errno.EEXIST:
# Overwrite existing file, emulating shutil.copyfile().
os.unlink(localFilePath)
# It would be very unlikely to fail again for same reason but possible
# nonetheless in which case we should just give up.
os.symlink(jobStoreFilePath, localFilePath)
# Now we succeeded and don't need to copy
return
else:
raise
# If we get here, symlinking isn't an option.
if os.stat(jobStoreFilePath).st_dev == os.stat(localDirPath).st_dev:
# It is possible that we can hard link the file.
# Note that even if the device numbers match, we can end up trying
# to create a "cross-device" link.
try:
os.link(jobStoreFilePath, localFilePath)
# It worked!
return
except OSError as e:
if e.errno == errno.EEXIST:
# Overwrite existing file, emulating shutil.copyfile().
os.unlink(localFilePath)
# It would be very unlikely to fail again for same reason but possible
# nonetheless in which case we should just give up.
os.link(jobStoreFilePath, localFilePath)
# Now we succeeded and don't need to copy
return
elif e.errno == errno.EXDEV:
# It's a cross-device link even though it didn't appear to be.
# Just keep going and hit the file copy case.
pass
else:
logger.critical(
'Unexpected OSError when reading file from job store')
logger.critical('jobStoreFilePath: ' + jobStoreFilePath +
' ' +
str(os.path.exists(jobStoreFilePath)))
logger.critical('localFilePath: ' + localFilePath + ' ' +
str(os.path.exists(localFilePath)))
raise
# If we get here, neither a symlink nor a hardlink will work.
# Make a complete copy.
atomic_copy(jobStoreFilePath, localFilePath)
def writeFile(self, localFilePath, jobStoreID=None, cleanup=False):
absPath = self._getUniqueFilePath(localFilePath, jobStoreID, cleanup)
relPath = self._getFileIdFromPath(absPath)
atomic_copy(localFilePath, absPath)
return relPath
