def _copyOrLink(self, srcURL, destPath, symlink=False):
# linking is not done be default because of issue #1755
srcPath = self._extractPathFromUrl(srcURL)
if self.linkImports or symlink:
os.symlink(os.path.realpath(srcPath), destPath)
else:
atomic_copy(srcPath, destPath)
def _copy_or_link(self, src_path, dst_path, symlink=False):
# linking is not done be default because of issue #1755
srcPath = self._extract_path_from_url(src_path)
if self.linkImports or symlink:
os.symlink(os.path.realpath(srcPath), dst_path)
else:
atomic_copy(srcPath, dst_path)
def updateFile(self, jobStoreFileID, localFilePath):
self._checkJobStoreFileID(jobStoreFileID)
jobStoreFilePath = self._getFilePathFromId(jobStoreFileID)
if os.path.samefile(jobStoreFilePath, localFilePath):
# The files are already the same file. We can't copy on eover the other.
return
atomic_copy(localFilePath, jobStoreFilePath)
def update_file(self, file_id, local_path):
self._check_job_store_file_id(file_id)
jobStoreFilePath = self._get_file_path_from_id(file_id)
if os.path.samefile(jobStoreFilePath, local_path):
# The files are already the same file. We can't copy on eover the other.
return
atomic_copy(local_path, jobStoreFilePath)
def _export_file(self, otherCls, file_id, uri):
if issubclass(otherCls, FileJobStore):
srcPath = self._get_file_path_from_id(file_id)
destPath = self._extract_path_from_url(uri)
executable = getattr(file_id, 'executable', False)
if self.moveExports:
self._move_and_linkback(srcPath,
destPath,
executable=executable)
else:
atomic_copy(srcPath, destPath, executable=executable)
else:
super()._default_export_file(otherCls, file_id, uri)
def _exportFile(self, otherCls, jobStoreFileID, url):
if issubclass(otherCls, FileJobStore):
srcPath = self._getFilePathFromId(jobStoreFileID)
destPath = self._extractPathFromUrl(url)
executable = getattr(jobStoreFileID, 'executable', False)
if self.moveExports:
self._move_and_linkback(srcPath,
destPath,
executable=executable)
else:
atomic_copy(srcPath, destPath, executable=executable)
else:
super(FileJobStore,
self)._defaultExportFile(otherCls, jobStoreFileID, url)
def readFile(self, jobStoreFileID, localFilePath, symlink=False):
self._checkJobStoreFileID(jobStoreFileID)
jobStoreFilePath = self._getFilePathFromId(jobStoreFileID)
localDirPath = os.path.dirname(localFilePath)
executable = getattr(jobStoreFileID, 'executable', False)
if not symlink and os.path.islink(localFilePath):
# We had a symlink and want to clobber it with a hardlink or copy.
os.unlink(localFilePath)
if os.path.exists(localFilePath) and os.path.samefile(
jobStoreFilePath, localFilePath):
# The files are already the same: same name, hardlinked, or
# symlinked. There is nothing to do, and trying to shutil.copyfile
# one over the other will fail.
return
if symlink:
# If the reader will accept a symlink, so always give them one.
# There's less that can go wrong.
try:
os.symlink(jobStoreFilePath, localFilePath)
# It worked!
return
except OSError as e:
if e.errno == errno.EEXIST:
# Overwrite existing file, emulating shutil.copyfile().
os.unlink(localFilePath)
# It would be very unlikely to fail again for same reason but possible
# nonetheless in which case we should just give up.
os.symlink(jobStoreFilePath, localFilePath)
# Now we succeeded and don't need to copy
return
else:
raise
# If we get here, symlinking isn't an option.
if os.stat(jobStoreFilePath).st_dev == os.stat(localDirPath).st_dev:
# It is possible that we can hard link the file.
# Note that even if the device numbers match, we can end up trying
# to create a "cross-device" link.
try:
os.link(jobStoreFilePath, localFilePath)
# It worked!
return
except OSError as e:
if e.errno == errno.EEXIST:
# Overwrite existing file, emulating shutil.copyfile().
os.unlink(localFilePath)
# It would be very unlikely to fail again for same reason but possible
# nonetheless in which case we should just give up.
os.link(jobStoreFilePath, localFilePath)
# Now we succeeded and don't need to copy
return
elif e.errno == errno.EXDEV:
# It's a cross-device link even though it didn't appear to be.
# Just keep going and hit the file copy case.
pass
else:
logger.critical(
'Unexpected OSError when reading file from job store')
logger.critical('jobStoreFilePath: ' + jobStoreFilePath +
' ' +
str(os.path.exists(jobStoreFilePath)))
logger.critical('localFilePath: ' + localFilePath + ' ' +
str(os.path.exists(localFilePath)))
raise
# If we get here, neither a symlink nor a hardlink will work.
# Make a complete copy.
atomic_copy(jobStoreFilePath, localFilePath, executable=executable)
def writeFile(self, localFilePath, jobStoreID=None, cleanup=False):
absPath = self._getUniqueFilePath(localFilePath, jobStoreID, cleanup)
relPath = self._getFileIdFromPath(absPath)
atomic_copy(localFilePath, absPath)
return relPath
def write_file(self, local_path, job_id=None, cleanup=False):
absPath = self._get_unique_file_path(local_path, job_id, cleanup)
relPath = self._get_file_id_from_path(absPath)
atomic_copy(local_path, absPath)
return relPath