def _checkJobStoreFileID(self, jobStoreFileID):
"""
Raises NoSuchFileException if the jobStoreFileID does not exist or is not a file.
"""
absPath = os.path.join(self.tempFilesDir, jobStoreFileID)
if not os.path.exists(absPath):
raise NoSuchFileException("File %s does not exist in jobStore" %
jobStoreFileID)
if not os.path.isfile(absPath):
raise NoSuchFileException("Path %s is not a file in the jobStore" %
jobStoreFileID)
def getPublicUrl(self, jobStoreFileID):
self._checkJobStoreFileID(jobStoreFileID)
jobStorePath = self._getAbsPath(jobStoreFileID)
if os.path.exists(jobStorePath):
return 'file:' + jobStorePath
else:
raise NoSuchFileException(jobStoreFileID)
def readFileStream(self, jobStoreFileID):
version = self._getFileVersion(jobStoreFileID)
if version is None: raise NoSuchFileException(jobStoreFileID)
log.debug("Reading version %s of file %s", version, jobStoreFileID)
with self._downloadStream(jobStoreFileID, version,
self.files) as readable:
yield readable
def file_exists(self, file_id):
absPath = self._get_file_path_from_id(file_id)
if (not absPath.startswith(self.jobsDir)
and not absPath.startswith(self.filesDir)
and not absPath.startswith(self.jobFilesDir)):
# Don't even look for it, it is out of bounds.
raise NoSuchFileException(file_id)
try:
st = os.stat(absPath)
except os.error:
return False
if not stat.S_ISREG(st.st_mode):
raise NoSuchFileException(file_id)
return True
def get_public_url(self, jobStoreFileID):
self._check_job_store_file_id(jobStoreFileID)
jobStorePath = self._get_file_path_from_id(jobStoreFileID)
if os.path.exists(jobStorePath):
return 'file:' + jobStorePath
else:
raise NoSuchFileException(jobStoreFileID)
def _checkJobStoreFileID(self, jobStoreFileID):
"""
:raise NoSuchFileException: if the jobStoreFileID does not exist or is not a file
"""
if not self.fileExists(jobStoreFileID):
raise NoSuchFileException("File %s does not exist in jobStore" %
jobStoreFileID)
def _downloadStream(self, fileName, encrypt=True):
"""
Yields a context manager that can be used to read from the bucket
with a stream. See :class:`~toil.jobStores.utils.WritablePipe` for an example.
:param fileName: name of file in bucket to be read
:type fileName: str
:param encrypt: whether or not the file is encrypted
:type encrypt: bool
:return: an instance of ReadablePipe.
:rtype: :class:`~toil.jobStores.utils.ReadablePipe`
"""
blob = self.bucket.get_blob(compat_bytes(fileName), encryption_key=self.sseKey if encrypt else None)
if blob is None:
raise NoSuchFileException(fileName)
class DownloadPipe(ReadablePipe):
def writeTo(self, writable):
try:
blob.download_to_file(writable)
finally:
writable.close()
with DownloadPipe() as readable:
yield readable
def getPublicUrl(self, fileName):
try:
key = self._getKey(fileName)
except:
raise NoSuchFileException(fileName)
return key.generate_url(
expires_in=self.publicUrlExpiration.total_seconds())
def fileExists(self, jobStoreFileID):
absPath = self._getFilePathFromId(jobStoreFileID)
if (not absPath.startswith(self.jobsDir)
and not absPath.startswith(self.filesDir)
and not absPath.startswith(self.jobFilesDir)):
# Don't even look for it, it is out of bounds.
raise NoSuchFileException(jobStoreFileID)
try:
st = os.stat(absPath)
except os.error:
return False
if not stat.S_ISREG(st.st_mode):
raise NoSuchFileException(jobStoreFileID)
return True
def _download_stream(self, fileName, encrypt=True, encoding=None, errors=None):
"""
Yields a context manager that can be used to read from the bucket
with a stream. See :class:`~toil.jobStores.utils.WritablePipe` for an example.
:param fileName: name of file in bucket to be read
:type fileName: str
:param encrypt: whether or not the file is encrypted
:type encrypt: bool
:param str encoding: the name of the encoding used to encode the file. Encodings are the same
as for encode(). Defaults to None which represents binary mode.
:param str errors: an optional string that specifies how encoding errors are to be handled. Errors
are the same as for open(). Defaults to 'strict' when an encoding is specified.
:return: an instance of ReadablePipe.
:rtype: :class:`~toil.jobStores.utils.ReadablePipe`
"""
blob = self.bucket.get_blob(compat_bytes(fileName), encryption_key=self.sseKey if encrypt else None)
if blob is None:
raise NoSuchFileException(fileName)
class DownloadPipe(ReadablePipe):
def writeTo(self, writable):
try:
blob.download_to_file(writable)
finally:
writable.close()
with DownloadPipe(encoding=encoding, errors=errors) as readable:
yield readable
def _checkJobStoreFileID(self, jobStoreFileID):
"""
:raise NoSuchFileException: if the file with ID jobStoreFileID does
not exist or is not a file
"""
if not self.fileExists(jobStoreFileID):
raise NoSuchFileException(jobStoreFileID)
def readSharedFileStream(self, sharedFileName):
assert self._validateSharedFileName(sharedFileName)
sharedFileID = self._newFileID(sharedFileName)
if not self.fileExists(sharedFileID):
raise NoSuchFileException(sharedFileID)
with self._downloadStream(sharedFileID, self.files) as fd:
yield fd
def readFile(self, jobStoreFileID, localFilePath, symlink=False):
# used on non-shared files which will be encrypted if available
# checking for JobStoreID existence
if not self.fileExists(jobStoreFileID):
raise NoSuchFileException(jobStoreFileID)
with open(localFilePath, 'w') as writeable:
blob = self.bucket.get_blob(bytes(jobStoreFileID), encryption_key=self.sseKey)
blob.download_to_file(writeable)
def fileExists(self, jobStoreFileID):
absPath = self._getAbsPath(jobStoreFileID)
if not os.path.exists(absPath):
return False
if not os.path.isfile(absPath):
raise NoSuchFileException("Path %s is not a file in the jobStore" %
jobStoreFileID)
return True
def _writeFile(self, jobStoreID, fileObj, update=False, encrypt=True):
blob = self.bucket.blob(compat_bytes(jobStoreID), encryption_key=self.sseKey if encrypt else None)
if not update:
# TODO: should probably raise a special exception and be added to all jobStores
assert not blob.exists()
else:
if not blob.exists():
raise NoSuchFileException(jobStoreID)
blob.upload_from_file(fileObj)
def fileExists(self, jobStoreFileID):
absPath = self._getAbsPath(jobStoreFileID)
try:
st = os.stat(absPath)
except os.error:
return False
if not stat.S_ISREG(st.st_mode):
raise NoSuchFileException("Path %s is not a file in the jobStore" % jobStoreFileID)
return True
def readFile(self, jobStoreFileID, localFilePath):
# used on non-shared files which will be encrypted if avaliable
headers = self.encryptedHeaders
# checking for JobStoreID existance
if not self.exists(jobStoreFileID):
raise NoSuchFileException(jobStoreFileID)
with open(localFilePath, 'w') as writeable:
self._getKey(jobStoreFileID,
headers).get_contents_to_file(writeable,
headers=headers)
def readFile(self, jobStoreFileID, localFilePath):
try:
with self._downloadStream(jobStoreFileID, self.files) as read_fd:
with open(localFilePath, 'w') as write_fd:
while True:
buf = read_fd.read(self._maxAzureBlockBytes)
write_fd.write(buf)
if not buf: break
except AzureMissingResourceHttpError:
raise NoSuchFileException(jobStoreFileID)
def readSharedFileStream(self, sharedFileName):
assert self._validateSharedFileName(sharedFileName)
jobStoreFileID = self._newFileID(sharedFileName)
version = self._getFileVersion(jobStoreFileID)
if version is None: raise NoSuchFileException(jobStoreFileID)
log.debug("Read version %s from shared file %s (%s)", version,
sharedFileName, jobStoreFileID)
with self._downloadStream(jobStoreFileID, version,
self.files) as readable:
yield readable
def readSharedFileStream(self, sharedFileName):
assert self._validateSharedFileName( sharedFileName )
try:
with open(os.path.join(self.jobStoreDir, sharedFileName), 'r') as f:
yield f
except IOError as e:
if e.errno == errno.ENOENT:
raise NoSuchFileException(sharedFileName,sharedFileName)
else:
raise
def readSharedFileStream(self, sharedFileName):
self._requireValidSharedFileName(sharedFileName)
try:
with open(self._getSharedFilePath(sharedFileName), 'rb') as f:
yield f
except IOError as e:
if e.errno == errno.ENOENT:
raise NoSuchFileException(sharedFileName)
else:
raise
def read_file(self, file_id, local_path, symlink=False):
# used on non-shared files which will be encrypted if available
# checking for JobStoreID existence
if not self.file_exists(file_id):
raise NoSuchFileException(file_id)
with AtomicFileCreate(local_path) as tmpPath:
with open(tmpPath, 'wb') as writeable:
blob = self.bucket.get_blob(compat_bytes(file_id), encryption_key=self.sseKey)
blob.download_to_file(writeable)
if getattr(file_id, 'executable', False):
os.chmod(local_path, os.stat(local_path).st_mode | stat.S_IXUSR)
def loadOrFail(cls, jobStoreFileID, customName=None):
"""
:rtype: FileInfo
:return: an instance of this class representing the file with the given ID
:raises NoSuchFileException: if given file does not exist
"""
self = cls.load(jobStoreFileID)
if self is None:
raise NoSuchFileException(jobStoreFileID, customName=customName)
else:
return self
def getPublicUrl(self, jobStoreFileID):
try:
self.files.get_blob_properties(blob_name=str(jobStoreFileID))
except AzureMissingResourceHttpError:
raise NoSuchFileException(jobStoreFileID)
startTime = (datetime.utcnow() - timedelta(minutes=5))
endTime = datetime.utcnow() + self.publicUrlExpiration
sas_token = self.files.generate_blob_shared_access_signature(blob_name=str(jobStoreFileID),
permission=BlobPermissions.READ,
start=startTime,
expiry=endTime)
return self.files.make_blob_url(blob_name=str(jobStoreFileID)) + '?' + sas_token
def getPublicUrl(self, jobStoreFileID):
try:
self.files.get_blob_properties(blob_name=jobStoreFileID)
except AzureMissingResourceHttpError:
raise NoSuchFileException(jobStoreFileID)
# Compensate of a little bit of clock skew
startTimeStr = (datetime.utcnow() - timedelta(minutes=5)).strftime(self._azureTimeFormat)
endTime = datetime.utcnow() + self.publicUrlExpiration
endTimeStr = endTime.strftime(self._azureTimeFormat)
sap = SharedAccessPolicy(AccessPolicy(startTimeStr, endTimeStr,
BlobSharedAccessPermissions.READ))
sas_token = self.files.generate_shared_access_signature(blob_name=jobStoreFileID,
shared_access_policy=sap)
return self.files.make_blob_url(blob_name=jobStoreFileID) + '?' + sas_token
def _getSharedFileID(self, sharedFileName):
try:
with self.conn.cursor() as cur:
cur.execute("SELECT id FROM file_store WHERE namespace = %s AND shared IS TRUE AND name = %s AND _deleted_at IS NULL;", (self.namespace, sharedFileName))
record = cur.fetchone()
if record is None:
raise NoSuchFileException(sharedFileName, sharedFileName)
else:
return record['id']
except RuntimeError as e:
# Handle known errors
raise e
self.conn.commit()
def readSharedFileStream(self, sharedFileName, encoding=None, errors=None):
self._requireValidSharedFileName(sharedFileName)
try:
with open(self._getSharedFilePath(sharedFileName),
'rb' if encoding == None else 'rt',
encoding=encoding,
errors=errors) as f:
yield f
except IOError as e:
if e.errno == errno.ENOENT:
raise NoSuchFileException(sharedFileName)
else:
raise
def get_file_size(self, file_id):
# Duplicate a bunch of fileExists to save on stat calls
absPath = self._get_file_path_from_id(file_id)
if (not absPath.startswith(self.jobsDir)
and not absPath.startswith(self.filesDir)
and not absPath.startswith(self.jobFilesDir)):
# Don't even look for it, it is out of bounds.
raise NoSuchFileException(file_id)
try:
st = os.stat(absPath)
except os.error:
return 0
return st.st_size
def getFileSize(self, jobStoreFileID):
# Duplicate a bunch of fileExists to save on stat calls
absPath = self._getFilePathFromId(jobStoreFileID)
if (not absPath.startswith(self.jobsDir)
and not absPath.startswith(self.filesDir)
and not absPath.startswith(self.jobFilesDir)):
# Don't even look for it, it is out of bounds.
raise NoSuchFileException(jobStoreFileID)
try:
st = os.stat(absPath)
except os.error:
return 0
return st.st_size
def _getKey(self, jobStoreID=None, headers=None):
# gets remote key, in contrast to self._newKey
key = None
try:
key = self.files.get_key(jobStoreID, headers=headers)
except boto.exception.GSDataError:
if headers == self.encryptedHeaders:
# https://github.com/boto/boto/issues/3518
# see self._writeFile for more
pass
else:
raise
if key is None:
raise NoSuchFileException(jobStoreID)
else:
return key
