def _checkJobStoreFileID(self, jobStoreFileID): """ Raises NoSuchFileException if the jobStoreFileID does not exist or is not a file. """ absPath = os.path.join(self.tempFilesDir, jobStoreFileID) if not os.path.exists(absPath): raise NoSuchFileException("File %s does not exist in jobStore" % jobStoreFileID) if not os.path.isfile(absPath): raise NoSuchFileException("Path %s is not a file in the jobStore" % jobStoreFileID)
def getPublicUrl(self, jobStoreFileID): self._checkJobStoreFileID(jobStoreFileID) jobStorePath = self._getAbsPath(jobStoreFileID) if os.path.exists(jobStorePath): return 'file:' + jobStorePath else: raise NoSuchFileException(jobStoreFileID)
def readFileStream(self, jobStoreFileID): version = self._getFileVersion(jobStoreFileID) if version is None: raise NoSuchFileException(jobStoreFileID) log.debug("Reading version %s of file %s", version, jobStoreFileID) with self._downloadStream(jobStoreFileID, version, self.files) as readable: yield readable
def file_exists(self, file_id): absPath = self._get_file_path_from_id(file_id) if (not absPath.startswith(self.jobsDir) and not absPath.startswith(self.filesDir) and not absPath.startswith(self.jobFilesDir)): # Don't even look for it, it is out of bounds. raise NoSuchFileException(file_id) try: st = os.stat(absPath) except os.error: return False if not stat.S_ISREG(st.st_mode): raise NoSuchFileException(file_id) return True
def get_public_url(self, jobStoreFileID): self._check_job_store_file_id(jobStoreFileID) jobStorePath = self._get_file_path_from_id(jobStoreFileID) if os.path.exists(jobStorePath): return 'file:' + jobStorePath else: raise NoSuchFileException(jobStoreFileID)
def _checkJobStoreFileID(self, jobStoreFileID): """ :raise NoSuchFileException: if the jobStoreFileID does not exist or is not a file """ if not self.fileExists(jobStoreFileID): raise NoSuchFileException("File %s does not exist in jobStore" % jobStoreFileID)
def _downloadStream(self, fileName, encrypt=True): """ Yields a context manager that can be used to read from the bucket with a stream. See :class:`~toil.jobStores.utils.WritablePipe` for an example. :param fileName: name of file in bucket to be read :type fileName: str :param encrypt: whether or not the file is encrypted :type encrypt: bool :return: an instance of ReadablePipe. :rtype: :class:`~toil.jobStores.utils.ReadablePipe` """ blob = self.bucket.get_blob(compat_bytes(fileName), encryption_key=self.sseKey if encrypt else None) if blob is None: raise NoSuchFileException(fileName) class DownloadPipe(ReadablePipe): def writeTo(self, writable): try: blob.download_to_file(writable) finally: writable.close() with DownloadPipe() as readable: yield readable
def getPublicUrl(self, fileName): try: key = self._getKey(fileName) except: raise NoSuchFileException(fileName) return key.generate_url( expires_in=self.publicUrlExpiration.total_seconds())
def fileExists(self, jobStoreFileID): absPath = self._getFilePathFromId(jobStoreFileID) if (not absPath.startswith(self.jobsDir) and not absPath.startswith(self.filesDir) and not absPath.startswith(self.jobFilesDir)): # Don't even look for it, it is out of bounds. raise NoSuchFileException(jobStoreFileID) try: st = os.stat(absPath) except os.error: return False if not stat.S_ISREG(st.st_mode): raise NoSuchFileException(jobStoreFileID) return True
def _download_stream(self, fileName, encrypt=True, encoding=None, errors=None): """ Yields a context manager that can be used to read from the bucket with a stream. See :class:`~toil.jobStores.utils.WritablePipe` for an example. :param fileName: name of file in bucket to be read :type fileName: str :param encrypt: whether or not the file is encrypted :type encrypt: bool :param str encoding: the name of the encoding used to encode the file. Encodings are the same as for encode(). Defaults to None which represents binary mode. :param str errors: an optional string that specifies how encoding errors are to be handled. Errors are the same as for open(). Defaults to 'strict' when an encoding is specified. :return: an instance of ReadablePipe. :rtype: :class:`~toil.jobStores.utils.ReadablePipe` """ blob = self.bucket.get_blob(compat_bytes(fileName), encryption_key=self.sseKey if encrypt else None) if blob is None: raise NoSuchFileException(fileName) class DownloadPipe(ReadablePipe): def writeTo(self, writable): try: blob.download_to_file(writable) finally: writable.close() with DownloadPipe(encoding=encoding, errors=errors) as readable: yield readable
def _checkJobStoreFileID(self, jobStoreFileID): """ :raise NoSuchFileException: if the file with ID jobStoreFileID does not exist or is not a file """ if not self.fileExists(jobStoreFileID): raise NoSuchFileException(jobStoreFileID)
def readSharedFileStream(self, sharedFileName): assert self._validateSharedFileName(sharedFileName) sharedFileID = self._newFileID(sharedFileName) if not self.fileExists(sharedFileID): raise NoSuchFileException(sharedFileID) with self._downloadStream(sharedFileID, self.files) as fd: yield fd
def readFile(self, jobStoreFileID, localFilePath, symlink=False): # used on non-shared files which will be encrypted if available # checking for JobStoreID existence if not self.fileExists(jobStoreFileID): raise NoSuchFileException(jobStoreFileID) with open(localFilePath, 'w') as writeable: blob = self.bucket.get_blob(bytes(jobStoreFileID), encryption_key=self.sseKey) blob.download_to_file(writeable)
def fileExists(self, jobStoreFileID): absPath = self._getAbsPath(jobStoreFileID) if not os.path.exists(absPath): return False if not os.path.isfile(absPath): raise NoSuchFileException("Path %s is not a file in the jobStore" % jobStoreFileID) return True
def _writeFile(self, jobStoreID, fileObj, update=False, encrypt=True): blob = self.bucket.blob(compat_bytes(jobStoreID), encryption_key=self.sseKey if encrypt else None) if not update: # TODO: should probably raise a special exception and be added to all jobStores assert not blob.exists() else: if not blob.exists(): raise NoSuchFileException(jobStoreID) blob.upload_from_file(fileObj)
def fileExists(self, jobStoreFileID): absPath = self._getAbsPath(jobStoreFileID) try: st = os.stat(absPath) except os.error: return False if not stat.S_ISREG(st.st_mode): raise NoSuchFileException("Path %s is not a file in the jobStore" % jobStoreFileID) return True
def readFile(self, jobStoreFileID, localFilePath): # used on non-shared files which will be encrypted if avaliable headers = self.encryptedHeaders # checking for JobStoreID existance if not self.exists(jobStoreFileID): raise NoSuchFileException(jobStoreFileID) with open(localFilePath, 'w') as writeable: self._getKey(jobStoreFileID, headers).get_contents_to_file(writeable, headers=headers)
def readFile(self, jobStoreFileID, localFilePath): try: with self._downloadStream(jobStoreFileID, self.files) as read_fd: with open(localFilePath, 'w') as write_fd: while True: buf = read_fd.read(self._maxAzureBlockBytes) write_fd.write(buf) if not buf: break except AzureMissingResourceHttpError: raise NoSuchFileException(jobStoreFileID)
def readSharedFileStream(self, sharedFileName): assert self._validateSharedFileName(sharedFileName) jobStoreFileID = self._newFileID(sharedFileName) version = self._getFileVersion(jobStoreFileID) if version is None: raise NoSuchFileException(jobStoreFileID) log.debug("Read version %s from shared file %s (%s)", version, sharedFileName, jobStoreFileID) with self._downloadStream(jobStoreFileID, version, self.files) as readable: yield readable
def readSharedFileStream(self, sharedFileName): assert self._validateSharedFileName( sharedFileName ) try: with open(os.path.join(self.jobStoreDir, sharedFileName), 'r') as f: yield f except IOError as e: if e.errno == errno.ENOENT: raise NoSuchFileException(sharedFileName,sharedFileName) else: raise
def readSharedFileStream(self, sharedFileName): self._requireValidSharedFileName(sharedFileName) try: with open(self._getSharedFilePath(sharedFileName), 'rb') as f: yield f except IOError as e: if e.errno == errno.ENOENT: raise NoSuchFileException(sharedFileName) else: raise
def read_file(self, file_id, local_path, symlink=False): # used on non-shared files which will be encrypted if available # checking for JobStoreID existence if not self.file_exists(file_id): raise NoSuchFileException(file_id) with AtomicFileCreate(local_path) as tmpPath: with open(tmpPath, 'wb') as writeable: blob = self.bucket.get_blob(compat_bytes(file_id), encryption_key=self.sseKey) blob.download_to_file(writeable) if getattr(file_id, 'executable', False): os.chmod(local_path, os.stat(local_path).st_mode | stat.S_IXUSR)
def loadOrFail(cls, jobStoreFileID, customName=None): """ :rtype: FileInfo :return: an instance of this class representing the file with the given ID :raises NoSuchFileException: if given file does not exist """ self = cls.load(jobStoreFileID) if self is None: raise NoSuchFileException(jobStoreFileID, customName=customName) else: return self
def getPublicUrl(self, jobStoreFileID): try: self.files.get_blob_properties(blob_name=str(jobStoreFileID)) except AzureMissingResourceHttpError: raise NoSuchFileException(jobStoreFileID) startTime = (datetime.utcnow() - timedelta(minutes=5)) endTime = datetime.utcnow() + self.publicUrlExpiration sas_token = self.files.generate_blob_shared_access_signature(blob_name=str(jobStoreFileID), permission=BlobPermissions.READ, start=startTime, expiry=endTime) return self.files.make_blob_url(blob_name=str(jobStoreFileID)) + '?' + sas_token
def getPublicUrl(self, jobStoreFileID): try: self.files.get_blob_properties(blob_name=jobStoreFileID) except AzureMissingResourceHttpError: raise NoSuchFileException(jobStoreFileID) # Compensate of a little bit of clock skew startTimeStr = (datetime.utcnow() - timedelta(minutes=5)).strftime(self._azureTimeFormat) endTime = datetime.utcnow() + self.publicUrlExpiration endTimeStr = endTime.strftime(self._azureTimeFormat) sap = SharedAccessPolicy(AccessPolicy(startTimeStr, endTimeStr, BlobSharedAccessPermissions.READ)) sas_token = self.files.generate_shared_access_signature(blob_name=jobStoreFileID, shared_access_policy=sap) return self.files.make_blob_url(blob_name=jobStoreFileID) + '?' + sas_token
def _getSharedFileID(self, sharedFileName): try: with self.conn.cursor() as cur: cur.execute("SELECT id FROM file_store WHERE namespace = %s AND shared IS TRUE AND name = %s AND _deleted_at IS NULL;", (self.namespace, sharedFileName)) record = cur.fetchone() if record is None: raise NoSuchFileException(sharedFileName, sharedFileName) else: return record['id'] except RuntimeError as e: # Handle known errors raise e self.conn.commit()
def readSharedFileStream(self, sharedFileName, encoding=None, errors=None): self._requireValidSharedFileName(sharedFileName) try: with open(self._getSharedFilePath(sharedFileName), 'rb' if encoding == None else 'rt', encoding=encoding, errors=errors) as f: yield f except IOError as e: if e.errno == errno.ENOENT: raise NoSuchFileException(sharedFileName) else: raise
def get_file_size(self, file_id): # Duplicate a bunch of fileExists to save on stat calls absPath = self._get_file_path_from_id(file_id) if (not absPath.startswith(self.jobsDir) and not absPath.startswith(self.filesDir) and not absPath.startswith(self.jobFilesDir)): # Don't even look for it, it is out of bounds. raise NoSuchFileException(file_id) try: st = os.stat(absPath) except os.error: return 0 return st.st_size
def getFileSize(self, jobStoreFileID): # Duplicate a bunch of fileExists to save on stat calls absPath = self._getFilePathFromId(jobStoreFileID) if (not absPath.startswith(self.jobsDir) and not absPath.startswith(self.filesDir) and not absPath.startswith(self.jobFilesDir)): # Don't even look for it, it is out of bounds. raise NoSuchFileException(jobStoreFileID) try: st = os.stat(absPath) except os.error: return 0 return st.st_size
def _getKey(self, jobStoreID=None, headers=None): # gets remote key, in contrast to self._newKey key = None try: key = self.files.get_key(jobStoreID, headers=headers) except boto.exception.GSDataError: if headers == self.encryptedHeaders: # https://github.com/boto/boto/issues/3518 # see self._writeFile for more pass else: raise if key is None: raise NoSuchFileException(jobStoreID) else: return key