def _createJobStore(self, config=None): from toil.jobStores.aws.jobStore import AWSJobStore partSize = self._partSize() for encrypted in (True, False): self.assertTrue(AWSJobStore.FileInfo.maxInlinedSize(encrypted) < partSize) AWSJobStore.FileInfo.defaultS3PartSize = partSize return AWSJobStore.loadOrCreateJobStore(self.testRegion + ':' + self.namePrefix, config=config, partSize=2**20*5)
def loadJobStore(jobStoreString, config=None): """ Loads a jobStore. :param jobStoreString: see exception message below :param config: see AbstractJobStore.__init__ :return: an instance of a concrete subclass of AbstractJobStore :rtype: jobStores.abstractJobStore.AbstractJobStore """ if jobStoreString[0] in '/.': jobStoreString = 'file:' + jobStoreString try: jobStoreName, jobStoreArgs = jobStoreString.split(':', 1) except ValueError: raise RuntimeError( 'Job store string must either be a path starting in . or / or a contain at least one ' 'colon separating the name of the job store implementation from an initialization ' 'string specific to that job store. If a path starting in . or / is passed, the file ' 'job store will be used for backwards compatibility.') if jobStoreName == 'file': from toil.jobStores.fileJobStore import FileJobStore return FileJobStore(jobStoreArgs, config=config) elif jobStoreName == 'aws': from toil.jobStores.aws.jobStore import AWSJobStore return AWSJobStore.createJobStore(jobStoreArgs, config=config) elif jobStoreName == 'azure': from toil.jobStores.azureJobStore import AzureJobStore account, namePrefix = jobStoreArgs.split(':', 1) return AzureJobStore(account, namePrefix, config=config) else: raise RuntimeError("Unknown job store implementation '%s'" % jobStoreName)
def getJobStore(cls, locator): """ Create an instance of the concrete job store implementation that matches the given locator. :param str locator: The location of the job store to be represent by the instance :return: an instance of a concrete subclass of AbstractJobStore :rtype: toil.jobStores.abstractJobStore.AbstractJobStore """ name, rest = cls.parseLocator(locator) if name == 'file': from toil.jobStores.fileJobStore import FileJobStore return FileJobStore(rest) elif name == 'aws': from toil.jobStores.aws.jobStore import AWSJobStore return AWSJobStore(rest) elif name == 'azure': from toil.jobStores.azureJobStore import AzureJobStore return AzureJobStore(rest) elif name == 'google': from toil.jobStores.googleJobStore import GoogleJobStore projectID, namePrefix = rest.split(':', 1) return GoogleJobStore(namePrefix, projectID) else: raise RuntimeError("Unknown job store implementation '%s'" % name)
def loadJobStore( jobStoreString, config=None ): """ Loads a jobStore. :param jobStoreString: see exception message below :param config: see AbstractJobStore.__init__ :return: an instance of a concrete subclass of AbstractJobStore :rtype: jobStores.abstractJobStore.AbstractJobStore """ if jobStoreString[ 0 ] in '/.': jobStoreString = 'file:' + jobStoreString try: jobStoreName, jobStoreArgs = jobStoreString.split( ':', 1 ) except ValueError: raise RuntimeError( 'Job store string must either be a path starting in . or / or a contain at least one ' 'colon separating the name of the job store implementation from an initialization ' 'string specific to that job store. If a path starting in . or / is passed, the file ' 'job store will be used for backwards compatibility.' ) if jobStoreName == 'file': from toil.jobStores.fileJobStore import FileJobStore return FileJobStore( jobStoreArgs, config=config ) elif jobStoreName == 'aws': from toil.jobStores.aws.jobStore import AWSJobStore return AWSJobStore.createJobStore( jobStoreArgs, config=config ) elif jobStoreName == 'azure': from toil.jobStores.azureJobStore import AzureJobStore account, namePrefix = jobStoreArgs.split( ':', 1 ) return AzureJobStore( account, namePrefix, config=config ) else: raise RuntimeError( "Unknown job store implementation '%s'" % jobStoreName )
def _hashTestFile(self, url): from toil.jobStores.aws.jobStore import AWSJobStore key = AWSJobStore._getKeyForUrl(urlparse.urlparse(url), existing=True) try: contents = key.get_contents_as_string() finally: key.bucket.connection.close() return hashlib.md5(contents).hexdigest()
def _createJobStore(self): from toil.jobStores.aws.jobStore import AWSJobStore partSize = self._partSize() for encrypted in (True, False): self.assertTrue( AWSJobStore.FileInfo.maxInlinedSize(encrypted) < partSize) return AWSJobStore(self.awsRegion() + ':' + self.namePrefix, partSize=partSize)
def _createJobStore(self, config=None): from toil.jobStores.aws.jobStore import AWSJobStore partSize = self._partSize() for encrypted in (True, False): self.assertTrue(AWSJobStore.FileInfo.maxInlinedSize(encrypted) < partSize) AWSJobStore.FileInfo.defaultS3PartSize = partSize return AWSJobStore.loadOrCreateJobStore(self.testRegion + ':' + self.namePrefix, config=config, partSize=5 * 2 ** 20)
def _cleanUpExternalStore(url): from toil.jobStores.aws.jobStore import AWSJobStore try: bucket, _ = AWSJobStore._extractKeyInfoFromUrl(urlparse.urlparse(url)) except boto.exception.S3ResponseError as ex: assert ex.error_code == 404 else: s3 = boto.connect_s3() for key in bucket.list(): key.delete() s3.delete_bucket(bucket)
def _cleanUpExternalStore(url): from toil.jobStores.aws.jobStore import AWSJobStore import boto try: bucket, _ = AWSJobStore._extractKeyInfoFromUrl( urlparse.urlparse(url)) except boto.exception.S3ResponseError as ex: assert ex.error_code == 404 else: s3 = boto.connect_s3() for key in bucket.list(): key.delete() s3.delete_bucket(bucket)
def loadOrCreateJobStore(jobStoreString, config=None): """ Loads an existing jobStore if it already exists. Otherwise a new instance of a jobStore is created and returned. :param str jobStoreString: see exception message below :param toil.common.Config config: see AbstractJobStore.__init__ :return: an instance of a concrete subclass of AbstractJobStore :rtype: toil.jobStores.abstractJobStore.AbstractJobStore """ if jobStoreString[0] in '/.': jobStoreString = 'file:' + jobStoreString try: jobStoreName, jobStoreArgs = jobStoreString.split(':', 1) except ValueError: raise RuntimeError( 'A job store string must either be a path starting in . or / or a contain at ' 'least one colon separating the name of the job store implementation from an ' 'initialization string specific to that job store. If a path starting in . or / ' 'is passed, the file job store will be used for backwards compatibility.' ) if jobStoreName == 'file': from toil.jobStores.fileJobStore import FileJobStore return FileJobStore(jobStoreArgs, config=config) elif jobStoreName == 'aws': from toil.jobStores.aws.jobStore import AWSJobStore return AWSJobStore.loadOrCreateJobStore(jobStoreArgs, config=config) elif jobStoreName == 'azure': from toil.jobStores.azureJobStore import AzureJobStore account, namePrefix = jobStoreArgs.split(':', 1) return AzureJobStore(account, namePrefix, config=config) elif jobStoreName == 'google': from toil.jobStores.googleJobStore import GoogleJobStore projectID, namePrefix = jobStoreArgs.split(':', 1) return GoogleJobStore(namePrefix, projectID, config=config) else: raise RuntimeError("Unknown job store implementation '%s'" % jobStoreName)
def loadOrCreateJobStore(locator, config=None): """ Loads an existing jobStore if it already exists. Otherwise a new instance of a jobStore is created and returned. :param str locator: The location of the job store. :param toil.common.Config config: see AbstractJobStore.__init__ :return: an instance of a concrete subclass of AbstractJobStore :rtype: toil.jobStores.abstractJobStore.AbstractJobStore """ if locator[0] in '/.': locator = 'file:' + locator try: jobStoreName, jobStoreArgs = locator.split(':', 1) except ValueError: raise RuntimeError('Invalid job store locator for proper formatting check locator ' 'documentation for each job store.') if jobStoreName == 'file': from toil.jobStores.fileJobStore import FileJobStore return FileJobStore(jobStoreArgs, config=config) elif jobStoreName == 'aws': from toil.jobStores.aws.jobStore import AWSJobStore return AWSJobStore.loadOrCreateJobStore(jobStoreArgs, config=config) elif jobStoreName == 'azure': from toil.jobStores.azureJobStore import AzureJobStore account, namePrefix = jobStoreArgs.split(':', 1) return AzureJobStore(account, namePrefix, config=config) elif jobStoreName == 'google': from toil.jobStores.googleJobStore import GoogleJobStore projectID, namePrefix = jobStoreArgs.split(':', 1) return GoogleJobStore(namePrefix, projectID, config=config) else: raise RuntimeError("Unknown job store implementation '%s'" % jobStoreName)
def getJobStore(locator): """ Create an instance of the concrete job store implementation that matches the given locator. :param str locator: The location of the job store to be represent by the instance :return: an instance of a concrete subclass of AbstractJobStore :rtype: toil.jobStores.abstractJobStore.AbstractJobStore """ if locator[0] in '/.': locator = 'file:' + locator try: name, rest = locator.split(':', 1) except ValueError: raise RuntimeError('Invalid job store locator syntax.') if name == 'file': from toil.jobStores.fileJobStore import FileJobStore return FileJobStore(rest) elif name == 'aws': from toil.jobStores.aws.jobStore import AWSJobStore return AWSJobStore(rest) elif name == 'azure': from toil.jobStores.azureJobStore import AzureJobStore account, namePrefix = rest.split(':', 1) return AzureJobStore(rest) elif name == 'google': from toil.jobStores.googleJobStore import GoogleJobStore projectID, namePrefix = rest.split(':', 1) return GoogleJobStore(namePrefix, projectID, config=config) else: raise RuntimeError("Unknown job store implementation '%s'" % name)
def tearDownClass(cls) -> None: if cls.bucket: AWSJobStore._delete_bucket(cls.bucket) super().tearDownClass()
def _hashUrl(url): from toil.jobStores.aws.jobStore import AWSJobStore bucket, key = AWSJobStore._extractKeyInfoFromUrl(urlparse.urlparse(url), existing=True) return hashlib.md5(key.get_contents_as_string()).hexdigest()
def _hashUrl(url): from toil.jobStores.aws.jobStore import AWSJobStore bucket, key = AWSJobStore._extractKeyInfoFromUrl( urlparse.urlparse(url), existing=True) return hashlib.md5(key.get_contents_as_string()).hexdigest()