def _createJobStore(self, config=None):
from toil.jobStores.aws.jobStore import AWSJobStore
partSize = self._partSize()
for encrypted in (True, False):
self.assertTrue(AWSJobStore.FileInfo.maxInlinedSize(encrypted) < partSize)
AWSJobStore.FileInfo.defaultS3PartSize = partSize
return AWSJobStore.loadOrCreateJobStore(self.testRegion + ':' + self.namePrefix, config=config, partSize=2**20*5)
def loadJobStore(jobStoreString, config=None):
"""
Loads a jobStore.
:param jobStoreString: see exception message below
:param config: see AbstractJobStore.__init__
:return: an instance of a concrete subclass of AbstractJobStore
:rtype: jobStores.abstractJobStore.AbstractJobStore
"""
if jobStoreString[0] in '/.':
jobStoreString = 'file:' + jobStoreString
try:
jobStoreName, jobStoreArgs = jobStoreString.split(':', 1)
except ValueError:
raise RuntimeError(
'Job store string must either be a path starting in . or / or a contain at least one '
'colon separating the name of the job store implementation from an initialization '
'string specific to that job store. If a path starting in . or / is passed, the file '
'job store will be used for backwards compatibility.')
if jobStoreName == 'file':
from toil.jobStores.fileJobStore import FileJobStore
return FileJobStore(jobStoreArgs, config=config)
elif jobStoreName == 'aws':
from toil.jobStores.aws.jobStore import AWSJobStore
return AWSJobStore.createJobStore(jobStoreArgs, config=config)
elif jobStoreName == 'azure':
from toil.jobStores.azureJobStore import AzureJobStore
account, namePrefix = jobStoreArgs.split(':', 1)
return AzureJobStore(account, namePrefix, config=config)
else:
raise RuntimeError("Unknown job store implementation '%s'" %
jobStoreName)
def getJobStore(cls, locator):
"""
Create an instance of the concrete job store implementation that matches the given locator.
:param str locator: The location of the job store to be represent by the instance
:return: an instance of a concrete subclass of AbstractJobStore
:rtype: toil.jobStores.abstractJobStore.AbstractJobStore
"""
name, rest = cls.parseLocator(locator)
if name == 'file':
from toil.jobStores.fileJobStore import FileJobStore
return FileJobStore(rest)
elif name == 'aws':
from toil.jobStores.aws.jobStore import AWSJobStore
return AWSJobStore(rest)
elif name == 'azure':
from toil.jobStores.azureJobStore import AzureJobStore
return AzureJobStore(rest)
elif name == 'google':
from toil.jobStores.googleJobStore import GoogleJobStore
projectID, namePrefix = rest.split(':', 1)
return GoogleJobStore(namePrefix, projectID)
else:
raise RuntimeError("Unknown job store implementation '%s'" % name)
def loadJobStore( jobStoreString, config=None ):
"""
Loads a jobStore.
:param jobStoreString: see exception message below
:param config: see AbstractJobStore.__init__
:return: an instance of a concrete subclass of AbstractJobStore
:rtype: jobStores.abstractJobStore.AbstractJobStore
"""
if jobStoreString[ 0 ] in '/.':
jobStoreString = 'file:' + jobStoreString
try:
jobStoreName, jobStoreArgs = jobStoreString.split( ':', 1 )
except ValueError:
raise RuntimeError(
'Job store string must either be a path starting in . or / or a contain at least one '
'colon separating the name of the job store implementation from an initialization '
'string specific to that job store. If a path starting in . or / is passed, the file '
'job store will be used for backwards compatibility.' )
if jobStoreName == 'file':
from toil.jobStores.fileJobStore import FileJobStore
return FileJobStore( jobStoreArgs, config=config )
elif jobStoreName == 'aws':
from toil.jobStores.aws.jobStore import AWSJobStore
return AWSJobStore.createJobStore( jobStoreArgs, config=config )
elif jobStoreName == 'azure':
from toil.jobStores.azureJobStore import AzureJobStore
account, namePrefix = jobStoreArgs.split( ':', 1 )
return AzureJobStore( account, namePrefix, config=config )
else:
raise RuntimeError( "Unknown job store implementation '%s'" % jobStoreName )
def _hashTestFile(self, url):
from toil.jobStores.aws.jobStore import AWSJobStore
key = AWSJobStore._getKeyForUrl(urlparse.urlparse(url), existing=True)
try:
contents = key.get_contents_as_string()
finally:
key.bucket.connection.close()
return hashlib.md5(contents).hexdigest()
def _hashTestFile(self, url):
from toil.jobStores.aws.jobStore import AWSJobStore
key = AWSJobStore._getKeyForUrl(urlparse.urlparse(url), existing=True)
try:
contents = key.get_contents_as_string()
finally:
key.bucket.connection.close()
return hashlib.md5(contents).hexdigest()
def _createJobStore(self):
from toil.jobStores.aws.jobStore import AWSJobStore
partSize = self._partSize()
for encrypted in (True, False):
self.assertTrue(
AWSJobStore.FileInfo.maxInlinedSize(encrypted) < partSize)
return AWSJobStore(self.awsRegion() + ':' + self.namePrefix,
partSize=partSize)
def _createJobStore(self, config=None):
from toil.jobStores.aws.jobStore import AWSJobStore
partSize = self._partSize()
for encrypted in (True, False):
self.assertTrue(AWSJobStore.FileInfo.maxInlinedSize(encrypted) < partSize)
AWSJobStore.FileInfo.defaultS3PartSize = partSize
return AWSJobStore.loadOrCreateJobStore(self.testRegion + ':' + self.namePrefix,
config=config,
partSize=5 * 2 ** 20)
def _cleanUpExternalStore(url):
from toil.jobStores.aws.jobStore import AWSJobStore
try:
bucket, _ = AWSJobStore._extractKeyInfoFromUrl(urlparse.urlparse(url))
except boto.exception.S3ResponseError as ex:
assert ex.error_code == 404
else:
s3 = boto.connect_s3()
for key in bucket.list():
key.delete()
s3.delete_bucket(bucket)
def _cleanUpExternalStore(url):
from toil.jobStores.aws.jobStore import AWSJobStore
import boto
try:
bucket, _ = AWSJobStore._extractKeyInfoFromUrl(
urlparse.urlparse(url))
except boto.exception.S3ResponseError as ex:
assert ex.error_code == 404
else:
s3 = boto.connect_s3()
for key in bucket.list():
key.delete()
s3.delete_bucket(bucket)
def loadOrCreateJobStore(jobStoreString, config=None):
"""
Loads an existing jobStore if it already exists. Otherwise a new instance of a jobStore is
created and returned.
:param str jobStoreString: see exception message below
:param toil.common.Config config: see AbstractJobStore.__init__
:return: an instance of a concrete subclass of AbstractJobStore
:rtype: toil.jobStores.abstractJobStore.AbstractJobStore
"""
if jobStoreString[0] in '/.':
jobStoreString = 'file:' + jobStoreString
try:
jobStoreName, jobStoreArgs = jobStoreString.split(':', 1)
except ValueError:
raise RuntimeError(
'A job store string must either be a path starting in . or / or a contain at '
'least one colon separating the name of the job store implementation from an '
'initialization string specific to that job store. If a path starting in . or / '
'is passed, the file job store will be used for backwards compatibility.'
)
if jobStoreName == 'file':
from toil.jobStores.fileJobStore import FileJobStore
return FileJobStore(jobStoreArgs, config=config)
elif jobStoreName == 'aws':
from toil.jobStores.aws.jobStore import AWSJobStore
return AWSJobStore.loadOrCreateJobStore(jobStoreArgs,
config=config)
elif jobStoreName == 'azure':
from toil.jobStores.azureJobStore import AzureJobStore
account, namePrefix = jobStoreArgs.split(':', 1)
return AzureJobStore(account, namePrefix, config=config)
elif jobStoreName == 'google':
from toil.jobStores.googleJobStore import GoogleJobStore
projectID, namePrefix = jobStoreArgs.split(':', 1)
return GoogleJobStore(namePrefix, projectID, config=config)
else:
raise RuntimeError("Unknown job store implementation '%s'" %
jobStoreName)
def loadOrCreateJobStore(jobStoreString, config=None):
"""
Loads an existing jobStore if it already exists. Otherwise a new instance of a jobStore is
created and returned.
:param str jobStoreString: see exception message below
:param toil.common.Config config: see AbstractJobStore.__init__
:return: an instance of a concrete subclass of AbstractJobStore
:rtype: toil.jobStores.abstractJobStore.AbstractJobStore
"""
if jobStoreString[0] in '/.':
jobStoreString = 'file:' + jobStoreString
try:
jobStoreName, jobStoreArgs = jobStoreString.split(':', 1)
except ValueError:
raise RuntimeError(
'A job store string must either be a path starting in . or / or a contain at '
'least one colon separating the name of the job store implementation from an '
'initialization string specific to that job store. If a path starting in . or / '
'is passed, the file job store will be used for backwards compatibility.' )
if jobStoreName == 'file':
from toil.jobStores.fileJobStore import FileJobStore
return FileJobStore(jobStoreArgs, config=config)
elif jobStoreName == 'aws':
from toil.jobStores.aws.jobStore import AWSJobStore
return AWSJobStore.loadOrCreateJobStore(jobStoreArgs, config=config)
elif jobStoreName == 'azure':
from toil.jobStores.azureJobStore import AzureJobStore
account, namePrefix = jobStoreArgs.split(':', 1)
return AzureJobStore(account, namePrefix, config=config)
elif jobStoreName == 'google':
from toil.jobStores.googleJobStore import GoogleJobStore
projectID, namePrefix = jobStoreArgs.split(':', 1)
return GoogleJobStore(namePrefix, projectID, config=config)
else:
raise RuntimeError("Unknown job store implementation '%s'" % jobStoreName)
def loadOrCreateJobStore(locator, config=None):
"""
Loads an existing jobStore if it already exists. Otherwise a new instance of a jobStore is
created and returned.
:param str locator: The location of the job store.
:param toil.common.Config config: see AbstractJobStore.__init__
:return: an instance of a concrete subclass of AbstractJobStore
:rtype: toil.jobStores.abstractJobStore.AbstractJobStore
"""
if locator[0] in '/.':
locator = 'file:' + locator
try:
jobStoreName, jobStoreArgs = locator.split(':', 1)
except ValueError:
raise RuntimeError('Invalid job store locator for proper formatting check locator '
'documentation for each job store.')
if jobStoreName == 'file':
from toil.jobStores.fileJobStore import FileJobStore
return FileJobStore(jobStoreArgs, config=config)
elif jobStoreName == 'aws':
from toil.jobStores.aws.jobStore import AWSJobStore
return AWSJobStore.loadOrCreateJobStore(jobStoreArgs, config=config)
elif jobStoreName == 'azure':
from toil.jobStores.azureJobStore import AzureJobStore
account, namePrefix = jobStoreArgs.split(':', 1)
return AzureJobStore(account, namePrefix, config=config)
elif jobStoreName == 'google':
from toil.jobStores.googleJobStore import GoogleJobStore
projectID, namePrefix = jobStoreArgs.split(':', 1)
return GoogleJobStore(namePrefix, projectID, config=config)
else:
raise RuntimeError("Unknown job store implementation '%s'" % jobStoreName)
def loadOrCreateJobStore(locator, config=None):
"""
Loads an existing jobStore if it already exists. Otherwise a new instance of a jobStore is
created and returned.
:param str locator: The location of the job store.
:param toil.common.Config config: see AbstractJobStore.__init__
:return: an instance of a concrete subclass of AbstractJobStore
:rtype: toil.jobStores.abstractJobStore.AbstractJobStore
"""
if locator[0] in '/.':
locator = 'file:' + locator
try:
jobStoreName, jobStoreArgs = locator.split(':', 1)
except ValueError:
raise RuntimeError('Invalid job store locator for proper formatting check locator '
'documentation for each job store.')
if jobStoreName == 'file':
from toil.jobStores.fileJobStore import FileJobStore
return FileJobStore(jobStoreArgs, config=config)
elif jobStoreName == 'aws':
from toil.jobStores.aws.jobStore import AWSJobStore
return AWSJobStore.loadOrCreateJobStore(jobStoreArgs, config=config)
elif jobStoreName == 'azure':
from toil.jobStores.azureJobStore import AzureJobStore
account, namePrefix = jobStoreArgs.split(':', 1)
return AzureJobStore(account, namePrefix, config=config)
elif jobStoreName == 'google':
from toil.jobStores.googleJobStore import GoogleJobStore
projectID, namePrefix = jobStoreArgs.split(':', 1)
return GoogleJobStore(namePrefix, projectID, config=config)
else:
raise RuntimeError("Unknown job store implementation '%s'" % jobStoreName)
def getJobStore(locator):
"""
Create an instance of the concrete job store implementation that matches the given locator.
:param str locator: The location of the job store to be represent by the instance
:return: an instance of a concrete subclass of AbstractJobStore
:rtype: toil.jobStores.abstractJobStore.AbstractJobStore
"""
if locator[0] in '/.':
locator = 'file:' + locator
try:
name, rest = locator.split(':', 1)
except ValueError:
raise RuntimeError('Invalid job store locator syntax.')
if name == 'file':
from toil.jobStores.fileJobStore import FileJobStore
return FileJobStore(rest)
elif name == 'aws':
from toil.jobStores.aws.jobStore import AWSJobStore
return AWSJobStore(rest)
elif name == 'azure':
from toil.jobStores.azureJobStore import AzureJobStore
account, namePrefix = rest.split(':', 1)
return AzureJobStore(rest)
elif name == 'google':
from toil.jobStores.googleJobStore import GoogleJobStore
projectID, namePrefix = rest.split(':', 1)
return GoogleJobStore(namePrefix, projectID, config=config)
else:
raise RuntimeError("Unknown job store implementation '%s'" % name)
def tearDownClass(cls) -> None:
if cls.bucket:
AWSJobStore._delete_bucket(cls.bucket)
super().tearDownClass()
def _hashUrl(url):
from toil.jobStores.aws.jobStore import AWSJobStore
bucket, key = AWSJobStore._extractKeyInfoFromUrl(urlparse.urlparse(url), existing=True)
return hashlib.md5(key.get_contents_as_string()).hexdigest()
def _hashUrl(url):
from toil.jobStores.aws.jobStore import AWSJobStore
bucket, key = AWSJobStore._extractKeyInfoFromUrl(
urlparse.urlparse(url), existing=True)
return hashlib.md5(key.get_contents_as_string()).hexdigest()
