コード例 #1
0
 def _hashTestFile(self, url):
     import boto
     from toil.jobStores.googleJobStore import GoogleJobStore
     projectID, uri = GoogleJobStore._getResources(urlparse.urlparse(url))
     uri = boto.storage_uri(uri)
     contents = uri.get_contents_as_string(headers=self.headers)
     return hashlib.md5(contents).hexdigest()
コード例 #2
0
ファイル: jobStoreTest.py プロジェクト: joelarmstrong/toil
 def _createExternalStore():
     import boto
     from toil.jobStores.googleJobStore import GoogleJobStore
     uriString = "gs://import-export-test-%s" % str(uuid.uuid4())
     uri = boto.storage_uri(uriString)
     return GoogleJobStore._retryCreateBucket(
         uri=uri, headers=GoogleJobStoreTest.headers)
コード例 #3
0
ファイル: common.py プロジェクト: xulong2005/toil
    def getJobStore(cls, locator):
        """
        Create an instance of the concrete job store implementation that matches the given locator.

        :param str locator: The location of the job store to be represent by the instance

        :return: an instance of a concrete subclass of AbstractJobStore
        :rtype: toil.jobStores.abstractJobStore.AbstractJobStore
        """
        name, rest = cls.parseLocator(locator)
        if name == 'file':
            from toil.jobStores.fileJobStore import FileJobStore
            return FileJobStore(rest)
        elif name == 'aws':
            from toil.jobStores.aws.jobStore import AWSJobStore
            return AWSJobStore(rest)
        elif name == 'azure':
            from toil.jobStores.azureJobStore import AzureJobStore
            return AzureJobStore(rest)
        elif name == 'google':
            from toil.jobStores.googleJobStore import GoogleJobStore
            projectID, namePrefix = rest.split(':', 1)
            return GoogleJobStore(namePrefix, projectID)
        else:
            raise RuntimeError("Unknown job store implementation '%s'" % name)
コード例 #4
0
 def _hashTestFile(self, url):
     import boto
     from toil.jobStores.googleJobStore import GoogleJobStore
     projectID, uri = GoogleJobStore._getResources(urlparse.urlparse(url))
     uri = boto.storage_uri(uri)
     contents = uri.get_contents_as_string(headers=self.headers)
     return hashlib.md5(contents).hexdigest()
コード例 #5
0
ファイル: jobStoreTest.py プロジェクト: joelarmstrong/toil
 def _hashUrl(url):
     import boto
     from toil.jobStores.googleJobStore import GoogleJobStore
     projectID, uri = GoogleJobStore._getResources(urlparse.urlparse(url))
     return hashlib.md5(
         boto.storage_uri(uri).get_contents_as_string(
             headers=GoogleJobStoreTest.headers)).hexdigest()
コード例 #6
0
ファイル: common.py プロジェクト: anukat2015/toil
    def loadOrCreateJobStore(jobStoreString, config=None):
        """
        Loads an existing jobStore if it already exists. Otherwise a new instance of a jobStore is
        created and returned.

        :param str jobStoreString: see exception message below
        :param toil.common.Config config: see AbstractJobStore.__init__
        :return: an instance of a concrete subclass of AbstractJobStore
        :rtype: toil.jobStores.abstractJobStore.AbstractJobStore
        """
        if jobStoreString[0] in '/.':
            jobStoreString = 'file:' + jobStoreString

        try:
            jobStoreName, jobStoreArgs = jobStoreString.split(':', 1)
        except ValueError:
            raise RuntimeError(
                'A job store string must either be a path starting in . or / or a contain at '
                'least one colon separating the name of the job store implementation from an '
                'initialization string specific to that job store. If a path starting in . or / '
                'is passed, the file job store will be used for backwards compatibility.'
            )

        if jobStoreName == 'file':
            from toil.jobStores.fileJobStore import FileJobStore
            return FileJobStore(jobStoreArgs, config=config)

        elif jobStoreName == 'aws':
            from toil.jobStores.aws.jobStore import AWSJobStore
            return AWSJobStore.loadOrCreateJobStore(jobStoreArgs,
                                                    config=config)

        elif jobStoreName == 'azure':
            from toil.jobStores.azureJobStore import AzureJobStore
            account, namePrefix = jobStoreArgs.split(':', 1)
            return AzureJobStore(account, namePrefix, config=config)

        elif jobStoreName == 'google':
            from toil.jobStores.googleJobStore import GoogleJobStore
            projectID, namePrefix = jobStoreArgs.split(':', 1)
            return GoogleJobStore(namePrefix, projectID, config=config)
        else:
            raise RuntimeError("Unknown job store implementation '%s'" %
                               jobStoreName)
コード例 #7
0
    def loadOrCreateJobStore(locator, config=None):
        """
        Loads an existing jobStore if it already exists. Otherwise a new instance of a jobStore is
        created and returned.

        :param str locator: The location of the job store.
        :param toil.common.Config config: see AbstractJobStore.__init__
        :return: an instance of a concrete subclass of AbstractJobStore
        :rtype: toil.jobStores.abstractJobStore.AbstractJobStore
        """
        if locator[0] in '/.':
            locator = 'file:' + locator

        try:
            jobStoreName, jobStoreArgs = locator.split(':', 1)
        except ValueError:
            raise RuntimeError('Invalid job store locator for proper formatting check locator '
                               'documentation for each job store.')

        if jobStoreName == 'file':
            from toil.jobStores.fileJobStore import FileJobStore
            return FileJobStore(jobStoreArgs, config=config)

        elif jobStoreName == 'aws':
            from toil.jobStores.aws.jobStore import AWSJobStore
            return AWSJobStore.loadOrCreateJobStore(jobStoreArgs, config=config)

        elif jobStoreName == 'azure':
            from toil.jobStores.azureJobStore import AzureJobStore
            account, namePrefix = jobStoreArgs.split(':', 1)
            return AzureJobStore(account, namePrefix, config=config)
        
        elif jobStoreName == 'google':
            from toil.jobStores.googleJobStore import GoogleJobStore
            projectID, namePrefix = jobStoreArgs.split(':', 1)
            return GoogleJobStore(namePrefix, projectID, config=config)
        else:
            raise RuntimeError("Unknown job store implementation '%s'" % jobStoreName)
コード例 #8
0
ファイル: jobStoreTest.py プロジェクト: broadinstitute/toil
 def _cleanUpExternalStore(url):
     import boto
     from toil.jobStores.googleJobStore import GoogleJobStore
     projectID, uri = GoogleJobStore._getResources(urlparse.urlparse(url))
     uri = boto.storage_uri(uri)
     headers = {"x-goog-project-id": projectID}
     bucket = uri.get_bucket(headers=headers, validate=True)
     if bucket is not None:
         while True:
             for key in bucket.list():
                 try:
                     key.delete()
                 except boto.exception.GSResponseError as e:
                     if e.status == 404:
                         pass
                     else:
                         raise e
             try:
                 uri.delete_bucket()
             except boto.exception.GSResponseError as e:
                     if e.status == 404:
                         break
                     else:
                         continue
コード例 #9
0
ファイル: jobStoreTest.py プロジェクト: joelarmstrong/toil
 def _cleanUpExternalStore(url):
     import boto
     from toil.jobStores.googleJobStore import GoogleJobStore
     projectID, uri = GoogleJobStore._getResources(urlparse.urlparse(url))
     uri = boto.storage_uri(uri)
     headers = {"x-goog-project-id": projectID}
     bucket = uri.get_bucket(headers=headers, validate=True)
     if bucket is not None:
         while True:
             for key in bucket.list():
                 try:
                     key.delete()
                 except boto.exception.GSResponseError as e:
                     if e.status == 404:
                         pass
                     else:
                         raise e
             try:
                 uri.delete_bucket()
             except boto.exception.GSResponseError as e:
                 if e.status == 404:
                     break
                 else:
                     continue
コード例 #10
0
ファイル: common.py プロジェクト: teaguesterling/toil
    def getJobStore(locator):
        """
        Create an instance of the concrete job store implementation that matches the given locator.

        :param str locator: The location of the job store to be represent by the instance

        :return: an instance of a concrete subclass of AbstractJobStore
        :rtype: toil.jobStores.abstractJobStore.AbstractJobStore
        """
        if locator[0] in '/.':
            locator = 'file:' + locator

        try:
            name, rest = locator.split(':', 1)
        except ValueError:
            raise RuntimeError('Invalid job store locator syntax.')

        if name == 'file':
            from toil.jobStores.fileJobStore import FileJobStore
            return FileJobStore(rest)

        elif name == 'aws':
            from toil.jobStores.aws.jobStore import AWSJobStore
            return AWSJobStore(rest)

        elif name == 'azure':
            from toil.jobStores.azureJobStore import AzureJobStore
            account, namePrefix = rest.split(':', 1)
            return AzureJobStore(rest)

        elif name == 'google':
            from toil.jobStores.googleJobStore import GoogleJobStore
            projectID, namePrefix = rest.split(':', 1)
            return GoogleJobStore(namePrefix, projectID, config=config)
        else:
            raise RuntimeError("Unknown job store implementation '%s'" % name)
コード例 #11
0
 def _createJobStore(self, config=None):
     from toil.jobStores.googleJobStore import GoogleJobStore
     return GoogleJobStore.createJobStore(self.namePrefix + ":" +
                                          GoogleJobStoreTest.projectID,
                                          config=config)
コード例 #12
0
 def _createJobStore(self):
     from toil.jobStores.googleJobStore import GoogleJobStore
     return GoogleJobStore.initialize(GoogleJobStoreTest.projectID + ":" + self.namePrefix)
コード例 #13
0
 def _createExternalStore(self):
     import boto
     from toil.jobStores.googleJobStore import GoogleJobStore
     uriString = "gs://import-export-test-%s" % str(uuid.uuid4())
     uri = boto.storage_uri(uriString)
     return GoogleJobStore._retryCreateBucket(uri=uri, headers=self.headers)
コード例 #14
0
 def _createJobStore(self):
     from toil.jobStores.googleJobStore import GoogleJobStore
     return GoogleJobStore.initialize(GoogleJobStoreTest.projectID + ":" + self.namePrefix)
コード例 #15
0
ファイル: jobStoreTest.py プロジェクト: anukat2015/toil
 def _createJobStore(self, config=None):
     from toil.jobStores.googleJobStore import GoogleJobStore
     return GoogleJobStore.createJobStore(self.namePrefix + ":" + GoogleJobStoreTest.projectID,
                                          config=config)
コード例 #16
0
ファイル: jobStoreTest.py プロジェクト: broadinstitute/toil
 def _hashUrl(url):
     import boto
     from toil.jobStores.googleJobStore import GoogleJobStore
     projectID, uri = GoogleJobStore._getResources(urlparse.urlparse(url))
     return hashlib.md5(boto.storage_uri(uri).get_contents_as_string(headers=GoogleJobStoreTest.headers)).hexdigest()