Exemplo n.º 1
0
def loadJobStore(jobStoreString, config=None):
    """
    Loads a jobStore.

    :param jobStoreString: see exception message below
    :param config: see AbstractJobStore.__init__
    :return: an instance of a concrete subclass of AbstractJobStore
    :rtype: jobStores.abstractJobStore.AbstractJobStore
    """
    if jobStoreString[0] in '/.':
        jobStoreString = 'file:' + jobStoreString

    try:
        jobStoreName, jobStoreArgs = jobStoreString.split(':', 1)
    except ValueError:
        raise RuntimeError(
            'Job store string must either be a path starting in . or / or a contain at least one '
            'colon separating the name of the job store implementation from an initialization '
            'string specific to that job store. If a path starting in . or / is passed, the file '
            'job store will be used for backwards compatibility.')

    if jobStoreName == 'file':
        from toil.jobStores.fileJobStore import FileJobStore
        return FileJobStore(jobStoreArgs, config=config)
    elif jobStoreName == 'aws':
        from toil.jobStores.aws.jobStore import AWSJobStore
        return AWSJobStore.createJobStore(jobStoreArgs, config=config)
    elif jobStoreName == 'azure':
        from toil.jobStores.azureJobStore import AzureJobStore
        account, namePrefix = jobStoreArgs.split(':', 1)
        return AzureJobStore(account, namePrefix, config=config)
    else:
        raise RuntimeError("Unknown job store implementation '%s'" %
                           jobStoreName)
Exemplo n.º 2
0
 def _hashTestFile(self, url):
     from toil.jobStores.azureJobStore import AzureJobStore
     url = urlparse.urlparse(url)
     blobService, containerName, blobName = AzureJobStore._extractBlobInfoFromUrl(
         url)
     content = blobService.get_blob_to_bytes(containerName, blobName)
     return hashlib.md5(content).hexdigest()
Exemplo n.º 3
0
    def getJobStore(cls, locator):
        """
        Create an instance of the concrete job store implementation that matches the given locator.

        :param str locator: The location of the job store to be represent by the instance

        :return: an instance of a concrete subclass of AbstractJobStore
        :rtype: toil.jobStores.abstractJobStore.AbstractJobStore
        """
        name, rest = cls.parseLocator(locator)
        if name == 'file':
            from toil.jobStores.fileJobStore import FileJobStore
            return FileJobStore(rest)
        elif name == 'aws':
            from toil.jobStores.aws.jobStore import AWSJobStore
            return AWSJobStore(rest)
        elif name == 'azure':
            from toil.jobStores.azureJobStore import AzureJobStore
            return AzureJobStore(rest)
        elif name == 'google':
            from toil.jobStores.googleJobStore import GoogleJobStore
            projectID, namePrefix = rest.split(':', 1)
            return GoogleJobStore(namePrefix, projectID)
        else:
            raise RuntimeError("Unknown job store implementation '%s'" % name)
Exemplo n.º 4
0
 def _hashTestFile(self, url):
     from toil.jobStores.azureJobStore import AzureJobStore, retry_azure
     url = urlparse.urlparse(url)
     blob = AzureJobStore._parseWasbUrl(url)
     for attempt in retry_azure():
         with attempt:
             content = blob.service.get_blob_to_bytes(blob.container, blob.name)
             return hashlib.md5(content).hexdigest()
Exemplo n.º 5
0
 def _hashTestFile(self, url):
     from toil.jobStores.azureJobStore import AzureJobStore, retry_azure
     url = urlparse.urlparse(url)
     blob = AzureJobStore._parseWasbUrl(url)
     for attempt in retry_azure():
         with attempt:
             content = blob.service.get_blob_to_bytes(blob.container, blob.name)
             return hashlib.md5(content).hexdigest()
Exemplo n.º 6
0
    def loadOrCreateJobStore(jobStoreString, config=None):
        """
        Loads an existing jobStore if it already exists. Otherwise a new instance of a jobStore is
        created and returned.

        :param str jobStoreString: see exception message below
        :param toil.common.Config config: see AbstractJobStore.__init__
        :return: an instance of a concrete subclass of AbstractJobStore
        :rtype: toil.jobStores.abstractJobStore.AbstractJobStore
        """
        if jobStoreString[0] in '/.':
            jobStoreString = 'file:' + jobStoreString

        try:
            jobStoreName, jobStoreArgs = jobStoreString.split(':', 1)
        except ValueError:
            raise RuntimeError(
                'A job store string must either be a path starting in . or / or a contain at '
                'least one colon separating the name of the job store implementation from an '
                'initialization string specific to that job store. If a path starting in . or / '
                'is passed, the file job store will be used for backwards compatibility.'
            )

        if jobStoreName == 'file':
            from toil.jobStores.fileJobStore import FileJobStore
            return FileJobStore(jobStoreArgs, config=config)

        elif jobStoreName == 'aws':
            from toil.jobStores.aws.jobStore import AWSJobStore
            return AWSJobStore.loadOrCreateJobStore(jobStoreArgs,
                                                    config=config)

        elif jobStoreName == 'azure':
            from toil.jobStores.azureJobStore import AzureJobStore
            account, namePrefix = jobStoreArgs.split(':', 1)
            return AzureJobStore(account, namePrefix, config=config)

        elif jobStoreName == 'google':
            from toil.jobStores.googleJobStore import GoogleJobStore
            projectID, namePrefix = jobStoreArgs.split(':', 1)
            return GoogleJobStore(namePrefix, projectID, config=config)
        else:
            raise RuntimeError("Unknown job store implementation '%s'" %
                               jobStoreName)
Exemplo n.º 7
0
    def loadOrCreateJobStore(locator, config=None):
        """
        Loads an existing jobStore if it already exists. Otherwise a new instance of a jobStore is
        created and returned.

        :param str locator: The location of the job store.
        :param toil.common.Config config: see AbstractJobStore.__init__
        :return: an instance of a concrete subclass of AbstractJobStore
        :rtype: toil.jobStores.abstractJobStore.AbstractJobStore
        """
        if locator[0] in '/.':
            locator = 'file:' + locator

        try:
            jobStoreName, jobStoreArgs = locator.split(':', 1)
        except ValueError:
            raise RuntimeError('Invalid job store locator for proper formatting check locator '
                               'documentation for each job store.')

        if jobStoreName == 'file':
            from toil.jobStores.fileJobStore import FileJobStore
            return FileJobStore(jobStoreArgs, config=config)

        elif jobStoreName == 'aws':
            from toil.jobStores.aws.jobStore import AWSJobStore
            return AWSJobStore.loadOrCreateJobStore(jobStoreArgs, config=config)

        elif jobStoreName == 'azure':
            from toil.jobStores.azureJobStore import AzureJobStore
            account, namePrefix = jobStoreArgs.split(':', 1)
            return AzureJobStore(account, namePrefix, config=config)
        
        elif jobStoreName == 'google':
            from toil.jobStores.googleJobStore import GoogleJobStore
            projectID, namePrefix = jobStoreArgs.split(':', 1)
            return GoogleJobStore(namePrefix, projectID, config=config)
        else:
            raise RuntimeError("Unknown job store implementation '%s'" % jobStoreName)
Exemplo n.º 8
0
    def getJobStore(locator):
        """
        Create an instance of the concrete job store implementation that matches the given locator.

        :param str locator: The location of the job store to be represent by the instance

        :return: an instance of a concrete subclass of AbstractJobStore
        :rtype: toil.jobStores.abstractJobStore.AbstractJobStore
        """
        if locator[0] in '/.':
            locator = 'file:' + locator

        try:
            name, rest = locator.split(':', 1)
        except ValueError:
            raise RuntimeError('Invalid job store locator syntax.')

        if name == 'file':
            from toil.jobStores.fileJobStore import FileJobStore
            return FileJobStore(rest)

        elif name == 'aws':
            from toil.jobStores.aws.jobStore import AWSJobStore
            return AWSJobStore(rest)

        elif name == 'azure':
            from toil.jobStores.azureJobStore import AzureJobStore
            account, namePrefix = rest.split(':', 1)
            return AzureJobStore(rest)

        elif name == 'google':
            from toil.jobStores.googleJobStore import GoogleJobStore
            projectID, namePrefix = rest.split(':', 1)
            return GoogleJobStore(namePrefix, projectID, config=config)
        else:
            raise RuntimeError("Unknown job store implementation '%s'" % name)
Exemplo n.º 9
0
 def _createJobStore(self, config=None):
     from toil.jobStores.azureJobStore import AzureJobStore
     return AzureJobStore(self.accountName, self.namePrefix, config=config)
Exemplo n.º 10
0
 def _createJobStore(self):
     from toil.jobStores.azureJobStore import AzureJobStore
     return AzureJobStore(self.accountName + ':' + self.namePrefix)
Exemplo n.º 11
0
 def _cleanUpExternalStore(url):
     from toil.jobStores.azureJobStore import AzureJobStore
     blobService, containerName, _ = AzureJobStore._extractBlobInfoFromUrl(urlparse.urlparse(url))
     blobService.delete_container(containerName)
Exemplo n.º 12
0
 def _hashUrl(url):
     from toil.jobStores.azureJobStore import AzureJobStore
     blobService, containerName, blobName = AzureJobStore._extractBlobInfoFromUrl(urlparse.urlparse(url))
     content = blobService.get_blob_to_bytes(containerName, blobName)
     return hashlib.md5(content).hexdigest()
Exemplo n.º 13
0
 def _cleanUpExternalStore(url):
     from toil.jobStores.azureJobStore import AzureJobStore
     blobService, containerName, _ = AzureJobStore._extractBlobInfoFromUrl(
         urlparse.urlparse(url))
     blobService.delete_container(containerName)
Exemplo n.º 14
0
 def _createJobStore(self, config=None):
     from toil.jobStores.azureJobStore import AzureJobStore
     return AzureJobStore('toiltest',
                          self.namePrefix,
                          config=config,
                          jobChunkSize=128)