コード例 #1
0
ファイル: instance_flow.py プロジェクト: carze/vappio
def runInstances(credClient,
                 ami,
                 key,
                 iType,
                 groups,
                 availZone,
                 bidPrice,
                 minInstances,
                 maxInstances,
                 userData):
    def _runInstances(num):
        if bidPrice:
            return credClient.runSpotInstances(bidPrice=bidPrice,
                                               ami=ami,
                                               key=key,
                                               instanceType=iType,
                                               groups=groups,
                                               availabilityZone=availZone,
                                               numInstances=num,
                                               userData=userData)
        else:
            return credClient.runInstances(ami=ami,
                                           key=key,
                                           instanceType=iType,
                                           groups=groups,
                                           availabilityZone=availZone,
                                           numInstances=num,
                                           userData=userData)

    instances = []
    @defer.inlineCallbacks
    def _startInstances():
        startedInstances = yield _runInstances(maxInstances - len(instances))
        instances.extend(startedInstances)
        if len(instances) < minInstances:
            raise InstanceStartError('Wanted %d instances got %d' %
                                     (maxInstances - len(instances),
                                      len(startedInstances)))


    try:
        yield defer_utils.tryUntil(RUN_INSTANCE_TRIES,
                                   _startInstances,
                                   onFailure=defer_utils.sleep(30))
    except Exception, err:
        ## If we got an exception then terminate any instances
        ## that were started and reraise exception.
        ## The last thing we want is to leak instances
        ##
        ## This is not completely safe!  We should probably
        ## raise an exception with the started instances in it
        ## and let the caller decide what to do with them
        log.err('Error starting instances')
        log.err(err)
        defer_utils.mapSerial(lambda iChunk :
                                  credClient.terminateInstances(iChunk),
                              func.chunk(5, instances))
コード例 #2
0
ファイル: instance_flow.py プロジェクト: carze/vappio
def runInstances(credClient, ami, key, iType, groups, availZone, bidPrice,
                 minInstances, maxInstances, userData):
    def _runInstances(num):
        if bidPrice:
            return credClient.runSpotInstances(bidPrice=bidPrice,
                                               ami=ami,
                                               key=key,
                                               instanceType=iType,
                                               groups=groups,
                                               availabilityZone=availZone,
                                               numInstances=num,
                                               userData=userData)
        else:
            return credClient.runInstances(ami=ami,
                                           key=key,
                                           instanceType=iType,
                                           groups=groups,
                                           availabilityZone=availZone,
                                           numInstances=num,
                                           userData=userData)

    instances = []

    @defer.inlineCallbacks
    def _startInstances():
        startedInstances = yield _runInstances(maxInstances - len(instances))
        instances.extend(startedInstances)
        if len(instances) < minInstances:
            raise InstanceStartError(
                'Wanted %d instances got %d' %
                (maxInstances - len(instances), len(startedInstances)))

    try:
        yield defer_utils.tryUntil(RUN_INSTANCE_TRIES,
                                   _startInstances,
                                   onFailure=defer_utils.sleep(30))
    except Exception, err:
        ## If we got an exception then terminate any instances
        ## that were started and reraise exception.
        ## The last thing we want is to leak instances
        ##
        ## This is not completely safe!  We should probably
        ## raise an exception with the started instances in it
        ## and let the caller decide what to do with them
        log.err('Error starting instances')
        log.err(err)
        defer_utils.mapSerial(
            lambda iChunk: credClient.terminateInstances(iChunk),
            func.chunk(5, instances))
コード例 #3
0
ファイル: tag_mq_transfer.py プロジェクト: carze/vappio
def _makeDirsOnCluster(cluster, dirNames):
    """
    Creates a series of directories on a cluster
    """
    @defer.inlineCallbacks
    def _createDir(d):
        yield ssh.runProcessSSH(cluster['master']['public_dns'],
                                'mkdir -p ' + d,
                                None,
                                log.err,
                                cluster['config']['ssh.user'],
                                cluster['config']['ssh.options'])
        try:
            if cluster['master']['instance_type'] is not None:
                yield ssh.runProcessSSH(cluster['master']['public_dns'],
                                        'chown -R %s %s' % (cluster['config']['vappio.user'],
                                                            d),
                                        None,
                                        log.err,
                                        cluster['config']['ssh.user'],
                                        cluster['config']['ssh.options'])
        except commands.Error:
            pass
        
    return defer_utils.mapSerial(_createDir, dirNames)
コード例 #4
0
ファイル: persist.py プロジェクト: carze/vappio
        def _deleteEmptyDirs(files):
            def _rmDir(d):
                return commands.runProcess(
                    ['rmdir', d], stderrf=log.err).addErrback(lambda _: None)

            dirs = set([os.path.dirname(f) for f in files])
            return defer_utils.mapSerial(_rmDir, dirs)
コード例 #5
0
def downloadMapSerial(urls):
    """
    Uses mapSerial to download all urls in serial
    """
    getPage = lambda url: http.getPage(url, connectionTimeout=30, timeout=30
                                       ).addCallback(lambda content:
                                                     (url, content))

    d = defer_utils.mapSerial(getPage, urls)
    d.addCallback(dict)
    return d
コード例 #6
0
ファイル: cluster_mq_terminate.py プロジェクト: carze/vappio
def terminateCluster(credClient, persistManager, clusterName, userName):
    cluster = yield persistManager.loadCluster(clusterName, userName)

    yield defer_utils.mapSerial(
        lambda instances: credClient.terminateInstances(instances),
        func.chunk(5, cluster.execNodes + cluster.dataNodes))

    if cluster.master:
        yield credClient.terminateInstances([cluster.master])

    defer.returnValue(cluster.setState(cluster.TERMINATED))
コード例 #7
0
ファイル: cluster_mq_terminate.py プロジェクト: carze/vappio
def terminateCluster(credClient, persistManager, clusterName, userName):
    cluster = yield persistManager.loadCluster(clusterName, userName)

    yield defer_utils.mapSerial(
        lambda instances: credClient.terminateInstances(instances), func.chunk(5, cluster.execNodes + cluster.dataNodes)
    )

    if cluster.master:
        yield credClient.terminateInstances([cluster.master])

    defer.returnValue(cluster.setState(cluster.TERMINATED))
コード例 #8
0
ファイル: download_urls.py プロジェクト: carze/vappio
def downloadMapSerial(urls):
    """
    Uses mapSerial to download all urls in serial
    """
    getPage = lambda url : http.getPage(url,
                                        connectionTimeout=30,
                                        timeout=30
                                        ).addCallback(lambda content : (url, content))

    d = defer_utils.mapSerial(getPage, urls)
    d.addCallback(dict)
    return d
コード例 #9
0
def terminateInstancesByAttribute(persistManager, credClient, clusterName,
                                  userName, byAttribute, attributeValues):

    cluster = yield persistManager.loadCluster(clusterName, userName)

    instances = [
        i for i in cluster.execNodes + cluster.dataNodes
        if i[byAttribute] in attributeValues
    ]

    yield defer_utils.mapSerial(credClient.terminateInstances,
                                func.chunk(5, instances))

    defer.returnValue(
        cluster.removeExecNodes(instances).removeDataNodes(instances))
コード例 #10
0
def terminateInstancesByAttribute(persistManager,
                                  credClient,
                                  clusterName,
                                  userName,
                                  byAttribute,
                                  attributeValues):
    
    cluster = yield persistManager.loadCluster(clusterName, userName)

    instances = [i
                 for i in cluster.execNodes + cluster.dataNodes
                 if i[byAttribute] in attributeValues]
    
    yield defer_utils.mapSerial(credClient.terminateInstances,
                                func.chunk(5, instances))

    defer.returnValue(cluster.removeExecNodes(instances).removeDataNodes(instances))
コード例 #11
0
ファイル: cluster_mq_terminate.py プロジェクト: carze/vappio
def removeTerminatedCluster(persistManager, credClient, clusterName, userName):
    yield defer_utils.sleep(REMOVE_CLUSTER_TIMEOUT)()
    cluster = yield persistManager.loadCluster(clusterName, userName)

    if cluster.state == cluster.TERMINATED:
        # Another check to make sure the instances have
        # really been terminated
        instances = ([cluster.master] + cluster.execNodes + cluster.dataNodes)

        instances = yield credClient.updateInstances(instances)

        undeadInstances = [i for i in instances if i['state'] != 'terminated']

        if undeadInstances:
            yield defer_utils.mapSerial(
                lambda instances: credClient.terminateInstances(instances),
                func.chunk(5, undeadInstances))

        yield persistManager.removeCluster(clusterName, userName)
コード例 #12
0
ファイル: cluster_mq_terminate.py プロジェクト: carze/vappio
def removeTerminatedCluster(persistManager, credClient, clusterName, userName):
    yield defer_utils.sleep(REMOVE_CLUSTER_TIMEOUT)()
    cluster = yield persistManager.loadCluster(clusterName, userName)

    if cluster.state == cluster.TERMINATED:
        # Another check to make sure the instances have
        # really been terminated
        instances = [cluster.master] + cluster.execNodes + cluster.dataNodes

        instances = yield credClient.updateInstances(instances)

        undeadInstances = [i for i in instances if i["state"] != "terminated"]

        if undeadInstances:
            yield defer_utils.mapSerial(
                lambda instances: credClient.terminateInstances(instances), func.chunk(5, undeadInstances)
            )

        yield persistManager.removeCluster(clusterName, userName)
コード例 #13
0
def _makeDirsOnCluster(cluster, dirNames):
    """
    Creates a series of directories on a cluster
    """
    @defer.inlineCallbacks
    def _createDir(d):
        yield ssh.runProcessSSH(cluster['master']['public_dns'],
                                'mkdir -p ' + d, None, log.err,
                                cluster['config']['ssh.user'],
                                cluster['config']['ssh.options'])
        try:
            if cluster['master']['instance_type'] is not None:
                yield ssh.runProcessSSH(
                    cluster['master']['public_dns'],
                    'chown -R %s %s' % (cluster['config']['vappio.user'], d),
                    None, log.err, cluster['config']['ssh.user'],
                    cluster['config']['ssh.options'])
        except commands.Error:
            pass

    return defer_utils.mapSerial(_createDir, dirNames)
コード例 #14
0
        try:
            self.cache.pop(credential)
        except Exception, e:
            log.err(e)

        return defer.succeed(None)

    @defer_utils.timeIt
    @defer.inlineCallbacks
    def refreshInstances(self):
        @defer.inlineCallbacks
        def _refresh(credDict):
            credInstance = credDict['cred_instance']

            try:
                instances = yield credInstance.listInstances()
                credDict['instances'] = instances
            except Exception, e:
                log.err(e)

        yield defer_utils.mapSerial(_refresh, self.cache.values())

        self.state.refreshInstancesDelayed = reactor.callLater(
            REFRESH_INTERVAL, self.refreshInstances)

    def getCredential(self, credName):
        return self.cache[credName]

    def getAllCredentials(self):
        return self.cache
コード例 #15
0
ファイル: persist.py プロジェクト: carze/vappio
        def _deleteEmptyDirs(files):
            def _rmDir(d):
                return commands.runProcess(["rmdir", d], stderrf=log.err).addErrback(lambda _: None)

            dirs = set([os.path.dirname(f) for f in files])
            return defer_utils.mapSerial(_rmDir, dirs)
コード例 #16
0
ファイル: credentials_cache.py プロジェクト: carze/vappio
    def deleteCredentialFromCache(self, credential):
        try:
            self.cache.pop(credential)
        except Exception, e:
            log.err(e)

        return defer.succeed(None)

    @defer_utils.timeIt
    @defer.inlineCallbacks
    def refreshInstances(self):
        @defer.inlineCallbacks
        def _refresh(credDict):
            credInstance = credDict["cred_instance"]

            try:
                instances = yield credInstance.listInstances()
                credDict["instances"] = instances
            except Exception, e:
                log.err(e)

        yield defer_utils.mapSerial(_refresh, self.cache.values())

        self.state.refreshInstancesDelayed = reactor.callLater(REFRESH_INTERVAL, self.refreshInstances)

    def getCredential(self, credName):
        return self.cache[credName]

    def getAllCredentials(self):
        return self.cache