Example #1
0
def test_node_run_tests(instance_name='ocgis-test-node', branch='next', failed='false'):
    am = AwsManager()
    instance = am.get_instance_by_name(instance_name)
    tcenv = 'test-ocgis'
    texclude = '!slow,!remote,!esmpy7'
    tgdal_data = '/home/ubuntu/anaconda/envs/{0}/share/gdal'.format(tcenv)
    tocgis_dir_shpcabinet = '/home/ubuntu/data/ocgis_test_data/shp'
    tocgis_dir_test_data = '/home/ubuntu/data/ocgis_test_data/'
    tsrc = '~/git/ocgis/src'

    def _run_():
        senv = dict(OCGIS_DIR_SHPCABINET=tocgis_dir_shpcabinet, OCGIS_DIR_TEST_DATA=tocgis_dir_test_data,
                    GDAL_DATA=tgdal_data)
        with shell_env(**senv):
            with prefix('source activate {0}'.format(tcenv)):
                with cd(tsrc):
                    run('git checkout next')
                    run('git pull')
                    if failed == 'true':
                        cmd = 'cp .noseids /tmp; git checkout {tbranch}; git pull; nosetests -vs --failed --with-id -a {texclude} ocgis/test'
                    else:
                        cmd = 'cp .noseids /tmp; rm .noseids; git checkout {tbranch}; git pull; nosetests -vs --with-id -a {texclude} ocgis/test'
                    cmd = cmd.format(tbranch=branch, texclude=texclude)
                    run(cmd)

    am.do_task(_run_, instance=instance)
Example #2
0
def test_node_run_tests(instance_name='ocgis-test-node',
                        branch='next',
                        failed='false'):
    am = AwsManager()
    instance = am.get_instance_by_name(instance_name)
    tcenv = 'test-ocgis'
    texclude = '!slow,!remote,!esmpy7'
    tgdal_data = '/home/ubuntu/anaconda/envs/{0}/share/gdal'.format(tcenv)
    tocgis_dir_shpcabinet = '/home/ubuntu/data/ocgis_test_data/shp'
    tocgis_dir_test_data = '/home/ubuntu/data/ocgis_test_data/'
    tsrc = '~/git/ocgis/src'

    def _run_():
        senv = dict(OCGIS_DIR_SHPCABINET=tocgis_dir_shpcabinet,
                    OCGIS_DIR_TEST_DATA=tocgis_dir_test_data,
                    GDAL_DATA=tgdal_data)
        with shell_env(**senv):
            with prefix('source activate {0}'.format(tcenv)):
                with cd(tsrc):
                    run('git checkout next')
                    run('git pull')
                    if failed == 'true':
                        cmd = 'cp .noseids /tmp; git checkout {tbranch}; git pull; nosetests -vs --failed --with-id -a {texclude} ocgis/test'
                    else:
                        cmd = 'cp .noseids /tmp; rm .noseids; git checkout {tbranch}; git pull; nosetests -vs --with-id -a {texclude} ocgis/test'
                    cmd = cmd.format(tbranch=branch, texclude=texclude)
                    run(cmd)

    am.do_task(_run_, instance=instance)
Example #3
0
def test_node_ebs_mount(instance_name='ocgis-test-node',
                        ebs_mount_name='/dev/xvdg',
                        ebs_mount_dir='~/data'):
    am = AwsManager()
    instance = am.get_instance_by_name(instance_name)
    kwargs = {'mount_name': ebs_mount_name, 'mount_dir': ebs_mount_dir}
    am.do_task(ebs_mount, instance=instance, kwargs=kwargs)
Example #4
0
def test_node_launch(instance_type='m3.xlarge'):
    am = AwsManager()
    instance_name = 'ocgis-test-node'
    image_id = 'ami-d1878fe1'
    ebs_mount_dir = '~/data'
    ebs_mount_name = '/dev/xvdf'
    instance = am.launch_new_instance(instance_name, image_id=image_id, instance_type=instance_type)
    test_node_ebs_mount(instance_name=instance_name, ebs_mount_name=ebs_mount_name, ebs_mount_dir=ebs_mount_dir)
    print am.get_ssh_command(instance=instance)
Example #5
0
def test_node_launch(instance_type='m3.xlarge'):
    am = AwsManager()
    instance_name = 'ocgis-test-node'
    image_id = 'ami-d1878fe1'
    ebs_mount_dir = '~/data'
    ebs_mount_name = '/dev/xvdf'
    instance = am.launch_new_instance(instance_name,
                                      image_id=image_id,
                                      instance_type=instance_type)
    test_node_ebs_mount(instance_name=instance_name,
                        ebs_mount_name=ebs_mount_name,
                        ebs_mount_dir=ebs_mount_dir)
    print am.get_ssh_command(instance=instance)
Example #6
0
    def test_run_bash_script(self):

        def _exists_():
            with cd('~/a-new-directory'):
                self.assertTrue(True)

        _, script = mkstemp()
        try:
            with open(script, 'w') as f:
                f.write('mkdir ~/a-new-directory\n')
            m = AwsManager(CONF_PATH)
            assert self.shared_instance
            m.do_task(run_bash_script, name=self.shared_instance_name, args=(script,))
            m.do_task(_exists_, name=self.shared_instance_name)
        finally:
            os.remove(script)
Example #7
0
class AbstractWorkerAWS(AbstractWorker):
    __metaclass__ = ABCMeta

    try:
        from saws import AwsManager
    except ImportError:
        pass
    else:
        manager = AwsManager()
        aws_key_path = manager.conf.aws_key_path
    password = None

    def __init__(self, *args, **kwargs):
        AbstractWorker.__init__(self, *args, **kwargs)
        env.key_filename = self.aws_key_path
        assert self.instance is not None

    @property
    def ip_address(self):
        self.instance = self.manager.get_instance_by_name(self.aws_name)
        return self.instance.ip_address

    @abstractproperty
    def aws_ami(self):
        return 'ami-65625655'

    @abstractproperty
    def aws_name(self):
        return 'conda-builder'

    @abstractproperty
    def aws_type(self):
        return 't2.micro'

    @abstractproperty
    def server_dir_data(self):
        return '/home/me/data'

    @classmethod
    def ebs_mount(cls):
        cmd = 'mount {0} {1}'.format(AWS_EBS_MOUNT_NAME, cls.server_dir_data)
        sudo(cmd)

    @classmethod
    def launch(cls):
        from helpers import get_ssh_cmd

        LOG.info('launching aws instance')
        if AWS_EBS_SHOULD_MOUNT:
            ebs_snapshot_id = AWS_EBS_SNAPSHOT_ID
        else:
            ebs_snapshot_id = None

        i = cls.manager.launch_new_instance(cls.aws_name, image_id=cls.aws_ami, instance_type=cls.aws_type,
                                            ebs_snapshot_id=ebs_snapshot_id)
        if AWS_EBS_SHOULD_MOUNT:
            with settings(host_string=i.ip_address, connection_attempts=10, user=cls.user):
                LOG.info('attaching volume')
                cls.run('mkdir -p {0}'.format(cls.server_dir_data))
                LOG.info('sleeping')
                time.sleep(5)
                cls.ebs_mount()
        LOG.info('instance launched')
        LOG.info(get_ssh_cmd(cls.aws_key_path, cls.user, i.ip_address))

    def terminate(self):
        self.instance.terminate()
        LOG.info('instance terminated')
Example #8
0
def test_node_terminate():
    am = AwsManager()
    instance_name = 'ocgis-test-node'
    instance = am.get_instance_by_name(instance_name)
    instance.terminate()
Example #9
0
def test_node_ebs_mount(instance_name='ocgis-test-node', ebs_mount_name='/dev/xvdg', ebs_mount_dir='~/data'):
    am = AwsManager()
    instance = am.get_instance_by_name(instance_name)
    kwargs = {'mount_name': ebs_mount_name, 'mount_dir': ebs_mount_dir}
    am.do_task(ebs_mount, instance=instance, kwargs=kwargs)
Example #10
0
def test_node_terminate():
    am = AwsManager()
    instance_name = 'ocgis-test-node'
    instance = am.get_instance_by_name(instance_name)
    instance.terminate()