def test_node_run_tests(instance_name='ocgis-test-node', branch='next', failed='false'):
am = AwsManager()
instance = am.get_instance_by_name(instance_name)
tcenv = 'test-ocgis'
texclude = '!slow,!remote,!esmpy7'
tgdal_data = '/home/ubuntu/anaconda/envs/{0}/share/gdal'.format(tcenv)
tocgis_dir_shpcabinet = '/home/ubuntu/data/ocgis_test_data/shp'
tocgis_dir_test_data = '/home/ubuntu/data/ocgis_test_data/'
tsrc = '~/git/ocgis/src'
def _run_():
senv = dict(OCGIS_DIR_SHPCABINET=tocgis_dir_shpcabinet, OCGIS_DIR_TEST_DATA=tocgis_dir_test_data,
GDAL_DATA=tgdal_data)
with shell_env(**senv):
with prefix('source activate {0}'.format(tcenv)):
with cd(tsrc):
run('git checkout next')
run('git pull')
if failed == 'true':
cmd = 'cp .noseids /tmp; git checkout {tbranch}; git pull; nosetests -vs --failed --with-id -a {texclude} ocgis/test'
else:
cmd = 'cp .noseids /tmp; rm .noseids; git checkout {tbranch}; git pull; nosetests -vs --with-id -a {texclude} ocgis/test'
cmd = cmd.format(tbranch=branch, texclude=texclude)
run(cmd)
am.do_task(_run_, instance=instance)
def test_node_run_tests(instance_name='ocgis-test-node',
branch='next',
failed='false'):
am = AwsManager()
instance = am.get_instance_by_name(instance_name)
tcenv = 'test-ocgis'
texclude = '!slow,!remote,!esmpy7'
tgdal_data = '/home/ubuntu/anaconda/envs/{0}/share/gdal'.format(tcenv)
tocgis_dir_shpcabinet = '/home/ubuntu/data/ocgis_test_data/shp'
tocgis_dir_test_data = '/home/ubuntu/data/ocgis_test_data/'
tsrc = '~/git/ocgis/src'
def _run_():
senv = dict(OCGIS_DIR_SHPCABINET=tocgis_dir_shpcabinet,
OCGIS_DIR_TEST_DATA=tocgis_dir_test_data,
GDAL_DATA=tgdal_data)
with shell_env(**senv):
with prefix('source activate {0}'.format(tcenv)):
with cd(tsrc):
run('git checkout next')
run('git pull')
if failed == 'true':
cmd = 'cp .noseids /tmp; git checkout {tbranch}; git pull; nosetests -vs --failed --with-id -a {texclude} ocgis/test'
else:
cmd = 'cp .noseids /tmp; rm .noseids; git checkout {tbranch}; git pull; nosetests -vs --with-id -a {texclude} ocgis/test'
cmd = cmd.format(tbranch=branch, texclude=texclude)
run(cmd)
am.do_task(_run_, instance=instance)
def test_node_ebs_mount(instance_name='ocgis-test-node',
ebs_mount_name='/dev/xvdg',
ebs_mount_dir='~/data'):
am = AwsManager()
instance = am.get_instance_by_name(instance_name)
kwargs = {'mount_name': ebs_mount_name, 'mount_dir': ebs_mount_dir}
am.do_task(ebs_mount, instance=instance, kwargs=kwargs)
def test_node_launch(instance_type='m3.xlarge'):
am = AwsManager()
instance_name = 'ocgis-test-node'
image_id = 'ami-d1878fe1'
ebs_mount_dir = '~/data'
ebs_mount_name = '/dev/xvdf'
instance = am.launch_new_instance(instance_name, image_id=image_id, instance_type=instance_type)
test_node_ebs_mount(instance_name=instance_name, ebs_mount_name=ebs_mount_name, ebs_mount_dir=ebs_mount_dir)
print am.get_ssh_command(instance=instance)
def test_node_launch(instance_type='m3.xlarge'):
am = AwsManager()
instance_name = 'ocgis-test-node'
image_id = 'ami-d1878fe1'
ebs_mount_dir = '~/data'
ebs_mount_name = '/dev/xvdf'
instance = am.launch_new_instance(instance_name,
image_id=image_id,
instance_type=instance_type)
test_node_ebs_mount(instance_name=instance_name,
ebs_mount_name=ebs_mount_name,
ebs_mount_dir=ebs_mount_dir)
print am.get_ssh_command(instance=instance)
def test_run_bash_script(self):
def _exists_():
with cd('~/a-new-directory'):
self.assertTrue(True)
_, script = mkstemp()
try:
with open(script, 'w') as f:
f.write('mkdir ~/a-new-directory\n')
m = AwsManager(CONF_PATH)
assert self.shared_instance
m.do_task(run_bash_script, name=self.shared_instance_name, args=(script,))
m.do_task(_exists_, name=self.shared_instance_name)
finally:
os.remove(script)
class AbstractWorkerAWS(AbstractWorker):
__metaclass__ = ABCMeta
try:
from saws import AwsManager
except ImportError:
pass
else:
manager = AwsManager()
aws_key_path = manager.conf.aws_key_path
password = None
def __init__(self, *args, **kwargs):
AbstractWorker.__init__(self, *args, **kwargs)
env.key_filename = self.aws_key_path
assert self.instance is not None
@property
def ip_address(self):
self.instance = self.manager.get_instance_by_name(self.aws_name)
return self.instance.ip_address
@abstractproperty
def aws_ami(self):
return 'ami-65625655'
@abstractproperty
def aws_name(self):
return 'conda-builder'
@abstractproperty
def aws_type(self):
return 't2.micro'
@abstractproperty
def server_dir_data(self):
return '/home/me/data'
@classmethod
def ebs_mount(cls):
cmd = 'mount {0} {1}'.format(AWS_EBS_MOUNT_NAME, cls.server_dir_data)
sudo(cmd)
@classmethod
def launch(cls):
from helpers import get_ssh_cmd
LOG.info('launching aws instance')
if AWS_EBS_SHOULD_MOUNT:
ebs_snapshot_id = AWS_EBS_SNAPSHOT_ID
else:
ebs_snapshot_id = None
i = cls.manager.launch_new_instance(cls.aws_name, image_id=cls.aws_ami, instance_type=cls.aws_type,
ebs_snapshot_id=ebs_snapshot_id)
if AWS_EBS_SHOULD_MOUNT:
with settings(host_string=i.ip_address, connection_attempts=10, user=cls.user):
LOG.info('attaching volume')
cls.run('mkdir -p {0}'.format(cls.server_dir_data))
LOG.info('sleeping')
time.sleep(5)
cls.ebs_mount()
LOG.info('instance launched')
LOG.info(get_ssh_cmd(cls.aws_key_path, cls.user, i.ip_address))
def terminate(self):
self.instance.terminate()
LOG.info('instance terminated')
def test_node_terminate():
am = AwsManager()
instance_name = 'ocgis-test-node'
instance = am.get_instance_by_name(instance_name)
instance.terminate()
def test_node_ebs_mount(instance_name='ocgis-test-node', ebs_mount_name='/dev/xvdg', ebs_mount_dir='~/data'):
am = AwsManager()
instance = am.get_instance_by_name(instance_name)
kwargs = {'mount_name': ebs_mount_name, 'mount_dir': ebs_mount_dir}
am.do_task(ebs_mount, instance=instance, kwargs=kwargs)
def test_node_terminate():
am = AwsManager()
instance_name = 'ocgis-test-node'
instance = am.get_instance_by_name(instance_name)
instance.terminate()