def ssh(name): """ Opens a shell connection to a host given by its EC2 name. """ instance = get_instance_by_tags({'Name': name}) if instance is None: return with settings(user='******', host_string=instance.public_dns_name, key_filename='/Users/aismail/.ssh/ami-keypair.pem'): open_shell()
def generate_hiera_datasources(): """ Hiera datasources are hierarchical configurations to be applied by puppet automatically in its template files. We need them for provisioning purposes. """ instances = get_all_instances() local('rm -rf /tmp/hiera') local('mkdir -p /tmp/hiera/node') common_context = { 'mongo_hostname': get_instance_by_tags({'Name': 'measurements'}).public_dns_name, 'redis_hostname': get_instance_by_tags({'Name': 'sessions'}).public_dns_name, 'kestrel_hostname': get_instance_by_tags({'Name': 'queues'}).public_dns_name } render_template('admin/templates/common.json', common_context, '/tmp/hiera/common.json') for instance in instances: context = {'hostname': instance.public_dns_name} render_template('admin/templates/node.json', context, '/tmp/hiera/node/%s.json' % instance.private_dns_name)
def generate_settings_local_file(): """ Given a crunch node, generate a settings.py file pointing the modules running on it to the correct resources (redis/kestrel/mongodb/etc). """ mongo = get_instance_by_tags({'Name': 'measurements'}) if not mongo: print("Could not find measurements DB in the cloud!") return redis = get_instance_by_tags({'Name': 'sessions'}) if not redis: print("Could not find sessions DB in the cloud!") return kestrel = get_instance_by_tags({'Name': 'queues'}) if not kestrel: print("Could not find queues machine in the cloud!") return # Render the settings_local.py file template to a string. context = { 'kestrel_server': kestrel.public_dns_name, 'kestrel_port': 22133, 'mongo_server': mongo.public_dns_name, 'mongo_port': 27017, 'redis_server': redis.public_dns_name, 'redis_port': 6379, } content = render_template('admin/templates/settings.py', context).split('\n') # Afterwards, string is written line by line using echo to the remote # host. file_path = '/home/ami/AmI-Platform/core/settings_local.py' run('echo "" > %s' % file_path) for line in content: run('echo "%s" >> %s' % (line, file_path))
def ssh(name): """ Opens a shell connection to a host given by its EC2 name. """ instance = get_instance_by_tags({'Name': name}) if instance is None: return # Use command-line SSH instead of fab's open_shell which is troublesome # and ignore host-related warnings. # http://stackoverflow.com/questions/9299651/warning-permanently-added-to-the-list-of-known-hosts-message-from-git local('ssh -i %s ' '-o StrictHostKeyChecking=no ' '-o UserKnownHostsFile=/dev/null ' '-o LogLevel=quiet ' 'ami@%s' % (env.key_filename, instance.public_dns_name))
def host(name): instance = get_instance_by_tags({'Name': name}) if instance is None: return print instance.public_dns_name
def play_experiment(name='duminica'): # File name where the experiment should be downloaded on the recorder # host. If it exists, it will just be played back, instead of re-downloaded. file_name = '/tmp/%s.txt' % name # URL from S3 where the experiment should be found. Note that for now # we are using S3 to host the experiments on a standard bucket. url = 'https://s3.amazonaws.com/ami-lab-experiments/%s.txt' % name # Connect to MongoDB host and erase all measurements so far. This will # prevent then from accumulating and filling up the disk mongo_instance = get_instance_by_tags({'Name': 'measurements'}) if mongo_instance is None: print("Could not find MongoDB instance where measurements are stored.") return with settings(host_string=mongo_instance.public_dns_name): run('mongo measurements --eval "db.docs.remove();"') # Connect to Redis host and clean up all the state gathered so far. # This will make sure that the experiment runs on a blank state. redis_instance = get_instance_by_tags({'Name': 'sessions'}) if redis_instance is None: print("Could not find Redis instance where sessions are stored.") return with settings(host_string=redis_instance.public_dns_name): run('redis-cli flushall') # Search among the crunch nodes the one on which ami-recorder is running recorder_hostname = get_crunch_running('ami-recorder') if recorder_hostname is None: print("Something is misconfigured. No crunch node is running the " "ami-recorder module, thus we have nowhere to run the experiment") return with settings(host_string=recorder_hostname): with cd('/home/ami/AmI-Platform'): # Only create the experiment record in MongoDB if it isn't there # already. Otherwise, just stay put :) # # We don't need to check whether the experiment actually corresponds # to the correct file, since the file name is currently generated # automatically from the experiment name. if str(run('python experiment.py list | grep %s | wc -l' % file_name)).strip() == '0': # Start and stop the experiment immediately just to create a # record in MongoDB in order to fool the experiment system :) run('python experiment.py --file %s start %s' % (file_name, name)) run('python experiment.py stop %s' % name) # Make sure we're only fetching the dump if the file doesn't # already exist. # TODO(andrei): check the integrity of the file as well via md5 if str(run('ls -la /tmp/ | grep "%s.txt" | wc -l' % name)).strip() == '0': # Fetch the dump from the remote location and place it just # where the experiment system thinks it recorded it. run('wget %s -O %s' % (url, file_name)) # This will cause the experiment measurements to be pumped # in the kestrel queues, thus triggering the whole processing # along the pipeline run('python experiment.py play %s' % name)