def default_application(appname):
"""The default application loading function."""
parts = appname.split(':')
if len(parts) in (1, 2) and env.manifest is None:
error.raise_error('No manifest found ({0}).\n'
'Please specify the fully qualified app name.\n'
"Use 'ravtest ps -a' for a list.",
manifest.manifest_name())
if len(parts) in (1, 2):
project = env.manifest['project']['name']
console.info('Project name is `{0}`.', project)
defname = parts[0]
instance = parts[1] if len(parts) == 2 else None
elif len(parts) == 3:
project, defname, instance = parts
else:
error.raise_error('Illegal application name: `{0}`.', appname)
apps = cache.find_applications(project, defname, instance)
if len(apps) == 0:
error.raise_error('No instances of application `{0}` exist.',
defname)
elif len(apps) > 1:
error.raise_error('Multiple instances of `{0}` exist.\n'
'Use `ravtest ps` to list the instances and then\n'
'specify the application with its instance id.',
defname)
app = cache.get_application(apps[0]['id'])
return app
def add_defaults(manifest):
"""Add defaults to manifest."""
directory = manifest['_directory']
manifest.setdefault('project', {})
name = manifest['project'].get('name')
if name is None:
_, name = os.path.split(directory)
console.info("Using '{0}' as the project name.", name)
manifest['project']['name'] = name
language = manifest['project'].get('language')
if language is None:
language = detect_language(manifest, directory)
if language:
console.info("Detected a {0} project.", language)
manifest['project']['language'] = language
if language and language not in manifest.get('languages', {}):
console.warning("Unknown language '{0}' in manifest.", language)
repo = manifest.setdefault('repository', {})
typ = repo.get('type')
if typ is None:
typ = versioncontrol.detect_type(directory)
if typ:
console.info('Detected a {0} repository.', typ)
repo['type'] = typ
url = repo.get('url')
if url is None and typ:
url = versioncontrol.get_origin(directory, typ)
if url:
console.info("Using remote origin '{0}'.", url)
repo['url'] = url
manifest.setdefault('defaults', {})
manifest.setdefault('languages', {})
manifest.setdefault('applications', [])
def do_login(args, env):
"""The "ravello login" command."""
username = args.username or args.user
password = args.password
if username is None:
console.writeln('Enter your Ravello credentials.')
if username is None:
username = console.prompt('Username: '******'Password: '******'Successfully logged in.')
return 0
def do_lint(args, env):
"""The "ravtest lint" command."""
if args.check_entities:
login.default_login()
console.info('Checking manifest...')
manif = manifest.load_manifest()
manifest.check_manifest(manif)
manifest.add_defaults(manif)
manifest.percolate_defaults(manif)
manifest.expand_shorthands(manif)
manifest.complete_data(manif)
if args.check_entities:
manifest.check_manifest_entities(manif)
console.info('Manifest OK')
if args.dump:
console.write(util.prettify(manif))
def create_keypair():
"""Create a new keypair and upload it to Ravello."""
cfgdir = util.get_config_dir()
privname = os.path.join(cfgdir, 'id_ravello')
pubname = privname + '.pub'
keyname = 'ravello@%s' % socket.gethostname()
# Prefer to generate the key locallly with ssh-keygen because
# that gives us more privacy. If ssh-keygen is not available, ask
# for a key through the API.
sshkeygen = util.which('ssh-keygen')
if sshkeygen:
try:
console.info("Generating keypair using 'ssh-keygen'...")
subprocess.call(['ssh-keygen', '-q', '-t', 'rsa', '-C', keyname,
'-b', '2048', '-N', '', '-f', privname])
except subprocess.CalledProcessError as e:
error.raise_error('ssh-keygen returned with error status {0}',
e.returncode)
with file(pubname) as fin:
pubkey = fin.read()
keyparts = pubkey.strip().split()
else:
keyname = 'ravello@api-generated'
console.info('Requesting a new keypair via the API...')
keypair = env.api.create_keypair()
with file(privname, 'w') as fout:
fout.write(keypair['privateKey'])
with file(pubname, 'w') as fout:
fout.write(keypair['publicKey'].rstrip())
fout.write(' {0} (generated remotely)\n'.format(keyname))
pubkey = keypair['publicKey'].rstrip()
keyparts = pubkey.split()
keyparts[2:] = [keyname]
# Create the pubkey in the API under a unique name
pubkeys = env.api.get_pubkeys()
keyname = util.get_unused_name(keyname, pubkeys)
keyparts[2] = keyname
keydata = '{0} {1} {2}\n'.format(*keyparts)
pubkey = {'name': keyname}
pubkey['publicKey'] = keydata
pubkey = env.api.create_pubkey(pubkey)
with file(pubname, 'w') as fout:
fout.write(keydata)
env.public_key = pubkey
env.private_key_file = privname
return pubkey
def do_run(args, env):
"""The "ravello run" command."""
login.default_login()
keypair.default_keypair()
manif = manifest.default_manifest()
appname = args.application
for appdef in manif.get('applications', []):
if appdef['name'] == appname:
break
else:
error.raise_error("Unknown application `{0}`.", appname)
vms = set((vm['name'] for vm in appdef.get('vms', [])))
if args.vms:
only = set((name for name in args.vms.split(',')))
if not only <= vms:
unknown = [name for name in only if name not in vms]
what = inflect.plural_noun('virtual machine', len(unknown))
error.raise_error("Unknown {0}: {1}", ', '.join(unknown), what)
vms = [name for name in vms if name in only]
if not vms:
error.raise_error('No virtual machines in application.')
app = application.create_or_reuse_application(appdef, args.new)
app = application.wait_for_application(app, vms)
if args.command:
for vm in appdef['vms']:
for task in vm['tasks']:
if task['name'] == 'execute':
task['commands'] = [args.command]
elif args.dry_run:
for vm in appdef['vms']:
vm['tasks'] = []
ret = tasks.run_all_tasks(app, vms)
console.info('\n== The following services will be available for {0} '
'minutes:\n', appdef['keepalive'])
for vm in app['vms']:
if vm['name'] not in vms:
continue
svcs = vm.get('suppliedServices')
if not svcs:
continue
console.info('On virtual machine `{0}`:', vm['name'])
for svc in svcs:
svc = svc['baseService']
addr = util.format_service(vm, svc)
console.info(' * {0}: {1}', svc['name'], addr)
console.info('')
return error.EX_OK if ret == 0 else error.EX_SOFTWARE
def do_save(args, env):
"""The "ravello save" command."""
with env.let(quiet=True):
login.default_login()
keypair.default_keypair()
manif = manifest.default_manifest(required=False)
app = application.default_application(args.application)
appname = app["name"]
project, defname, instance = appname.split(":")
state = application.get_application_state(app)
if state not in ("STOPPED", "STARTED"):
error.raise_error(
"Application `{0}:{1}` is currently in state {2}.\n" "Can only create blueprint when STOPPED or STARTED.",
defname,
instance,
state,
)
if state == "STARTED" and not env.always_confirm:
console.info("Application `{0}:{1}` is currently running.", defname, instance)
result = console.confirm("Do you want to continue with a live snapshot")
if not result:
console.info("Not confirmed.")
return error.EX_OK
template = "{0}:{1}".format(project, defname)
bpname = util.get_unused_name(template, cache.get_blueprints())
parts = bpname.split(":")
console.info("Saving blueprint as `{0}:{1}`.", parts[1], parts[2])
blueprint = env.api.create_blueprint(bpname, app)
env.blueprint = blueprint # for tests
console.info("Blueprint creation process started.")
console.info("Use 'ravtest ps -b' to monitor progress.")
return error.EX_OK
def create_or_reuse_application(appdef, force_new):
"""Create a new application or re-use a suitable existing one."""
app = None
if not force_new:
app = reuse_existing_application(appdef)
if app is not None:
state = get_application_state(app)
parts = app['name'].split(':')
console.info('Re-using {0} application `{1}:{2}`.',
state.lower(), parts[1], parts[2])
app = start_application(app)
if app is None:
app = create_new_application(appdef)
app = publish_application(app)
parts = app['name'].split(':')
console.info('Created new application `{1}:{2}`.', *parts)
console.info('Published to {0[cloud]}/{0[regionName]}.', app)
return app
def do_ps(args, env):
"""The "ravello ps" command."""
with env.let(quiet=True):
login.default_login()
pubkey = keypair.default_keypair()
manif = manifest.default_manifest(required=False)
if manif is None and not args.all:
error.raise_error('Project manifest ({0}) not found.\n'
"Use 'ravtest ps -a' to list all applications.",
manifest.manifest_name())
if args.all:
project = None
else:
project = manif['project']['name']
console.info('Project name is `{0}`.', project)
if args.blueprint:
apps = cache.find_blueprints(project)
what = 'blueprint'
else:
apps = cache.find_applications(project)
what = 'application'
apps = sorted(apps, key=lambda app: app['name'])
objs = inflect.plural_noun(what)
console.writeln('Currently available {0}:\n', objs)
current_project = None
for app in apps:
parts = app['name'].split(':')
if parts[0] != project and not args.all:
continue
if args.all and current_project != parts[0]:
console.writeln("== Project: `{0}`", parts[0])
current_project = parts[0]
if args.full and not args.blueprint:
app = cache.get_application(app['id'])
cloud = app.get('cloud')
region = app.get('regionName')
started = app.get('totalStartedVms')
publish_time = app.get('publishStartTime')
creation_time = app.get('creationTime')
created = publish_time or creation_time
if created:
now = time.time()
created = util.format_timedelta(now - created/1000)
created = '{0} ago'.format(created)
else:
created = ''
if args.full and not args.blueprint:
vms = [ vm['name'] for vm in application.get_vms(app) ]
vms = '`{0}`'.format('`, `'.join(vms))
state = application.get_application_state(app)
else:
state = app.get('state') or ''
console.writeln('=== {0}: `{1}:{2}`', what.title(), parts[1], parts[2])
what2 = inflect.plural_noun('VM', started)
if state:
console.writeln(' state: {0}', state)
if started is not None:
console.writeln(' {0} {1} running', started, what2)
if cloud is not None:
console.writeln(' published to {0}/{1}', cloud, region)
if created:
console.writeln(' created: {0}', created)
if args.full and not args.blueprint:
console.writeln(' VMs: {0}', vms)
console.writeln()
return error.EX_OK
def run_all_tasks(app, vms):
"""Run the runbook for an application ``app``."""
hosts = []
host_info = {}
appname = app['name'].split(':')[1]
for appdef in env.manifest['applications']:
if appdef['name'] == appname:
break
else:
error.raise_error('Application definition not found?')
for vm in app['vms']:
if vm['name'] not in vms:
continue
ipaddr = vm['dynamicMetadata']['externalIp']
hosts.append(ipaddr)
host_info[ipaddr] = vm['name']
env.test_id = os.urandom(16).encode('hex')
console.info('Starting run `{0}`.', env.test_id)
env.host_info = host_info
env.start_time = int(time.time())
env.lock = multiprocessing.Lock()
manager = multiprocessing.Manager()
env.shared_state = manager.dict()
for vmname in vms:
vmstate = {}
vmstate['exited'] = False
vmstate['completed_tasks'] = {}
vmstate['shell_env_update'] = {}
env.shared_state[vmname] = vmstate
env.appdef = appdef
env.application = app
env.vms = vms
fab.env.user = '******'
fab.env.key_filename = env.private_key_file
fab.env.disable_known_hosts = True
fab.env.remote_interrupt = True
fab.env.hosts = hosts
fab.env.parallel = True
fab.env.output_prefix = env.debug
fabric.state.output.running = env.debug
fabric.state.output.output = True
fabric.state.output.status = env.debug
# This is where it all happens...
noun = inflect.plural_noun('virtual machine', len(vms))
console.info('Executing tasks on {0} {1}...', len(vms), noun)
fabric.tasks.execute(run_tasklist, env)
errors = set()
for vmname in vms:
vmstate = env.shared_state[vmname]
for taskname,status in vmstate['completed_tasks'].items():
if status != 0:
errors.add('`{0}` on `{1}`'.format(taskname, vmname))
if not errors:
console.info('All tasks were executed succesfully!')
else:
what = inflect.plural_noun('task', len(errors))
errapps = ', '.join(errors)
console.error('The following {0} failed: {1}', what, errapps)
fabric.network.disconnect_all()
return len(errors)
def do_ssh(args, env):
"""The "ravello ssh" command."""
with env.let(quiet=True):
login.default_login()
keypair.default_keypair()
if manifest.manifest_exists():
with env.let(quiet=True):
manif = manifest.default_manifest()
else:
manif = None
parts = args.application.split(':')
if len(parts) in (1, 2) and manif is None:
error.raise_error('No manifest found ({0}).\n'
'Please specify the fully qualified app name.\n'
'Use `ravtest ps --all` for a list.',
manifest.manifest_name())
if len(parts) in (1, 2):
project = manif['project']['name']
console.info('Project name is `{0}`.', project)
defname = parts[0]
instance = parts[1] if len(parts) == 2 else None
elif len(parts) == 3:
project, defname, instance = parts
else:
error.raise_error('Illegal application name: `{0}`.', appname)
apps = cache.find_applications(project, defname, instance)
if len(apps) == 0:
error.raise_error('No instances of application `{0}` exist.',
defname)
elif len(apps) > 1:
error.raise_error('Multiple instances of `{0}` exist.\n'
'Use `ravtest ps` to list the instances and then\n'
'specify the application with its instance id.',
defname)
app = cache.get_application(apps[0]['id'])
appname = app['name']
_, _, instance = appname.split(':')
vmname = args.vm
vm = application.get_vm(app, vmname)
if vm is None:
error.raise_error('Application `{0}:{1}` has no VM named `{2}`.\n'
'Use `ravtest ps --full` to see a list of VMs.',
defname, instance, vmname)
console.info("Connecting to VM `{0}` of application `{1}:{2}`...",
vmname, defname, instance)
# Start up the application and wait for it if we need to.
state = application.get_application_state(app)
if state not in ('PUBLISHING', 'STARTING', 'STOPPED', 'STARTED'):
error.raise_error("VM `{0}` is in an unknown state.", vmname)
userdata = vm.get('customVmConfigurationData', {})
vmkey = userdata.get('keypair', {})
if vmkey.get('id') != env.public_key['id']:
error.raise_error("VM uses unknown public key `{0}`.",
vmkey.get('name'))
application.start_application(app)
application.wait_for_application(app, [vmname])
# Now run ssh. Prefer openssh but fall back to using Fabric/Paramiko.
host = 'ravello@{0}'.format(vm['dynamicMetadata']['externalIp'])
command = '~/bin/run {0}'.format(args.testid)
openssh = util.find_openssh()
interactive = os.isatty(sys.stdin.fileno())
if interactive and openssh:
if not sys.platform.startswith('win'):
# On Unix use execve(). This is the most efficient.
argv = ['ssh', '-i', env.private_key_file,
'-o', 'UserKnownHostsFile=/dev/null',
'-o', 'StrictHostKeyChecking=no',
'-o', 'LogLevel=quiet',
'-t', host, command]
console.debug('Starting {0}', ' '.join(argv))
os.execve(openssh, argv, os.environ)
else:
# Windows has execve() but for some reason it does not work
# well with arguments with spaces in it. So use subprocess
# instead.
command = [openssh, '-i', env.private_key_file,
'-o', 'UserKnownHostsFile=NUL',
'-o', 'StrictHostKeyChecking=no',
'-o', 'LogLevel=quiet',
'-t', host, command]
ssh = subprocess.Popen(command)
ret = ssh.wait()
error.exit(ret)
# TODO: should also support PuTTY on Windows
console.info(textwrap.dedent("""\
Warning: no local openssh installation found.
Falling back to Fabric/Paramiko for an interactive shell.
However, please note:
* CTRL-C and terminal resize signals may not work.
* Output of programs that repaint the screen may
be garbled (e.g. progress bars).
"""))
fab.env.host_string = host
fab.env.key_filename = env.private_key_file
fab.env.disable_known_hosts = True
fab.env.remote_interrupt = True
fab.env.output_prefix = None
fabric.state.output.running = None
fabric.state.output.status = None
ret = fab.run(command, warn_only=True)
return ret.return_code