def __init__(self, controller):
self.name = "OpenStack launcher"
self.docker_registry = None
self.docker_user = None
self.heat_server = None
self.openstack_key = None
self.system = System()
self.installer = Installer()
self.controller = controller
self._init_open_stack()
self._init_app_db()
def __init__(self):
self.cmds = CmdHelp()
self.cmds.description = "Development environment for big data applications"
self.cmds.version = ferry.__version__
self.cmds.usage = "ferry COMMAND [arg...]"
self.cmds.add_option("-m", "--mode", "Deployment mode")
self.cmds.add_option("-c", "--conf", "Deployment configuration")
self.cmds.add_cmd("clean", "Clean zombie Ferry processes")
self.cmds.add_cmd("server", "Start all the servers")
self.cmds.add_cmd("deploy", "Deploy a service to the cloud")
self.cmds.add_cmd("help", "Print this help message")
self.cmds.add_cmd("info", "Print version information")
self.cmds.add_cmd("inspect", "Return low-level information on a service")
self.cmds.add_cmd("install", "Install all the Ferry images")
self.cmds.add_cmd("logs", "Copy over the logs to the host")
self.cmds.add_cmd("ps", "List deployed and running services")
self.cmds.add_cmd("rm", "Remove a service or snapshot")
self.cmds.add_cmd("snapshot", "Take a snapshot")
self.cmds.add_cmd("snapshots", "List all snapshots")
self.cmds.add_cmd("ssh", "Connect to a client/connector")
self.cmds.add_cmd("start", "Start a new service or snapshot")
self.cmds.add_cmd("stop", "Stop a running service")
self.cmds.add_cmd("quit", "Stop the Ferry servers")
self.ferry_server = "http://127.0.0.1:4000"
self.default_user = "******"
self.installer = Installer()
def __init__(self):
self.cmds = CmdHelp()
self.cmds.description = "Development environment for big data applications"
self.cmds.version = ferry.__version__
self.cmds.usage = "ferry COMMAND [arg...]"
self.cmds.add_option("-b", "--build", "Build Ferry default images")
self.cmds.add_option("-c", "--conf", "Deployment configuration")
self.cmds.add_option("-d", "--dns", "Specify DNS servers")
self.cmds.add_option("-k", "--key", "Specify key directory")
self.cmds.add_option("-l", "--log", "Log configuration file")
self.cmds.add_option("-m", "--mode", "Deployment mode")
self.cmds.add_option("-n", "--naked", "Start in naked mode")
self.cmds.add_option("-r", "--retry", "Retry action on failure")
self.cmds.add_option("-t", "--net", "Use host network device")
self.cmds.add_option("-u", "--upgrade", "Upgrade Ferry")
self.cmds.add_cmd("build", "Build a Dockerfile")
self.cmds.add_cmd("clean", "Clean zombie Ferry processes")
self.cmds.add_cmd("help", "Print this help message")
self.cmds.add_cmd("info", "Print version information")
self.cmds.add_cmd("inspect",
"Return low-level information on a service")
self.cmds.add_cmd("install", "Install the Ferry images")
self.cmds.add_cmd("login", "Login to Ferry servers")
self.cmds.add_cmd("logs", "Copy over the logs to the host")
self.cmds.add_cmd("ls", "View installed applications")
self.cmds.add_cmd("ls-images", "View installed images")
self.cmds.add_cmd("ps", "List deployed and running services")
self.cmds.add_cmd("pull", "Pull a remote image")
self.cmds.add_cmd("push", "Push an image to a remote registry")
self.cmds.add_cmd("rm", "Remove a service or snapshot")
self.cmds.add_cmd("server", "Start all the servers")
self.cmds.add_cmd("snapshot", "Take a snapshot")
self.cmds.add_cmd("snapshots", "List all snapshots")
self.cmds.add_cmd("ssh", "Connect to a client/connector")
self.cmds.add_cmd("start", "Start a new service or snapshot")
self.cmds.add_cmd("stop", "Stop a running service")
self.cmds.add_cmd("quit", "Stop the Ferry servers")
self.ferry_server = 'http://127.0.0.1:4000'
self.default_user = '******'
self.installer = Installer(self)
def __init__(self, controller):
self.name = "OpenStack launcher"
self.docker_registry = None
self.docker_user = None
self.heat_server = None
self.openstack_key = None
self.system = System()
self.installer = Installer()
self.controller = controller
self._init_open_stack()
self._init_app_db()
def __init__(self):
self.cmds = CmdHelp()
self.cmds.description = "Development environment for big data applications"
self.cmds.version = ferry.__version__
self.cmds.usage = "ferry COMMAND [arg...]"
self.cmds.add_option("-b", "--build", "Build Ferry default images")
self.cmds.add_option("-c", "--conf", "Deployment configuration")
self.cmds.add_option("-d", "--dns", "Specify DNS servers")
self.cmds.add_option("-k", "--key", "Specify key directory")
self.cmds.add_option("-l", "--log", "Log configuration file")
self.cmds.add_option("-m", "--mode", "Deployment mode")
self.cmds.add_option("-n", "--naked", "Start in naked mode")
self.cmds.add_option("-r", "--retry", "Retry action on failure")
self.cmds.add_option("-t", "--net", "Use host network device")
self.cmds.add_option("-u", "--upgrade", "Upgrade Ferry")
self.cmds.add_cmd("build", "Build a Dockerfile")
self.cmds.add_cmd("clean", "Clean zombie Ferry processes")
self.cmds.add_cmd("help", "Print this help message")
self.cmds.add_cmd("info", "Print version information")
self.cmds.add_cmd("inspect", "Return low-level information on a service")
self.cmds.add_cmd("install", "Install the Ferry images")
self.cmds.add_cmd("login", "Login to Ferry servers")
self.cmds.add_cmd("logs", "Copy over the logs to the host")
self.cmds.add_cmd("ls", "View installed applications")
self.cmds.add_cmd("ls-images", "View installed images")
self.cmds.add_cmd("ps", "List deployed and running services")
self.cmds.add_cmd("pull", "Pull a remote image")
self.cmds.add_cmd("push", "Push an image to a remote registry")
self.cmds.add_cmd("rm", "Remove a service or snapshot")
self.cmds.add_cmd("server", "Start all the servers")
self.cmds.add_cmd("snapshot", "Take a snapshot")
self.cmds.add_cmd("snapshots", "List all snapshots")
self.cmds.add_cmd("ssh", "Connect to a client/connector")
self.cmds.add_cmd("start", "Start a new service or snapshot")
self.cmds.add_cmd("stop", "Stop a running service")
self.cmds.add_cmd("quit", "Stop the Ferry servers")
self.ferry_server = 'http://127.0.0.1:4000'
self.default_user = '******'
self.installer = Installer(self)
class CLI(object):
def __init__(self):
self.cmds = CmdHelp()
self.cmds.description = "Development environment for big data applications"
self.cmds.version = ferry.__version__
self.cmds.usage = "ferry COMMAND [arg...]"
self.cmds.add_option("-m", "--mode", "Deployment mode")
self.cmds.add_option("-c", "--conf", "Deployment configuration")
self.cmds.add_cmd("clean", "Clean zombie Ferry processes")
self.cmds.add_cmd("server", "Start all the servers")
self.cmds.add_cmd("deploy", "Deploy a service to the cloud")
self.cmds.add_cmd("help", "Print this help message")
self.cmds.add_cmd("info", "Print version information")
self.cmds.add_cmd("inspect", "Return low-level information on a service")
self.cmds.add_cmd("install", "Install all the Ferry images")
self.cmds.add_cmd("logs", "Copy over the logs to the host")
self.cmds.add_cmd("ps", "List deployed and running services")
self.cmds.add_cmd("rm", "Remove a service or snapshot")
self.cmds.add_cmd("snapshot", "Take a snapshot")
self.cmds.add_cmd("snapshots", "List all snapshots")
self.cmds.add_cmd("ssh", "Connect to a client/connector")
self.cmds.add_cmd("start", "Start a new service or snapshot")
self.cmds.add_cmd("stop", "Stop a running service")
self.cmds.add_cmd("quit", "Stop the Ferry servers")
self.ferry_server = "http://127.0.0.1:4000"
self.default_user = "******"
self.installer = Installer()
"""
Create a new stack.
"""
def _create_stack(self, stack_description, args):
mode = self._parse_deploy_arg("mode", args, default="local")
conf = self._parse_deploy_arg("conf", args, default="default")
payload = {"payload": json.dumps(stack_description), "mode": mode, "conf": conf}
try:
res = requests.post(self.ferry_server + "/create", data=payload)
return str(res.text)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _format_snapshots_query(self, json_data):
bases = []
date = []
for uuid in json_data.keys():
bases.append(json_data[uuid]["base"])
if "snapshot_ts" in json_data[uuid]:
date.append(json_data[uuid]["snapshot_ts"])
else:
date.append(" ")
t = PrettyTable()
t.add_column("UUID", json_data.keys())
t.add_column("Base", bases)
t.add_column("Date", date)
return t.get_string(sortby="Date", padding_width=2)
def _format_table_query(self, json_data):
storage = []
compute = []
connectors = []
status = []
base = []
time = []
# Each additional row should include the actual data.
for uuid in json_data.keys():
for c in json_data[uuid]["connectors"]:
connectors.append(c)
backends = json_data[uuid]["backends"]
for b in backends:
if b["storage"]:
storage.append(b["storage"])
else:
storage.append(" ")
if b["compute"]:
compute.append(b["compute"])
else:
compute.append(" ")
status.append(json_data[uuid]["status"])
base.append(json_data[uuid]["base"])
time.append(json_data[uuid]["ts"])
t = PrettyTable()
t.add_column("UUID", json_data.keys())
t.add_column("Storage", storage)
t.add_column("Compute", compute)
t.add_column("Connectors", connectors)
t.add_column("Status", status)
t.add_column("Base", base)
t.add_column("Time", time)
return t.get_string(sortby="UUID", padding_width=2)
def _stop_all(self):
try:
constraints = {"status": "running"}
payload = {"constraints": json.dumps(constraints)}
res = requests.get(self.ferry_server + "/query", params=payload)
stacks = json.loads(res.text)
for uuid in stacks.keys():
self._manage_stacks({"uuid": uuid, "action": "stop"})
except ConnectionError:
logging.error("could not connect to ferry server")
def _read_stacks(self, show_all=False, args=None):
try:
res = requests.get(self.ferry_server + "/query")
query_reply = json.loads(res.text)
deployed_reply = {}
if show_all:
mode = self._parse_deploy_arg("mode", args, default="local")
conf = self._parse_deploy_arg("conf", args, default="default")
payload = {"mode": mode, "conf": conf}
res = requests.get(self.ferry_server + "/deployed", params=payload)
deployed_reply = json.loads(res.text)
# Merge the replies and format.
return self._format_table_query(dict(query_reply.items() + deployed_reply.items()))
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _list_snapshots(self):
try:
res = requests.get(self.ferry_server + "/snapshots")
json_reply = json.loads(res.text)
return self._format_snapshots_query(json_reply)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _format_stack_inspect(self, json_data):
return json.dumps(json_data, sort_keys=True, indent=2, separators=(",", ":"))
"""
Inspect a specific stack.
"""
def _inspect_stack(self, stack_id):
payload = {"uuid": stack_id}
try:
res = requests.get(self.ferry_server + "/stack", params=payload)
json_value = json.loads(str(res.text))
return self._format_stack_inspect(json_value)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
"""
Copy over the logs.
"""
def _copy_logs(self, stack_id, to_dir):
payload = {"uuid": stack_id, "dir": to_dir}
try:
res = requests.get(self.ferry_server + "/logs", params=payload)
json_value = json.loads(str(res.text))
return self._format_stack_inspect(json_value)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
"""
Read the location of the directory containing the keys
used to communicate with the containers.
"""
def _read_key_dir(self):
f = open(ferry.install.DEFAULT_DOCKER_KEY, "r")
return f.read().strip()
"""
Connector a specific client/connector via ssh.
"""
def _connect_stack(self, stack_id, connector_id):
# Get the IP and default user information for this connector.
payload = {"uuid": stack_id}
try:
res = requests.get(self.ferry_server + "/stack", params=payload)
json_value = json.loads(str(res.text))
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
connector_ip = None
for cg in json_value["connectors"]:
if not connector_id:
connector_ip = cg["entry"]["ip"]
break
elif connector_id == cg["uniq"]:
connector_ip = cg["entry"]["ip"]
break
else:
logging.warning("no match: %s %s" % (connector_id, cg["uniq"]))
# Now form the ssh command. This just executes in the same shell.
if connector_ip:
key_opt = "-o StrictHostKeyChecking=no"
host_opt = "-o UserKnownHostsFile=/dev/null"
ident = "-i %s/id_rsa" % self._read_key_dir()
dest = "%s@%s" % (self.default_user, connector_ip)
cmd = "ssh %s %s %s %s" % (key_opt, host_opt, ident, dest)
logging.warning(cmd)
os.execv("/usr/bin/ssh", cmd.split())
def _parse_deploy_arg(self, param, args, default):
pattern = re.compile("--%s=(\w+)" % param)
for a in args:
m = pattern.match(a)
if m and m.group(0) != "":
return m.group(1)
return default
"""
Deploy the stack.
"""
def _deploy_stack(self, stack_id, args):
mode = self._parse_deploy_arg("mode", args, default="local")
conf = self._parse_deploy_arg("conf", args, default="default")
payload = {"uuid": stack_id, "mode": mode, "conf": conf}
try:
res = requests.post(self.ferry_server + "/deploy", data=payload)
return res.text
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
"""
Manage the stack.
"""
def _manage_stacks(self, stack_info):
try:
res = requests.post(self.ferry_server + "/manage/stack", data=stack_info)
return str(res.text)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
"""
Output the help message.
"""
def _print_help(self):
return self.cmds.print_help()
"""
Output version information.
"""
def _print_info(self):
try:
res = requests.get(self.ferry_server + "/version")
s = self.cmds.description + "\n"
s += "Version: %s\n" % self.cmds.version
s += "Docker: %s\n" % res.text.strip()
return s
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
"""
Helper method to read a file.
"""
def _read_file_arg(self, file_name):
json_file = open(os.path.abspath(file_name), "r")
json_text = ""
for line in json_file:
json_text += line.strip()
return json_text
def _check_ssh_key(self):
keydir = open(ferry.install.DEFAULT_DOCKER_KEY, "r").read().strip()
if keydir == ferry.install.DEFAULT_KEY_DIR:
try:
ferry.install.GLOBAL_KEY_DIR = os.environ["HOME"] + "/.ssh/ferry"
os.makedirs(os.environ["HOME"] + "/.ssh/ferry")
shutil.copy(ferry.install.DEFAULT_KEY_DIR + "/id_rsa", ferry.install.GLOBAL_KEY_DIR + "/id_rsa")
os.chmod(ferry.install.GLOBAL_KEY_DIR + "/id_rsa", 0400)
except OSError as e:
if e.errno != errno.EEXIST:
logging.error("Could not create private key (%s)" % os.strerror(e.errno))
ferry.install._touch_file(ferry.install.DEFAULT_DOCKER_KEY, ferry.install.GLOBAL_KEY_DIR)
logging.warning("Copied private key to " + ferry.install.GLOBAL_KEY_DIR)
"""
This is the command dispatch table.
"""
def dispatch_cmd(self, cmd, args, options):
if cmd == "start":
# Check if the user is using the global key directory.
# If so, we need to make a copy of the key.
self._check_ssh_key()
arg = args.pop(0)
json_arg = {}
if os.path.exists(arg):
json_string = self._read_file_arg(arg)
json_arg = json.loads(json_string)
json_arg["_file"] = arg
json_arg["_file_path"] = arg
else:
# Check if the user wants to use one of the global plans.
global_path = FERRY_HOME + "/data/plans/" + arg
# Check if the user has passed in a file extension.
# If not go ahead and add one.
file_path = global_path
n, e = os.path.splitext(global_path)
if e == "":
file_path += ".json"
if os.path.exists(file_path):
json_string = self._read_file_arg(file_path)
json_arg = json.loads(json_string)
json_arg["_file_path"] = file_path
json_arg["_file"] = arg
return self._create_stack(json_arg, args)
elif cmd == "ps":
if len(args) > 0 and args[0] == "-a":
opt = args.pop(0)
return self._read_stacks(show_all=True, args=args)
else:
return self._read_stacks(show_all=False, args=args)
elif cmd == "snapshots":
return self._list_snapshots()
elif cmd == "install":
msg = self.installer.install(args)
self.installer._stop_docker_daemon()
return msg
elif cmd == "clean":
self.installer._stop_docker_daemon(force=True)
return "cleaned ferry daemon"
elif cmd == "inspect":
return self._inspect_stack(args[0])
elif cmd == "logs":
return self._copy_logs(args[0], args[1])
elif cmd == "server":
self.installer.start_web()
return "started ferry"
elif cmd == "ssh":
# Check if the user is using the global key directory.
# If so, we need to make a copy of the key.
self._check_ssh_key()
# Go ahead and connect to the stack.
stack_id = args[0]
connector_id = None
if len(args) > 1:
connector_id = args[1]
return self._connect_stack(stack_id, connector_id)
elif cmd == "quit":
self._stop_all()
self.installer.stop_web()
self.installer._stop_docker_daemon()
return "stopped ferry"
elif cmd == "deploy":
stack_id = args.pop(0)
return self._deploy_stack(stack_id, args)
elif cmd == "info":
return self._print_info()
elif cmd == "help":
return self._print_help()
else:
stack_info = {"uuid": args[0], "action": cmd}
return self._manage_stacks(stack_info)
class CLI(object):
def __init__(self):
self.cmds = CmdHelp()
self.cmds.description = "Development environment for big data applications"
self.cmds.version = ferry.__version__
self.cmds.usage = "ferry COMMAND [arg...]"
self.cmds.add_option("-c", "--conf", "Deployment configuration")
self.cmds.add_option("-l", "--log", "Log configuration file")
self.cmds.add_option("-k", "--key", "Specify key directory")
self.cmds.add_option("-m", "--mode", "Deployment mode")
self.cmds.add_option("-u", "--upgrade", "Upgrade Ferry")
self.cmds.add_option("-b", "--build", "Manually build Ferry images")
self.cmds.add_cmd("clean", "Clean zombie Ferry processes")
self.cmds.add_cmd("server", "Start all the servers")
self.cmds.add_cmd("deploy", "Deploy a service to the cloud")
self.cmds.add_cmd("help", "Print this help message")
self.cmds.add_cmd("info", "Print version information")
self.cmds.add_cmd("inspect", "Return low-level information on a service")
self.cmds.add_cmd("install", "Install all the Ferry images")
self.cmds.add_cmd("logs", "Copy over the logs to the host")
self.cmds.add_cmd("ps", "List deployed and running services")
self.cmds.add_cmd("rm", "Remove a service or snapshot")
self.cmds.add_cmd("snapshot", "Take a snapshot")
self.cmds.add_cmd("snapshots", "List all snapshots")
self.cmds.add_cmd("ssh", "Connect to a client/connector")
self.cmds.add_cmd("start", "Start a new service or snapshot")
self.cmds.add_cmd("stop", "Stop a running service")
self.cmds.add_cmd("quit", "Stop the Ferry servers")
self.ferry_server = 'http://127.0.0.1:4000'
self.default_user = '******'
self.installer = Installer()
"""
Create a new stack.
"""
def _create_stack(self, stack_description, args):
mode = self._parse_deploy_arg('mode', args, default='local')
conf = self._parse_deploy_arg('conf', args, default='default')
payload = { 'payload' : json.dumps(stack_description),
'mode' : mode,
'conf' : conf }
try:
res = requests.post(self.ferry_server + '/create', data=payload)
return str(res.text)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _format_snapshots_query(self, json_data):
bases = []
date = []
for uuid in json_data.keys():
bases.append(json_data[uuid]['base'])
if 'snapshot_ts' in json_data[uuid]:
date.append(json_data[uuid]['snapshot_ts'])
else:
date.append(' ')
t = PrettyTable()
t.add_column("UUID", json_data.keys())
t.add_column("Base", bases)
t.add_column("Date", date)
return t.get_string(sortby="Date",
padding_width=2)
def _format_table_query(self, json_data):
storage = []
compute = []
connectors = []
status = []
base = []
time = []
# Each additional row should include the actual data.
for uuid in json_data.keys():
for c in json_data[uuid]['connectors']:
connectors.append(c)
backends = json_data[uuid]['backends']
for b in backends:
if b['storage']:
storage.append(b['storage'])
else:
storage.append(' ')
if b['compute']:
compute.append(b['compute'])
else:
compute.append(' ')
status.append(json_data[uuid]['status'])
base.append(json_data[uuid]['base'])
time.append(json_data[uuid]['ts'])
t = PrettyTable()
t.add_column("UUID", json_data.keys())
t.add_column("Storage", storage)
t.add_column("Compute", compute)
t.add_column("Connectors", connectors)
t.add_column("Status", status)
t.add_column("Base", base)
t.add_column("Time", time)
return t.get_string(sortby="UUID",
padding_width=2)
def _stop_all(self):
try:
constraints = { 'status' : 'running' }
payload = { 'constraints' : json.dumps(constraints) }
res = requests.get(self.ferry_server + '/query', params=payload)
stacks = json.loads(res.text)
for uuid in stacks.keys():
self._manage_stacks({'uuid' : uuid,
'action' : 'stop'})
except ConnectionError:
logging.error("could not connect to ferry server")
def _read_stacks(self, show_all=False, args=None):
try:
res = requests.get(self.ferry_server + '/query')
query_reply = json.loads(res.text)
deployed_reply = {}
if show_all:
mode = self._parse_deploy_arg('mode', args, default='local')
conf = self._parse_deploy_arg('conf', args, default='default')
payload = { 'mode' : mode,
'conf' : conf }
res = requests.get(self.ferry_server + '/deployed', params=payload)
deployed_reply = json.loads(res.text)
# Merge the replies and format.
return self._format_table_query(dict(query_reply.items() + deployed_reply.items()))
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _list_snapshots(self):
try:
res = requests.get(self.ferry_server + '/snapshots')
json_reply = json.loads(res.text)
return self._format_snapshots_query(json_reply)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _format_stack_inspect(self, json_data):
return json.dumps(json_data,
sort_keys=True,
indent=2,
separators=(',',':'))
"""
Inspect a specific stack.
"""
def _inspect_stack(self, stack_id):
payload = { 'uuid':stack_id }
try:
res = requests.get(self.ferry_server + '/stack', params=payload)
json_value = json.loads(str(res.text))
return self._format_stack_inspect(json_value)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
"""
Copy over the logs.
"""
def _copy_logs(self, stack_id, to_dir):
payload = {'uuid':stack_id,
'dir':to_dir}
try:
res = requests.get(self.ferry_server + '/logs', params=payload)
json_value = json.loads(str(res.text))
return self._format_stack_inspect(json_value)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
"""
Connector a specific client/connector via ssh.
"""
def _connect_stack(self, stack_id, connector_id):
# Get the IP and default user information for this connector.
payload = {'uuid':stack_id}
try:
res = requests.get(self.ferry_server + '/stack', params=payload)
json_value = json.loads(str(res.text))
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
connector_ip = None
for cg in json_value['connectors']:
if not connector_id:
connector_ip = cg['entry']['ip']
break
elif connector_id == cg['uniq']:
connector_ip = cg['entry']['ip']
break
else:
logging.warning("no match: %s %s" % (connector_id, cg['uniq']))
# Now form the ssh command. This just executes in the same shell.
if connector_ip:
keydir, _ = self._read_key_dir()
key_opt = '-o StrictHostKeyChecking=no'
host_opt = '-o UserKnownHostsFile=/dev/null'
ident = '-i %s/id_rsa' % keydir
dest = '%s@%s' % (self.default_user, connector_ip)
cmd = "ssh %s %s %s %s" % (key_opt, host_opt, ident, dest)
logging.warning(cmd)
os.execv('/usr/bin/ssh', cmd.split())
def _parse_deploy_arg(self, param, args, default):
pattern = re.compile('--%s=(\w+)' % param)
for a in args:
m = pattern.match(a)
if m and m.group(0) != '':
return m.group(1)
return default
"""
Deploy the stack.
"""
def _deploy_stack(self, stack_id, args):
mode = self._parse_deploy_arg('mode', args, default='local')
conf = self._parse_deploy_arg('conf', args, default='default')
payload = { 'uuid' : stack_id,
'mode' : mode,
'conf' : conf }
try:
res = requests.post(self.ferry_server + '/deploy', data=payload)
return res.text
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
"""
Manage the stack.
"""
def _manage_stacks(self, stack_info):
try:
res = requests.post(self.ferry_server + '/manage/stack', data=stack_info)
return str(res.text)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
"""
Output the help message.
"""
def _print_help(self):
return self.cmds.print_help()
"""
Output version information.
"""
def _print_info(self):
try:
res = requests.get(self.ferry_server + '/version')
s = self.cmds.description + '\n'
s += "Version: %s\n" % self.cmds.version
s += "Docker: %s\n" % res.text.strip()
return s
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
"""
Helper method to read a file.
"""
def _read_file_arg(self, file_name):
json_file = open(os.path.abspath(file_name), 'r')
json_text = ''
for line in json_file:
json_text += line.strip()
return json_text
"""
Read the location of the directory containing the keys
used to communicate with the containers.
"""
def _read_key_dir(self):
f = open(ferry.install.DEFAULT_DOCKER_KEY, 'r')
k = f.read().strip().split("://")
return k[1], k[0]
def _using_tmp_ssh_key(self):
keydir, tmp = self._read_key_dir()
return tmp == "tmp"
def _check_ssh_key(self):
keydir, _ = self._read_key_dir()
if keydir == ferry.install.DEFAULT_KEY_DIR:
keydir = os.environ['HOME'] + '/.ssh/tmp-ferry'
ferry.install.GLOBAL_KEY_DIR = 'tmp://' + keydir
try:
os.makedirs(keydir)
except OSError as e:
if e.errno != errno.EEXIST:
logging.error("Could not create ssh directory %s" % keydir)
exit(1)
try:
shutil.copy(ferry.install.DEFAULT_KEY_DIR + '/id_rsa', keydir + '/id_rsa')
shutil.copy(ferry.install.DEFAULT_KEY_DIR + '/id_rsa.pub', keydir + '/id_rsa.pub')
uid = pwd.getpwnam(os.environ['USER']).pw_uid
gid = grp.getgrnam(os.environ['USER']).gr_gid
os.chown(keydir + '/id_rsa', uid, gid)
os.chmod(keydir + '/id_rsa', 0400)
os.chown(keydir + '/id_rsa.pub', uid, gid)
os.chmod(keydir + '/id_rsa.pub', 0444)
except OSError as e:
logging.error("Could not copy ssh keys (%s)" % str(e))
exit(1)
except IOError as e:
if e.errno == errno.EACCES:
logging.error("Could not override keys in %s, please type \'ferry clean\' and try again." % keydir)
exit(1)
ferry.install._touch_file(ferry.install.DEFAULT_DOCKER_KEY,
ferry.install.GLOBAL_KEY_DIR)
logging.warning("Copied ssh keys to " + ferry.install.GLOBAL_KEY_DIR)
"""
This is the command dispatch table.
"""
def dispatch_cmd(self, cmd, args, options):
if(cmd == 'start'):
self._check_ssh_key()
arg = args.pop(0)
json_arg = {}
if os.path.exists(arg):
json_string = self._read_file_arg(arg)
json_arg = json.loads(json_string)
json_arg['_file'] = arg
json_arg['_file_path'] = arg
else:
# Check if the user wants to use one of the global plans.
global_path = FERRY_HOME + '/data/plans/' + arg
# If the user has not supplied a file extension, look for the
# file with a YAML extension
file_path = global_path
n, e = os.path.splitext(global_path)
if e == '':
file_path += '.yaml'
e = 'yaml'
if os.path.exists(file_path):
if e == 'json':
json_string = self._read_file_arg(file_path)
json_arg = json.loads(json_string)
elif e == 'yaml':
yaml_file = open(file_path, 'r')
json_arg = yaml.load(yaml_file)
json_arg['_file_path'] = file_path
json_arg['_file'] = arg
return self._create_stack(json_arg, args)
elif(cmd == 'ps'):
if len(args) > 0 and args[0] == '-a':
opt = args.pop(0)
return self._read_stacks(show_all=True, args = args)
else:
return self._read_stacks(show_all=False, args = args)
elif(cmd == 'snapshots'):
return self._list_snapshots()
elif(cmd == 'install'):
msg = self.installer.install(args, options)
self.installer._stop_docker_daemon()
return msg
elif(cmd == 'clean'):
self._check_ssh_key()
self.installer._start_docker_daemon()
self.installer._clean_web()
self.installer._stop_docker_daemon(force=True)
self.installer._reset_ssh_key()
return 'cleaned ferry'
elif(cmd == 'inspect'):
return self._inspect_stack(args[0])
elif(cmd == 'logs'):
return self._copy_logs(args[0], args[1])
elif(cmd == 'server'):
self.installer.start_web(options)
return 'started ferry'
elif(cmd == 'ssh'):
self._check_ssh_key()
stack_id = args[0]
connector_id = None
if len(args) > 1:
connector_id = args[1]
return self._connect_stack(stack_id, connector_id)
elif(cmd == 'quit'):
self._check_ssh_key()
self._stop_all()
self.installer.stop_web()
self.installer._stop_docker_daemon()
if self._using_tmp_ssh_key():
self.installer._reset_ssh_key()
return 'stopped ferry'
elif(cmd == 'deploy'):
self._check_ssh_key()
stack_id = args.pop(0)
return self._deploy_stack(stack_id, args)
elif(cmd == 'info'):
return self._print_info()
elif(cmd == 'help'):
return self._print_help()
else:
stack_info = {'uuid' : args[0],
'action' : cmd}
return self._manage_stacks(stack_info)
class CLI(object):
def __init__(self):
self.cmds = CmdHelp()
self.cmds.description = "Development environment for big data applications"
self.cmds.version = ferry.__version__
self.cmds.usage = "ferry COMMAND [arg...]"
self.cmds.add_option("-b", "--build", "Build Ferry default images")
self.cmds.add_option("-c", "--conf", "Deployment configuration")
self.cmds.add_option("-d", "--dns", "Specify DNS servers")
self.cmds.add_option("-k", "--key", "Specify key directory")
self.cmds.add_option("-l", "--log", "Log configuration file")
self.cmds.add_option("-m", "--mode", "Deployment mode")
self.cmds.add_option("-n", "--naked", "Start in naked mode")
self.cmds.add_option("-r", "--retry", "Retry action on failure")
self.cmds.add_option("-t", "--net", "Use host network device")
self.cmds.add_option("-u", "--upgrade", "Upgrade Ferry")
self.cmds.add_cmd("build", "Build a Dockerfile")
self.cmds.add_cmd("clean", "Clean zombie Ferry processes")
self.cmds.add_cmd("help", "Print this help message")
self.cmds.add_cmd("info", "Print version information")
self.cmds.add_cmd("inspect", "Return low-level information on a service")
self.cmds.add_cmd("install", "Install the Ferry images")
self.cmds.add_cmd("login", "Login to Ferry servers")
self.cmds.add_cmd("logs", "Copy over the logs to the host")
self.cmds.add_cmd("ls", "View installed applications")
self.cmds.add_cmd("ps", "List deployed and running services")
self.cmds.add_cmd("pull", "Pull a remote image")
self.cmds.add_cmd("push", "Push an image to a remote registry")
self.cmds.add_cmd("rm", "Remove a service or snapshot")
self.cmds.add_cmd("server", "Start all the servers")
self.cmds.add_cmd("snapshot", "Take a snapshot")
self.cmds.add_cmd("snapshots", "List all snapshots")
self.cmds.add_cmd("ssh", "Connect to a client/connector")
self.cmds.add_cmd("start", "Start a new service or snapshot")
self.cmds.add_cmd("stop", "Stop a running service")
self.cmds.add_cmd("quit", "Stop the Ferry servers")
self.ferry_server = 'http://127.0.0.1:4000'
self.default_user = '******'
self.installer = Installer(self)
def _pull_image(self, image):
"""
Pull a remote image to the local registry.
"""
try:
payload = { 'image' : image }
res = requests.get(self.ferry_server + '/image', params=payload)
return str(res.text)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _pull_app(self, app):
"""
Pull a local application to Ferry servers.
"""
# Now download the application description
# from the Ferry servers.
account, key, server = self.installer.get_ferry_account()
if account:
# Read in the contents of the application and
# generate the API key.
req = { 'action' : 'fetch',
'app' : app,
'account' : account }
sig = self.installer.create_signature(json.dumps(req), key)
try:
payload = { 'id' : account,
'app' : app,
'sig' : sig }
res = requests.get(server + '/app', params=payload)
status = json.loads(res.text)
file_name = self.installer.store_app(app, status['ext'], status['content'])
if file_name:
content = self._read_app_content(file_name)
images = self._get_user_images(content)
for i in images:
self._pull_image(i)
return app
else:
return "failed"
except ConnectionError:
logging.error("could not connect to application server")
return "failed"
def _pull(self, image):
"""
Pull a remote application/image to the local registry.
"""
logging.warning("pulling " + image)
# Figure out if we're pushing a Ferry application or
# plain Docker image.
s = image.split("://")
if len(s) > 1:
proto = s[0]
image = s[1]
else:
proto = "image"
image = s[0]
if proto == "image":
return self._pull_image(image)
else:
return self._pull_app(image)
def _push_image(self, image, registry):
"""
Push a local image to a remote registry.
"""
try:
payload = { 'image' : image,
'server' : registry }
res = requests.post(self.ferry_server + '/image', data=payload)
return str(res.text)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _builtin_image(self, image):
"""
Indicates whether the image is a pre-built Ferry image.
Right now we only verify the client images.
"""
return image in ["ferry/hadoop-client",
"ferry/spark-client",
"ferry/cassandra-client",
"ferry/openmpi-client"]
def _get_user_images(self, content):
"""
Get the user-defined images.
"""
images = set()
for c in content['connectors']:
p = c['personality']
if not self._builtin_image(p):
images.add(p)
return images
def _read_app_content(self, file_path):
"""
Read the content of the application.
"""
json_arg = None
with open(file_path, "r") as f:
n, e = os.path.splitext(file_path)
if e == '.json':
json_string = self._read_file_arg(file_path)
json_arg = json.loads(json_string)
elif e == '.yaml' or e == '.yml':
yaml_file = open(file_path, 'r')
json_arg = yaml.load(yaml_file)
return json_arg
def _push_app(self, app, registry):
"""
Push a local application to Ferry servers.
"""
# First find all the images that need to
# be pushed to the Docker registry.
content = self._read_app_content(app)
if content:
images = self._get_user_images(content)
for i in images:
self._push_image(i, registry)
# Register the application in the Ferry database.
account, key, server = self.installer.get_ferry_account()
if account:
# Read in the contents of the application and
# generate the API key.
with open(app, "r") as f:
name = account + '/' + os.path.basename(app)
name, ext = os.path.splitext(name)
content = f.read()
req = { 'action' : 'register',
'app' : name,
'account' : account }
sig = self.installer.create_signature(json.dumps(req), key)
try:
payload = { 'id' : account,
'app' : name,
'ext' : ext,
'content' : content,
'sig' : sig }
res = requests.post(server + '/app', data=payload)
status = json.loads(res.text)
if status['status'] == 'fail':
logging.error("failed to register app " + app)
return "Failed to register app " + app
else:
return status['name']
except ConnectionError:
logging.error("could not connect to application server")
return "Could not register the application."
except ValueError as e:
logging.error(str(e))
return "Registration server sent back unknown reply"
else:
logging.error("could not read account information")
return "Could not read account information."
def _push(self, image, registry):
"""
Push a local appliation/image to a remote registry.
"""
logging.warning("pushing " + image)
# Figure out if we're pushing a Ferry application or
# plain Docker image.
s = image.split("://")
if len(s) > 1:
proto = s[0]
image = s[1]
else:
proto = "image"
image = s[0]
if proto == "image":
return self._push_image(image, registry)
else:
return self._push_app(image, registry)
def _build(self, dockerfile):
"""
Build a new local image.
"""
logging.warning("building " + dockerfile)
names = self.installer._get_image(dockerfile)
name = names.pop().split("/")
if len(name) == 1:
repo = GUEST_DOCKER_REPO
image = name[0]
else:
repo = name[0]
image = name[1]
build_dir = os.path.dirname(dockerfile)
self.installer._compile_image(image, repo, build_dir, build=True)
if len(names) > 0:
self.installer._tag_images(image, repo, names)
def _login(self):
"""
Login to a remote registry
"""
try:
res = requests.post(self.ferry_server + '/login')
return str(res.text)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _create_stack(self, stack_description, args, private_key):
"""
Create a new stack.
"""
payload = { 'payload' : json.dumps(stack_description),
'key' : private_key }
try:
res = requests.post(self.ferry_server + '/create', data=payload)
return True, str(res.text)
except ConnectionError:
logging.error("could not connect to ferry server")
return False, "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _ask_question(self, question_text):
question = colored(question_text, 'red')
prompt = colored(' >> ', 'green')
return raw_input(question + prompt)
def _format_apps_query(self, json_data):
authors = []
versions = []
descriptions = []
for app in json_data.keys():
authors.append(json_data[app]['author'])
versions.append(json_data[app]['version'])
descriptions.append(json_data[app]['short'])
t = PrettyTable()
t.add_column("App", json_data.keys())
t.add_column("Author", authors)
t.add_column("Version", versions)
t.add_column("Description", descriptions)
return t.get_string(sortby="App",
padding_width=2)
def _format_snapshots_query(self, json_data):
bases = []
date = []
for uuid in json_data.keys():
bases.append(json_data[uuid]['base'])
if 'snapshot_ts' in json_data[uuid]:
date.append(json_data[uuid]['snapshot_ts'])
else:
date.append(' ')
t = PrettyTable()
t.add_column("UUID", json_data.keys())
t.add_column("Base", bases)
t.add_column("Date", date)
return t.get_string(sortby="Date",
padding_width=2)
def _format_table_query(self, json_data):
storage = []
compute = []
connectors = []
status = []
base = []
time = []
# Each additional row should include the actual data.
for uuid in json_data.keys():
csstore = []
bstore = []
cstore = []
for c in json_data[uuid]['connectors']:
csstore.append(c)
backends = json_data[uuid]['backends']
for b in backends:
if b['storage']:
bstore.append(b['storage'])
else:
bstore.append(' ')
if b['compute']:
cstore.append(b['compute'])
else:
cstore.append(' ')
storage.append(bstore)
compute.append(cstore)
connectors.append(csstore)
status.append(json_data[uuid]['status'])
base.append(json_data[uuid]['base'])
time.append(json_data[uuid]['ts'])
t = PrettyTable()
t.add_column("UUID", json_data.keys())
t.add_column("Storage", storage)
t.add_column("Compute", compute)
t.add_column("Connectors", connectors)
t.add_column("Status", status)
t.add_column("Base", base)
t.add_column("Time", time)
return t.get_string(sortby="UUID",
padding_width=2)
def _stop_all(self, private_key):
try:
# We used to stop all the services when quitting
# Ferry, but this does not seem safe since a user
# may want to keep running a Hadoop cluster even if
# Ferry is shutdown.
# constraints = { 'status' : 'running' }
# payload = { 'constraints' : json.dumps(constraints) }
# res = requests.get(self.ferry_server + '/query', params=payload)
# stacks = json.loads(res.text)
# for uuid in stacks.keys():
# self._manage_stacks({'uuid' : uuid,
# 'key' : private_key,
# 'action' : 'stop'})
# Shutdown any backend services that may have
# been started (e.g., OpenStack Heat server, etc.).
res = requests.post(self.ferry_server + '/quit')
logging.info(res.text)
except ConnectionError:
logging.error("could not connect to ferry server")
def _read_stacks(self, show_all=False, args=None):
try:
res = requests.get(self.ferry_server + '/query')
query_reply = json.loads(res.text)
deployed_reply = {}
if show_all:
mode = self._parse_deploy_arg('mode', args, default='local')
conf = self._parse_deploy_arg('conf', args, default='default')
payload = { 'mode' : mode,
'conf' : conf }
res = requests.get(self.ferry_server + '/deployed', params=payload)
deployed_reply = json.loads(res.text)
# Merge the replies and format.
return self._format_table_query(dict(query_reply.items() + deployed_reply.items()))
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _list_apps(self):
"""
List all installed applications including the built-in applications.
"""
try:
res = requests.get(self.ferry_server + '/apps')
json_reply = json.loads(res.text)
return self._format_apps_query(json_reply)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _list_snapshots(self):
"""
List all snapshots.
"""
try:
res = requests.get(self.ferry_server + '/snapshots')
json_reply = json.loads(res.text)
return self._format_snapshots_query(json_reply)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _format_stack_inspect(self, json_data):
return json.dumps(json_data,
sort_keys=True,
indent=2,
separators=(',',':'))
def _inspect_stack(self, stack_id):
"""
Inspect a specific stack.
"""
payload = { 'uuid':stack_id }
try:
res = requests.get(self.ferry_server + '/stack', params=payload)
return res.text
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _copy_logs(self, stack_id, to_dir):
"""
Copy over the logs.
"""
payload = {'uuid':stack_id,
'dir':to_dir}
try:
res = requests.get(self.ferry_server + '/logs', params=payload)
json_value = json.loads(str(res.text))
return self._format_stack_inspect(json_value)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
"""
Connector a specific client/connector via ssh.
"""
def _connect_stack(self, stack_id, connector_id, options):
# Get the IP and default user information for this connector.
payload = {'uuid':stack_id}
try:
res = requests.get(self.ferry_server + '/stack', params=payload)
json_value = json.loads(str(res.text))
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
except ValueError:
json_value = None
connector_ip = None
if json_value and 'connectors' in json_value:
for cg in json_value['connectors']:
if not connector_id and 'ip' in cg['entry']:
connector_ip = cg['entry']['ip']
break
elif connector_id == cg['uniq'] and 'ip' in cg['entry']:
connector_ip = cg['entry']['ip']
break
# Now form the ssh command. This just executes in the same shell.
if connector_ip:
private_key = self._get_ssh_key(options=options)
key_opt = '-o StrictHostKeyChecking=no'
host_opt = '-o UserKnownHostsFile=/dev/null'
ident = '-i %s' % private_key
dest = '%s@%s' % (self.default_user, connector_ip)
cmd = "ssh %s %s %s %s" % (key_opt, host_opt, ident, dest)
logging.warning(cmd)
os.execv('/usr/bin/ssh', cmd.split())
else:
logging.warning("could not find connector %s" % connector_id)
return "could not connect to " + str(stack_id)
def _parse_deploy_arg(self, param, args, default):
pattern = re.compile('--%s=(\w+)' % param)
for a in args:
m = pattern.match(a)
if m and m.group(0) != '':
return m.group(1)
return default
def _manage_stacks(self, stack_info):
"""
Manage the stack.
"""
try:
res = requests.post(self.ferry_server + '/manage/stack', data=stack_info)
return str(res.text)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _print_help(self):
"""
Output the help message.
"""
return self.cmds.print_help()
def _print_info(self):
"""
Output version information.
"""
try:
res = requests.get(self.ferry_server + '/version')
s = self.cmds.description + '\n'
s += "Version: %s\n" % self.cmds.version
s += "Docker: %s\n" % res.text.strip()
return s
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _read_file_arg(self, file_name):
"""
Helper method to read a file.
"""
json_file = open(os.path.abspath(file_name), 'r')
json_text = ''
for line in json_file:
json_text += line.strip()
return json_text
def _get_ssh_key(self, options=None):
if options and '-k' in options:
return options['-k'][0]
else:
return ferry.install.DEFAULT_SSH_KEY
def _find_installed_app(self, app):
"""
Help find the path to the application. Check both the built-in
global directory and the user-installed directory.
"""
file_path = None
for item in os.listdir(FERRY_HOME + '/data/plans/'):
if app == os.path.splitext(item)[0]:
return FERRY_HOME + '/data/plans/' + item
if not file_path:
for user in os.listdir(DEFAULT_FERRY_APPS):
if os.path.isdir(DEFAULT_FERRY_APPS + '/' + user):
for item in os.listdir(DEFAULT_FERRY_APPS + '/' + user):
if app == user + '/' + os.path.splitext(item)[0]:
return DEFAULT_FERRY_APPS + '/' + user + '/' + item
def _format_output(self, reply):
output = reply['text'] + "\n"
if 'msgs' in reply:
for c in reply['msgs'].keys():
output += "%s: %s\n" % (c, reply['msgs'][c])
return output
def _start_stack(self, options, args):
private_key = self._get_ssh_key(options=options)
# Check if we need to build the image before running.
if '-b' in options:
build_dir = options['-b'][0]
self._build(build_dir + '/Dockerfile')
# Try to figure out what application the user is staring.
# This could be a new stack or an existing stopped stack.
arg = args.pop(0)
json_arg = {}
if not os.path.exists(arg):
file_path = self._find_installed_app(arg)
else:
file_path = arg
# Looks like the user is trying to start a brand
# new application (specified by a filename).
if file_path and os.path.exists(file_path):
file_path = os.path.abspath(file_path)
json_arg = self._read_app_content(file_path)
if not json_arg:
logging.error("could not load file " + file_path)
exit(1)
else:
# Check if there are any questions associated with
# this application stack. If so, we should prompt the
# user and include the answers.
if 'questions' in json_arg:
for q in json_arg['questions']:
question = q['question']
q['_answer'] = self._ask_question(question)
json_arg['_file_path'] = file_path
json_arg['_file'] = arg
# Create the application stack and print
# the status message.
posted, reply = self._create_stack(json_arg, args, private_key)
if posted:
try:
reply = json.loads(reply)
if reply['status'] == 'failed':
return 'could not create application'
else:
return self._format_output(reply)
except ValueError as e:
logging.error(reply)
def dispatch_cmd(self, cmd, args, options):
"""
This is the command dispatch table.
"""
if(cmd == 'start'):
return self._start_stack(options, args)
elif(cmd == 'ps'):
if len(args) > 0 and args[0] == '-a':
opt = args.pop(0)
return self._read_stacks(show_all=True, args = args)
else:
return self._read_stacks(show_all=False, args = args)
elif(cmd == 'snapshots'):
return self._list_snapshots()
elif(cmd == 'install'):
msg = self.installer.install(args, options)
self.installer._stop_docker_daemon()
return msg
elif(cmd == 'clean'):
self.installer._force_stop_web()
self.installer._stop_docker_daemon(force=True)
return 'cleaned ferry'
elif(cmd == 'inspect'):
return self._inspect_stack(args[0])
elif(cmd == 'logs'):
return self._copy_logs(args[0], args[1])
elif(cmd == 'server'):
self.installer.start_web(options)
return 'started ferry'
elif(cmd == 'ssh'):
stack_id = args[0]
connector_id = None
if len(args) > 1:
connector_id = args[1]
return self._connect_stack(stack_id, connector_id, options)
elif(cmd == 'quit'):
private_key = self._get_ssh_key(options=options)
self._stop_all(private_key)
self.installer.stop_web(private_key)
self.installer._stop_docker_daemon()
return 'stopped ferry'
elif(cmd == 'ls'):
return self._list_apps()
elif(cmd == 'info'):
return self._print_info()
elif(cmd == 'build'):
return self._build(args[0])
elif(cmd == 'pull'):
return self._pull(args[0])
elif(cmd == 'push'):
image = args.pop(0)
if len(args) > 0:
registry = args.pop(0)
else:
registry = None
return self._push(image, registry)
elif(cmd == 'login'):
return self._login()
elif(cmd == 'help'):
return self._print_help()
else:
# The user wants to perform some management function
# over the stack.
private_key = self._get_ssh_key(options=options)
stack_info = {'uuid' : args[0],
'key' : private_key,
'action' : cmd}
return self._manage_stacks(stack_info)
class SingleLauncher(object):
"""
Launches new Ferry containers on an OpenStack cluster.
Unlike the multi-launcher, containers use a single pre-assigned
network for all communication. This makes it suitable for OpenStack
environments that only support a single network (i.e., HP Cloud).
"""
def __init__(self, controller):
self.name = "OpenStack launcher"
self.docker_registry = None
self.docker_user = None
self.heat_server = None
self.openstack_key = None
self.system = System()
self.installer = Installer()
self.controller = controller
self._init_open_stack()
self._init_app_db()
def support_proxy(self):
"""
The OpenStack backend supports proxy mode by assigning all the
machines a floating IP.
"""
return True
def _init_app_db(self):
self.mongo = MongoClient(os.environ['MONGODB'],
27017,
connectTimeoutMS=6000)
self.apps = self.mongo['cloud']['openstack']
def _init_open_stack(self):
conf = ferry.install.read_ferry_config()
# First we need to know the deployment system
# we are using.
self.data_device = conf['system']['network']
provider = conf['system']['provider']
# Now get some basic OpenStack information
params = conf[provider]['params']
self.default_dc = params['dc']
self.default_zone = params['zone']
# Some OpenStack login credentials.
if self._check_openstack_credentials():
self.openstack_user = os.environ['OS_USERNAME']
self.openstack_pass = os.environ['OS_PASSWORD']
self.tenant_id = os.environ['OS_TENANT_ID']
self.tenant_name = os.environ['OS_TENANT_NAME']
else:
logging.error("Missing OpenStack credentials")
raise ValueError("Missing OpenStack credentials")
# Some information regarding OpenStack
# networking. Necessary for
servers = conf[provider][self.default_dc]
self.manage_network = servers['network']
self.external_network = servers['extnet']
# OpenStack API endpoints.
self.region = servers['region']
self.keystone_server = servers['keystone']
self.nova_server = servers['nova']
self.neutron_server = servers['neutron']
# Check if the user has provided a Heat
# server. Not all OpenStack clusters provide
# Heat. If not, we'll need to start a local instance.
self.heatuuid = None
if 'HEAT_URL' in os.environ:
self.heat_server = os.environ['HEAT_URL']
elif 'heat' in servers:
self.heat_server = servers['heat']
else:
self.heat_server = self._check_and_start_heat(self.tenant_id)
logging.warning("using heat server " + str(self.heat_server))
# This gives us information about the image to use
# for the supplied provider.
deploy = conf[provider]['deploy']
self.default_image = deploy['image']
self.default_personality = deploy['personality']
self.default_user = deploy['default-user']
self.ssh_key = deploy['ssh']
self.ssh_user = deploy['ssh-user']
# Make sure that the ssh key is actually present.
keypath = self._get_host_key()
if not os.path.exists(keypath):
logging.error("could not find ssh key (%s)" % self.ssh_key)
raise ValueError("Missing ssh keys")
# Initialize the OpenStack clients and also
# download some networking information (subnet ID,
# cidr, gateway, etc.)
self._init_openstack_clients()
self._collect_subnet_info()
def _get_host_key(self):
"""
Get the location of the private ssh key.
"""
p = self.ssh_key.split("/")
if len(p) == 1:
return "/ferry/keys/" + self.ssh_key + ".pem"
else:
return self.ssh_key + ".pem"
def _check_and_start_heat(self, tenant_id):
"""
Check and start the Ferry Heat image.
"""
# Check if the image is downloaded locally.
# If not, it will automatically pull it.
logging.info("Check for Heat image")
self.installer._check_and_pull_image("ferry/heatserver")
# Check if the Heat log directory exists yet. If not
# go ahead and create it.
heatlogs = ferry.install.DOCKER_DIR + "/heatlog"
try:
if not os.path.isdir(heatlogs):
os.makedirs(heatlogs)
self.installer._change_permission(heatlogs)
except OSError as e:
logging.error(e.strerror)
sys.exit(1)
# Start the Heat image and capture the IP address. We can
# then hand over this IP to the rest of the configuration.
volumes = {heatlogs: "/var/log/heat"}
heatplan = {
'image': 'ferry/heatserver',
'type': 'ferry/heatserver',
'keydir': {},
'keyname': None,
'privatekey': None,
'volumes': volumes,
'volume_user': ferry.install.DEFAULT_FERRY_OWNER,
'ports': [],
'exposed': ["8004", "8000"],
'internal': [],
'hostname': 'heatserver',
'netenable': True,
'default_cmd': '',
'args': 'trust'
}
self.heatuuid = 'fht-' + str(uuid.uuid4()).split('-')[0]
self.heatbox = self.installer.fabric.alloc(self.heatuuid,
self.heatuuid, [heatplan],
"HEAT")[0]
if not self.heatbox:
logging.error("Could not start Heat server")
sys.exit(1)
else:
return "http://%s:8004/v1/%s" % (str(
self.heatbox.internal_ip), tenant_id)
def _check_openstack_credentials(self):
envs = ['OS_USERNAME', 'OS_PASSWORD', 'OS_TENANT_ID', 'OS_TENANT_NAME']
for e in envs:
if not e in os.environ:
return False
return True
def _init_openstack_clients(self):
# Instantiate the Heat client.
if 'HEAT_API_VERSION' in os.environ:
heat_api_version = os.environ['HEAT_API_VERSION']
else:
heat_api_version = '1'
kwargs = {
'username': self.openstack_user,
'password': self.openstack_pass,
'include_pass': True,
'tenant_id': self.tenant_id,
'tenant_name': self.tenant_name,
'auth_url': self.keystone_server
}
self.heat = heat_client.Client(heat_api_version, self.heat_server,
**kwargs)
# Check to make sure that the Heat client can actually
# connect to the Heat server. This is because we may
# have just started the Heat server, so it can a while to refresh.
for i in range(0, 10):
try:
stacks = self.heat.stacks.list()
for s in stacks:
logging.warning("found Heat stack: " + str(s))
connected = True
break
except:
time.sleep(12)
connected = False
if not connected:
raise ValueError("Could not connect to Heat")
# Instantiate the Neutron client.
# There should be a better way of figuring out the API version.
neutron_api_version = "2.0"
kwargs['endpoint_url'] = self.neutron_server
self.neutron = neutron_client.Client(neutron_api_version, **kwargs)
# Instantiate the Nova client. The Nova client is used
# to stop/restart instances.
nova_api_version = "1.1"
kwargs = {
'username': self.openstack_user,
'api_key': self.openstack_pass,
'tenant_id': self.tenant_id,
'auth_url': self.keystone_server,
'service_type': 'compute',
'region_name': self.region
}
self.nova = nova_client.Client(nova_api_version, **kwargs)
def _create_floating_ip(self, name, port):
"""
Create and attach a floating IP to the supplied port.
"""
plan = {
name: {
"Type": "OS::Neutron::FloatingIP",
"Properties": {
"floating_network_id": self.external_network
}
},
name + "_assoc": {
"Type": "OS::Neutron::FloatingIPAssociation",
"Properties": {
"floatingip_id": {
"Ref": name
},
"port_id": {
"Ref": port
}
}
}
}
desc = {"type": "OS::Neutron::FloatingIP"}
return plan, desc
def _create_security_group(self, group_name, ports, internal):
"""
Create and assign a security group to the supplied server.
"""
# Create the basic security group.
# This only includes SSH. We can later update the group
# to include additional ports.
desc = {
group_name: {
"Type": "OS::Neutron::SecurityGroup",
"Properties": {
"name":
group_name,
"description":
"Ferry firewall rules",
"rules": [{
"protocol": "icmp",
"remote_ip_prefix": "0.0.0.0/0"
}, {
"protocol": "tcp",
"remote_ip_prefix": "0.0.0.0/0",
"port_range_min": 22,
"port_range_max": 22
}]
}
}
}
# Additional ports for the security group. These
# port values can be accessible from anywhere.
for p in ports:
min_port = p[0]
max_port = p[1]
desc[group_name]["Properties"]["rules"].append({
"protocol":
"tcp",
"remote_ip_prefix":
"0.0.0.0/0",
"port_range_min":
min_port,
"port_range_max":
max_port
})
# Additional ports for the security group. These
# port values can only be accessed from within the same network.
for p in internal:
min_port = p[0]
max_port = p[1]
desc[group_name]["Properties"]["rules"].append({
"protocol":
"tcp",
"remote_ip_prefix":
self.subnet["cidr"],
"port_range_min":
min_port,
"port_range_max":
max_port
})
return desc
def _create_storage_volume(self, volume_name, server_name, size_gb):
"""
Create and attach a storage volume to the supplied server.
"""
desc = {
volume_name: {
"Type": "OS::Cinder::Volume",
"Properties": {
"size": size_db,
"availability_zone": self.default_zone
}
},
volume_name + "_attachment": {
"Type": "OS::Cinder::VolumeAttachment",
"Properties": {
"volume_id": {
"Ref": volume_name
},
"instance_uuid": {
"Ref": server_name
},
"mount_point": "/dev/vdc"
}
}
}
return desc
def _create_port(self, name, network, sec_group, ref=True):
desc = {
name: {
"Type": "OS::Neutron::Port",
"Properties": {
"name": name,
"security_groups": [{
"Ref": sec_group
}]
}
}
}
if ref:
desc[name]["Properties"]["network"] = {"Ref": network}
else:
desc[name]["Properties"]["network"] = network
return desc
def _create_server_init(self):
"""
Create the server init process. These commands are run on the
host after the host has booted up.
"""
user_data = {
"Fn::Base64": {
"Fn::Join": [
"",
[
"#!/bin/bash -v\n", "umount /mnt\n",
"parted --script /dev/vdb mklabel gpt\n",
"parted --script /dev/vdb mkpart primary xfs 0% 100%\n",
"mkfs.xfs /dev/vdb1\n", "mkdir /ferry/data\n",
"mkdir /ferry/keys\n", "mkdir /ferry/containers\n",
"mount -o noatime /dev/vdb1 /ferry/data\n",
"export FERRY_SCRATCH=/ferry/data\n",
"export FERRY_DIR=/ferry/master\n",
"echo export FERRY_SCRATCH=/ferry/data >> /etc/profile\n",
"echo export FERRY_DIR=/ferry/master >> /etc/profile\n",
"export HOME=/root\n", "export USER=root\n",
"mkdir /home/ferry/.ssh\n",
"cp /home/%s/.ssh/authorized_keys /home/ferry/.ssh/\n"
% self.default_user,
"cp /home/%s/.ssh/authorized_keys /root/.ssh/\n" %
self.default_user,
"chown -R ferry:ferry /home/ferry/.ssh\n",
"chown -R ferry:ferry /ferry/data\n",
"chown -R ferry:ferry /ferry/keys\n",
"chown -R ferry:ferry /ferry/containers\n",
"ferry server -n\n", "sleep 3\n"
]
]
}
}
return user_data
def _create_volume_attachment(self, iface, instance, volume_id):
plan = {
iface: {
"Type": "OS::Cinder::VolumeAttachment",
"Properties": {
"instance_uuid": {
"Ref": instance
},
"mountpoint": "/dev/vdc",
"volume_id": volume_id
}
}
}
desc = {"type": "OS::Cinder::VolumeAttachment"}
return plan, desc
def _create_instance(self, name, image, size, manage_network, sec_group):
"""
Create a new instance
"""
plan = {
name: {
"Type": "OS::Nova::Server",
"Properties": {
"name": name,
"image": image,
"key_name": self.ssh_key,
"flavor": size,
"availability_zone": self.default_zone,
"networks": []
}
}
}
desc = {
name: {
"type": "OS::Nova::Server",
"name": name,
"ports": [],
"volumes": []
}
}
# Create a port for the manage network.
port_descs = []
port_name = "ferry-port-%s" % name
port_descs.append(
self._create_port(port_name, manage_network, sec_group, ref=False))
plan[name]["Properties"]["networks"].append({
"port": {
"Ref": port_name
},
"network": manage_network
})
desc[name]["ports"].append(port_name)
desc[port_name] = {"type": "OS::Neutron::Port", "role": "manage"}
# Combine all the port descriptions.
for d in port_descs:
plan = dict(plan.items() + d.items())
# Now add the user script.
user_data = self._create_server_init()
plan[name]["Properties"]["user_data"] = user_data
return plan, desc
def _create_floatingip_plan(self, cluster_uuid, ifaces):
"""
Assign floating IPs to the supplied interfaces/ports.
"""
plan = {
"AWSTemplateFormatVersion": "2010-09-09",
"Description": "Ferry generated Heat plan",
"Resources": {}
}
desc = {}
for i in range(0, len(ifaces)):
ip_name = "ferry-ip-%s-%d" % (cluster_uuid, i)
ip_plan, desc[ip_name] = self._create_floating_ip(
ip_name, ifaces[i])
plan["Resources"] = dict(plan["Resources"].items() +
ip_plan.items())
return plan, desc
def _create_security_plan(self, cluster_uuid, ports, internal, ctype):
"""
Update the security group.
"""
sec_group_name = "ferry-sec-%s-%s" % (cluster_uuid, ctype)
plan = {
"AWSTemplateFormatVersion":
"2010-09-09",
"Description":
"Ferry generated Heat plan",
"Resources":
self._create_security_group(sec_group_name, ports, internal)
}
desc = {sec_group_name: {"type": "OS::Neutron::SecurityGroup"}}
return plan, desc
def _create_instance_plan(self, cluster_uuid, num_instances, image, size,
sec_group_name, ctype):
plan = {
"AWSTemplateFormatVersion": "2010-09-09",
"Description": "Ferry generated Heat plan",
"Resources": {},
"Outputs": {}
}
desc = {}
for i in range(0, num_instances):
# Create the actual instances.
instance_name = "ferry-instance-%s-%s-%d" % (cluster_uuid, ctype,
i)
instance_plan, instance_desc = self._create_instance(
instance_name, image, size, self.manage_network,
sec_group_name)
plan["Resources"] = dict(plan["Resources"].items() +
instance_plan.items())
desc = dict(desc.items() + instance_desc.items())
return plan, desc
def _launch_heat_plan(self, stack_name, heat_plan, stack_desc):
"""
Launch the cluster plan.
"""
logging.info("launching heat plan: " + str(heat_plan))
try:
# Try to create the application stack.
resp = self.heat.stacks.create(stack_name=stack_name,
template=heat_plan)
except HTTPBadRequest as e:
logging.error(e.strerror)
return None
except:
# We could not create the stack. This probably
# means that either the Heat server is down or the
# OpenStack cluster is down.
logging.error("could not create Heat stack")
return None
# Now wait for the stack to be in a completed state
# before returning. That way we'll know if the stack creation
# has failed or not.
if not self._wait_for_stack(resp["stack"]["id"]):
logging.warning("Heat plan %s CREATE_FAILED" % resp["stack"]["id"])
return None
# Now find the physical IDs of all the resources.
resources = self._collect_resources(resp["stack"]["id"])
for r in resources:
if r["logical_resource_id"] in stack_desc:
stack_desc[
r["logical_resource_id"]]["id"] = r["physical_resource_id"]
# Record the Stack ID in the description so that
# we can refer back to it later.
stack_desc[stack_name] = {
"id": resp["stack"]["id"],
"type": "OS::Heat::Stack"
}
return stack_desc
def _wait_for_stack(self, stack_id):
"""
Wait for stack completion.
"""
while (True):
try:
stack = self.heat.stacks.get(stack_id)
if stack.status == "COMPLETE":
return True
elif stack.status == "FAILED":
return False
else:
time.sleep(4)
except:
logging.error("could not fetch stack status (%s)" %
str(stack_id))
def _collect_resources(self, stack_id):
"""
Collect all the stack resources so that we can create
additional plans and use IDs.
"""
try:
resources = self.heat.resources.list(stack_id)
descs = [r.to_dict() for r in resources]
return descs
except:
return []
def _collect_subnet_info(self):
"""
Collect the data network subnet info (ID, CIDR, and gateway).
"""
subnets = self.neutron.list_subnets()
for s in subnets['subnets']:
if s['network_id'] == self.manage_network:
self.subnet = {
"id": s['id'],
"cidr": s['cidr'],
"gateway": s['gateway_ip']
}
def _collect_network_info(self, stack_desc):
"""
Collect all the networking information.
"""
# First get the floating IP information.
ip_map = {}
floatingips = self.neutron.list_floatingips()
for f in floatingips['floatingips']:
if f['fixed_ip_address']:
ip_map[f['fixed_ip_address']] = f['floating_ip_address']
# Now fill in the various networking information, including
# subnet, IP address, and floating address. We should also
# probably collect MAC addresseses..
ports = self.neutron.list_ports()
for p in ports['ports']:
if p['name'] != "" and p['name'] in stack_desc:
port_desc = stack_desc[p['name']]
port_desc["subnet"] = p['fixed_ips'][0]['subnet_id']
port_desc["ip_address"] = p['fixed_ips'][0]['ip_address']
# Not all ports are associated with a floating IP, so
# we need to check first.
if port_desc["ip_address"] in ip_map:
port_desc["floating_ip"] = ip_map[port_desc["ip_address"]]
return stack_desc
def _collect_instance_info(self, stack_desc):
"""
Collect all the instance information.
"""
servers = self.nova.servers.list()
for s in servers:
if s.name != "" and s.name in stack_desc:
instance_desc = stack_desc[s.name]
instance_desc["id"] = s.id
return stack_desc
def _create_app_stack(self, cluster_uuid, num_instances,
security_group_ports, internal_ports,
assign_floating_ip, ctype):
"""
Create an empty application stack. This includes the instances,
security groups, and floating IPs.
"""
logging.info("creating security group for %s" % cluster_uuid)
sec_group_plan, sec_group_desc = self._create_security_plan(
cluster_uuid=cluster_uuid,
ports=security_group_ports,
internal=internal_ports,
ctype=ctype)
logging.info("creating instances for %s" % cluster_uuid)
stack_plan, stack_desc = self._create_instance_plan(
cluster_uuid=cluster_uuid,
num_instances=num_instances,
image=self.default_image,
size=self.default_personality,
sec_group_name=sec_group_desc.keys()[0],
ctype=ctype)
# See if we need to assign any floating IPs
# for this stack. We need the references to the neutron
# port which is contained in the description.
if assign_floating_ip:
logging.info("creating floating IPs for %s" % cluster_uuid)
ifaces = []
for k in stack_desc.keys():
if stack_desc[k]["type"] == "OS::Neutron::Port" and stack_desc[
k]["role"] == "manage":
ifaces.append(k)
ip_plan, ip_desc = self._create_floatingip_plan(
cluster_uuid=cluster_uuid, ifaces=ifaces)
else:
ip_plan = {"Resources": {}}
ip_desc = {}
# Now we need to combine all these plans and
# launch the cluster.
stack_plan["Resources"] = dict(sec_group_plan["Resources"].items() +
ip_plan["Resources"].items() +
stack_plan["Resources"].items())
stack_desc = dict(stack_desc.items() + sec_group_desc.items() +
ip_desc.items())
stack_desc = self._launch_heat_plan(
"ferry-app-%s-%s" % (ctype.upper(), cluster_uuid), stack_plan,
stack_desc)
# Now find all the IP addresses of the various machines.
if stack_desc:
stack_desc = self._collect_instance_info(stack_desc)
return self._collect_network_info(stack_desc)
else:
return None
def _get_private_ip(self, server, subnet_id, resources):
"""
Get the IP address associated with the supplied server.
"""
for port_name in server["ports"]:
port_desc = resources[port_name]
if port_desc["subnet"] == subnet_id:
return port_desc["ip_address"]
def _get_public_ip(self, server, resources):
"""
Get the IP address associated with the supplied server.
"""
for port_name in server["ports"]:
port_desc = resources[port_name]
if "floating_ip" in port_desc:
return port_desc["floating_ip"]
def _get_servers(self, resources):
servers = []
for r in resources.values():
if type(r) is dict and r["type"] == "OS::Nova::Server":
servers.append(r)
return servers
def _get_net_info(self, server_info, subnet, resources):
"""
Look up the IP address, gateway, and subnet range.
"""
cidr = subnet["cidr"].split("/")[1]
ip = self._get_private_ip(server_info, subnet["id"], resources)
# We want to use the host NIC, so modify LXC to use phys networking, and
# then start the docker containers on the server.
lxc_opts = [
"lxc.network.type = phys",
"lxc.network.ipv4 = %s/%s" % (ip, cidr),
"lxc.network.ipv4.gateway = %s" % subnet["gateway"],
"lxc.network.link = %s" % self.data_device,
"lxc.network.name = eth0", "lxc.network.flags = up"
]
return lxc_opts, ip
def _update_app_db(self, cluster_uuid, service_uuid, heat_plan):
# Make a copy of the plan before inserting into
# mongo, otherwise the "_id" field will be added
# silently.
heat_plan["_cluster_uuid"] = cluster_uuid
heat_plan["_service_uuid"] = service_uuid
self.apps.insert(copy.deepcopy(heat_plan))
def alloc(self, cluster_uuid, service_uuid, container_info, ctype, proxy):
"""
Allocate a new cluster.
"""
# Now take the cluster and create the security group
# to expose all the right ports.
sec_group_ports = []
internal_ports = []
if ctype == "connector":
# Since this is a connector, we need to expose
# the public ports. For now, we ignore the host port.
floating_ip = True
for c in container_info:
for p in c['ports']:
s = str(p).split(":")
if len(s) > 1:
sec_group_ports.append((s[1], s[1]))
else:
sec_group_ports.append((s[0], s[0]))
else:
if proxy:
# Otherwise, the backend should also get floating IPs
# so that the controller can access it.
floating_ip = True
else:
# If the controller is acting as a proxy, then it has
# direct access to the VMs, so the backend shouldn't
# get any floating IPs.
floating_ip = False
# We need to create a range tuple, so check if
# the exposed port is a range.
for p in container_info[0]['exposed']:
s = p.split("-")
if len(s) == 1:
sec_group_ports.append((s[0], s[0]))
else:
sec_group_ports.append((s[0], s[1]))
# Also see if there are any ports that should be
# open within the cluster (but not outside). Typically
# used for IPC (where ports may be assigned within a random range).
for p in container_info[0]['internal']:
s = p.split("-")
if len(s) == 1:
internal_ports.append((s[0], s[0]))
else:
internal_ports.append((s[0], s[1]))
# Tell OpenStack to allocate the cluster.
resources = self._create_app_stack(
cluster_uuid=cluster_uuid,
num_instances=len(container_info),
security_group_ports=sec_group_ports,
internal_ports=internal_ports,
assign_floating_ip=floating_ip,
ctype=ctype)
# Now we need to ask the cluster to start the
# Docker containers.
containers = []
mounts = {}
if resources:
# Store the resources cluster ID.
self._update_app_db(cluster_uuid, service_uuid, resources)
servers = self._get_servers(resources)
for i in range(0, len(container_info)):
# Fetch a server to run the Docker commands.
server = servers[i]
# Get the LXC networking options
lxc_opts, private_ip = self._get_net_info(
server, self.subnet, resources)
# Now get an addressable IP address. If we're acting as a proxy within
# the same cluster, we can just use the private address. Otherwise
# we'll need to route via the public IP address.
if proxy:
server_ip = private_ip
else:
server_ip = self._get_public_ip(server, resources)
# Verify that the user_data processes all started properly
# and that the docker daemon is actually running. If it is
# not running, try re-executing.
if not self.controller._verify_ferry_server(server_ip):
self.controller._execute_server_init(server_ip)
# Copy over the public keys, but also verify that it does
# get copied over properly.
self.controller._copy_public_keys(container_info[i], server_ip)
if self.controller._verify_public_keys(server_ip):
container, cmounts = self.controller.execute_docker_containers(
container_info[i], lxc_opts, private_ip, server_ip)
if container:
mounts = dict(mounts.items() + cmounts.items())
containers.append(container)
else:
logging.error("could not copy over ssh key!")
return None
# Check if we need to set the file permissions
# for the mounted volumes.
for c, i in mounts.items():
for _, v in i['vols']:
self.controller.cmd([c], 'chown -R %s %s' % (i['user'], v))
return containers
else:
# OpenStack failed to launch the application stack.
# This can be caused by improper OpenStack credentials
# or if the OpenStack cluster is under heavy load (i.e.,
# requests are getting timed out).
return None
def _delete_stack(self, cluster_uuid, service_uuid):
# Find the relevant stack information.
ips = []
stacks = self.apps.find({
"_cluster_uuid": cluster_uuid,
"_service_uuid": service_uuid
})
logging.warning("Deleting cluster %s" % str(cluster_uuid))
for stack in stacks:
for s in stack.values():
if type(s) is dict and s["type"] == "OS::Heat::Stack":
stack_id = s["id"]
# To delete the stack properly, we first need to disassociate
# the floating IPs.
resources = self._collect_resources(stack_id)
for r in resources:
if r["resource_type"] == "OS::Neutron::FloatingIP":
self.neutron.update_floatingip(
r["physical_resource_id"],
{'floatingip': {
'port_id': None
}})
# Now try to delete the stack. Wrap this in a try-block so that
# we don't completely fail even if the stack doesn't exist.
try:
logging.warning("Deleting stack %s" % str(stack_id))
self.heat.stacks.delete(stack_id)
except HTTPNotFound as e:
logging.warning(e)
self.apps.remove({
"_cluster_uuid": cluster_uuid,
"_service_uuid": service_uuid
})
def _stop_stack(self, cluster_uuid, service_uuid):
stacks = self.apps.find({
"_cluster_uuid": cluster_uuid,
"_service_uuid": service_uuid
})
for stack in stacks:
servers = self._get_servers(stack)
for s in servers:
self.nova.servers.stop(s["id"])
def _restart_stack(self, cluster_uuid, service_uuid):
ips = []
stacks = self.apps.find({
"_cluster_uuid": cluster_uuid,
"_service_uuid": service_uuid
})
for stack in stacks:
# Find the set of servers and restart them
# one by one. It would be nicer if Heat had a way to
# restart them all at once, but not sure how to do that...
servers = self._get_servers(stack)
for s in servers:
self.nova.servers.start(s["id"])
ips.append(self._get_public_ip(s, stack))
# Wait for the servers to actually be in the
# "running" status before returning.
for s in servers:
while (True):
found = self.nova.servers.list(
search_opts={"name": s["name"]})
for f in found["servers"]:
if f["status"] == "ACTIVE":
break
time.sleep(4)
return ips
def quit(self):
"""
Check if the Heat server is running, and if so
go ahead and stop it.
"""
if self.heatuuid:
self.installer.fabric.stop(self.heatuuid, self.heatuuid,
[self.heatbox])
class CLI(object):
def __init__(self):
self.cmds = CmdHelp()
self.cmds.description = "Development environment for big data applications"
self.cmds.version = ferry.__version__
self.cmds.usage = "ferry COMMAND [arg...]"
self.cmds.add_option("-c", "--conf", "Deployment configuration")
self.cmds.add_option("-n", "--dns", "Use custom DNS")
self.cmds.add_option("-l", "--log", "Log configuration file")
self.cmds.add_option("-k", "--key", "Specify key directory")
self.cmds.add_option("-m", "--mode", "Deployment mode")
self.cmds.add_option("-u", "--upgrade", "Upgrade Ferry")
self.cmds.add_option("-b", "--build", "Build Ferry default images")
self.cmds.add_cmd("build", "Build a Dockerfile")
self.cmds.add_cmd("clean", "Clean zombie Ferry processes")
self.cmds.add_cmd("server", "Start all the servers")
self.cmds.add_cmd("deploy", "Deploy a service to the cloud")
self.cmds.add_cmd("help", "Print this help message")
self.cmds.add_cmd("info", "Print version information")
self.cmds.add_cmd("inspect", "Return low-level information on a service")
self.cmds.add_cmd("install", "Install the Ferry images")
self.cmds.add_cmd("login", "Login to Ferry servers")
self.cmds.add_cmd("logs", "Copy over the logs to the host")
self.cmds.add_cmd("ls", "View installed applications")
self.cmds.add_cmd("ps", "List deployed and running services")
self.cmds.add_cmd("pull", "Pull a remote image")
self.cmds.add_cmd("push", "Push an image to a remote registry")
self.cmds.add_cmd("rm", "Remove a service or snapshot")
self.cmds.add_cmd("snapshot", "Take a snapshot")
self.cmds.add_cmd("snapshots", "List all snapshots")
self.cmds.add_cmd("ssh", "Connect to a client/connector")
self.cmds.add_cmd("start", "Start a new service or snapshot")
self.cmds.add_cmd("stop", "Stop a running service")
self.cmds.add_cmd("quit", "Stop the Ferry servers")
self.ferry_server = 'http://127.0.0.1:4000'
self.default_user = '******'
self.installer = Installer(self)
def _pull_image(self, image):
"""
Pull a remote image to the local registry.
"""
try:
payload = { 'image' : image }
res = requests.get(self.ferry_server + '/image', params=payload)
return str(res.text)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _pull_app(self, app):
"""
Pull a local application to Ferry servers.
"""
# Now download the application description
# from the Ferry servers.
account, key, server = self.installer.get_ferry_account()
if account:
# Read in the contents of the application and
# generate the API key.
req = { 'action' : 'fetch',
'app' : app,
'account' : account }
sig = self.installer.create_signature(json.dumps(req), key)
try:
payload = { 'id' : account,
'app' : app,
'sig' : sig }
res = requests.get(server + '/app', params=payload)
status = json.loads(res.text)
file_name = self.installer.store_app(app, status['ext'], status['content'])
if file_name:
content = self._read_app_content(file_name)
images = self._get_user_images(content)
for i in images:
self._pull_image(i)
return app
else:
return "failed"
except ConnectionError:
logging.error("could not connect to application server")
return "failed"
def _pull(self, image):
"""
Pull a remote application/image to the local registry.
"""
logging.warning("pulling " + image)
# Figure out if we're pushing a Ferry application or
# plain Docker image.
s = image.split("://")
if len(s) > 1:
proto = s[0]
image = s[1]
else:
proto = "image"
image = s[0]
if proto == "image":
return self._pull_image(image, registry)
else:
return self._pull_app(image)
def _push_image(self, image, registry):
"""
Push a local image to a remote registry.
"""
try:
payload = { 'image' : image,
'server' : registry }
res = requests.post(self.ferry_server + '/image', data=payload)
return str(res.text)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _builtin_image(self, image):
"""
Indicates whether the image is a pre-built Ferry image.
Right now we only verify the client images.
"""
return image in ["ferry/hadoop-client",
"ferry/spark-client",
"ferry/cassandra-client",
"ferry/openmpi-client"]
def _get_user_images(self, content):
"""
Get the user-defined images.
"""
images = set()
for c in content['connectors']:
p = c['personality']
if not self._builtin_image(p):
images.add(p)
return images
def _read_app_content(self, file_path):
"""
Read the content of the application.
"""
json_arg = None
with open(file_path, "r") as f:
n, e = os.path.splitext(file_path)
if e == '.json':
json_string = self._read_file_arg(file_path)
json_arg = json.loads(json_string)
elif e == '.yaml' or e == '.yml':
yaml_file = open(file_path, 'r')
json_arg = yaml.load(yaml_file)
return json_arg
def _push_app(self, app, registry):
"""
Push a local application to Ferry servers.
"""
# First find all the images that need to
# be pushed to the Docker registry.
content = self._read_app_content(app)
if content:
images = self._get_user_images(content)
for i in images:
self._push_image(i, registry)
# Register the application in the Ferry database.
account, key, server = self.installer.get_ferry_account()
if account:
# Read in the contents of the application and
# generate the API key.
with open(app, "r") as f:
name = account + '/' + os.path.basename(app)
name, ext = os.path.splitext(name)
content = f.read()
req = { 'action' : 'register',
'app' : name,
'account' : account }
sig = self.installer.create_signature(json.dumps(req), key)
try:
payload = { 'id' : account,
'app' : name,
'ext' : ext,
'content' : content,
'sig' : sig }
res = requests.post(server + '/app', data=payload)
status = json.loads(res.text)
if status['status'] == 'fail':
logging.error("failed to register app " + app)
return "Failed to register app " + app
else:
return status['name']
except ConnectionError:
logging.error("could not connect to application server")
return "Could not register the application."
except ValueError as e:
logging.error(str(e))
return "Registration server sent back unknown reply"
else:
logging.error("could not read account information")
return "Could not read account information."
def _push(self, image, registry):
"""
Push a local appliation/image to a remote registry.
"""
logging.warning("pushing " + image)
# Figure out if we're pushing a Ferry application or
# plain Docker image.
s = image.split("://")
if len(s) > 1:
proto = s[0]
image = s[1]
else:
proto = "image"
image = s[0]
if proto == "image":
return self._push_image(image, registry)
else:
return self._push_app(image, registry)
def _build(self, dockerfile):
"""
Build a new local image.
"""
logging.warning("building " + dockerfile)
names = self.installer._get_image(dockerfile)
name = names.pop().split("/")
if len(name) == 1:
repo = GUEST_DOCKER_REPO
image = name[0]
else:
repo = name[0]
image = name[1]
build_dir = os.path.dirname(dockerfile)
self.installer._compile_image(image, repo, build_dir, build=True)
if len(names) > 0:
self.installer._tag_images(image, repo, names)
def _login(self):
"""
Login to a remote registry
"""
try:
res = requests.post(self.ferry_server + '/login')
return str(res.text)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _create_stack(self, stack_description, args):
"""
Create a new stack.
"""
mode = self._parse_deploy_arg('mode', args, default='local')
conf = self._parse_deploy_arg('conf', args, default='default')
payload = { 'payload' : json.dumps(stack_description),
'mode' : mode,
'conf' : conf }
try:
res = requests.post(self.ferry_server + '/create', data=payload)
return True, str(res.text)
except ConnectionError:
logging.error("could not connect to ferry server")
return False, "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _resubmit_create(self, reply, stack_description, args):
values = {}
for q in reply['questions']:
question = colored(q['query']['text'], 'red')
prompt = colored(' >> ', 'green')
v = raw_input(question + prompt)
values[q['query']['id']] = v
stack_description['iparams'] = values
posted, reply = self._create_stack(stack_description, args)
if posted:
try:
reply = json.loads(reply)
return reply['text']
except ValueError as e:
logging.error(reply)
def _format_apps_query(self, json_data):
authors = []
versions = []
descriptions = []
for app in json_data.keys():
authors.append(json_data[app]['author'])
versions.append(json_data[app]['version'])
descriptions.append(json_data[app]['description'])
t = PrettyTable()
t.add_column("App", json_data.keys())
t.add_column("Author", authors)
t.add_column("Version", versions)
t.add_column("Description", descriptions)
return t.get_string(sortby="App",
padding_width=2)
def _format_snapshots_query(self, json_data):
bases = []
date = []
for uuid in json_data.keys():
bases.append(json_data[uuid]['base'])
if 'snapshot_ts' in json_data[uuid]:
date.append(json_data[uuid]['snapshot_ts'])
else:
date.append(' ')
t = PrettyTable()
t.add_column("UUID", json_data.keys())
t.add_column("Base", bases)
t.add_column("Date", date)
return t.get_string(sortby="Date",
padding_width=2)
def _format_table_query(self, json_data):
storage = []
compute = []
connectors = []
status = []
base = []
time = []
# Each additional row should include the actual data.
for uuid in json_data.keys():
csstore = []
bstore = []
cstore = []
for c in json_data[uuid]['connectors']:
csstore.append(c)
backends = json_data[uuid]['backends']
for b in backends:
if b['storage']:
bstore.append(b['storage'])
else:
bstore.append(' ')
if b['compute']:
cstore.append(b['compute'])
else:
cstore.append(' ')
storage.append(bstore)
compute.append(cstore)
connectors.append(csstore)
status.append(json_data[uuid]['status'])
base.append(json_data[uuid]['base'])
time.append(json_data[uuid]['ts'])
t = PrettyTable()
t.add_column("UUID", json_data.keys())
t.add_column("Storage", storage)
t.add_column("Compute", compute)
t.add_column("Connectors", connectors)
t.add_column("Status", status)
t.add_column("Base", base)
t.add_column("Time", time)
return t.get_string(sortby="UUID",
padding_width=2)
def _stop_all(self):
try:
constraints = { 'status' : 'running' }
payload = { 'constraints' : json.dumps(constraints) }
res = requests.get(self.ferry_server + '/query', params=payload)
stacks = json.loads(res.text)
for uuid in stacks.keys():
self._manage_stacks({'uuid' : uuid,
'action' : 'stop'})
except ConnectionError:
logging.error("could not connect to ferry server")
def _read_stacks(self, show_all=False, args=None):
try:
res = requests.get(self.ferry_server + '/query')
query_reply = json.loads(res.text)
deployed_reply = {}
if show_all:
mode = self._parse_deploy_arg('mode', args, default='local')
conf = self._parse_deploy_arg('conf', args, default='default')
payload = { 'mode' : mode,
'conf' : conf }
res = requests.get(self.ferry_server + '/deployed', params=payload)
deployed_reply = json.loads(res.text)
# Merge the replies and format.
return self._format_table_query(dict(query_reply.items() + deployed_reply.items()))
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _list_apps(self):
"""
List all installed applications including the built-in applications.
"""
try:
res = requests.get(self.ferry_server + '/apps')
json_reply = json.loads(res.text)
return self._format_apps_query(json_reply)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _list_snapshots(self):
"""
List all snapshots.
"""
try:
res = requests.get(self.ferry_server + '/snapshots')
json_reply = json.loads(res.text)
return self._format_snapshots_query(json_reply)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _format_stack_inspect(self, json_data):
return json.dumps(json_data,
sort_keys=True,
indent=2,
separators=(',',':'))
def _inspect_stack(self, stack_id):
"""
Inspect a specific stack.
"""
payload = { 'uuid':stack_id }
try:
res = requests.get(self.ferry_server + '/stack', params=payload)
return res.text
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _copy_logs(self, stack_id, to_dir):
"""
Copy over the logs.
"""
payload = {'uuid':stack_id,
'dir':to_dir}
try:
res = requests.get(self.ferry_server + '/logs', params=payload)
json_value = json.loads(str(res.text))
return self._format_stack_inspect(json_value)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
"""
Connector a specific client/connector via ssh.
"""
def _connect_stack(self, stack_id, connector_id):
# Get the IP and default user information for this connector.
payload = {'uuid':stack_id}
try:
res = requests.get(self.ferry_server + '/stack', params=payload)
json_value = json.loads(str(res.text))
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
connector_ip = None
for cg in json_value['connectors']:
if not connector_id:
connector_ip = cg['entry']['ip']
break
elif connector_id == cg['uniq']:
connector_ip = cg['entry']['ip']
break
# Now form the ssh command. This just executes in the same shell.
if connector_ip:
keydir, _ = self._read_key_dir()
key_opt = '-o StrictHostKeyChecking=no'
host_opt = '-o UserKnownHostsFile=/dev/null'
ident = '-i %s/id_rsa' % keydir
dest = '%s@%s' % (self.default_user, connector_ip)
cmd = "ssh %s %s %s %s" % (key_opt, host_opt, ident, dest)
logging.warning(cmd)
os.execv('/usr/bin/ssh', cmd.split())
else:
logging.warning("could not find connector %s" % connector_id)
def _parse_deploy_arg(self, param, args, default):
pattern = re.compile('--%s=(\w+)' % param)
for a in args:
m = pattern.match(a)
if m and m.group(0) != '':
return m.group(1)
return default
def _deploy_stack(self, stack_id, args):
"""
Deploy the stack.
"""
mode = self._parse_deploy_arg('mode', args, default='local')
conf = self._parse_deploy_arg('conf', args, default='default')
payload = { 'uuid' : stack_id,
'mode' : mode,
'conf' : conf }
try:
res = requests.post(self.ferry_server + '/deploy', data=payload)
return res.text
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _manage_stacks(self, stack_info):
"""
Manage the stack.
"""
try:
res = requests.post(self.ferry_server + '/manage/stack', data=stack_info)
return str(res.text)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _print_help(self):
"""
Output the help message.
"""
return self.cmds.print_help()
def _print_info(self):
"""
Output version information.
"""
try:
res = requests.get(self.ferry_server + '/version')
s = self.cmds.description + '\n'
s += "Version: %s\n" % self.cmds.version
s += "Docker: %s\n" % res.text.strip()
return s
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _read_file_arg(self, file_name):
"""
Helper method to read a file.
"""
json_file = open(os.path.abspath(file_name), 'r')
json_text = ''
for line in json_file:
json_text += line.strip()
return json_text
def _read_key_dir(self, options=None, root=False, server=False):
"""
Read the location of the directory containing the keys
used to communicate with the containers.
"""
if root:
keydir = ferry.install._get_key_dir(root=True, server=server)
else:
keydir = ferry.install._get_key_dir(root=False, server=server)
# May need to generate the user key.
if not os.path.isfile(keydir):
self.installer._process_ssh_key(options=options, root=False)
# Read the key directory
with open(keydir, 'r') as f:
k = f.read().strip().split("://")
return k[1], k[0]
def _using_tmp_ssh_key(self):
"""
Check if the key being used is a temporary key.
"""
keydir, tmp = self._read_key_dir(root=True, server=True)
return tmp == "tmp"
def _check_ssh_key(self, options=None, root=False, server=False):
"""
Check if the user has supplied a custom key
directory. If not copies over a temporary key.
"""
keydir, _ = self._read_key_dir(options=options,
root=root)
if keydir == ferry.install.DEFAULT_KEY_DIR:
if root:
keydir = os.environ['HOME'] + '/.ssh/tmp-root'
ferry.install.GLOBAL_ROOT_DIR = 'tmp://' + keydir
else:
keydir = os.environ['HOME'] + '/.ssh/tmp-ferry'
ferry.install.GLOBAL_KEY_DIR = 'tmp://' + keydir
try:
os.makedirs(keydir)
except OSError as e:
if e.errno != errno.EEXIST:
logging.error("Could not create ssh directory %s" % keydir)
exit(1)
try:
shutil.copy(ferry.install.DEFAULT_KEY_DIR + '/id_rsa', keydir + '/id_rsa')
shutil.copy(ferry.install.DEFAULT_KEY_DIR + '/id_rsa.pub', keydir + '/id_rsa.pub')
uid = pwd.getpwnam(os.environ['USER']).pw_uid
gid = grp.getgrnam(os.environ['USER']).gr_gid
os.chown(keydir + '/id_rsa', uid, gid)
os.chmod(keydir + '/id_rsa', 0600)
os.chown(keydir + '/id_rsa.pub', uid, gid)
os.chmod(keydir + '/id_rsa.pub', 0644)
except OSError as e:
logging.error("Could not copy ssh keys (%s)" % str(e))
exit(1)
except IOError as e:
if e.errno == errno.EACCES:
logging.error("Could not override keys in %s, please delete those keys and try again." % keydir)
exit(1)
if root:
ferry.install._touch_file(ferry.install._get_key_dir(root=True, server=server),
ferry.install.GLOBAL_ROOT_DIR)
logging.warning("Copied ssh keys to " + ferry.install.GLOBAL_ROOT_DIR)
else:
ferry.install._touch_file(ferry.install._get_key_dir(root=False, server=server),
ferry.install.GLOBAL_KEY_DIR)
logging.warning("Copied ssh keys to " + ferry.install.GLOBAL_KEY_DIR)
def _find_installed_app(self, app):
"""
Help find the path to the application. Check both the built-in
global directory and the user-installed directory.
"""
file_path = None
for item in os.listdir(FERRY_HOME + '/data/plans/'):
if app == os.path.splitext(item)[0]:
return FERRY_HOME + '/data/plans/' + item
if not file_path:
for user in os.listdir(DEFAULT_FERRY_APPS):
if os.path.isdir(DEFAULT_FERRY_APPS + '/' + user):
for item in os.listdir(DEFAULT_FERRY_APPS + '/' + user):
if app == user + '/' + os.path.splitext(item)[0]:
return DEFAULT_FERRY_APPS + '/' + user + '/' + item
def dispatch_cmd(self, cmd, args, options):
"""
This is the command dispatch table.
"""
if(cmd == 'start'):
self._check_ssh_key(options=options)
# Check if we need to build the image before running.
if '-b' in options:
build_dir = options['-b'][0]
self._build(build_dir + '/Dockerfile')
arg = args.pop(0)
json_arg = {}
if not os.path.exists(arg):
file_path = self._find_installed_app(arg)
else:
file_path = arg
if file_path and os.path.exists(file_path):
file_path = os.path.abspath(file_path)
json_arg = self._read_app_content(file_path)
if not json_arg:
logging.error("could not load file " + file_path)
exit(1)
json_arg['_file_path'] = file_path
json_arg['_file'] = arg
posted, reply = self._create_stack(json_arg, args)
if posted:
try:
reply = json.loads(reply)
if reply['status'] == 'query':
return self._resubmit_create(reply, json_arg, args)
elif reply['status'] == 'failed':
return 'could not create application'
else:
return reply['text']
except ValueError as e:
logging.error(reply)
elif(cmd == 'ps'):
if len(args) > 0 and args[0] == '-a':
opt = args.pop(0)
return self._read_stacks(show_all=True, args = args)
else:
return self._read_stacks(show_all=False, args = args)
elif(cmd == 'snapshots'):
return self._list_snapshots()
elif(cmd == 'install'):
msg = self.installer.install(args, options)
self.installer._stop_docker_daemon()
return msg
elif(cmd == 'clean'):
self._check_ssh_key(options=options, server=True)
self.installer.start_web(clean=True)
self.installer._clean_rules()
self.installer.stop_web()
self.installer._stop_docker_daemon(force=True)
self.installer._reset_ssh_key(root=True)
return 'cleaned ferry'
elif(cmd == 'inspect'):
return self._inspect_stack(args[0])
elif(cmd == 'logs'):
return self._copy_logs(args[0], args[1])
elif(cmd == 'server'):
self.installer.start_web(options)
return 'started ferry'
elif(cmd == 'ssh'):
self._check_ssh_key(options=options)
stack_id = args[0]
connector_id = None
if len(args) > 1:
connector_id = args[1]
return self._connect_stack(stack_id, connector_id)
elif(cmd == 'quit'):
self._check_ssh_key(options=options, root=True, server=True)
self._stop_all()
self.installer.stop_web()
self.installer._stop_docker_daemon()
if self._using_tmp_ssh_key():
self.installer._reset_ssh_key(root=True)
return 'stopped ferry'
elif(cmd == 'deploy'):
self._check_ssh_key(options=options)
stack_id = args.pop(0)
return self._deploy_stack(stack_id, args)
elif(cmd == 'ls'):
return self._list_apps()
elif(cmd == 'info'):
return self._print_info()
elif(cmd == 'build'):
return self._build(args[0])
elif(cmd == 'pull'):
return self._pull(args[0])
elif(cmd == 'push'):
image = args.pop(0)
if len(args) > 0:
registry = args.pop(0)
else:
registry = None
return self._push(image, registry)
elif(cmd == 'login'):
return self._login()
elif(cmd == 'help'):
return self._print_help()
else:
stack_info = {'uuid' : args[0],
'action' : cmd}
return self._manage_stacks(stack_info)
from ferry.docker.manager import DockerManager
from ferry.docker.docker import DockerInstance
import os
import Queue
import sys
import threading2
import time
from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop
# Initialize Flask
app = Flask(__name__)
# Initialize the storage driver
installer = Installer()
docker = DockerManager()
def _stack_worker():
"""
Worker thread.
"""
while (True):
payload = _new_queue.get()
if payload["_action"] == "new":
_allocate_new_worker(payload["_uuid"], payload)
elif payload["_action"] == "stopped":
_allocate_stopped_worker(payload)
elif payload["_action"] == "snapshotted":
class CLI(object):
def __init__(self):
self.cmds = CmdHelp()
self.cmds.description = "Development environment for big data applications"
self.cmds.version = ferry.__version__
self.cmds.usage = "ferry COMMAND [arg...]"
self.cmds.add_option("-b", "--build", "Build Ferry default images")
self.cmds.add_option("-c", "--conf", "Deployment configuration")
self.cmds.add_option("-d", "--dns", "Specify DNS servers")
self.cmds.add_option("-k", "--key", "Specify key directory")
self.cmds.add_option("-l", "--log", "Log configuration file")
self.cmds.add_option("-m", "--mode", "Deployment mode")
self.cmds.add_option("-n", "--naked", "Start in naked mode")
self.cmds.add_option("-r", "--retry", "Retry action on failure")
self.cmds.add_option("-t", "--net", "Use host network device")
self.cmds.add_option("-u", "--upgrade", "Upgrade Ferry")
self.cmds.add_cmd("build", "Build a Dockerfile")
self.cmds.add_cmd("clean", "Clean zombie Ferry processes")
self.cmds.add_cmd("help", "Print this help message")
self.cmds.add_cmd("info", "Print version information")
self.cmds.add_cmd("inspect",
"Return low-level information on a service")
self.cmds.add_cmd("install", "Install the Ferry images")
self.cmds.add_cmd("login", "Login to Ferry servers")
self.cmds.add_cmd("logs", "Copy over the logs to the host")
self.cmds.add_cmd("ls", "View installed applications")
self.cmds.add_cmd("ls-images", "View installed images")
self.cmds.add_cmd("ps", "List deployed and running services")
self.cmds.add_cmd("pull", "Pull a remote image")
self.cmds.add_cmd("push", "Push an image to a remote registry")
self.cmds.add_cmd("rm", "Remove a service or snapshot")
self.cmds.add_cmd("server", "Start all the servers")
self.cmds.add_cmd("snapshot", "Take a snapshot")
self.cmds.add_cmd("snapshots", "List all snapshots")
self.cmds.add_cmd("ssh", "Connect to a client/connector")
self.cmds.add_cmd("start", "Start a new service or snapshot")
self.cmds.add_cmd("stop", "Stop a running service")
self.cmds.add_cmd("quit", "Stop the Ferry servers")
self.ferry_server = 'http://127.0.0.1:4000'
self.default_user = '******'
self.installer = Installer(self)
def _pull_image(self, image):
"""
Pull a remote image to the local registry.
"""
try:
payload = {'image': image}
res = requests.get(self.ferry_server + '/image', params=payload)
return str(res.text)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _pull_app(self, app):
"""
Pull a local application to Ferry servers.
"""
# Now download the application description
# from the Ferry servers.
account, key, server = self.installer.get_ferry_account()
if account:
# Read in the contents of the application and
# generate the API key.
req = {'action': 'fetch', 'app': app, 'account': account}
sig = self.installer.create_signature(json.dumps(req), key)
try:
payload = {'id': account, 'app': app, 'sig': sig}
res = requests.get(server + '/app', params=payload)
status = json.loads(res.text)
file_name = self.installer.store_app(app, status['ext'],
status['content'])
if file_name:
content = self._read_app_content(file_name)
images = self._get_user_images(content)
for i in images:
self._pull_image(i)
return app
else:
return "failed"
except ConnectionError:
logging.error("could not connect to application server")
return "failed"
def _pull(self, image):
"""
Pull a remote application/image to the local registry.
"""
logging.warning("pulling " + image)
# Figure out if we're pushing a Ferry application or
# plain Docker image.
s = image.split("://")
if len(s) > 1:
proto = s[0]
image = s[1]
else:
proto = "image"
image = s[0]
if proto == "image":
return self._pull_image(image)
else:
return self._pull_app(image)
def _push_image(self, image, registry):
"""
Push a local image to a remote registry.
"""
try:
payload = {'image': image, 'server': registry}
res = requests.post(self.ferry_server + '/image', data=payload)
return str(res.text)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _builtin_image(self, image):
"""
Indicates whether the image is a pre-built Ferry image.
Right now we only verify the client images.
"""
return image in [
"ferry/hadoop-client", "ferry/spark-client",
"ferry/cassandra-client", "ferry/openmpi-client"
]
def _get_user_images(self, content):
"""
Get the user-defined images.
"""
images = set()
for c in content['connectors']:
p = c['personality']
if not self._builtin_image(p):
images.add(p)
return images
def _read_app_content(self, file_path):
"""
Read the content of the application.
"""
json_arg = None
with open(file_path, "r") as f:
n, e = os.path.splitext(file_path)
if e == '.json':
json_string = self._read_file_arg(file_path)
json_arg = json.loads(json_string)
elif e == '.yaml' or e == '.yml':
yaml_file = open(file_path, 'r')
json_arg = yaml.load(yaml_file)
return json_arg
def _push_app(self, app, registry):
"""
Push a local application to Ferry servers.
"""
# First find all the images that need to
# be pushed to the Docker registry.
content = self._read_app_content(app)
if content:
images = self._get_user_images(content)
for i in images:
self._push_image(i, registry)
# Register the application in the Ferry database.
account, key, server = self.installer.get_ferry_account()
if account:
# Read in the contents of the application and
# generate the API key.
with open(app, "r") as f:
name = account + '/' + os.path.basename(app)
name, ext = os.path.splitext(name)
content = f.read()
req = {'action': 'register', 'app': name, 'account': account}
sig = self.installer.create_signature(json.dumps(req), key)
try:
payload = {
'id': account,
'app': name,
'ext': ext,
'content': content,
'sig': sig
}
res = requests.post(server + '/app', data=payload)
status = json.loads(res.text)
if status['status'] == 'fail':
logging.error("failed to register app " + app)
return "Failed to register app " + app
else:
return status['name']
except ConnectionError:
logging.error("could not connect to application server")
return "Could not register the application."
except ValueError as e:
logging.error(str(e))
return "Registration server sent back unknown reply"
else:
logging.error("could not read account information")
return "Could not read account information."
def _push(self, image, registry):
"""
Push a local appliation/image to a remote registry.
"""
logging.warning("pushing " + image)
# Figure out if we're pushing a Ferry application or
# plain Docker image.
s = image.split("://")
if len(s) > 1:
proto = s[0]
image = s[1]
else:
proto = "image"
image = s[0]
if proto == "image":
return self._push_image(image, registry)
else:
return self._push_app(image, registry)
def _build(self, dockerfile):
"""
Build a new local image.
"""
logging.warning("building " + dockerfile)
names = self.installer._get_image(dockerfile)
name = names.pop().split("/")
if len(name) == 1:
repo = GUEST_DOCKER_REPO
image = name[0]
else:
repo = name[0]
image = name[1]
build_dir = os.path.dirname(dockerfile)
self.installer._compile_image(image, repo, build_dir, build=True)
if len(names) > 0:
self.installer._tag_images(image, repo, names)
def _login(self):
"""
Login to a remote registry
"""
try:
res = requests.post(self.ferry_server + '/login')
return str(res.text)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _create_stack(self, stack_description, args, private_key):
"""
Create a new stack.
"""
payload = {
'payload': json.dumps(stack_description),
'key': private_key
}
try:
res = requests.post(self.ferry_server + '/create', data=payload)
return True, str(res.text)
except ConnectionError:
logging.error("could not connect to ferry server")
return False, "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _ask_question(self, question_text):
question = colored(question_text, 'red')
prompt = colored(' >> ', 'green')
return raw_input(question + prompt)
def _format_images_query(self, json_data):
t = PrettyTable()
unique = sets.Set()
for j in json_data:
unique.add(j)
t.add_column("Image", list(unique))
return t.get_string(sortby="Image", padding_width=2)
def _format_apps_query(self, json_data):
authors = []
versions = []
descriptions = []
for app in json_data.keys():
authors.append(json_data[app]['author'])
versions.append(json_data[app]['version'])
descriptions.append(json_data[app]['short'])
t = PrettyTable()
t.add_column("App", json_data.keys())
t.add_column("Author", authors)
t.add_column("Version", versions)
t.add_column("Description", descriptions)
return t.get_string(sortby="App", padding_width=2)
def _format_snapshots_query(self, json_data):
bases = []
date = []
for uuid in json_data.keys():
bases.append(json_data[uuid]['base'])
if 'snapshot_ts' in json_data[uuid]:
date.append(json_data[uuid]['snapshot_ts'])
else:
date.append(' ')
t = PrettyTable()
t.add_column("UUID", json_data.keys())
t.add_column("Base", bases)
t.add_column("Date", date)
return t.get_string(sortby="Date", padding_width=2)
def _format_table_query(self, json_data):
storage = []
compute = []
connectors = []
status = []
base = []
time = []
# Each additional row should include the actual data.
for uuid in json_data.keys():
csstore = []
bstore = []
cstore = []
for c in json_data[uuid]['connectors']:
csstore.append(c)
backends = json_data[uuid]['backends']
for b in backends:
if b['storage']:
bstore.append(b['storage'])
else:
bstore.append(' ')
if b['compute']:
cstore.append(b['compute'])
else:
cstore.append(' ')
storage.append(bstore)
compute.append(cstore)
connectors.append(csstore)
status.append(json_data[uuid]['status'])
base.append(json_data[uuid]['base'])
time.append(json_data[uuid]['ts'])
t = PrettyTable()
t.add_column("UUID", json_data.keys())
t.add_column("Storage", storage)
t.add_column("Compute", compute)
t.add_column("Connectors", connectors)
t.add_column("Status", status)
t.add_column("Base", base)
t.add_column("Time", time)
return t.get_string(sortby="UUID", padding_width=2)
def _stop_all(self, private_key):
try:
# We used to stop all the services when quitting
# Ferry, but this does not seem safe since a user
# may want to keep running a Hadoop cluster even if
# Ferry is shutdown.
# constraints = { 'status' : 'running' }
# payload = { 'constraints' : json.dumps(constraints) }
# res = requests.get(self.ferry_server + '/query', params=payload)
# stacks = json.loads(res.text)
# for uuid in stacks.keys():
# self._manage_stacks({'uuid' : uuid,
# 'key' : private_key,
# 'action' : 'stop'})
# Shutdown any backend services that may have
# been started (e.g., OpenStack Heat server, etc.).
res = requests.post(self.ferry_server + '/quit')
logging.info(res.text)
except ConnectionError:
logging.error("could not connect to ferry server")
def _read_stacks(self, show_all=False, args=None):
try:
res = requests.get(self.ferry_server + '/query')
query_reply = json.loads(res.text)
deployed_reply = {}
if show_all:
mode = self._parse_deploy_arg('mode', args, default='local')
conf = self._parse_deploy_arg('conf', args, default='default')
payload = {'mode': mode, 'conf': conf}
res = requests.get(self.ferry_server + '/deployed',
params=payload)
deployed_reply = json.loads(res.text)
# Merge the replies and format.
return self._format_table_query(
dict(query_reply.items() + deployed_reply.items()))
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _list_apps(self):
"""
List all installed applications including the built-in applications.
"""
try:
res = requests.get(self.ferry_server + '/apps')
json_reply = json.loads(res.text)
return self._format_apps_query(json_reply)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _list_images(self):
"""
List all installed Docker images.
"""
try:
res = requests.get(self.ferry_server + '/images')
json_reply = json.loads(res.text)
return self._format_images_query(json_reply)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _list_snapshots(self):
"""
List all snapshots.
"""
try:
res = requests.get(self.ferry_server + '/snapshots')
json_reply = json.loads(res.text)
return self._format_snapshots_query(json_reply)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _format_stack_inspect(self, json_data):
return json.dumps(json_data,
sort_keys=True,
indent=2,
separators=(',', ':'))
def _inspect_stack(self, stack_id):
"""
Inspect a specific stack.
"""
payload = {'uuid': stack_id}
try:
res = requests.get(self.ferry_server + '/stack', params=payload)
return res.text
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _copy_logs(self, stack_id, to_dir):
"""
Copy over the logs.
"""
payload = {'uuid': stack_id, 'dir': to_dir}
try:
res = requests.get(self.ferry_server + '/logs', params=payload)
json_value = json.loads(str(res.text))
return self._format_stack_inspect(json_value)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
"""
Connector a specific client/connector via ssh.
"""
def _connect_stack(self, stack_id, connector_id, options):
# Get the IP and default user information for this connector.
payload = {'uuid': stack_id}
try:
res = requests.get(self.ferry_server + '/stack', params=payload)
json_value = json.loads(str(res.text))
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
except ValueError:
json_value = None
connector_ip = None
if json_value and 'connectors' in json_value:
for cg in json_value['connectors']:
if not connector_id and 'ip' in cg['entry']:
connector_ip = cg['entry']['ip']
break
elif connector_id == cg['uniq'] and 'ip' in cg['entry']:
connector_ip = cg['entry']['ip']
break
# Now form the ssh command. This just executes in the same shell.
if connector_ip:
private_key = self._get_ssh_key(options=options)
key_opt = '-o StrictHostKeyChecking=no'
host_opt = '-o UserKnownHostsFile=/dev/null'
ident = '-i %s' % private_key
dest = '%s@%s' % (self.default_user, connector_ip)
cmd = "ssh %s %s %s %s" % (key_opt, host_opt, ident, dest)
logging.warning(cmd)
os.execv('/usr/bin/ssh', cmd.split())
else:
logging.warning("could not find connector %s" % connector_id)
return "could not connect to " + str(stack_id)
def _parse_deploy_arg(self, param, args, default):
pattern = re.compile('--%s=(\w+)' % param)
for a in args:
m = pattern.match(a)
if m and m.group(0) != '':
return m.group(1)
return default
def _manage_stacks(self, stack_info):
"""
Manage the stack.
"""
try:
res = requests.post(self.ferry_server + '/manage/stack',
data=stack_info)
return str(res.text)
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _print_help(self):
"""
Output the help message.
"""
return self.cmds.print_help()
def _print_info(self):
"""
Output version information.
"""
try:
res = requests.get(self.ferry_server + '/version')
s = self.cmds.description + '\n'
s += "Version: %s\n" % self.cmds.version
s += "Docker: %s\n" % res.text.strip()
return s
except ConnectionError:
logging.error("could not connect to ferry server")
return "It appears Ferry servers are not running.\nType sudo ferry server and try again."
def _read_file_arg(self, file_name):
"""
Helper method to read a file.
"""
json_file = open(os.path.abspath(file_name), 'r')
json_text = ''
for line in json_file:
json_text += line.strip()
return json_text
def _get_ssh_key(self, options=None):
if options and '-k' in options:
return options['-k'][0]
else:
return ferry.install.DEFAULT_SSH_KEY
def _find_installed_app(self, app):
"""
Help find the path to the application. Check both the built-in
global directory and the user-installed directory.
"""
file_path = None
for item in os.listdir(FERRY_HOME + '/data/plans/'):
if app == os.path.splitext(item)[0]:
return FERRY_HOME + '/data/plans/' + item
if not file_path:
for user in os.listdir(DEFAULT_FERRY_APPS):
if os.path.isdir(DEFAULT_FERRY_APPS + '/' + user):
for item in os.listdir(DEFAULT_FERRY_APPS + '/' + user):
if app == user + '/' + os.path.splitext(item)[0]:
return DEFAULT_FERRY_APPS + '/' + user + '/' + item
def _format_output(self, reply):
output = reply['text'] + "\n"
if 'msgs' in reply:
for c in reply['msgs'].keys():
output += "%s: %s\n" % (c, reply['msgs'][c])
return output
def _start_stack(self, options, args):
private_key = self._get_ssh_key(options=options)
# Check if we need to build the image before running.
if '-b' in options:
build_dir = options['-b'][0]
self._build(build_dir + '/Dockerfile')
# Try to figure out what application the user is staring.
# This could be a new stack or an existing stopped stack.
arg = args.pop(0)
json_arg = {}
if not os.path.exists(arg):
file_path = self._find_installed_app(arg)
else:
file_path = arg
# Looks like the user is trying to start a brand
# new application (specified by a filename).
if file_path and os.path.exists(file_path):
file_path = os.path.abspath(file_path)
json_arg = self._read_app_content(file_path)
if not json_arg:
logging.error("could not load file " + file_path)
exit(1)
else:
# Check if there are any questions associated with
# this application stack. If so, we should prompt the
# user and include the answers.
if 'questions' in json_arg:
for q in json_arg['questions']:
question = q['question']
q['_answer'] = self._ask_question(question)
json_arg['_file_path'] = file_path
json_arg['_file'] = arg
# Create the application stack and print
# the status message.
posted, reply = self._create_stack(json_arg, args, private_key)
if posted:
try:
reply = json.loads(reply)
if reply['status'] == 'failed':
return 'could not create application'
else:
return self._format_output(reply)
except ValueError as e:
logging.error(reply)
def dispatch_cmd(self, cmd, args, options):
"""
This is the command dispatch table.
"""
if (cmd == 'start'):
return self._start_stack(options, args)
elif (cmd == 'ps'):
if len(args) > 0 and args[0] == '-a':
opt = args.pop(0)
return self._read_stacks(show_all=True, args=args)
else:
return self._read_stacks(show_all=False, args=args)
elif (cmd == 'snapshots'):
return self._list_snapshots()
elif (cmd == 'install'):
msg = self.installer.install(args, options)
self.installer._stop_docker_daemon()
return msg
elif (cmd == 'clean'):
self.installer._force_stop_web()
self.installer._stop_docker_daemon(force=True)
return 'cleaned ferry'
elif (cmd == 'inspect'):
return self._inspect_stack(args[0])
elif (cmd == 'logs'):
return self._copy_logs(args[0], args[1])
elif (cmd == 'server'):
self.installer.start_web(options)
return 'started ferry'
elif (cmd == 'ssh'):
stack_id = args[0]
connector_id = None
if len(args) > 1:
connector_id = args[1]
return self._connect_stack(stack_id, connector_id, options)
elif (cmd == 'quit'):
private_key = self._get_ssh_key(options=options)
self._stop_all(private_key)
self.installer.stop_web(private_key)
self.installer._stop_docker_daemon()
return 'stopped ferry'
elif (cmd == 'ls'):
return self._list_apps()
elif (cmd == 'ls-images'):
return self._list_images()
elif (cmd == 'info'):
return self._print_info()
elif (cmd == 'build'):
return self._build(args[0])
elif (cmd == 'pull'):
return self._pull(args[0])
elif (cmd == 'push'):
image = args.pop(0)
if len(args) > 0:
registry = args.pop(0)
else:
registry = None
return self._push(image, registry)
elif (cmd == 'login'):
return self._login()
elif (cmd == 'help'):
return self._print_help()
else:
# The user wants to perform some management function
# over the stack.
private_key = self._get_ssh_key(options=options)
stack_info = {'uuid': args[0], 'key': private_key, 'action': cmd}
return self._manage_stacks(stack_info)
class SingleLauncher(object):
"""
Launches new Ferry containers on an OpenStack cluster.
Unlike the multi-launcher, containers use a single pre-assigned
network for all communication. This makes it suitable for OpenStack
environments that only support a single network (i.e., HP Cloud).
"""
def __init__(self, controller):
self.name = "OpenStack launcher"
self.docker_registry = None
self.docker_user = None
self.heat_server = None
self.openstack_key = None
self.system = System()
self.installer = Installer()
self.controller = controller
self._init_open_stack()
self._init_app_db()
def support_proxy(self):
"""
The OpenStack backend supports proxy mode by assigning all the
machines a floating IP.
"""
return True
def _init_app_db(self):
self.mongo = MongoClient(os.environ['MONGODB'], 27017, connectTimeoutMS=6000)
self.apps = self.mongo['cloud']['openstack']
def _init_open_stack(self):
conf = ferry.install.read_ferry_config()
# First we need to know the deployment system
# we are using.
self.data_device = conf['system']['network']
provider = conf['system']['provider']
# Now get some basic OpenStack information
params = conf[provider]['params']
self.default_dc = params['dc']
self.default_zone = params['zone']
# Some OpenStack login credentials.
if self._check_openstack_credentials():
self.openstack_user = os.environ['OS_USERNAME']
self.openstack_pass = os.environ['OS_PASSWORD']
self.tenant_id = os.environ['OS_TENANT_ID']
self.tenant_name = os.environ['OS_TENANT_NAME']
else:
logging.error("Missing OpenStack credentials")
raise ValueError("Missing OpenStack credentials")
# Some information regarding OpenStack
# networking. Necessary for
servers = conf[provider][self.default_dc]
self.manage_network = servers['network']
self.external_network = servers['extnet']
# OpenStack API endpoints.
self.region = servers['region']
self.keystone_server = servers['keystone']
self.nova_server = servers['nova']
self.neutron_server = servers['neutron']
# Check if the user has provided a Heat
# server. Not all OpenStack clusters provide
# Heat. If not, we'll need to start a local instance.
self.heatuuid = None
if 'HEAT_URL' in os.environ:
self.heat_server = os.environ['HEAT_URL']
elif 'heat' in servers:
self.heat_server = servers['heat']
else:
self.heat_server = self._check_and_start_heat(self.tenant_id)
logging.warning("using heat server " + str(self.heat_server))
# This gives us information about the image to use
# for the supplied provider.
deploy = conf[provider]['deploy']
self.default_image = deploy['image']
self.default_personality = deploy['personality']
self.default_user = deploy['default-user']
self.ssh_key = deploy['ssh']
self.ssh_user = deploy['ssh-user']
# Make sure that the ssh key is actually present.
keypath = self._get_host_key()
if not os.path.exists(keypath):
logging.error("could not find ssh key (%s)" % self.ssh_key)
raise ValueError("Missing ssh keys")
# Initialize the OpenStack clients and also
# download some networking information (subnet ID,
# cidr, gateway, etc.)
self._init_openstack_clients()
self._collect_subnet_info()
def _get_host_key(self):
"""
Get the location of the private ssh key.
"""
p = self.ssh_key.split("/")
if len(p) == 1:
return "/ferry/keys/" + self.ssh_key + ".pem"
else:
return self.ssh_key + ".pem"
def _check_and_start_heat(self, tenant_id):
"""
Check and start the Ferry Heat image.
"""
# Check if the image is downloaded locally.
# If not, it will automatically pull it.
logging.info("Check for Heat image")
self.installer._check_and_pull_image("ferry/heatserver")
# Check if the Heat log directory exists yet. If not
# go ahead and create it.
heatlogs = ferry.install.DOCKER_DIR + "/heatlog"
try:
if not os.path.isdir(heatlogs):
os.makedirs(heatlogs)
self.installer._change_permission(heatlogs)
except OSError as e:
logging.error(e.strerror)
sys.exit(1)
# Start the Heat image and capture the IP address. We can
# then hand over this IP to the rest of the configuration.
volumes = { heatlogs : "/var/log/heat" }
heatplan = {'image':'ferry/heatserver',
'type':'ferry/heatserver',
'keydir': {},
'keyname': None,
'privatekey': None,
'volumes':volumes,
'volume_user':ferry.install.DEFAULT_FERRY_OWNER,
'ports':[],
'exposed':["8004","8000"],
'internal':[],
'hostname':'heatserver',
'netenable':True,
'default_cmd' : '',
'args': 'trust'
}
self.heatuuid = 'fht-' + str(uuid.uuid4()).split('-')[0]
self.heatbox = self.installer.fabric.alloc(self.heatuuid, self.heatuuid, [heatplan], "HEAT")[0]
if not self.heatbox:
logging.error("Could not start Heat server")
sys.exit(1)
else:
return "http://%s:8004/v1/%s" % (str(self.heatbox.internal_ip),
tenant_id)
def _check_openstack_credentials(self):
envs = ['OS_USERNAME', 'OS_PASSWORD',
'OS_TENANT_ID', 'OS_TENANT_NAME']
for e in envs:
if not e in os.environ:
return False
return True
def _init_openstack_clients(self):
# Instantiate the Heat client.
if 'HEAT_API_VERSION' in os.environ:
heat_api_version = os.environ['HEAT_API_VERSION']
else:
heat_api_version = '1'
kwargs = {
'username' : self.openstack_user,
'password' : self.openstack_pass,
'include_pass' : True,
'tenant_id': self.tenant_id,
'tenant_name': self.tenant_name,
'auth_url' : self.keystone_server
}
self.heat = heat_client.Client(heat_api_version,
self.heat_server,
**kwargs)
# Check to make sure that the Heat client can actually
# connect to the Heat server. This is because we may
# have just started the Heat server, so it can a while to refresh.
for i in range(0, 10):
try:
stacks = self.heat.stacks.list()
for s in stacks:
logging.warning("found Heat stack: " + str(s))
connected = True
break
except:
time.sleep(12)
connected = False
if not connected:
raise ValueError("Could not connect to Heat")
# Instantiate the Neutron client.
# There should be a better way of figuring out the API version.
neutron_api_version = "2.0"
kwargs['endpoint_url'] = self.neutron_server
self.neutron = neutron_client.Client(neutron_api_version, **kwargs)
# Instantiate the Nova client. The Nova client is used
# to stop/restart instances.
nova_api_version = "1.1"
kwargs = {
'username' : self.openstack_user,
'api_key' : self.openstack_pass,
'tenant_id': self.tenant_id,
'auth_url' : self.keystone_server,
'service_type' : 'compute',
'region_name' : self.region
}
self.nova = nova_client.Client(nova_api_version, **kwargs)
def _create_floating_ip(self, name, port):
"""
Create and attach a floating IP to the supplied port.
"""
plan = { name : { "Type": "OS::Neutron::FloatingIP",
"Properties": { "floating_network_id": self.external_network }},
name + "_assoc" : { "Type": "OS::Neutron::FloatingIPAssociation",
"Properties": { "floatingip_id": { "Ref" : name },
"port_id": { "Ref" : port }}}}
desc = { "type" : "OS::Neutron::FloatingIP" }
return plan, desc
def _create_security_group(self, group_name, ports, internal):
"""
Create and assign a security group to the supplied server.
"""
# Create the basic security group.
# This only includes SSH. We can later update the group
# to include additional ports.
desc = { group_name : { "Type" : "OS::Neutron::SecurityGroup",
"Properties" : { "name" : group_name,
"description" : "Ferry firewall rules",
"rules" : [ { "protocol" : "icmp",
"remote_ip_prefix": "0.0.0.0/0" },
{ "protocol" : "tcp",
"remote_ip_prefix": "0.0.0.0/0",
"port_range_min" : 22,
"port_range_max" : 22 }]}}}
# Additional ports for the security group. These
# port values can be accessible from anywhere.
for p in ports:
min_port = p[0]
max_port = p[1]
desc[group_name]["Properties"]["rules"].append({ "protocol" : "tcp",
"remote_ip_prefix": "0.0.0.0/0",
"port_range_min" : min_port,
"port_range_max" : max_port })
# Additional ports for the security group. These
# port values can only be accessed from within the same network.
for p in internal:
min_port = p[0]
max_port = p[1]
desc[group_name]["Properties"]["rules"].append({ "protocol" : "tcp",
"remote_ip_prefix": self.subnet["cidr"],
"port_range_min" : min_port,
"port_range_max" : max_port })
return desc
def _create_storage_volume(self, volume_name, server_name, size_gb):
"""
Create and attach a storage volume to the supplied server.
"""
desc = { volume_name : { "Type" : "OS::Cinder::Volume",
"Properties": { "size" : size_db,
"availability_zone": self.default_zone
}},
volume_name + "_attachment" : { "Type" : "OS::Cinder::VolumeAttachment",
"Properties": { "volume_id" : { "Ref" : volume_name },
"instance_uuid": { "Ref" : server_name },
"mount_point": "/dev/vdc"
}}}
return desc
def _create_port(self, name, network, sec_group, ref=True):
desc = { name : { "Type" : "OS::Neutron::Port",
"Properties" : { "name" : name,
"security_groups" : [{ "Ref" : sec_group }]}}}
if ref:
desc[name]["Properties"]["network"] = { "Ref" : network }
else:
desc[name]["Properties"]["network"] = network
return desc
def _create_server_init(self):
"""
Create the server init process. These commands are run on the
host after the host has booted up.
"""
user_data = {
"Fn::Base64": {
"Fn::Join": [
"",
[
"#!/bin/bash -v\n",
"umount /mnt\n",
"parted --script /dev/vdb mklabel gpt\n",
"parted --script /dev/vdb mkpart primary xfs 0% 100%\n",
"mkfs.xfs /dev/vdb1\n",
"mkdir /ferry/data\n",
"mkdir /ferry/keys\n",
"mkdir /ferry/containers\n",
"mount -o noatime /dev/vdb1 /ferry/data\n",
"export FERRY_SCRATCH=/ferry/data\n",
"export FERRY_DIR=/ferry/master\n",
"echo export FERRY_SCRATCH=/ferry/data >> /etc/profile\n",
"echo export FERRY_DIR=/ferry/master >> /etc/profile\n",
"export HOME=/root\n",
"export USER=root\n",
"mkdir /home/ferry/.ssh\n",
"cp /home/%s/.ssh/authorized_keys /home/ferry/.ssh/\n" % self.default_user,
"cp /home/%s/.ssh/authorized_keys /root/.ssh/\n" % self.default_user,
"chown -R ferry:ferry /home/ferry/.ssh\n",
"chown -R ferry:ferry /ferry/data\n",
"chown -R ferry:ferry /ferry/keys\n",
"chown -R ferry:ferry /ferry/containers\n",
"ferry server -n\n",
"sleep 3\n"
]
]
}}
return user_data
def _create_volume_attachment(self, iface, instance, volume_id):
plan = { iface: { "Type": "OS::Cinder::VolumeAttachment",
"Properties": { "instance_uuid": { "Ref" : instance },
"mountpoint": "/dev/vdc",
"volume_id": volume_id}}}
desc = { "type" : "OS::Cinder::VolumeAttachment" }
return plan, desc
def _create_instance(self, name, image, size, manage_network, sec_group):
"""
Create a new instance
"""
plan = { name : { "Type" : "OS::Nova::Server",
"Properties" : { "name" : name,
"image" : image,
"key_name" : self.ssh_key,
"flavor" : size,
"availability_zone" : self.default_zone,
"networks" : []}}}
desc = { name : { "type" : "OS::Nova::Server",
"name" : name,
"ports" : [],
"volumes" : [] }}
# Create a port for the manage network.
port_descs = []
port_name = "ferry-port-%s" % name
port_descs.append(self._create_port(port_name, manage_network, sec_group, ref=False))
plan[name]["Properties"]["networks"].append({ "port" : { "Ref" : port_name },
"network" : manage_network})
desc[name]["ports"].append(port_name)
desc[port_name] = { "type" : "OS::Neutron::Port",
"role" : "manage" }
# Combine all the port descriptions.
for d in port_descs:
plan = dict(plan.items() + d.items())
# Now add the user script.
user_data = self._create_server_init()
plan[name]["Properties"]["user_data"] = user_data
return plan, desc
def _create_floatingip_plan(self, cluster_uuid, ifaces):
"""
Assign floating IPs to the supplied interfaces/ports.
"""
plan = { "AWSTemplateFormatVersion" : "2010-09-09",
"Description" : "Ferry generated Heat plan",
"Resources" : {} }
desc = {}
for i in range(0, len(ifaces)):
ip_name = "ferry-ip-%s-%d" % (cluster_uuid, i)
ip_plan, desc[ip_name] = self._create_floating_ip(ip_name, ifaces[i])
plan["Resources"] = dict(plan["Resources"].items() + ip_plan.items())
return plan, desc
def _create_security_plan(self, cluster_uuid, ports, internal, ctype):
"""
Update the security group.
"""
sec_group_name = "ferry-sec-%s-%s" % (cluster_uuid, ctype)
plan = { "AWSTemplateFormatVersion" : "2010-09-09",
"Description" : "Ferry generated Heat plan",
"Resources" : self._create_security_group(sec_group_name, ports, internal)}
desc = { sec_group_name : { "type" : "OS::Neutron::SecurityGroup" }}
return plan, desc
def _create_instance_plan(self, cluster_uuid, num_instances, image, size, sec_group_name, ctype):
plan = { "AWSTemplateFormatVersion" : "2010-09-09",
"Description" : "Ferry generated Heat plan",
"Resources" : {},
"Outputs" : {} }
desc = {}
for i in range(0, num_instances):
# Create the actual instances.
instance_name = "ferry-instance-%s-%s-%d" % (cluster_uuid, ctype, i)
instance_plan, instance_desc = self._create_instance(instance_name, image, size, self.manage_network, sec_group_name)
plan["Resources"] = dict(plan["Resources"].items() + instance_plan.items())
desc = dict(desc.items() + instance_desc.items())
return plan, desc
def _launch_heat_plan(self, stack_name, heat_plan, stack_desc):
"""
Launch the cluster plan.
"""
logging.info("launching heat plan: " + str(heat_plan))
try:
# Try to create the application stack.
resp = self.heat.stacks.create(stack_name=stack_name, template=heat_plan)
except HTTPBadRequest as e:
logging.error(e.strerror)
return None
except:
# We could not create the stack. This probably
# means that either the Heat server is down or the
# OpenStack cluster is down.
logging.error("could not create Heat stack")
return None
# Now wait for the stack to be in a completed state
# before returning. That way we'll know if the stack creation
# has failed or not.
if not self._wait_for_stack(resp["stack"]["id"]):
logging.warning("Heat plan %s CREATE_FAILED" % resp["stack"]["id"])
return None
# Now find the physical IDs of all the resources.
resources = self._collect_resources(resp["stack"]["id"])
for r in resources:
if r["logical_resource_id"] in stack_desc:
stack_desc[r["logical_resource_id"]]["id"] = r["physical_resource_id"]
# Record the Stack ID in the description so that
# we can refer back to it later.
stack_desc[stack_name] = { "id" : resp["stack"]["id"],
"type": "OS::Heat::Stack" }
return stack_desc
def _wait_for_stack(self, stack_id):
"""
Wait for stack completion.
"""
while(True):
try:
stack = self.heat.stacks.get(stack_id)
if stack.status == "COMPLETE":
return True
elif stack.status == "FAILED":
return False
else:
time.sleep(4)
except:
logging.error("could not fetch stack status (%s)" % str(stack_id))
def _collect_resources(self, stack_id):
"""
Collect all the stack resources so that we can create
additional plans and use IDs.
"""
try:
resources = self.heat.resources.list(stack_id)
descs = [r.to_dict() for r in resources]
return descs
except:
return []
def _collect_subnet_info(self):
"""
Collect the data network subnet info (ID, CIDR, and gateway).
"""
subnets = self.neutron.list_subnets()
for s in subnets['subnets']:
if s['network_id'] == self.manage_network:
self.subnet = { "id" : s['id'],
"cidr" : s['cidr'],
"gateway" : s['gateway_ip'] }
def _collect_network_info(self, stack_desc):
"""
Collect all the networking information.
"""
# First get the floating IP information.
ip_map = {}
floatingips = self.neutron.list_floatingips()
for f in floatingips['floatingips']:
if f['fixed_ip_address']:
ip_map[f['fixed_ip_address']] = f['floating_ip_address']
# Now fill in the various networking information, including
# subnet, IP address, and floating address. We should also
# probably collect MAC addresseses..
ports = self.neutron.list_ports()
for p in ports['ports']:
if p['name'] != "" and p['name'] in stack_desc:
port_desc = stack_desc[p['name']]
port_desc["subnet"] = p['fixed_ips'][0]['subnet_id']
port_desc["ip_address"] = p['fixed_ips'][0]['ip_address']
# Not all ports are associated with a floating IP, so
# we need to check first.
if port_desc["ip_address"] in ip_map:
port_desc["floating_ip"] = ip_map[port_desc["ip_address"]]
return stack_desc
def _collect_instance_info(self, stack_desc):
"""
Collect all the instance information.
"""
servers = self.nova.servers.list()
for s in servers:
if s.name != "" and s.name in stack_desc:
instance_desc = stack_desc[s.name]
instance_desc["id"] = s.id
return stack_desc
def _create_app_stack(self, cluster_uuid, num_instances, security_group_ports, internal_ports, assign_floating_ip, ctype):
"""
Create an empty application stack. This includes the instances,
security groups, and floating IPs.
"""
logging.info("creating security group for %s" % cluster_uuid)
sec_group_plan, sec_group_desc = self._create_security_plan(cluster_uuid = cluster_uuid,
ports = security_group_ports,
internal = internal_ports,
ctype = ctype)
logging.info("creating instances for %s" % cluster_uuid)
stack_plan, stack_desc = self._create_instance_plan(cluster_uuid = cluster_uuid,
num_instances = num_instances,
image = self.default_image,
size = self.default_personality,
sec_group_name = sec_group_desc.keys()[0],
ctype = ctype)
# See if we need to assign any floating IPs
# for this stack. We need the references to the neutron
# port which is contained in the description.
if assign_floating_ip:
logging.info("creating floating IPs for %s" % cluster_uuid)
ifaces = []
for k in stack_desc.keys():
if stack_desc[k]["type"] == "OS::Neutron::Port" and stack_desc[k]["role"] == "manage":
ifaces.append(k)
ip_plan, ip_desc = self._create_floatingip_plan(cluster_uuid = cluster_uuid,
ifaces = ifaces)
else:
ip_plan = { "Resources" : {}}
ip_desc = {}
# Now we need to combine all these plans and
# launch the cluster.
stack_plan["Resources"] = dict(sec_group_plan["Resources"].items() +
ip_plan["Resources"].items() +
stack_plan["Resources"].items())
stack_desc = dict(stack_desc.items() +
sec_group_desc.items() +
ip_desc.items())
stack_desc = self._launch_heat_plan("ferry-app-%s-%s" % (ctype.upper(), cluster_uuid), stack_plan, stack_desc)
# Now find all the IP addresses of the various machines.
if stack_desc:
stack_desc = self._collect_instance_info(stack_desc)
return self._collect_network_info(stack_desc)
else:
return None
def _get_private_ip(self, server, subnet_id, resources):
"""
Get the IP address associated with the supplied server.
"""
for port_name in server["ports"]:
port_desc = resources[port_name]
if port_desc["subnet"] == subnet_id:
return port_desc["ip_address"]
def _get_public_ip(self, server, resources):
"""
Get the IP address associated with the supplied server.
"""
for port_name in server["ports"]:
port_desc = resources[port_name]
if "floating_ip" in port_desc:
return port_desc["floating_ip"]
def _get_servers(self, resources):
servers = []
for r in resources.values():
if type(r) is dict and r["type"] == "OS::Nova::Server":
servers.append(r)
return servers
def _get_net_info(self, server_info, subnet, resources):
"""
Look up the IP address, gateway, and subnet range.
"""
cidr = subnet["cidr"].split("/")[1]
ip = self._get_private_ip(server_info, subnet["id"], resources)
# We want to use the host NIC, so modify LXC to use phys networking, and
# then start the docker containers on the server.
lxc_opts = ["lxc.network.type = phys",
"lxc.network.ipv4 = %s/%s" % (ip, cidr),
"lxc.network.ipv4.gateway = %s" % subnet["gateway"],
"lxc.network.link = %s" % self.data_device,
"lxc.network.name = eth0",
"lxc.network.flags = up"]
return lxc_opts, ip
def _update_app_db(self, cluster_uuid, service_uuid, heat_plan):
# Make a copy of the plan before inserting into
# mongo, otherwise the "_id" field will be added
# silently.
heat_plan["_cluster_uuid"] = cluster_uuid
heat_plan["_service_uuid"] = service_uuid
self.apps.insert(copy.deepcopy(heat_plan))
def alloc(self, cluster_uuid, service_uuid, container_info, ctype, proxy):
"""
Allocate a new cluster.
"""
# Now take the cluster and create the security group
# to expose all the right ports.
sec_group_ports = []
internal_ports = []
if ctype == "connector":
# Since this is a connector, we need to expose
# the public ports. For now, we ignore the host port.
floating_ip = True
for c in container_info:
for p in c['ports']:
s = str(p).split(":")
if len(s) > 1:
sec_group_ports.append( (s[1], s[1]) )
else:
sec_group_ports.append( (s[0], s[0]) )
else:
if proxy:
# Otherwise, the backend should also get floating IPs
# so that the controller can access it.
floating_ip = True
else:
# If the controller is acting as a proxy, then it has
# direct access to the VMs, so the backend shouldn't
# get any floating IPs.
floating_ip = False
# We need to create a range tuple, so check if
# the exposed port is a range.
for p in container_info[0]['exposed']:
s = p.split("-")
if len(s) == 1:
sec_group_ports.append( (s[0], s[0]) )
else:
sec_group_ports.append( (s[0], s[1]) )
# Also see if there are any ports that should be
# open within the cluster (but not outside). Typically
# used for IPC (where ports may be assigned within a random range).
for p in container_info[0]['internal']:
s = p.split("-")
if len(s) == 1:
internal_ports.append( (s[0], s[0]) )
else:
internal_ports.append( (s[0], s[1]) )
# Tell OpenStack to allocate the cluster.
resources = self._create_app_stack(cluster_uuid = cluster_uuid,
num_instances = len(container_info),
security_group_ports = sec_group_ports,
internal_ports = internal_ports,
assign_floating_ip = floating_ip,
ctype = ctype)
# Now we need to ask the cluster to start the
# Docker containers.
containers = []
mounts = {}
if resources:
# Store the resources cluster ID.
self._update_app_db(cluster_uuid, service_uuid, resources)
servers = self._get_servers(resources)
for i in range(0, len(container_info)):
# Fetch a server to run the Docker commands.
server = servers[i]
# Get the LXC networking options
lxc_opts, private_ip = self._get_net_info(server, self.subnet, resources)
# Now get an addressable IP address. If we're acting as a proxy within
# the same cluster, we can just use the private address. Otherwise
# we'll need to route via the public IP address.
if proxy:
server_ip = private_ip
else:
server_ip = self._get_public_ip(server, resources)
# Verify that the user_data processes all started properly
# and that the docker daemon is actually running. If it is
# not running, try re-executing.
if not self.controller._verify_ferry_server(server_ip):
self.controller._execute_server_init(server_ip)
# Copy over the public keys, but also verify that it does
# get copied over properly.
self.controller._copy_public_keys(container_info[i], server_ip)
if self.controller._verify_public_keys(server_ip):
container, cmounts = self.controller.execute_docker_containers(container_info[i], lxc_opts, private_ip, server_ip)
if container:
mounts = dict(mounts.items() + cmounts.items())
containers.append(container)
else:
logging.error("could not copy over ssh key!")
return None
# Check if we need to set the file permissions
# for the mounted volumes.
for c, i in mounts.items():
for _, v in i['vols']:
self.controller.cmd([c], 'chown -R %s %s' % (i['user'], v))
return containers
else:
# OpenStack failed to launch the application stack.
# This can be caused by improper OpenStack credentials
# or if the OpenStack cluster is under heavy load (i.e.,
# requests are getting timed out).
return None
def _delete_stack(self, cluster_uuid, service_uuid):
# Find the relevant stack information.
ips = []
stacks = self.apps.find( { "_cluster_uuid" : cluster_uuid,
"_service_uuid" : service_uuid } )
logging.warning("Deleting cluster %s" % str(cluster_uuid))
for stack in stacks:
for s in stack.values():
if type(s) is dict and s["type"] == "OS::Heat::Stack":
stack_id = s["id"]
# To delete the stack properly, we first need to disassociate
# the floating IPs.
resources = self._collect_resources(stack_id)
for r in resources:
if r["resource_type"] == "OS::Neutron::FloatingIP":
self.neutron.update_floatingip(r["physical_resource_id"], {'floatingip': {'port_id': None}})
# Now try to delete the stack. Wrap this in a try-block so that
# we don't completely fail even if the stack doesn't exist.
try:
logging.warning("Deleting stack %s" % str(stack_id))
self.heat.stacks.delete(stack_id)
except HTTPNotFound as e:
logging.warning(e)
self.apps.remove( { "_cluster_uuid" : cluster_uuid,
"_service_uuid" : service_uuid } )
def _stop_stack(self, cluster_uuid, service_uuid):
stacks = self.apps.find( { "_cluster_uuid" : cluster_uuid,
"_service_uuid" : service_uuid } )
for stack in stacks:
servers = self._get_servers(stack)
for s in servers:
self.nova.servers.stop(s["id"])
def _restart_stack(self, cluster_uuid, service_uuid):
ips = []
stacks = self.apps.find( { "_cluster_uuid" : cluster_uuid,
"_service_uuid" : service_uuid } )
for stack in stacks:
# Find the set of servers and restart them
# one by one. It would be nicer if Heat had a way to
# restart them all at once, but not sure how to do that...
servers = self._get_servers(stack)
for s in servers:
self.nova.servers.start(s["id"])
ips.append(self._get_public_ip(s, stack))
# Wait for the servers to actually be in the
# "running" status before returning.
for s in servers:
while(True):
found = self.nova.servers.list(search_opts = { "name" : s["name"] })
for f in found["servers"]:
if f["status"] == "ACTIVE":
break
time.sleep(4)
return ips
def quit(self):
"""
Check if the Heat server is running, and if so
go ahead and stop it.
"""
if self.heatuuid:
self.installer.fabric.stop(self.heatuuid, self.heatuuid, [self.heatbox])
