def __init__(self, cli=None): self.network = DHCPClient() self.fabric = LocalFabric(bootstrap=True) self.mongo = MongoInitializer(self.fabric.system) self.mongo.fabric = self.fabric self.mongo.template_dir = DEFAULT_TEMPLATE_DIR + '/mongo/' self.cli = cli self.config = read_ferry_config()
class Installer(object): def __init__(self, cli=None): self.network = DHCPClient() self.fabric = LocalFabric(bootstrap=True) self.mongo = MongoInitializer(self.fabric.system) self.mongo.fabric = self.fabric self.mongo.template_dir = DEFAULT_TEMPLATE_DIR + '/mongo/' self.cli = cli self.config = read_ferry_config() def get_ferry_account(self): """ Get the Ferry authentication information. """ if 'ferry' in self.config: args = self.config['ferry'] if all(k in args for k in ("user", "key", "server")): return args['user'], args['key'], args['server'] return None, None, None def _get_worker_info(self): """ Get information regarding how to start the remote HTTP API. """ args = self.config['web'] return int(args['workers']), args['bind'], args['port'] def create_signature(self, request, key): """ Generated a signed request. """ return hmac.new(key, request, hashlib.sha256).hexdigest() def store_app(self, app, ext, content): """ Store the application in the global directory. """ try: # We may need to create the parent directory # if this is the first time an application from this user # is downloaded. file_name = os.path.join(DEFAULT_FERRY_APPS, app + ext) os.makedirs(os.path.dirname(file_name)) with open(file_name, "w") as f: f.write(content) return file_name except IOError as e: logging.error(e) return None except OSError as os: logging.error(os) return None def _clean_rules(self): """ Get rid of all the forwarding rules. """ self.network.clean_rules() def _install_perform_build(self, options, num_tries): # Normally we don't want to build the Dockerfiles, # but sometimes we may for testing, etc. build = False if options and '-b' in options: build = True if options and '-u' in options: if len(options['-u']) > 0 and options['-u'][0] != True: logging.warning("performing select rebuild (%s)" % str(options['-u'])) self.build_from_list(options['-u'], DEFAULT_IMAGE_DIR, DEFAULT_DOCKER_REPO, build, recurse=False) else: logging.warning("performing forced rebuild") self.build_from_dir(DEFAULT_IMAGE_DIR, DEFAULT_DOCKER_REPO, build) else: # We want to be selective about which images # to rebuild. Useful if one image breaks, etc. to_build = self.check_images(DEFAULT_IMAGE_DIR, DEFAULT_DOCKER_REPO) if len(to_build) > 0: logging.warning("performing select rebuild (%s)" % str(to_build)) self.build_from_list(to_build, DEFAULT_IMAGE_DIR, DEFAULT_DOCKER_REPO, build) # Check that all the images were built. not_installed = self._check_all_images() if len(not_installed) == 0: return 'installed ferry' else: logging.error('images not built: ' + str(not_installed)) if num_tries == 0: return 'Some images were not installed. Please type \'ferry install\' again.' else: # Try building the images again. logging.info("retrying install (%d)" % num_tries) return self._install_perform_build(options, num_tries - 1) def install(self, args, options): # Check if the host is actually 64-bit. If not raise a warning and quit. if not _supported_arch(): return 'Your architecture appears to be 32-bit.\nOnly 64-bit architectures are supported at the moment.' if not _supported_python(): return 'You appear to be running Python3.\nOnly Python2 is supported at the moment.' if not _supported_lxc(): return 'You appear to be running an older version of LXC.\nOnly versions > 0.7.5 are supported.' if not _supported_docker(): return 'You appear to be running an older version of Docker.\nOnly versions > 1.2 are supported.' if not _has_ferry_user(): return 'You do not appear to have the \'docker\' group configured. Please create the \'docker\' group and try again.' # Create the various directories. try: if not os.path.isdir(DOCKER_DIR): os.makedirs(DOCKER_DIR) self._change_permission(DOCKER_DIR) except OSError as e: logging.error("Could not install Ferry.\n") logging.error(e.strerror) sys.exit(1) # Make sure that the Ferry keys have the correct # ownership & permission. self._check_and_change_ssh_keyperm() # Start the Ferry docker daemon. If it does not successfully # start, print out a msg. logging.warning("all prerequisites met...") start, msg = self._start_docker_daemon(options) if not start: logging.error('ferry docker daemon not started') return msg # Perform the actual build/download. Sometimes the download # may fail so give the user a chance to automatically retry # a few times before giving up. if options and '-r' in options: num_tries = int(options['-r']) else: num_tries = 0 return self._install_perform_build(options, num_tries) def _check_all_images(self): not_installed = [] images = [ 'mongodb', 'ferry-base', 'hadoop-base', 'hadoop', 'hadoop-client', 'hive-metastore', 'gluster', 'openmpi', 'openmpi-client', 'cassandra', 'cassandra-client', 'titan', 'spark' ] for i in images: if not self._check_image_installed("%s/%s" % (DEFAULT_DOCKER_REPO, i)): not_installed.append(i) return not_installed def _check_and_pull_image(self, image_name): if not self._check_image_installed(image_name): self._pull_image(image_name, on_client=False) return self._check_image_installed(image_name) def _check_image_installed(self, image_name): cmd = DOCKER_CMD + ' -H=' + DOCKER_SOCK + ' inspect %s 2> /dev/null' % image_name output = Popen(cmd, stdout=PIPE, shell=True).stdout.read() if output.strip() == '[]': return False else: return True def _transfer_config(self, config_dirs): """ Transfer the configuration to the containers. """ for c in config_dirs: container = c[0] from_dir = c[1] to_dir = c[2] self.fabric.copy([container], from_dir, to_dir) def _read_public_key(self, private_key): s = private_key.split("/") p = os.path.splitext(s[len(s) - 1])[0] return p def _check_and_change_ssh_keyperm(self): os.chmod(DEFAULT_SSH_KEY, 0600) uid, gid = _get_ferry_user() os.chown(DEFAULT_SSH_KEY, uid, gid) def start_web(self, options=None, clean=False): start, msg = self._start_docker_daemon(options) if not clean and not start: # We are trying to start the web services but the Docker # daemon won't start. If we're cleaning, it's not a big deal. logging.error(msg) sys.exit(1) # Check if the ssh key permission is properly set. self._check_and_change_ssh_keyperm() # Check if we're operating in naked mode. If so, # we just needed to start the docker daemon, so we're all done! if options and '-n' in options: return # Check if the user-application directory exists. # If not, create it. try: if not os.path.isdir(DEFAULT_FERRY_APPS): os.makedirs(DEFAULT_FERRY_APPS) self._change_permission(DEFAULT_FERRY_APPS) except OSError as e: logging.error("Could not create application directory.\n") logging.error(e.strerror) sys.exit(1) # Check if the Mongo directory exists yet. If not # go ahead and create it. try: if not os.path.isdir(DEFAULT_MONGO_DB): os.makedirs(DEFAULT_MONGO_DB) self._change_permission(DEFAULT_MONGO_DB) if not os.path.isdir(DEFAULT_MONGO_LOG): os.makedirs(DEFAULT_MONGO_LOG) self._change_permission(DEFAULT_MONGO_LOG) except OSError as e: logging.error("Could not start ferry servers.\n") logging.error(e.strerror) sys.exit(1) # Check if the Mongo image is built. if not self._check_image_installed('%s/mongodb' % DEFAULT_DOCKER_REPO): logging.error("Could not start ferry servers.\n") logging.error( "MongoDB images not found. Try executing 'ferry install'.") sys.exit(1) # Check if there are any other Mongo instances runnig. self._clean_web() # Start the Mongo server. Create a new configuration and # manually start the container. private_key = self.cli._get_ssh_key(options) volumes = { DEFAULT_MONGO_LOG: self.mongo.container_log_dir, DEFAULT_MONGO_DB: self.mongo.container_data_dir } mongoplan = { 'image': 'ferry/mongodb', 'type': 'ferry/mongodb', 'keydir': { '/service/keys': DEFAULT_KEY_DIR }, 'keyname': self._read_public_key(private_key), 'privatekey': private_key, 'volumes': volumes, 'volume_user': DEFAULT_FERRY_OWNER, 'ports': [], 'exposed': self.mongo.get_working_ports(1), 'internal': self.mongo.get_internal_ports(1), 'hostname': 'ferrydb', 'netenable': True, 'args': 'trust' } mongoconf = self.mongo.generate(1) mongoconf.uuid = 'fdb-' + str(uuid.uuid4()).split('-')[0] containers = self.fabric.alloc(mongoconf.uuid, mongoconf.uuid, [mongoplan], "MONGO") if containers and len(containers) > 0: mongobox = containers[0] else: logging.error("Could not start MongoDB image") sys.exit(1) ip = mongobox.internal_ip _touch_file('/tmp/ferry/mongodb.ip', ip, root=True) # Once the container is started, we'll need to copy over the # configuration files, and then manually send the 'start' command. s = { 'container': mongobox, 'data_dev': 'eth0', 'data_ip': mongobox.internal_ip, 'manage_ip': mongobox.internal_ip, 'host_name': mongobox.host_name, 'type': mongobox.service_type, 'args': mongobox.args } config_dirs, entry_point = self.mongo.apply(mongoconf, [s]) self._transfer_config(config_dirs) self.mongo.start_service([mongobox], entry_point, self.fabric) # Set the MongoDB env. variable. my_env = os.environ.copy() my_env['MONGODB'] = ip # Sleep a little while to let Mongo start receiving. time.sleep(2) # Start the DHCP server logging.warning("starting dhcp server") # cmd = 'gunicorn -t 3600 -b 127.0.0.1:5000 -w 1 ferry.ip.dhcp:app &' cmd = 'python %s/ip/dhcp.py 127.0.0.1 5000 &' % FERRY_HOME Popen(cmd, stdout=PIPE, shell=True, env=my_env) time.sleep(2) # Reserve the Mongo IP. self.network.reserve_ip(ip) # Start the Ferry HTTP server. Read in the web configuration # so that we know how many workers to start, etc. workers, bind, port = self._get_worker_info() logging.warning("starting API servers on (%s:%s) and (mongo:%s)" % (bind, port, ip)) # cmd = 'gunicorn -e FERRY_HOME=%s -t 3600 -w %d -b %s:%s ferry.http.httpapi:app &' % (FERRY_HOME, workers, bind, port) cmd = 'export FERRY_HOME=%s && python %s/http/httpapi.py %s %s &' % ( FERRY_HOME, FERRY_HOME, bind, port) Popen(cmd, shell=True, env=my_env) def _force_stop_web(self): logging.warning("stopping docker http servers") cmd = 'pkill -f gunicorn' Popen(cmd, stdout=PIPE, shell=True) def stop_web(self, key): # Shutdown the mongo instance if os.path.exists('/tmp/ferry/mongodb.ip'): f = open('/tmp/ferry/mongodb.ip', 'r') ip = f.read().strip() f.close() cmd = 'LC_ALL=C && ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s /service/sbin/startnode stop' % ( key, ip) logging.warning(cmd) output = Popen(cmd, stdout=PIPE, shell=True).stdout.read() logging.warning(output) cmd = 'LC_ALL=C && ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s /service/sbin/startnode halt' % ( key, ip) logging.warning(cmd) output = Popen(cmd, stdout=PIPE, shell=True).stdout.read() logging.warning(output) os.remove('/tmp/ferry/mongodb.ip') # Kill all the gunicorn instances. logging.warning("stopping http servers") cmd = 'ps -eaf | grep httpapi | awk \'{print $2}\' | xargs kill -15' Popen(cmd, stdout=PIPE, shell=True) cmd = 'ps -eaf | grep ferry.ip.dhcp | awk \'{print $2}\' | xargs kill -15' Popen(cmd, stdout=PIPE, shell=True) def _clean_web(self): docker = DOCKER_CMD + ' -H=' + DOCKER_SOCK cmd = docker + ' ps | grep ferry/mongodb | awk \'{print $1}\' | xargs ' + docker + ' stop ' logging.warning("cleaning previous mongo resources") logging.warning(cmd) child = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) child.stdout.read() child.stderr.read() def _copytree(self, src, dst): for item in os.listdir(src): s = os.path.join(src, item) d = os.path.join(dst, item) if os.path.isdir(s): shutil.copytree(s, d) else: shutil.copy2(s, d) def _change_permission(self, location): uid, gid = _get_ferry_user() os.chown(location, uid, gid) if os.path.isdir(location): os.chmod(location, 0774) for entry in os.listdir(location): self._change_permission(os.path.join(location, entry)) else: # Check if this file has a file extension. If not, # then assume it's a binary. s = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH if len(location.split(".")) == 1: s |= stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH os.chmod(location, s) """ Check if the dockerfiles are already built. """ def check_images(self, image_dir, repo): if self._docker_running(): build_images = [] for f in os.listdir(image_dir): dockerfile = image_dir + '/' + f + '/Dockerfile' image_names = self._check_dockerfile(dockerfile, repo) if len(image_names) > 0: build_images += image_names return build_images else: logging.error("ferry daemon not started") """ Build the docker images """ def build_from_list(self, to_build, image_dir, repo, build=False, recurse=True): if self._docker_running(): built_images = {} for f in os.listdir(image_dir): logging.warning("transforming dockerfile") self._transform_dockerfile(image_dir, f, repo) for f in os.listdir("/tmp/dockerfiles/"): dockerfile = '/tmp/dockerfiles/' + f + '/Dockerfile' images = self._get_image(dockerfile) intersection = [i for i in images if i in to_build] if len(intersection) > 0: image = images.pop(0) logging.warning("building image " + image) self._build_image(image, dockerfile, repo, built_images, recurse=recurse, build=build) if len(images) > 0: logging.warning("tagging images " + image) self._tag_images(image, repo, images) # After building everything, get rid of the temp dir. shutil.rmtree("/tmp/dockerfiles") else: logging.error("ferry daemon not started") """ Build the docker images """ def build_from_dir(self, image_dir, repo, build=False): if self._docker_running(): built_images = {} for f in os.listdir(image_dir): self._transform_dockerfile(image_dir, f, repo) for f in os.listdir("/tmp/dockerfiles/"): dockerfile = "/tmp/dockerfiles/" + f + "/Dockerfile" images = self._get_image(dockerfile) image = images.pop(0) self._build_image(image, dockerfile, repo, built_images, recurse=True, build=build) if len(images) > 0: logging.warning("tagging images " + image) self._tag_images(image, repo, images) # After building everything, get rid of the temp dir. # shutil.rmtree("/tmp/dockerfiles") else: logging.error("ferry daemon not started") def _docker_running(self): return os.path.exists('/var/run/ferry.sock') def _check_dockerimage(self, image, repo): qualified = repo + '/' + image cmd = DOCKER_CMD + ' -H=' + DOCKER_SOCK + ' inspect ' + qualified + ' 2> /dev/null' output = Popen(cmd, stdout=PIPE, shell=True).stdout.read() if output.strip() == '[]': return image else: return None def _check_dockerfile(self, dockerfile, repo): not_installed = [] images = self._get_image(dockerfile) for image in images: i = self._check_dockerimage(image, DEFAULT_DOCKER_REPO) if i: not_installed.append(image) return not_installed def _transform_dockerfile(self, image_dir, f, repo): if not os.path.exists("/tmp/dockerfiles/" + f): shutil.copytree(image_dir + '/' + f, '/tmp/dockerfiles/' + f) out_file = "/tmp/dockerfiles/" + f + "/Dockerfile" out = open(out_file, "w+") uid, gid = _get_ferry_user() download_url = _get_download_url() changes = {"USER": repo, "DOWNLOAD_URL": download_url, "DOCKER": gid} for line in open(image_dir + '/' + f + '/Dockerfile', "r"): s = Template(line).substitute(changes) out.write(s) out.close() def _build_image(self, image, f, repo, built_images, recurse=False, build=False): base = self._get_base(f) if recurse and base != "ubuntu:14.04": image_dir = os.path.dirname(os.path.dirname(f)) dockerfile = image_dir + '/' + base + '/Dockerfile' self._build_image(base, dockerfile, repo, built_images, recurse, build) if not image in built_images: if base == "ubuntu:14.04": self._pull_image(base) built_images[image] = True self._compile_image(image, repo, os.path.dirname(f), build) def _get_image(self, dockerfile): names = [] for l in open(dockerfile, 'r'): if l.strip() != '': s = l.split() if len(s) > 0: if s[0].upper() == 'NAME': names.append(s[1].strip()) return names def _get_base(self, dockerfile): base = None for l in open(dockerfile, 'r'): s = l.split() if len(s) > 0: if s[0].upper() == 'FROM': base = s[1].strip().split("/") return base[-1] return base def _continuous_print(self, process, on_client=True): while True: try: out = process.stdout.read(15) if out == '': break else: if on_client: sys.stdout.write(out) sys.stdout.flush() else: logging.warning("downloading image...") except IOError as e: logging.warning(e) try: errmsg = process.stderr.readline() if errmsg and errmsg != '': logging.warning(errmsg) else: logging.warning("downloaded image!") except IOError: pass def _pull_image(self, image, tag=None, on_client=True): if not tag: cmd = DOCKER_CMD + ' -H=' + DOCKER_SOCK + ' pull %s' % image else: cmd = DOCKER_CMD + ' -H=' + DOCKER_SOCK + ' pull %s:%s' % (image, tag) logging.warning(cmd) child = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) self._continuous_print(child, on_client=on_client) # Now tag the image with the 'latest' tag. if tag and tag != 'latest': cmd = DOCKER_CMD + ' -H=' + DOCKER_SOCK + ' tag' + ' %s:%s %s:%s' % ( image, tag, image, 'latest') logging.warning(cmd) Popen(cmd, stdout=PIPE, shell=True) def _compile_image(self, image, repo, image_dir, build=False): # Now build the image. if build: cmd = DOCKER_CMD + ' -H=' + DOCKER_SOCK + ' build --rm=true -t' + ' %s/%s %s' % ( repo, image, image_dir) logging.warning(cmd) child = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) self._continuous_print(child) # Now tag the image. cmd = DOCKER_CMD + ' -H=' + DOCKER_SOCK + ' tag' + ' %s/%s %s/%s:%s' % ( repo, image, repo, image, ferry.__version__) logging.warning(cmd) child = Popen(cmd, stdout=PIPE, shell=True) else: # Just pull the image from the public repo. image_name = "%s/%s" % (repo, image) self._pull_image(image_name, tag=ferry.__version__) def _tag_images(self, image, repo, alternatives): for a in alternatives: cmd = DOCKER_CMD + ' -H=' + DOCKER_SOCK + ' tag' + ' %s/%s:%s %s/%s:%s' % ( repo, image, ferry.__version__, repo, a, ferry.__version__) logging.warning(cmd) child = Popen(cmd, stdout=PIPE, shell=True) cmd = DOCKER_CMD + ' -H=' + DOCKER_SOCK + ' tag' + ' %s/%s:latest %s/%s:latest' % ( repo, image, repo, a) logging.warning(cmd) child = Popen(cmd, stdout=PIPE, shell=True) def _clean_images(self): cmd = DOCKER_CMD + ' -H=' + DOCKER_SOCK + ' | grep none | awk \'{print $1}\' | xargs ' + DOCKER_CMD + ' -H=' + DOCKER_SOCK + ' rmi' Popen(cmd, stdout=PIPE, shell=True) def _is_parent_dir(self, pdir, cdir): pdirs = pdir.split("/") cdirs = cdir.split("/") # Parent directory can never be longer than # the child directory. if len(pdirs) > len(cdirs): return False for i in range(0, len(pdirs)): # The parent directory shoudl always match # the child directory. Ignore the start and end # blank spaces caused by "split". if pdirs[i] != "" and pdirs[i] != cdirs[i]: return False return True def _is_running_btrfs(self): logging.warning("checking for btrfs") cmd = 'cat /etc/mtab | grep btrfs | awk \'{print $2}\'' output = Popen(cmd, stdout=PIPE, shell=True).stdout.read() if output.strip() != "": dirs = output.strip().split("\n") for d in dirs: if self._is_parent_dir(d, DOCKER_DIR): return True return False def _start_docker_daemon(self, options=None): # Check if the Ferry bridge has been created. _create_bridge() # Check if the docker daemon is already running try: if not self._docker_running(): # Use the ferry0 bridge. nflag = ' -b ferry0' # Use the LXC backend. This backend option must be specified # for Docker versions greater than 0.9.0. _, ver = _get_docker_version() if ver > (0, 9, 0): lflag = ' -e lxc' else: lflag = '' # Figure out which storage backend to use. Right # now we only support BTRFS or DeviceMapper, since # AUFS seems to break on some occasions. if self._is_running_btrfs(): logging.warning("using btrfs backend") bflag = ' -s btrfs' else: logging.warning("using devmapper backend") bflag = ' -s devicemapper' # Explicitly supply the DNS. if options and '-d' in options: logging.warning("using custom dns") dflag = '' for d in options['-d']: dflag += ' --dns %s' % d else: logging.warning("using public dns") dflag = ' --dns 8.8.8.8 --dns 8.8.4.4' # We need to fix this so that ICC is set to false. icc = ' --icc=true' cmd = 'nohup ' + DOCKER_CMD + ' -d' + ' -H=' + DOCKER_SOCK + ' -g=' + DOCKER_DIR + ' -p=' + DOCKER_PID + nflag + dflag + lflag + bflag + icc + ' 1>%s 2>&1 &' % DEFAULT_DOCKER_LOG logging.warning(cmd) Popen(cmd, stdout=PIPE, shell=True) # Wait a second to let the docker daemon do its thing. time.sleep(3) return True, "Ferry daemon running on /var/run/ferry.sock" else: return False, "Ferry appears to be already running. If this is an error, please type \'ferry clean\' and try again." except OSError as e: logging.error("could not start docker daemon.\n") logging.error(e.strerror) sys.exit(1) def _stop_docker_daemon(self, force=False): if force or self._docker_running(): logging.warning("stopping docker daemon") cmd = 'pkill -f ' + DOCKER_CMD Popen(cmd, stdout=PIPE, shell=True) try: os.remove('/var/run/ferry.sock') except OSError: pass
class Installer(object): def __init__(self, cli=None): self.network = DHCPClient() self.fabric = LocalFabric(bootstrap=True) self.mongo = MongoInitializer(self.fabric.system) self.mongo.fabric = self.fabric self.mongo.template_dir = DEFAULT_TEMPLATE_DIR + '/mongo/' self.cli = cli self.config = read_ferry_config() def get_ferry_account(self): """ Get the Ferry authentication information. """ if 'ferry' in self.config: args = self.config['ferry'] if all(k in args for k in ("user","key","server")): return args['user'], args['key'], args['server'] return None, None, None def _get_worker_info(self): """ Get information regarding how to start the remote HTTP API. """ args = self.config['web'] return int(args['workers']), args['bind'], args['port'] def create_signature(self, request, key): """ Generated a signed request. """ return hmac.new(key, request, hashlib.sha256).hexdigest() def store_app(self, app, ext, content): """ Store the application in the global directory. """ try: # We may need to create the parent directory # if this is the first time an application from this user # is downloaded. file_name = os.path.join(DEFAULT_FERRY_APPS, app + ext) os.makedirs(os.path.dirname(file_name)) with open(file_name, "w") as f: f.write(content) return file_name except IOError as e: logging.error(e) return None except OSError as os: logging.error(os) return None def _clean_rules(self): """ Get rid of all the forwarding rules. """ self.network.clean_rules() def _install_perform_build(self, options, num_tries): # Normally we don't want to build the Dockerfiles, # but sometimes we may for testing, etc. build = False if options and '-b' in options: build = True if options and '-u' in options: if len(options['-u']) > 0 and options['-u'][0] != True: logging.warning("performing select rebuild (%s)" % str(options['-u'])) self.build_from_list(options['-u'], DEFAULT_IMAGE_DIR, DEFAULT_DOCKER_REPO, build, recurse=False) else: logging.warning("performing forced rebuild") self.build_from_dir(DEFAULT_IMAGE_DIR, DEFAULT_DOCKER_REPO, build) else: # We want to be selective about which images # to rebuild. Useful if one image breaks, etc. to_build = self.check_images(DEFAULT_IMAGE_DIR, DEFAULT_DOCKER_REPO) if len(to_build) > 0: logging.warning("performing select rebuild (%s)" % str(to_build)) self.build_from_list(to_build, DEFAULT_IMAGE_DIR, DEFAULT_DOCKER_REPO, build) # Check that all the images were built. not_installed = self._check_all_images() if len(not_installed) == 0: return 'installed ferry' else: logging.error('images not built: ' + str(not_installed)) if num_tries == 0: return 'Some images were not installed. Please type \'ferry install\' again.' else: # Try building the images again. logging.info("retrying install (%d)" % num_tries) return self._install_perform_build(options, num_tries - 1) def install(self, args, options): # Check if the host is actually 64-bit. If not raise a warning and quit. if not _supported_arch(): return 'Your architecture appears to be 32-bit.\nOnly 64-bit architectures are supported at the moment.' if not _supported_python(): return 'You appear to be running Python3.\nOnly Python2 is supported at the moment.' if not _supported_lxc(): return 'Either LXC is not installed or you are running an older version of LXC.\nOnly versions > 0.7.5 are supported.' if not _supported_docker(): return 'Either Docker is not installed or you are running an older version of Docker.\nOnly versions >= 1.2.0 are supported.' if not _supported_bridge(): return 'The brctl command was not found. Please install the network bridge utilities.' if not _has_ferry_user(): return 'You do not appear to have the \'docker\' group configured. Please create the \'docker\' group and try again.' # Create the various directories. try: if not os.path.isdir(DOCKER_DIR): os.makedirs(DOCKER_DIR) self._change_permission(DOCKER_DIR) except OSError as e: logging.error("Could not install Ferry.\n") logging.error(e.strerror) sys.exit(1) # Make sure that the Ferry keys have the correct # ownership & permission. self._check_and_change_ssh_keyperm() # Start the Ferry docker daemon. If it does not successfully # start, print out a msg. logging.warning("all prerequisites met...") start, msg = self._start_docker_daemon(options) if not start: logging.error('ferry docker daemon not started') return msg # Perform the actual build/download. Sometimes the download # may fail so give the user a chance to automatically retry # a few times before giving up. if options and '-r' in options: num_tries = int(options['-r']) else: num_tries = 0 return self._install_perform_build(options, num_tries) def _check_all_images(self): not_installed = [] images = ['mongodb', 'ferry-base', 'hadoop-base', 'hadoop', 'hadoop-client', 'hive-metastore', 'gluster', 'openmpi', 'openmpi-client', 'cassandra', 'cassandra-client', 'titan', 'spark'] for i in images: if not self._check_image_installed("%s/%s" % (DEFAULT_DOCKER_REPO, i)): not_installed.append(i) return not_installed def _check_and_pull_image(self, image_name): if not self._check_image_installed(image_name): self._pull_image(image_name, on_client=False) return self._check_image_installed(image_name) def _check_image_installed(self, image_name): cmd = DOCKER_CMD + ' -H=' + DOCKER_SOCK + ' inspect %s 2> /dev/null' % image_name output = Popen(cmd, stdout=PIPE, shell=True).stdout.read() if output.strip() == '[]': return False else: return True def _transfer_config(self, config_dirs): """ Transfer the configuration to the containers. """ for c in config_dirs: container = c[0] from_dir = c[1] to_dir = c[2] self.fabric.copy([container], from_dir, to_dir) def _read_public_key(self, private_key): s = private_key.split("/") p = os.path.splitext(s[len(s) - 1])[0] return p def _check_and_change_ssh_keyperm(self): os.chmod(DEFAULT_SSH_KEY, 0600) uid, gid = _get_ferry_user() os.chown(DEFAULT_SSH_KEY, uid, gid) def start_web(self, options=None, clean=False): start, msg = self._start_docker_daemon(options) if not clean and not start: # We are trying to start the web services but the Docker # daemon won't start. If we're cleaning, it's not a big deal. logging.error(msg) sys.exit(1) # Check if the ssh key permission is properly set. self._check_and_change_ssh_keyperm() # Check if we're operating in naked mode. If so, # we just needed to start the docker daemon, so we're all done! if options and '-n' in options: return # Check if the user-application directory exists. # If not, create it. try: if not os.path.isdir(DEFAULT_FERRY_APPS): os.makedirs(DEFAULT_FERRY_APPS) self._change_permission(DEFAULT_FERRY_APPS) except OSError as e: logging.error("Could not create application directory.\n") logging.error(e.strerror) sys.exit(1) # Check if the Mongo directory exists yet. If not # go ahead and create it. try: if not os.path.isdir(DEFAULT_MONGO_DB): os.makedirs(DEFAULT_MONGO_DB) self._change_permission(DEFAULT_MONGO_DB) if not os.path.isdir(DEFAULT_MONGO_LOG): os.makedirs(DEFAULT_MONGO_LOG) self._change_permission(DEFAULT_MONGO_LOG) except OSError as e: logging.error("Could not start ferry servers.\n") logging.error(e.strerror) sys.exit(1) # Check if the Mongo image is built. if not self._check_image_installed('%s/mongodb' % DEFAULT_DOCKER_REPO): logging.error("Could not start ferry servers.\n") logging.error("MongoDB images not found. Try executing 'ferry install'.") sys.exit(1) # Check if there are any other Mongo instances runnig. self._clean_web() # Start the Mongo server. Create a new configuration and # manually start the container. private_key = self.cli._get_ssh_key(options) volumes = { DEFAULT_MONGO_LOG : self.mongo.container_log_dir, DEFAULT_MONGO_DB : self.mongo.container_data_dir } mongoplan = {'image':'ferry/mongodb', 'type':'ferry/mongodb', 'keydir': { '/service/keys' : DEFAULT_KEY_DIR }, 'keyname': self._read_public_key(private_key), 'privatekey': private_key, 'volumes':volumes, 'volume_user':DEFAULT_FERRY_OWNER, 'ports':[], 'exposed':self.mongo.get_working_ports(1), 'internal':self.mongo.get_internal_ports(1), 'hostname':'ferrydb', 'netenable':True, 'args': 'trust' } mongoconf = self.mongo.generate(1) mongoconf.uuid = 'fdb-' + str(uuid.uuid4()).split('-')[0] containers = self.fabric.alloc(mongoconf.uuid, mongoconf.uuid, [mongoplan], "MONGO") if containers and len(containers) > 0: mongobox = containers[0] else: logging.error("Could not start MongoDB image") sys.exit(1) ip = mongobox.internal_ip _touch_file('/tmp/ferry/mongodb.ip', ip, root=True) # Once the container is started, we'll need to copy over the # configuration files, and then manually send the 'start' command. s = { 'container':mongobox, 'data_dev':'eth0', 'data_ip':mongobox.internal_ip, 'manage_ip':mongobox.internal_ip, 'host_name':mongobox.host_name, 'type':mongobox.service_type, 'args':mongobox.args } config_dirs, entry_point = self.mongo.apply(mongoconf, [s]) self._transfer_config(config_dirs) self.mongo.start_service([mongobox], entry_point, self.fabric) # Set the MongoDB env. variable. my_env = os.environ.copy() my_env['MONGODB'] = ip # Sleep a little while to let Mongo start receiving. time.sleep(2) # Start the DHCP server logging.warning("starting dhcp server") # cmd = 'gunicorn -t 3600 -b 127.0.0.1:5000 -w 1 ferry.ip.dhcp:app &' cmd = 'python %s/ip/dhcp.py 127.0.0.1 5000 &' % FERRY_HOME Popen(cmd, stdout=PIPE, shell=True, env=my_env) time.sleep(2) # Reserve the Mongo IP. self.network.reserve_ip(ip) # Start the Ferry HTTP server. Read in the web configuration # so that we know how many workers to start, etc. workers, bind, port = self._get_worker_info() logging.warning("starting API servers on (%s:%s) and (mongo:%s)" % (bind, port, ip)) # cmd = 'gunicorn -e FERRY_HOME=%s -t 3600 -w %d -b %s:%s ferry.http.httpapi:app &' % (FERRY_HOME, workers, bind, port) cmd = 'export FERRY_HOME=%s && python %s/http/httpapi.py %s %s &' % (FERRY_HOME, FERRY_HOME, bind, port) Popen(cmd, shell=True, env=my_env) def _force_stop_web(self): logging.warning("stopping docker http servers") cmd = 'pkill -f gunicorn' Popen(cmd, stdout=PIPE, shell=True) def stop_web(self, key): # Shutdown the mongo instance if os.path.exists('/tmp/ferry/mongodb.ip'): f = open('/tmp/ferry/mongodb.ip', 'r') ip = f.read().strip() f.close() cmd = 'LC_ALL=C && ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s /service/sbin/startnode stop' % (key, ip) logging.warning(cmd) output = Popen(cmd, stdout=PIPE, shell=True).stdout.read() logging.warning(output) cmd = 'LC_ALL=C && ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s /service/sbin/startnode halt' % (key, ip) logging.warning(cmd) output = Popen(cmd, stdout=PIPE, shell=True).stdout.read() logging.warning(output) os.remove('/tmp/ferry/mongodb.ip') # Kill all the gunicorn instances. logging.warning("stopping http servers") cmd = 'ps -eaf | grep httpapi | awk \'{print $2}\' | xargs kill -15' Popen(cmd, stdout=PIPE, shell=True) cmd = 'ps -eaf | grep ferry.ip.dhcp | awk \'{print $2}\' | xargs kill -15' Popen(cmd, stdout=PIPE, shell=True) def _clean_web(self): docker = DOCKER_CMD + ' -H=' + DOCKER_SOCK cmd = docker + ' ps | grep ferry/mongodb | awk \'{print $1}\' | xargs ' + docker + ' stop ' logging.warning("cleaning previous mongo resources") logging.warning(cmd) child = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) child.stdout.read() child.stderr.read() def _copytree(self, src, dst): for item in os.listdir(src): s = os.path.join(src, item) d = os.path.join(dst, item) if os.path.isdir(s): shutil.copytree(s, d) else: shutil.copy2(s, d) def _change_permission(self, location): uid, gid = _get_ferry_user() os.chown(location, uid, gid) if os.path.isdir(location): os.chmod(location, 0774) for entry in os.listdir(location): self._change_permission(os.path.join(location, entry)) else: # Check if this file has a file extension. If not, # then assume it's a binary. s = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH if len(location.split(".")) == 1: s |= stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH os.chmod(location, s) """ Check if the dockerfiles are already built. """ def check_images(self, image_dir, repo): if self._docker_running(): build_images = [] for f in os.listdir(image_dir): dockerfile = image_dir + '/' + f + '/Dockerfile' image_names = self._check_dockerfile(dockerfile, repo) if len(image_names) > 0: build_images += image_names return build_images else: logging.error("ferry daemon not started") """ Build the docker images """ def build_from_list(self, to_build, image_dir, repo, build=False, recurse=True): if self._docker_running(): built_images = {} for f in os.listdir(image_dir): logging.warning("transforming dockerfile") self._transform_dockerfile(image_dir, f, repo) for f in os.listdir("/tmp/dockerfiles/"): dockerfile = '/tmp/dockerfiles/' + f + '/Dockerfile' images = self._get_image(dockerfile) intersection = [i for i in images if i in to_build] if len(intersection) > 0: image = images.pop(0) logging.warning("building image " + image) self._build_image(image, dockerfile, repo, built_images, recurse=recurse, build=build) if len(images) > 0: logging.warning("tagging images " + image) self._tag_images(image, repo, images) # After building everything, get rid of the temp dir. shutil.rmtree("/tmp/dockerfiles") else: logging.error("ferry daemon not started") """ Build the docker images """ def build_from_dir(self, image_dir, repo, build=False): if self._docker_running(): built_images = {} for f in os.listdir(image_dir): self._transform_dockerfile(image_dir, f, repo) for f in os.listdir("/tmp/dockerfiles/"): dockerfile = "/tmp/dockerfiles/" + f + "/Dockerfile" images = self._get_image(dockerfile) image = images.pop(0) self._build_image(image, dockerfile, repo, built_images, recurse=True, build=build) if len(images) > 0: logging.warning("tagging images " + image) self._tag_images(image, repo, images) # After building everything, get rid of the temp dir. # shutil.rmtree("/tmp/dockerfiles") else: logging.error("ferry daemon not started") def _docker_running(self): return os.path.exists('/var/run/ferry.sock') def _check_dockerimage(self, image, repo): qualified = repo + '/' + image cmd = DOCKER_CMD + ' -H=' + DOCKER_SOCK + ' inspect ' + qualified + ' 2> /dev/null' output = Popen(cmd, stdout=PIPE, shell=True).stdout.read() if output.strip() == '[]': return image else: return None def _check_dockerfile(self, dockerfile, repo): not_installed = [] images = self._get_image(dockerfile) for image in images: i = self._check_dockerimage(image, DEFAULT_DOCKER_REPO) if i: not_installed.append(image) return not_installed def _transform_dockerfile(self, image_dir, f, repo): if not os.path.exists("/tmp/dockerfiles/" + f): shutil.copytree(image_dir + '/' + f, '/tmp/dockerfiles/' + f) out_file = "/tmp/dockerfiles/" + f + "/Dockerfile" out = open(out_file, "w+") uid, gid = _get_ferry_user() download_url = _get_download_url() changes = { "USER" : repo, "DOWNLOAD_URL" : download_url, "DOCKER" : gid } for line in open(image_dir + '/' + f + '/Dockerfile', "r"): s = Template(line).substitute(changes) out.write(s) out.close() def _build_image(self, image, f, repo, built_images, recurse=False, build=False): base = self._get_base(f) if recurse and base != "ubuntu:14.04": image_dir = os.path.dirname(os.path.dirname(f)) dockerfile = image_dir + '/' + base + '/Dockerfile' self._build_image(base, dockerfile, repo, built_images, recurse, build) if not image in built_images: if base == "ubuntu:14.04": self._pull_image(base) built_images[image] = True self._compile_image(image, repo, os.path.dirname(f), build) def _get_image(self, dockerfile): names = [] for l in open(dockerfile, 'r'): if l.strip() != '': s = l.split() if len(s) > 0: if s[0].upper() == 'NAME': names.append(s[1].strip()) return names def _get_base(self, dockerfile): base = None for l in open(dockerfile, 'r'): s = l.split() if len(s) > 0: if s[0].upper() == 'FROM': base = s[1].strip().split("/") return base[-1] return base def _continuous_print(self, process, on_client=True): while True: try: out = process.stdout.read(15) if out == '': break else: if on_client: sys.stdout.write(out) sys.stdout.flush() else: logging.warning("downloading image...") except IOError as e: logging.warning(e) try: errmsg = process.stderr.readline() if errmsg and errmsg != '': logging.warning(errmsg) else: logging.warning("downloaded image!") except IOError: pass def _pull_image(self, image, tag=None, on_client=True): if not tag: cmd = DOCKER_CMD + ' -H=' + DOCKER_SOCK + ' pull %s' % image else: cmd = DOCKER_CMD + ' -H=' + DOCKER_SOCK + ' pull %s:%s' % (image, tag) logging.warning(cmd) child = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) self._continuous_print(child, on_client=on_client) # Now tag the image with the 'latest' tag. if tag and tag != 'latest': cmd = DOCKER_CMD + ' -H=' + DOCKER_SOCK + ' tag' + ' %s:%s %s:%s' % (image, tag, image, 'latest') logging.warning(cmd) Popen(cmd, stdout=PIPE, shell=True) def _compile_image(self, image, repo, image_dir, build=False): # Now build the image. if build: cmd = DOCKER_CMD + ' -H=' + DOCKER_SOCK + ' build --rm=true -t' + ' %s/%s %s' % (repo, image, image_dir) logging.warning(cmd) child = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) self._continuous_print(child) # Now tag the image. cmd = DOCKER_CMD + ' -H=' + DOCKER_SOCK + ' tag' + ' %s/%s %s/%s:%s' % (repo, image, repo, image, ferry.__version__) logging.warning(cmd) child = Popen(cmd, stdout=PIPE, shell=True) else: # Just pull the image from the public repo. image_name = "%s/%s" % (repo, image) self._pull_image(image_name, tag=ferry.__version__) def _tag_images(self, image, repo, alternatives): for a in alternatives: cmd = DOCKER_CMD + ' -H=' + DOCKER_SOCK + ' tag' + ' %s/%s:%s %s/%s:%s' % (repo, image, ferry.__version__, repo, a, ferry.__version__) logging.warning(cmd) child = Popen(cmd, stdout=PIPE, shell=True) cmd = DOCKER_CMD + ' -H=' + DOCKER_SOCK + ' tag' + ' %s/%s:latest %s/%s:latest' % (repo, image, repo, a) logging.warning(cmd) child = Popen(cmd, stdout=PIPE, shell=True) def _clean_images(self): cmd = DOCKER_CMD + ' -H=' + DOCKER_SOCK + ' | grep none | awk \'{print $1}\' | xargs ' + DOCKER_CMD + ' -H=' + DOCKER_SOCK + ' rmi' Popen(cmd, stdout=PIPE, shell=True) def _is_parent_dir(self, pdir, cdir): pdirs = pdir.split("/") cdirs = cdir.split("/") # Parent directory can never be longer than # the child directory. if len(pdirs) > len(cdirs): return False for i in range(0, len(pdirs)): # The parent directory shoudl always match # the child directory. Ignore the start and end # blank spaces caused by "split". if pdirs[i] != "" and pdirs[i] != cdirs[i]: return False return True def _is_running_btrfs(self): logging.warning("checking for btrfs") cmd = 'cat /etc/mtab | grep btrfs | awk \'{print $2}\'' output = Popen(cmd, stdout=PIPE, shell=True).stdout.read() if output.strip() != "": dirs = output.strip().split("\n") for d in dirs: if self._is_parent_dir(d, DOCKER_DIR): return True return False def _start_docker_daemon(self, options=None): # Check if the Ferry bridge has been created. _create_bridge() # Check if the docker daemon is already running try: if not self._docker_running(): # Use the ferry0 bridge. nflag = ' -b ferry0' # Use the LXC backend. This backend option must be specified # for Docker versions greater than 0.9.0. _, ver = _get_docker_version() if ver > (0, 9, 0): lflag = ' -e lxc' else: lflag = '' # Figure out which storage backend to use. Right # now we only support BTRFS or DeviceMapper, since # AUFS seems to break on some occasions. if self._is_running_btrfs(): logging.warning("using btrfs backend") bflag = ' -s btrfs' else: logging.warning("using devmapper backend") bflag = ' -s devicemapper' # Explicitly supply the DNS. if options and '-d' in options: logging.warning("using custom dns") dflag = '' for d in options['-d']: dflag += ' --dns %s' % d else: logging.warning("using public dns") dflag = ' --dns 8.8.8.8 --dns 8.8.4.4' # We need to fix this so that ICC is set to false. icc = ' --icc=true' cmd = 'nohup ' + DOCKER_CMD + ' -d' + ' -H=' + DOCKER_SOCK + ' -g=' + DOCKER_DIR + ' -p=' + DOCKER_PID + nflag + dflag + lflag + bflag + icc + ' 1>%s 2>&1 &' % DEFAULT_DOCKER_LOG logging.warning(cmd) Popen(cmd, stdout=PIPE, shell=True) # Wait a second to let the docker daemon do its thing. time.sleep(3) return True, "Ferry daemon running on /var/run/ferry.sock" else: return False, "Ferry appears to be already running. If this is an error, please type \'ferry clean\' and try again." except OSError as e: logging.error("could not start docker daemon.\n") logging.error(e.strerror) sys.exit(1) def _stop_docker_daemon(self, force=False): if force or self._docker_running(): logging.warning("stopping docker daemon") cmd = 'pkill -f ' + DOCKER_CMD Popen(cmd, stdout=PIPE, shell=True) try: os.remove('/var/run/ferry.sock') except OSError: pass