def delete(args, options): validate_common_options(options) with open(options.password_file, 'r') as f: password = f.read().strip() if not password: app.error("Empty password file") url = 'http://%s:%s/clusters/%s' % (options.api_host, options.api_port, options.cluster_name) values = dict(password=password) req = urllib2.Request(url, urllib.urlencode(values)) req.get_method = lambda: 'DELETE' try: response = urllib2.urlopen(req).read() except urllib2.HTTPError as e: log.error("DELETE request failed: %s, %s, %s" % ( e.code, BaseHTTPServer.BaseHTTPRequestHandler.responses[e.code], e.read())) app.quit(1) try: result = json.loads(response) if not isinstance(result, dict): raise ValueError() except ValueError: log.error("Invalid response: %s" % response) app.quit(1) log.info("Cluster deletion result: %s" % result) log.info("Waiting for the cluster to terminate...") wait_for_termination(result['cluster_url']) log.info("Cluster terminated/deleted")
def create(args, options): validate_common_options(options) if not options.num_nodes: app.error("--num_nodes is required") if not options.cluster_user: app.error("--cluster_user is required") url = 'http://%s:%s/clusters/%s' % (options.api_host, options.api_port, options.cluster_name) values = dict( num_nodes=int(options.num_nodes), cluster_user=options.cluster_user, size=options.size if options.size else '', backup_id=options.backup_id if options.backup_id else '') req = urllib2.Request(url, urllib.urlencode(values)) try: response = urllib2.urlopen(req).read() except urllib2.HTTPError as e: log.error("POST request failed: %s, %s, %s" % ( e.code, BaseHTTPServer.BaseHTTPRequestHandler.responses[e.code], e.read())) app.quit(1) try: result = json.loads(response) if not isinstance(result, dict): raise ValueError() except ValueError: log.error("Invalid response: %s" % response) app.quit(1) log.info("Cluster created. Cluster info: %s" % str(result)) with open(options.password_file, 'w') as f: f.write(result["cluster_password"]) log.info("Waiting for the master for this cluster to be elected...") master_endpoint = wait_for_master(result['cluster_url']).service_endpoint connection_str = "mysql://%s:%s@%s:%d/" % ( options.cluster_user, result["cluster_password"], master_endpoint.host, master_endpoint.port) log.info("Connecting to the MySQL cluster master: %s" % connection_str) engine = create_engine(connection_str) for i in range(5): # Loop for 5 times/seconds to wait for the master to be promoted. try: # TODO(jyx): Test writing to the master and reading from the slave. result = engine.execute("SELECT 1;").scalar() assert 1 == int(result), "Expecting result to be 1 but got %s" % result break except OperationalError: if i == 4: raise log.debug("MySQL master not ready yet. Sleep for 1 second...") time.sleep(1) log.info("Cluster successfully started")
def help_command(args, options): """Get help about a specific command. """ if len(args) == 0: app.help() for (command, doc) in app.get_commands_and_docstrings(): if args[0] == command: print('command %s:' % command) print(doc) app.quit(0) print('unknown command: %s' % args[0], file=sys.stderr)
def test_app_registry_exit_functions(self): self.factory.new_module('first') self.factory.new_module('second', dependencies='first') self.factory.new_module('third', dependencies=['first', 'second']) app.init(force_args=[]) app.quit(None, exit_function=sys.stdout.write) assert self.factory.value('third_exit') > 0 and ( self.factory.value('second_exit') > 0 and self.factory.value('first_exit') > 0), \ 'all exit callbacks should have been called' assert self.factory.value('third_exit') < self.factory.value('second_exit') assert self.factory.value('third_exit') < self.factory.value('first_exit')
def test_app_registry_exit_functions(self): self.factory.new_module('first') self.factory.new_module('second', dependencies='first') self.factory.new_module('third', dependencies=['first', 'second']) app.init(force_args=[]) def exit_function(*args): pass app.quit(None, exit_function=exit_function) assert self.factory.value('third_exit') > 0 and ( self.factory.value('second_exit') > 0 and self.factory.value('first_exit') > 0), \ 'all exit callbacks should have been called' assert self.factory.value('third_exit') < self.factory.value('second_exit') assert self.factory.value('third_exit') < self.factory.value('first_exit')
def delete(args, options): validate_common_options(options) with open(options.password_file, 'r') as f: password = f.read().strip() if not password: app.error("Empty password file") url = 'http://%s:%s/clusters/%s' % (options.api_host, options.api_port, options.cluster_name) values = dict(password=password) req = urllib2.Request(url, urllib.urlencode(values)) req.get_method = lambda: 'DELETE' try: response = urllib2.urlopen(req).read() except urllib2.HTTPError as e: log.error("DELETE request failed: %s, %s, %s" % (e.code, BaseHTTPServer.BaseHTTPRequestHandler.responses[e.code], e.read())) app.quit(1) try: result = json.loads(response) if not isinstance(result, dict): raise ValueError() except ValueError: log.error("Invalid response: %s" % response) app.quit(1) log.info("Cluster deletion result: %s" % result) log.info("Waiting for the cluster to terminate...") wait_for_termination(result['cluster_url']) log.info("Cluster terminated/deleted")
def create_workernode(args, options): """Bootstrap a new Hadoop Worker node Usage: create_workernode <zookeeper_url> """ try: zookeeper_ensemble_url = args[0] log.debug("Using %s" % zookeeper_ensemble_url) except IndexError: print("Please specify a url to perform Service Discovery") app.quit(1) log.debug('Identifying headnode') headnode_endpoints = Discovery(zookeeper_ensemble_url).retrieve_headnode_endpoint( options.headnode_path) log.debug('Found endpoints: %s' % headnode_endpoints) configured_xml = Configure(**headnode_endpoints).generate_xml(options.nm_web_port, options.nm_main_port, options.nm_loc_port, options.nm_shuffle_port, options.dn_web_port, options.dn_ipc_port, options.dn_rpc_port) with open(options.output, 'w') as fp: fp.write(configured_xml) os.symlink(options.output, 'core-site.xml') os.symlink(options.output, 'yarn-site.xml') os.symlink(options.output, 'hdfs-site.xml') os.symlink(options.output, 'mapred-site.xml') with open('fair-scheduler.xml', 'w') as fp: fp.write('<allocations><queue name=\'root\'></queue></allocations>') with open('hadoop-release', 'w') as fp: fp.write('hadoop-client-2.4.0.t04') shutil.copy('./etc/hadoop/log4j.properties', '.')
def create(args, options): validate_common_options(options) if not options.num_nodes: app.error("--num_nodes is required") if not options.cluster_user: app.error("--cluster_user is required") url = 'http://%s:%s/clusters/%s' % (options.api_host, options.api_port, options.cluster_name) values = dict( num_nodes=int(options.num_nodes), cluster_user=options.cluster_user, size=options.size if options.size else '', # 'urlencode()' doesn't accept None. backup_id=options.backup_id if options.backup_id else '', cluster_password=options.cluster_password if options.cluster_password else '') req = urllib2.Request(url, urllib.urlencode(values)) try: response = urllib2.urlopen(req).read() except urllib2.HTTPError as e: log.error("POST request failed: %s, %s, %s" % (e.code, BaseHTTPServer.BaseHTTPRequestHandler.responses[e.code], e.read())) app.quit(1) try: result = json.loads(response) if not isinstance(result, dict): raise ValueError() except ValueError: log.error("Invalid response: %s" % response) app.quit(1) log.info("Cluster created. Cluster info: %s" % str(result)) with open(options.password_file, 'w') as f: f.write(result["cluster_password"]) log.info("Waiting for the master for this cluster to be elected...") master_endpoint = wait_for_master( result['cluster_url']).service_endpoint connection_str = "mysql://%s:%s@%s:%d/" % ( options.cluster_user, result["cluster_password"], master_endpoint.host, master_endpoint.port) log.info("Connecting to the MySQL cluster master: %s" % connection_str) engine = create_engine(connection_str) for i in range( 5 ): # Loop for 5 times/seconds to wait for the master to be promoted. try: # TODO(jyx): Test writing to the master and reading from the slave. result = engine.execute("SELECT 1;").scalar() assert 1 == int( result), "Expecting result to be 1 but got %s" % result break except OperationalError: if i == 4: raise log.debug("MySQL master not ready yet. Sleep for 1 second...") time.sleep(1) log.info("Cluster successfully started")