def delete(args, options):
validate_common_options(options)
with open(options.password_file, 'r') as f:
password = f.read().strip()
if not password:
app.error("Empty password file")
url = 'http://%s:%s/clusters/%s' % (options.api_host, options.api_port, options.cluster_name)
values = dict(password=password)
req = urllib2.Request(url, urllib.urlencode(values))
req.get_method = lambda: 'DELETE'
try:
response = urllib2.urlopen(req).read()
except urllib2.HTTPError as e:
log.error("DELETE request failed: %s, %s, %s" % (
e.code, BaseHTTPServer.BaseHTTPRequestHandler.responses[e.code], e.read()))
app.quit(1)
try:
result = json.loads(response)
if not isinstance(result, dict):
raise ValueError()
except ValueError:
log.error("Invalid response: %s" % response)
app.quit(1)
log.info("Cluster deletion result: %s" % result)
log.info("Waiting for the cluster to terminate...")
wait_for_termination(result['cluster_url'])
log.info("Cluster terminated/deleted")
def create(args, options):
validate_common_options(options)
if not options.num_nodes:
app.error("--num_nodes is required")
if not options.cluster_user:
app.error("--cluster_user is required")
url = 'http://%s:%s/clusters/%s' % (options.api_host, options.api_port, options.cluster_name)
values = dict(
num_nodes=int(options.num_nodes),
cluster_user=options.cluster_user,
size=options.size if options.size else '',
backup_id=options.backup_id if options.backup_id else '')
req = urllib2.Request(url, urllib.urlencode(values))
try:
response = urllib2.urlopen(req).read()
except urllib2.HTTPError as e:
log.error("POST request failed: %s, %s, %s" % (
e.code, BaseHTTPServer.BaseHTTPRequestHandler.responses[e.code], e.read()))
app.quit(1)
try:
result = json.loads(response)
if not isinstance(result, dict):
raise ValueError()
except ValueError:
log.error("Invalid response: %s" % response)
app.quit(1)
log.info("Cluster created. Cluster info: %s" % str(result))
with open(options.password_file, 'w') as f:
f.write(result["cluster_password"])
log.info("Waiting for the master for this cluster to be elected...")
master_endpoint = wait_for_master(result['cluster_url']).service_endpoint
connection_str = "mysql://%s:%s@%s:%d/" % (
options.cluster_user,
result["cluster_password"],
master_endpoint.host,
master_endpoint.port)
log.info("Connecting to the MySQL cluster master: %s" % connection_str)
engine = create_engine(connection_str)
for i in range(5): # Loop for 5 times/seconds to wait for the master to be promoted.
try:
# TODO(jyx): Test writing to the master and reading from the slave.
result = engine.execute("SELECT 1;").scalar()
assert 1 == int(result), "Expecting result to be 1 but got %s" % result
break
except OperationalError:
if i == 4:
raise
log.debug("MySQL master not ready yet. Sleep for 1 second...")
time.sleep(1)
log.info("Cluster successfully started")
def help_command(args, options):
"""Get help about a specific command.
"""
if len(args) == 0:
app.help()
for (command, doc) in app.get_commands_and_docstrings():
if args[0] == command:
print('command %s:' % command)
print(doc)
app.quit(0)
print('unknown command: %s' % args[0], file=sys.stderr)
def test_app_registry_exit_functions(self):
self.factory.new_module('first')
self.factory.new_module('second', dependencies='first')
self.factory.new_module('third', dependencies=['first', 'second'])
app.init(force_args=[])
app.quit(None, exit_function=sys.stdout.write)
assert self.factory.value('third_exit') > 0 and (
self.factory.value('second_exit') > 0 and self.factory.value('first_exit') > 0), \
'all exit callbacks should have been called'
assert self.factory.value('third_exit') < self.factory.value('second_exit')
assert self.factory.value('third_exit') < self.factory.value('first_exit')
def test_app_registry_exit_functions(self):
self.factory.new_module('first')
self.factory.new_module('second', dependencies='first')
self.factory.new_module('third', dependencies=['first', 'second'])
app.init(force_args=[])
def exit_function(*args):
pass
app.quit(None, exit_function=exit_function)
assert self.factory.value('third_exit') > 0 and (
self.factory.value('second_exit') > 0 and self.factory.value('first_exit') > 0), \
'all exit callbacks should have been called'
assert self.factory.value('third_exit') < self.factory.value('second_exit')
assert self.factory.value('third_exit') < self.factory.value('first_exit')
def delete(args, options):
validate_common_options(options)
with open(options.password_file, 'r') as f:
password = f.read().strip()
if not password:
app.error("Empty password file")
url = 'http://%s:%s/clusters/%s' % (options.api_host, options.api_port,
options.cluster_name)
values = dict(password=password)
req = urllib2.Request(url, urllib.urlencode(values))
req.get_method = lambda: 'DELETE'
try:
response = urllib2.urlopen(req).read()
except urllib2.HTTPError as e:
log.error("DELETE request failed: %s, %s, %s" %
(e.code,
BaseHTTPServer.BaseHTTPRequestHandler.responses[e.code],
e.read()))
app.quit(1)
try:
result = json.loads(response)
if not isinstance(result, dict):
raise ValueError()
except ValueError:
log.error("Invalid response: %s" % response)
app.quit(1)
log.info("Cluster deletion result: %s" % result)
log.info("Waiting for the cluster to terminate...")
wait_for_termination(result['cluster_url'])
log.info("Cluster terminated/deleted")
def create_workernode(args, options):
"""Bootstrap a new Hadoop Worker node
Usage: create_workernode <zookeeper_url>
"""
try:
zookeeper_ensemble_url = args[0]
log.debug("Using %s" % zookeeper_ensemble_url)
except IndexError:
print("Please specify a url to perform Service Discovery")
app.quit(1)
log.debug('Identifying headnode')
headnode_endpoints = Discovery(zookeeper_ensemble_url).retrieve_headnode_endpoint(
options.headnode_path)
log.debug('Found endpoints: %s' % headnode_endpoints)
configured_xml = Configure(**headnode_endpoints).generate_xml(options.nm_web_port,
options.nm_main_port, options.nm_loc_port, options.nm_shuffle_port, options.dn_web_port,
options.dn_ipc_port, options.dn_rpc_port)
with open(options.output, 'w') as fp:
fp.write(configured_xml)
os.symlink(options.output, 'core-site.xml')
os.symlink(options.output, 'yarn-site.xml')
os.symlink(options.output, 'hdfs-site.xml')
os.symlink(options.output, 'mapred-site.xml')
with open('fair-scheduler.xml', 'w') as fp:
fp.write('<allocations><queue name=\'root\'></queue></allocations>')
with open('hadoop-release', 'w') as fp:
fp.write('hadoop-client-2.4.0.t04')
shutil.copy('./etc/hadoop/log4j.properties', '.')
def create(args, options):
validate_common_options(options)
if not options.num_nodes:
app.error("--num_nodes is required")
if not options.cluster_user:
app.error("--cluster_user is required")
url = 'http://%s:%s/clusters/%s' % (options.api_host, options.api_port,
options.cluster_name)
values = dict(
num_nodes=int(options.num_nodes),
cluster_user=options.cluster_user,
size=options.size
if options.size else '', # 'urlencode()' doesn't accept None.
backup_id=options.backup_id if options.backup_id else '',
cluster_password=options.cluster_password
if options.cluster_password else '')
req = urllib2.Request(url, urllib.urlencode(values))
try:
response = urllib2.urlopen(req).read()
except urllib2.HTTPError as e:
log.error("POST request failed: %s, %s, %s" %
(e.code,
BaseHTTPServer.BaseHTTPRequestHandler.responses[e.code],
e.read()))
app.quit(1)
try:
result = json.loads(response)
if not isinstance(result, dict):
raise ValueError()
except ValueError:
log.error("Invalid response: %s" % response)
app.quit(1)
log.info("Cluster created. Cluster info: %s" % str(result))
with open(options.password_file, 'w') as f:
f.write(result["cluster_password"])
log.info("Waiting for the master for this cluster to be elected...")
master_endpoint = wait_for_master(
result['cluster_url']).service_endpoint
connection_str = "mysql://%s:%s@%s:%d/" % (
options.cluster_user, result["cluster_password"],
master_endpoint.host, master_endpoint.port)
log.info("Connecting to the MySQL cluster master: %s" % connection_str)
engine = create_engine(connection_str)
for i in range(
5
): # Loop for 5 times/seconds to wait for the master to be promoted.
try:
# TODO(jyx): Test writing to the master and reading from the slave.
result = engine.execute("SELECT 1;").scalar()
assert 1 == int(
result), "Expecting result to be 1 but got %s" % result
break
except OperationalError:
if i == 4:
raise
log.debug("MySQL master not ready yet. Sleep for 1 second...")
time.sleep(1)
log.info("Cluster successfully started")
