def main():
common.setup()
# CHANGE ME
zone = 'us-central1-a'
machtype = 'n1-standard-4-d'
image = 'projects/google/images/ubuntu-12-04-v20120621'
# Persistent disk, if any.
disk = ''
# If this is set, only this slave will have the disk mounted, and it'll be rw.
# Otherwise, all slaves get the disk mounted ro
rw_disk_instance = ''
print 'Packaging up the stuff the coordinator will need...'
# tar will insert directories, so flatten the view a bit
subprocess.call(['cp', 'coordinator/coordinator.py', '.'])
subprocess.call(['cp', 'coordinator/hadoop_cluster.py', '.'])
subprocess.call(['tar', 'czf', 'coordinator.tgz', 'hadoop', 'gcelib',
'hadoop-tools.jar', 'cfg.py', 'util.py', 'coordinator.py',
'hadoop_cluster.py', 'start_setup.sh'])
subprocess.call(['rm', 'coordinator.py', 'hadoop_cluster.py'])
# Push to a fixed place for now
subprocess.call(['gsutil', 'cp', 'coordinator.tgz',
cfg.gs_coordinators_tarball])
subprocess.call(['rm', 'coordinator.tgz'])
print
print 'Launching coordinator...'
util.api.insert_instance(
name=cfg.coordinator, zone=zone,
machineType=machtype, image=image,
serviceAccounts=gce_shortcuts.service_accounts([cfg.compute_scope,
cfg.rw_storage_scope]),
networkInterfaces=gce_shortcuts.network(),
metadata=gce_shortcuts.metadata({
'startup-script': open('start_setup.sh').read(),
'bootstrap.sh': open('coordinator/bootstrap.sh').read(),
'tarball': cfg.gs_coordinators_tarball,
'gs_bucket': cfg.gs_bucket,
'zone': zone,
'machine_type': machtype,
'image': image,
'disk': disk,
'rw_disk_instance': rw_disk_instance,
'secret': cfg.secret
}),
blocking=True
)
print
print 'Waiting for coordinator to come online...'
while True:
status, _ = util.get_status(cfg.coordinator)
print status[1]
if status == util.InstanceState.SNITCH_READY:
break
time.sleep(cfg.poll_delay_secs)
print
print 'Controller is ready to receive commands.'
def test_service_accounts(self):
self.assertEqual(
shortcuts.service_accounts(),
[{'scopes': [],
'email': 'default'}])
self.assertEqual(
shortcuts.service_accounts(['a', 'b']),
[{'scopes': ['a', 'b'],
'email': 'default'}])
self.assertEqual(
shortcuts.service_accounts(['a', 'b'],
email='*****@*****.**'),
[{'scopes': ['a', 'b'],
'email': '*****@*****.**'}])
def test_service_accounts(self):
self.assertEqual(shortcuts.service_accounts(), [{
'scopes': [],
'email': 'default'
}])
self.assertEqual(shortcuts.service_accounts(['a', 'b']),
[{
'scopes': ['a', 'b'],
'email': 'default'
}])
self.assertEqual(
shortcuts.service_accounts(
['a', 'b'], email='*****@*****.**'),
[{
'scopes': ['a', 'b'],
'email': '*****@*****.**'
}])
def spawn_instance(self, name, snitch):
"""Create an instance with the specified snitch."""
disks = []
if cfg.disk:
# Can't mount rw if others have already mounted it ro... so just only
# mount it on one instance
if cfg.rw_disk_instance:
if cfg.rw_disk_instance == name:
disks = gce_shortcuts.rw_disks([cfg.disk])
# Otherwise, don't mount
else:
# Everyone gets it ro
disks = gce_shortcuts.ro_disks([cfg.disk])
network = []
# Always give the JobTracker and NameNode an external IP.
if (not name.startswith('hadoop-slave-')) or cfg.external_ips:
network = gce_shortcuts.network()
else:
network = gce_shortcuts.network(use_access_config=False)
if name == cfg.hadoop_namenode:
# This instance handles transfers from HDFS to GS.
scope = cfg.rw_storage_scope
else:
scope = cfg.ro_storage_scope
try:
resp = util.api.insert_instance(
name=name,
zone=cfg.zone,
machineType=cfg.machine_type,
image=cfg.image,
serviceAccounts=gce_shortcuts.service_accounts([scope]),
disks=disks,
metadata=gce_shortcuts.metadata({
# Key modified to avoid dots, which are disallowed in v1beta13.
'gs_bucket':
cfg.gs_bucket,
'snitch-tarball_tgz':
cfg.gs_snitch_tarball,
'startup-script':
open('start_setup.sh').read(),
'bootstrap_sh':
open('hadoop/bootstrap.sh').read(),
'snitch_py':
open(snitch).read()
}),
networkInterfaces=network,
blocking=True)
except gce.GceError as e:
logging.info('GCE exception inserting instance ' + name + ': ' +
str(e))
except Exception as e:
logging.info('exception inserting instance ' + name + ': ' +
str(e))
return True
def spawn_instance(self, name, snitch):
"""Create an instance with the specified snitch."""
disks = []
if cfg.disk:
# Can't mount rw if others have already mounted it ro... so just only
# mount it on one instance
if cfg.rw_disk_instance:
if cfg.rw_disk_instance == name:
disks = gce_shortcuts.rw_disks([cfg.disk])
# Otherwise, don't mount
else:
# Everyone gets it ro
disks = gce_shortcuts.ro_disks([cfg.disk])
network = []
# Always give the JobTracker and NameNode an external IP.
if (not name.startswith('hadoop-slave-')) or cfg.external_ips:
network = gce_shortcuts.network()
else:
network = gce_shortcuts.network(use_access_config=False)
if name == cfg.hadoop_namenode:
# This instance handles transfers from HDFS to GS.
scope = cfg.rw_storage_scope
else:
scope = cfg.ro_storage_scope
try:
resp = util.api.insert_instance(
name=name, zone=cfg.zone,
machineType=cfg.machine_type, image=cfg.image,
serviceAccounts=gce_shortcuts.service_accounts([scope]),
disks=disks,
metadata=gce_shortcuts.metadata({
# Key modified to avoid dots, which are disallowed in v1beta13.
'gs_bucket': cfg.gs_bucket,
'snitch-tarball_tgz': cfg.gs_snitch_tarball,
'startup-script': open('start_setup.sh').read(),
'bootstrap_sh': open('hadoop/bootstrap.sh').read(),
'snitch_py': open(snitch).read()
}),
networkInterfaces=network,
blocking=True
)
except gce.GceError as e:
logging.info('GCE exception inserting instance ' + name + ': ' + str(e))
except Exception as e:
logging.info('exception inserting instance ' + name + ': ' + str(e))
return True
def main():
common.setup()
# CHANGE ME
zone = 'us-central1-a'
machtype = 'n1-standard-4-d'
image = 'projects/google/images/ubuntu-12-04-v20120621'
# Persistent disk, if any.
disk = ''
# If this is set, only this slave will have the disk mounted, and it'll be rw.
# Otherwise, all slaves get the disk mounted ro
rw_disk_instance = ''
print 'Packaging up the stuff the coordinator will need...'
# tar will insert directories, so flatten the view a bit
subprocess.call(['cp', 'coordinator/coordinator.py', '.'])
subprocess.call(['cp', 'coordinator/hadoop_cluster.py', '.'])
subprocess.call([
'tar', 'czf', 'coordinator.tgz', 'hadoop', 'gcelib',
'hadoop-tools.jar', 'cfg.py', 'util.py', 'coordinator.py',
'hadoop_cluster.py', 'start_setup.sh'
])
subprocess.call(['rm', 'coordinator.py', 'hadoop_cluster.py'])
# Push to a fixed place for now
subprocess.call(
['gsutil', 'cp', 'coordinator.tgz', cfg.gs_coordinators_tarball])
subprocess.call(['rm', 'coordinator.tgz'])
print
print 'Launching coordinator...'
util.api.insert_instance(
name=cfg.coordinator,
zone=zone,
machineType=machtype,
image=image,
serviceAccounts=gce_shortcuts.service_accounts(
[cfg.compute_scope, cfg.rw_storage_scope]),
networkInterfaces=gce_shortcuts.network(),
metadata=gce_shortcuts.metadata({
# Key modified to avoid dots, which are disallowed in v1beta13.
'startup-script':
open('start_setup.sh').read(),
'bootstrap_sh':
open('coordinator/bootstrap.sh').read(),
'tarball':
cfg.gs_coordinators_tarball,
'gs_bucket':
cfg.gs_bucket,
'zone':
zone,
'machine_type':
machtype,
'image':
image,
'disk':
disk,
'rw_disk_instance':
rw_disk_instance,
'secret':
cfg.secret
}),
blocking=True)
print
print 'Waiting for coordinator to come online...'
while True:
status, _ = util.get_status(cfg.coordinator)
print status[1]
if status == util.InstanceState.SNITCH_READY:
break
time.sleep(cfg.poll_delay_secs)
print
print 'Controller is ready to receive commands.'