def main():
  common.setup()

  # CHANGE ME
  zone = 'us-central1-a'
  machtype = 'n1-standard-4-d'
  image = 'projects/google/images/ubuntu-12-04-v20120621'
  # Persistent disk, if any.
  disk = ''
  # If this is set, only this slave will have the disk mounted, and it'll be rw.
  # Otherwise, all slaves get the disk mounted ro
  rw_disk_instance = ''

  print 'Packaging up the stuff the coordinator will need...'
  # tar will insert directories, so flatten the view a bit
  subprocess.call(['cp', 'coordinator/coordinator.py', '.'])
  subprocess.call(['cp', 'coordinator/hadoop_cluster.py', '.'])
  subprocess.call(['tar', 'czf', 'coordinator.tgz', 'hadoop', 'gcelib',
                   'hadoop-tools.jar', 'cfg.py', 'util.py', 'coordinator.py',
                   'hadoop_cluster.py', 'start_setup.sh'])
  subprocess.call(['rm', 'coordinator.py', 'hadoop_cluster.py'])
  # Push to a fixed place for now
  subprocess.call(['gsutil', 'cp', 'coordinator.tgz',
                   cfg.gs_coordinators_tarball])
  subprocess.call(['rm', 'coordinator.tgz'])
  print

  print 'Launching coordinator...'
  util.api.insert_instance(
      name=cfg.coordinator, zone=zone,
      machineType=machtype, image=image,
      serviceAccounts=gce_shortcuts.service_accounts([cfg.compute_scope,
                                                      cfg.rw_storage_scope]),
      networkInterfaces=gce_shortcuts.network(),
      metadata=gce_shortcuts.metadata({
          'startup-script': open('start_setup.sh').read(),
          'bootstrap.sh': open('coordinator/bootstrap.sh').read(),
          'tarball': cfg.gs_coordinators_tarball,
          'gs_bucket': cfg.gs_bucket,
          'zone': zone,
          'machine_type': machtype,
          'image': image,
          'disk': disk,
          'rw_disk_instance': rw_disk_instance,
          'secret': cfg.secret
      }),
      blocking=True
  )
  print

  print 'Waiting for coordinator to come online...'
  while True:
    status, _ = util.get_status(cfg.coordinator)
    print status[1]
    if status == util.InstanceState.SNITCH_READY:
      break
    time.sleep(cfg.poll_delay_secs)
  print

  print 'Controller is ready to receive commands.'
예제 #2
0
  def test_service_accounts(self):
    self.assertEqual(
        shortcuts.service_accounts(),
        [{'scopes': [],
          'email': 'default'}])

    self.assertEqual(
        shortcuts.service_accounts(['a', 'b']),
        [{'scopes': ['a', 'b'],
          'email': 'default'}])

    self.assertEqual(
        shortcuts.service_accounts(['a', 'b'],
                                   email='*****@*****.**'),
        [{'scopes': ['a', 'b'],
          'email': '*****@*****.**'}])
예제 #3
0
    def test_service_accounts(self):
        self.assertEqual(shortcuts.service_accounts(), [{
            'scopes': [],
            'email': 'default'
        }])

        self.assertEqual(shortcuts.service_accounts(['a', 'b']),
                         [{
                             'scopes': ['a', 'b'],
                             'email': 'default'
                         }])

        self.assertEqual(
            shortcuts.service_accounts(
                ['a', 'b'], email='*****@*****.**'),
            [{
                'scopes': ['a', 'b'],
                'email': '*****@*****.**'
            }])
 def spawn_instance(self, name, snitch):
     """Create an instance with the specified snitch."""
     disks = []
     if cfg.disk:
         # Can't mount rw if others have already mounted it ro... so just only
         # mount it on one instance
         if cfg.rw_disk_instance:
             if cfg.rw_disk_instance == name:
                 disks = gce_shortcuts.rw_disks([cfg.disk])
             # Otherwise, don't mount
         else:
             # Everyone gets it ro
             disks = gce_shortcuts.ro_disks([cfg.disk])
     network = []
     # Always give the JobTracker and NameNode an external IP.
     if (not name.startswith('hadoop-slave-')) or cfg.external_ips:
         network = gce_shortcuts.network()
     else:
         network = gce_shortcuts.network(use_access_config=False)
     if name == cfg.hadoop_namenode:
         # This instance handles transfers from HDFS to GS.
         scope = cfg.rw_storage_scope
     else:
         scope = cfg.ro_storage_scope
     try:
         resp = util.api.insert_instance(
             name=name,
             zone=cfg.zone,
             machineType=cfg.machine_type,
             image=cfg.image,
             serviceAccounts=gce_shortcuts.service_accounts([scope]),
             disks=disks,
             metadata=gce_shortcuts.metadata({
                 # Key modified to avoid dots, which are disallowed in v1beta13.
                 'gs_bucket':
                 cfg.gs_bucket,
                 'snitch-tarball_tgz':
                 cfg.gs_snitch_tarball,
                 'startup-script':
                 open('start_setup.sh').read(),
                 'bootstrap_sh':
                 open('hadoop/bootstrap.sh').read(),
                 'snitch_py':
                 open(snitch).read()
             }),
             networkInterfaces=network,
             blocking=True)
     except gce.GceError as e:
         logging.info('GCE exception inserting instance ' + name + ': ' +
                      str(e))
     except Exception as e:
         logging.info('exception inserting instance ' + name + ': ' +
                      str(e))
     return True
 def spawn_instance(self, name, snitch):
   """Create an instance with the specified snitch."""
   disks = []
   if cfg.disk:
     # Can't mount rw if others have already mounted it ro... so just only
     # mount it on one instance
     if cfg.rw_disk_instance:
       if cfg.rw_disk_instance == name:
         disks = gce_shortcuts.rw_disks([cfg.disk])
       # Otherwise, don't mount
     else:
       # Everyone gets it ro
       disks = gce_shortcuts.ro_disks([cfg.disk])
   network = []
   # Always give the JobTracker and NameNode an external IP.
   if (not name.startswith('hadoop-slave-')) or cfg.external_ips:
     network = gce_shortcuts.network()
   else:
     network = gce_shortcuts.network(use_access_config=False)
   if name == cfg.hadoop_namenode:
     # This instance handles transfers from HDFS to GS.
     scope = cfg.rw_storage_scope
   else:
     scope = cfg.ro_storage_scope
   try:
     resp = util.api.insert_instance(
         name=name, zone=cfg.zone,
         machineType=cfg.machine_type, image=cfg.image,
         serviceAccounts=gce_shortcuts.service_accounts([scope]),
         disks=disks,
         metadata=gce_shortcuts.metadata({
             # Key modified to avoid dots, which are disallowed in v1beta13.
             'gs_bucket': cfg.gs_bucket,
             'snitch-tarball_tgz': cfg.gs_snitch_tarball,
             'startup-script': open('start_setup.sh').read(),
             'bootstrap_sh': open('hadoop/bootstrap.sh').read(),
             'snitch_py': open(snitch).read()
         }),
         networkInterfaces=network,
         blocking=True
     )
   except gce.GceError as e:
     logging.info('GCE exception inserting instance ' + name + ': ' + str(e))
   except Exception as e:
     logging.info('exception inserting instance ' + name + ': ' + str(e))
   return True
예제 #6
0
def main():
    common.setup()

    # CHANGE ME
    zone = 'us-central1-a'
    machtype = 'n1-standard-4-d'
    image = 'projects/google/images/ubuntu-12-04-v20120621'
    # Persistent disk, if any.
    disk = ''
    # If this is set, only this slave will have the disk mounted, and it'll be rw.
    # Otherwise, all slaves get the disk mounted ro
    rw_disk_instance = ''

    print 'Packaging up the stuff the coordinator will need...'
    # tar will insert directories, so flatten the view a bit
    subprocess.call(['cp', 'coordinator/coordinator.py', '.'])
    subprocess.call(['cp', 'coordinator/hadoop_cluster.py', '.'])
    subprocess.call([
        'tar', 'czf', 'coordinator.tgz', 'hadoop', 'gcelib',
        'hadoop-tools.jar', 'cfg.py', 'util.py', 'coordinator.py',
        'hadoop_cluster.py', 'start_setup.sh'
    ])
    subprocess.call(['rm', 'coordinator.py', 'hadoop_cluster.py'])
    # Push to a fixed place for now
    subprocess.call(
        ['gsutil', 'cp', 'coordinator.tgz', cfg.gs_coordinators_tarball])
    subprocess.call(['rm', 'coordinator.tgz'])
    print

    print 'Launching coordinator...'
    util.api.insert_instance(
        name=cfg.coordinator,
        zone=zone,
        machineType=machtype,
        image=image,
        serviceAccounts=gce_shortcuts.service_accounts(
            [cfg.compute_scope, cfg.rw_storage_scope]),
        networkInterfaces=gce_shortcuts.network(),
        metadata=gce_shortcuts.metadata({
            # Key modified to avoid dots, which are disallowed in v1beta13.
            'startup-script':
            open('start_setup.sh').read(),
            'bootstrap_sh':
            open('coordinator/bootstrap.sh').read(),
            'tarball':
            cfg.gs_coordinators_tarball,
            'gs_bucket':
            cfg.gs_bucket,
            'zone':
            zone,
            'machine_type':
            machtype,
            'image':
            image,
            'disk':
            disk,
            'rw_disk_instance':
            rw_disk_instance,
            'secret':
            cfg.secret
        }),
        blocking=True)
    print

    print 'Waiting for coordinator to come online...'
    while True:
        status, _ = util.get_status(cfg.coordinator)
        print status[1]
        if status == util.InstanceState.SNITCH_READY:
            break
        time.sleep(cfg.poll_delay_secs)
    print

    print 'Controller is ready to receive commands.'