Exemple #1
0
 def create_data_snapshot(self,
                          node,
                          prefix,
                          zone='us-central1-c',
                          devel=False):
     """
     Snapshot the data disk on the given machine.  Typically used for
     backing up very important data.
     """
     zone = self.expand_zone(zone)
     instance_name = self.instance_name(node, prefix, zone, devel=devel)
     info = json.loads(
         cmd([
             'gcloud', 'compute', 'instances', 'describe', instance_name,
             '--zone', zone, '--format=json'
         ],
             verbose=0))
     for disk in info['disks']:
         if disk.get('boot', False):
             continue
         src = disk['deviceName']
         target = 'data-%s-%s' % (src, time.strftime(TIMESTAMP_FORMAT))
         log("%s --> %s", src, target)
         try:
             cmd([
                 'gcloud', 'compute', 'disks', 'snapshot', '--project',
                 self.project, src, '--snapshot-names', target, '--zone',
                 zone
             ],
                 system=True)
         except Exception, mesg:
             log("WARNING: issue making snapshot %s -- %s", target, mesg)
Exemple #2
0
    def create_all_boot_snapshots(self):
        v = []

        v.append(('kubectl', ''))

        v.append(('postgres', 0))

        #for i in [0,1,2,3,4,5]:
        #    v.append(('db', i))

        #for name in self.dev_instances():
        #    node = name.split('v')[1]
        #    v.append(('dev', node))

        #for i in [0,1,2]:
        #    v.append(('web',i))
        v.append(('admin',0))
        log("snapshotting storage machine boot images")
        for i in [0,1,2,3,4,5]:
            v.append(('storage', i))
        log("snapshotting compute machine boot images")
        for i in [0,1,2,3,4,5,6,7,8]:
            v.append(('compute', i))

        errors = []
        log("snapshotting boot images: %s"%v)
        for prefix, node in v:
            try:
                log('snapshotting %s%s'%(prefix, node))
                self.create_boot_snapshot(node=node, prefix=prefix, zone='us-central1-c', devel=False)
            except Exception, mesg:
                errors.append(mesg)
                log("WARNING: issue making snapshot -- %s", mesg)
Exemple #3
0
    def create_data_snapshot(self, node, prefix, zone='us-central1-c', devel=False):
        """
        Snapshot the data disk on the given machine.  Typically used for
        backing up very important data.
        """
        zone = self.expand_zone(zone)
        instance_name = self.instance_name(node, prefix, zone, devel=devel)
        info = json.loads(cmd(['gcloud', 'compute', 'instances', 'describe',
                               instance_name, '--zone', zone, '--format=json'], verbose=0))

        errors = []
        for disk in info['disks']:
            # ignore boot disks (would be True)
            if disk.get('boot', False):
                continue
            # ignore read-only disks (like for globally mounted data volumes)
            # they would be snapshotted manually
            if disk.get('mode', 'READ_WRITE') == 'READ_ONLY':
                continue

            src = disk['source'].split('/')[-1]
            if 'swap' in src: continue
            if 'tmp' in src: continue
            target = 'data-%s-%s'%(src, time.strftime(TIMESTAMP_FORMAT))
            log("%s --> %s", src, target)
            try:
                cmd(['gcloud', 'compute', 'disks', 'snapshot',
                     '--project', self.project,
                    src,
                     '--snapshot-names', target,
                     '--zone', zone], system=True)
            except Exception, mesg:
                log("WARNING: issue making snapshot %s -- %s", target, mesg)
                errors.append(mesg)
Exemple #4
0
 def network_costs(self):
     # These are estimates based on usage during March and April.  May be lower in future
     # do to moving everything to GCE.  Not sure.
     us = 700; aus = 150; china = 20
     costs = 700 * PRICING['egress'] + 150*PRICING['egress-australia'] + 20*PRICING['egress-china']
     log("NETWORK      : %8s/month -- approx. %sGB Americas, %sGB EUR, %sGB CHINA", money(costs), us, aus, china)
     return costs
Exemple #5
0
    def create_data_snapshot(self, node, prefix, zone='us-central1-c', devel=False):
        """
        Snapshot the data disk on the given machine.  Typically used for
        backing up very important data.
        """
        zone = self.expand_zone(zone)
        instance_name = self.instance_name(node, prefix, zone, devel=devel)
        info = json.loads(cmd(['gcloud', 'compute', 'instances', 'describe',
                               instance_name, '--zone', zone, '--format=json'], verbose=0))

        errors = []
        for disk in info['disks']:
            if disk.get('boot', False):
                continue
            src = disk['deviceName']
            if 'swap' in src: continue
            target = 'data-%s-%s'%(src, time.strftime(TIMESTAMP_FORMAT))
            log("%s --> %s", src, target)
            try:
                cmd(['gcloud', 'compute', 'disks', 'snapshot',
                     '--project', self.project,
                    src,
                     '--snapshot-names', target,
                     '--zone', zone], system=True)
            except Exception, mesg:
                log("WARNING: issue making snapshot %s -- %s", target, mesg)
                errors.append(mesg)
Exemple #6
0
 def create_all_boot_snapshots(self):
     log("snapshotting storage boot image")
     self.create_boot_snapshot(node=0,
                               prefix='storage',
                               zone='us-central1-c',
                               devel=False)
     log("snapshotting backup boot image")
     self.create_boot_snapshot(node=0,
                               prefix='backup',
                               zone='us-central1-c',
                               devel=False)
     log("snapshotting admin boot image")
     self.create_boot_snapshot(node='',
                               prefix='admin',
                               zone='us-central1-c',
                               devel=False)
     log("snapshotting SMC server boot image")
     self.create_boot_snapshot(node=0,
                               prefix='smc',
                               zone='us-central1-c',
                               devel=False)
     log("snapshotting compute machine boot image")
     self.create_boot_snapshot(node=0,
                               prefix='compute',
                               zone='us-central1-c',
                               devel=False)
Exemple #7
0
 def instance_costs(self):
     cost = cost_upper = 0
     n_compute = 0
     n_smc = 0
     n_other = 0
     for x in cmd(['gcloud', 'compute', 'instances', 'list'], verbose=0).splitlines()[1:]:
         v = x.split()
         zone         = v[1]
         machine_type = v[2]
         status       = v[-1]
         if v[0].startswith('compute'):
             n_compute += 1
         elif v[0].startswith('smc'):
             n_smc += 1
         else:
             n_other += 1
         if status == "RUNNING":
             t = machine_type.split('-')
             if len(t) == 3:
                 b = '-'.join(t[:2])
                 cpus = int(t[2])
             else:
                 b = machine_type
                 cpus = 1
             cost += PRICING[b+'-month'] * cpus * PRICING[zone.split('-')[0]]
             cost_upper += PRICING[b+'-hour'] *30.5*24* cpus * PRICING[zone.split('-')[0]]
     log("INSTANCES    : compute=%s, smc=%s, other=%s: %s/month (or %s/month with sustained use)",
         n_compute, n_smc, n_other, money(cost_upper), money(cost))
     return cost_upper
Exemple #8
0
    def create_all_boot_snapshots(self):
        v = []

        v.append(('kubectl', ''))

        for i in [0,1,2,3,4,5]:
            v.append(('db', i))

        #for name in self.dev_instances():
        #    node = name.split('v')[1]
        #    v.append(('dev', node))

        for i in [0,1,2]:
            v.append(('web',i))
        v.append(('admin',0))
        log("snapshotting storage machine boot images")
        for i in [0,1,2,3,4,5]:
            v.append(('storage', i))
        log("snapshotting compute machine boot images")
        for i in [0,1,2,3,4,5,6,7]:
            v.append(('compute', i))

        errors = []
        log("snapshotting boot images: %s"%v)
        for prefix, node in v:
            try:
                log('snapshotting %s%s'%(prefix, node))
                self.create_boot_snapshot(node=node, prefix=prefix, zone='us-central1-c', devel=False)
            except Exception, mesg:
                errors.append(mesg)
                log("WARNING: issue making snapshot -- %s", mesg)
Exemple #9
0
    def _create_compute_server(self,
                               node,
                               zone='us-central1-c',
                               machine_type='n1-highmem-4',
                               network='default',
                               projects_ssd=False,
                               base_ssd=False,
                               projects_size=150,
                               devel=False,
                               address=None,
                               preemptible=False):
        zone = self.expand_zone(zone)
        name = self.instance_name(node=node,
                                  prefix='compute',
                                  zone=zone,
                                  devel=devel)

        log("creating root filesystem image")
        try:
            opts = [
                'gcloud', 'compute', '--project', self.project, 'disks',
                'create', name, '--zone', zone, '--source-snapshot',
                self.newest_snapshot('compute')
            ]
            if base_ssd:
                opts.extend(['--type', 'pd-ssd'])
            cmd(opts)
        except Exception, mesg:
            if 'already exists' not in str(mesg):
                raise
            log("%s already exists", name)
Exemple #10
0
 def instance_costs(self):
     cost = cost_upper = 0
     n_compute = 0
     n_smc = 0
     n_other = 0
     for x in cmd(['gcloud', 'compute', 'instances', 'list'],
                  verbose=0).splitlines()[1:]:
         v = x.split()
         zone = v[1]
         machine_type = v[2]
         status = v[-1]
         if v[0].startswith('compute'):
             n_compute += 1
         elif v[0].startswith('smc'):
             n_smc += 1
         else:
             n_other += 1
         if status == "RUNNING":
             t = machine_type.split('-')
             if len(t) == 3:
                 b = '-'.join(t[:2])
                 cpus = int(t[2])
             else:
                 b = machine_type
                 cpus = 1
             cost += PRICING[b +
                             '-month'] * cpus * PRICING[zone.split('-')[0]]
             cost_upper += PRICING[b +
                                   '-hour'] * 30.5 * 24 * cpus * PRICING[
                                       zone.split('-')[0]]
     log(
         "INSTANCES    : compute=%s, smc=%s, other=%s: %s/month (or %s/month with sustained use)",
         n_compute, n_smc, n_other, money(cost_upper), money(cost))
     return cost_upper
Exemple #11
0
    def create_data_snapshot(self, node, prefix, zone='us-central1-c', devel=False):
        """
        Snapshot the data disk on the given machine.  Typically used for
        backing up very important data.
        """
        zone = self.expand_zone(zone)
        instance_name = self.instance_name(node, prefix, zone, devel=devel)
        info = json.loads(cmd(['gcloud', 'compute', 'instances', 'describe',
                               instance_name, '--zone', zone, '--format=json'], verbose=0))

        errors = []
        for disk in info['disks']:
            # ignore boot disks (would be True)
            if disk.get('boot', False):
                continue
            # ignore read-only disks (like for globally mounted data volumes)
            # they would be snapshotted manually
            if disk.get('mode', 'READ_WRITE') == 'READ_ONLY':
                continue

            src = disk['source'].split('/')[-1]
            if 'swap' in src: continue
            if 'tmp' in src: continue
            target = 'data-%s-%s'%(src, time.strftime(TIMESTAMP_FORMAT))
            log("%s --> %s", src, target)
            try:
                cmd(['gcloud', 'compute', 'disks', 'snapshot',
                     '--project', self.project,
                    src,
                     '--snapshot-names', target,
                     '--zone', zone], system=True)
            except Exception, mesg:
                log("WARNING: issue making snapshot %s -- %s", target, mesg)
                errors.append(mesg)
Exemple #12
0
 def network_costs(self):
     # These are estimates based on usage during March and April.  May be lower in future
     # do to moving everything to GCE.  Not sure.
     us = 700; aus = 150; china = 20
     costs = 700 * PRICING['egress'] + 150*PRICING['egress-australia'] + 20*PRICING['egress-china']
     log("NETWORK      : %8s/month -- approx. %sGB Americas, %sGB EUR, %sGB CHINA", money(costs), us, aus, china)
     return costs
Exemple #13
0
 def network_costs(self):
     # These are estimates based on usage during March and April.  May be lower in future
     # do to moving everything to GCE.  Not sure.
     costs = 800 * PRICING['egress'] + 15 * PRICING[
         'egress-australia'] + 15 * PRICING['egress-china']
     log("NETWORK      : approx. %s/month", money(costs))
     return costs
Exemple #14
0
 def set_boot_auto_delete(self, name, zone):
     log("set boot disk of %s to auto-delete" % name)
     cmd([
         'gcloud', 'compute', '--project', self.project, 'instances',
         'set-disk-auto-delete', name, '--zone', zone, '--disk', name,
         '--auto-delete'
     ])
Exemple #15
0
 def costs(self):
     costs = {}
     total = 0
     for t in ['snapshot', 'disk', 'instance', 'network', 'gcs']:
         costs[t] = getattr(self, '%s_costs' % t)()
         total += costs[t]
     log("TOTAL        : %s/month", money(total))
     return costs
Exemple #16
0
 def costs(self):
     costs = {}
     total = 0
     for t in ['snapshot', 'disk', 'instance', 'network', 'gcs']:
         costs[t] = getattr(self, '%s_costs'%t)()
         total += costs[t]
     log("TOTAL        : %s/month", money(total))
     return costs
Exemple #17
0
    def create_all_data_snapshots(self, zone='us-central1-c'):
        for i in range(6):
            log("snapshotting storage%s storage data"%i)
            self.create_data_snapshot(node=i, prefix='storage', zone=zone, devel=False)

        log("snapshotting live user data")
        for n in self.compute_nodes(zone):
            self.create_data_snapshot(node=n, prefix='compute', zone=zone, devel=False)
Exemple #18
0
 def instance_costs(self):
     cost_lower = cost_upper = 0
     n_compute = 0
     n_web = 0
     n_db = 0
     n_other = 0
     n_dev = 0
     n_admin = 0
     n_storage = 0
     n_preempt = 0
     for x in cmd(['gcloud', 'compute', 'instances', 'list'],
                  verbose=0).splitlines()[1:]:
         v = x.split()
         zone = v[1]
         machine_type = v[2]
         status = v[-1]
         if status != 'RUNNING':
             continue
         if len(v) == 7:
             preempt = (v[3] == 'true')
             n_preempt += 1
         else:
             preempt = False
         if v[0].startswith('compute'):
             n_compute += 1
         elif v[0].startswith('web'):
             n_web += 1
         elif v[0].startswith('db'):
             n_db += 1
         elif v[0].startswith('dev'):
             n_dev += 1
         elif v[0].startswith('admin'):
             n_admin += 1
         elif v[0].startswith('storage'):
             n_storage += 1
         else:
             n_other += 1
         t = machine_type.split('-')
         if len(t) == 3:
             b = '-'.join(t[:2])
             cpus = int(t[2])
         else:
             b = machine_type
             cpus = 1
         if preempt:
             pricing_hour = PRICING[b + '-hour-pre']
             pricing_month = pricing_hour * 24 * 30.5
         else:
             pricing_hour = PRICING[b + '-hour']
             pricing_month = PRICING[b + '-month']
         cost_lower += pricing_month * cpus * PRICING[zone.split('-')[0]]
         cost_upper += pricing_hour * 30.5 * 24 * cpus * PRICING[zone.split(
             '-')[0]]
     log(
         "INSTANCES    : %8s/month -- (or %8s/month without sustained!); compute=%s, web=%s, db=%s, dev=%s, admin=%s, storage=%s, other=%s (preempt=%s)",
         money(cost_lower), money(cost_upper), n_compute, n_web, n_db,
         n_dev, n_admin, n_storage, n_other, n_preempt)
     return {'lower': cost_lower, 'upper': cost_upper}
Exemple #19
0
 def delete_devel_instances(self):
     for x in cmd(['gcloud', 'compute', 'instances', 'list'], verbose=0).splitlines()[1:]:
         v = x.split()
         name         = v[0]
         if '-devel-' in name:
             zone         = v[1]
             status       = v[-1]
             log("deleting devel instance: %s"%name)
             cmd(['gcloud', 'compute', 'instances', 'delete', '--zone', zone, name], system=True)
Exemple #20
0
def setup_projects_path():
    log("create paths")
    cmd("sudo mkdir -p /projects")
    cmd("sudo chmod a+x /projects")
    cmd("sudo touch /projects/snapshots; sudo chmod a+r /projects/snapshots")
    cmd("sudo mkdir -p /projects/conf")
    cmd("sudo chown salvus. /projects/conf")
    cmd("sudo mkdir -p /projects/sagemathcloud")
    cmd("sudo rsync -LrxH --delete /home/salvus/salvus/salvus/local_hub_template/ /projects/sagemathcloud/")
Exemple #21
0
 def instance_costs(self):
     cost_lower = cost_upper = 0
     n_compute = 0
     n_web = 0
     n_db = 0
     n_other = 0
     n_dev =0
     n_admin =0
     n_storage =0
     n_preempt = 0
     for x in cmd(['gcloud', 'compute', 'instances', 'list'], verbose=0).splitlines()[1:]:
         v = x.split()
         zone         = v[1]
         machine_type = v[2]
         status       = v[-1]
         if status != 'RUNNING':
             continue
         if len(v) == 7:
             preempt = (v[3] == 'true')
             n_preempt += 1
         else:
             preempt = False
         if v[0].startswith('compute'):
             n_compute += 1
         elif v[0].startswith('web'):
             n_web += 1
         elif v[0].startswith('db'):
             n_db += 1
         elif v[0].startswith('dev'):
             n_dev += 1
         elif v[0].startswith('admin'):
             n_admin += 1
         elif v[0].startswith('storage'):
             n_storage += 1
         else:
             n_other += 1
         t = machine_type.split('-')
         if len(t) == 3:
             b = '-'.join(t[:2])
             cpus = int(t[2])
         else:
             b = machine_type
             cpus = 1
         if b == 'custom':
             print("warning -custom machine types not supported; skipping ", x)
             continue
         if preempt:
             pricing_hour  = PRICING[b+'-hour-pre']
             pricing_month = pricing_hour*24*30.5
         else:
             pricing_hour  = PRICING[b+'-hour']
             pricing_month = PRICING[b+'-month']
         cost_lower += pricing_month * cpus * PRICING[zone.split('-')[0]]
         cost_upper += pricing_hour *30.5*24* cpus * PRICING[zone.split('-')[0]]
     log("INSTANCES    : %8s/month -- (or %8s/month without sustained!); compute=%s, web=%s, db=%s, dev=%s, admin=%s, storage=%s, other=%s (preempt=%s)",
          money(cost_lower), money(cost_upper), n_compute, n_web, n_db, n_dev, n_admin, n_storage,  n_other, n_preempt)
     return {'lower':cost_lower, 'upper':cost_upper}
Exemple #22
0
 def delete_devel_instances(self):
     for x in cmd(['gcloud', 'compute', 'instances', 'list'], verbose=0).splitlines()[1:]:
         v = x.split()
         name         = v[0]
         if '-devel-' in name:
             zone         = v[1]
             status       = v[-1]
             log("deleting devel instance: %s"%name)
             cmd(['gcloud', 'compute', 'instances', 'delete', '--zone', zone, name], system=True)
Exemple #23
0
def setup_projects_path():
    log("create paths")
    cmd("sudo mkdir -p /projects")
    cmd("sudo chmod a+x /projects")
    cmd("sudo touch /projects/snapshots; sudo chmod a+r /projects/snapshots")
    cmd("sudo mkdir -p /projects/conf")
    cmd("sudo chown salvus. /projects/conf")
    cmd("sudo mkdir -p /projects/sagemathcloud")
    cmd("sudo rsync -LrxH --delete /home/salvus/salvus/salvus/local_hub_template/ /projects/sagemathcloud/"
        )
Exemple #24
0
 def start_devel_instances(self):
     for x in cmd(['gcloud', 'compute', 'instances', 'list']).splitlines()[1:]:
         v = x.split()
         name         = v[0]
         if '-devel-' in name:
             zone         = v[1]
             status       = v[-1]
             if status == "TERMINATED":
                 log("starting %s"%name)
                 cmd(['gcloud', 'compute', 'instances', 'start', '--zone', zone, name])
Exemple #25
0
 def start_devel_instances(self):
     for x in cmd(['gcloud', 'compute', 'instances', 'list']).splitlines()[1:]:
         v = x.split()
         name         = v[0]
         if '-devel-' in name:
             zone         = v[1]
             status       = v[-1]
             if status == "TERMINATED":
                 log("starting %s"%name)
                 cmd(['gcloud', 'compute', 'instances', 'start', '--zone', zone, name])
Exemple #26
0
def delete_secrets():
    log("delete any possible sensitive info from the production install")
    log("wipe root ssh keys")
    cmd("sudo rm -f /root/.ssh/id_rsa /root/.ssh/id_rsa.pub")
    log("wipe salvus ssh keys")
    cmd("sudo rm -rf /home/salvus/.ssh/id_rsa*")
    log("wipe salvus secrets")
    cmd("sudo rm -rf /home/salvus/salvus/salvus/data/secrets/")
    log("wipe production logs")
    cmd("sudo rm -rf /home/salvus/logs/*")
Exemple #27
0
def create_data_secrets():
    cmd("mkdir -p %s"%SECRETS)
    log("sendgrid fake password (will not work)")
    cmd("echo 'will-not-work' > %s/sendgrid_email_password"%SECRETS)
    log("generate cassandra passwords")
    cmd("mkdir -p %s/cassandra"%SECRETS)
    cmd("makepasswd -q > %s/cassandra/hub"%SECRETS)
    cmd("makepasswd -q > %s/cassandra/salvus"%SECRETS)
    cmd("mkdir -p %s/sagemath.com"%SECRETS)
    cmd("yes US | openssl req -x509 -newkey rsa:2048 -keyout key.pem -out cert.pem -nodes -days 10000 && cat key.pem cert.pem > %s/sagemath.com/nopassphrase.pem"%SECRETS)
Exemple #28
0
def delete_secrets():
    log("delete any possible sensitive info from the production install")
    log("wipe root ssh keys")
    cmd("sudo rm -f /root/.ssh/id_rsa /root/.ssh/id_rsa.pub")
    log("wipe salvus ssh keys")
    cmd("sudo rm -rf /home/salvus/.ssh/id_rsa*")
    log("wipe salvus secrets")
    cmd("sudo rm -rf /home/salvus/salvus/salvus/data/secrets/")
    log("wipe production logs")
    cmd("sudo rm -rf /home/salvus/logs/*")
Exemple #29
0
    def create_all_data_snapshots(self, zone='us-central1-c'):
        # database backup disk
        #self.create_data_snapshot(node='-backup', prefix='db', zone=zone, devel=False)

        for i in range(6):
            log("snapshotting storage%s storage data"%i)
            self.create_data_snapshot(node=i, prefix='storage', zone=zone, devel=False)

        log("snapshotting live user data")
        for n in self.compute_nodes(zone):
            self.create_data_snapshot(node=n, prefix='compute', zone=zone, devel=False)
Exemple #30
0
 def create_all_boot_snapshots(self):
     log("snapshotting storage boot image")
     self.create_boot_snapshot(node=0, prefix='storage', zone='us-central1-c', devel=False)
     log("snapshotting backup boot image")
     self.create_boot_snapshot(node=0, prefix='backup', zone='us-central1-c', devel=False)
     log("snapshotting admin boot image")
     self.create_boot_snapshot(node='',prefix='admin', zone='us-central1-c', devel=False)
     log("snapshotting SMC server boot image")
     self.create_boot_snapshot(node=0, prefix='smc', zone='us-central1-c', devel=False)
     log("snapshotting compute machine boot image")
     self.create_boot_snapshot(node=0, prefix='compute', zone='us-central1-c', devel=False)
Exemple #31
0
def create_data_secrets():
    cmd("mkdir -p %s" % SECRETS)
    log("sendgrid fake password (will not work)")
    cmd("echo 'will-not-work' > %s/sendgrid_email_password" % SECRETS)
    log("generate cassandra passwords")
    cmd("mkdir -p %s/cassandra" % SECRETS)
    cmd("makepasswd -q > %s/cassandra/hub" % SECRETS)
    cmd("makepasswd -q > %s/cassandra/salvus" % SECRETS)
    cmd("mkdir -p %s/sagemath.com" % SECRETS)
    cmd("yes US | openssl req -x509 -newkey rsa:2048 -keyout key.pem -out cert.pem -nodes -days 10000 && cat key.pem cert.pem > %s/sagemath.com/nopassphrase.pem"
        % SECRETS)
Exemple #32
0
 def create_all_data_snapshots(self, zone='us-central1-c'):
     log("snapshotting storage data")
     self.create_data_snapshot(node=0,
                               prefix='storage',
                               zone=zone,
                               devel=False)
     log("snapshotting live user data")
     for n in self.compute_nodes(zone):
         self.create_data_snapshot(node=n,
                                   prefix='compute',
                                   zone=zone,
                                   devel=False)
Exemple #33
0
    def gcs_costs(self):
        # usage based on running "time gsutil du -sch" every once in a while, since it takes
        # quite a while to run.
        cassandra = 200
        database_backup	= 200
        gb_archive = 650  # delete in a few weeks...
        projects_backup = 1500

        usage = (database_backup + gb_archive + projects_backup)
        costs = usage * PRICING['gcs-nearline']
        log("CLOUD STORAGE: approx. %sGB nearline: %s/month", usage, money(costs))
        return costs
Exemple #34
0
    def create_dev(self, node, zone='us-central1-c', machine_type='n1-standard-1', size=30, preemptible=True, address=''):
        zone = self.expand_zone(zone)
        name = self.instance_name(node=node, prefix='dev', zone=zone)

        log("creating %sGB hard disk root filesystem image based on last smc snapshot", size)
        try:
            cmd(['gcloud', 'compute', '--project', self.project, 'disks', 'create', name,
                 '--zone', zone, '--source-snapshot', self.newest_snapshot('smc'),
                 '--size', size, '--type', 'pd-standard'])
        except Exception, mesg:
            if 'already exists' not in str(mesg):
                raise
Exemple #35
0
    def create_dev(self, node, zone='us-central1-c', machine_type='n1-standard-1', size=30, preemptible=True, address=''):
        zone = self.expand_zone(zone)
        name = self.instance_name(node=node, prefix='dev', zone=zone)

        log("creating %sGB hard disk root filesystem image based on last smc snapshot", size)
        try:
            cmd(['gcloud', 'compute', '--project', self.project, 'disks', 'create', name,
                 '--zone', zone, '--source-snapshot', self.newest_snapshot('smc'),
                 '--size', size, '--type', 'pd-standard'])
        except Exception, mesg:
            if 'already exists' not in str(mesg):
                raise
Exemple #36
0
 def autostart(self, instance):
     """
     Ensure that each instance in the input is running.
     """
     for x in cmd(['gcloud', 'compute', 'instances', 'list'], verbose=0).splitlines()[1:]:
         v = x.split()
         if len(v) > 2 and v[-1] != 'RUNNING':
             name = v[0]; zone = v[1]
             for x in instance:
                 if name.startswith(x):
                     log("Starting %s... at %s", name, time.asctime())
                     cmd(' '.join(['gcloud', 'compute', 'instances', 'start', '--zone', zone, name]) + '&', system=True)
                     break
Exemple #37
0
    def gcs_costs(self):
        # usage based on running "time gsutil du -sch" every once in a while, since it takes
        # quite a while to run.
        smc_db_backup = 50
        smc_projects_backup = 1300
        # This takes about 15 minutes and gives the usage above
        #     time gsutil du -sch gs://smc-projects-bup
        #     time gsutil du -sch gs://smc-db-backup

        usage = (smc_db_backup + smc_projects_backup)
        costs = usage * PRICING['gcs-nearline']
        log("CLOUD STORAGE: %8s/month -- approx. %sGB nearline", money(costs), usage)
        return costs
Exemple #38
0
    def gcs_costs(self):
        # usage based on running "time gsutil du -sch" every once in a while, since it takes
        # quite a while to run.
        cassandra = 200
        database_backup = 200
        gb_archive = 650  # delete in a few weeks...
        projects_backup = 1500

        usage = (database_backup + gb_archive + projects_backup)
        costs = usage * PRICING['gcs-nearline']
        log("CLOUD STORAGE: approx. %sGB nearline: %s/month", usage,
            money(costs))
        return costs
Exemple #39
0
 def costs(self):
     costs = {}
     total_lower = 0
     total_upper = 0
     for t in ['snapshot', 'disk', 'instance', 'network', 'gcs']:
         costs[t] = getattr(self, '%s_costs'%t)()
         if isinstance(costs[t], dict):
             total_lower += costs[t]['lower']
             total_upper += costs[t]['upper']
         else:
             total_lower += costs[t]
             total_upper += costs[t]
     log("TOTAL        : between %s/month and %s/month", money(total_lower), money(total_upper))
Exemple #40
0
 def costs(self):
     costs = {}
     total_lower = 0
     total_upper = 0
     for t in ['snapshot', 'disk', 'instance', 'network', 'gcs']:
         costs[t] = getattr(self, '%s_costs'%t)()
         if isinstance(costs[t], dict):
             total_lower += costs[t]['lower']
             total_upper += costs[t]['upper']
         else:
             total_lower += costs[t]
             total_upper += costs[t]
     log("TOTAL        : %8s/month -- up to as worse as %8s/month without sustained", money(total_lower), money(total_upper))
Exemple #41
0
 def autostart(self, instance):
     """
     Ensure that each instance in the input is running.
     """
     for x in cmd(['gcloud', 'compute', 'instances', 'list'], verbose=0).splitlines()[1:]:
         v = x.split()
         if len(v) > 2 and v[-1] != 'RUNNING':
             name = v[0]; zone = v[1]
             for x in instance:
                 if name.startswith(x):
                     log("Starting %s... at %s", name, time.asctime())
                     cmd(' '.join(['gcloud', 'compute', 'instances', 'start', '--zone', zone, name]) + '&', system=True)
                     break
Exemple #42
0
def init_cassandra_users():
    pw_hub = open("%s/cassandra/hub"%SECRETS).read()
    cmd("""echo "CREATE USER hub WITH PASSWORD '%s' SUPERUSER;" | cqlsh localhost -u cassandra -p cassandra"""%pw_hub, verbose=0)
    rc = "%s/.cqlshrc"%os.environ['HOME']
    log("writing %s", rc)
    open(rc, 'w').write("""
[authentication]
username=hub
password=%s
"""%pw_hub)
    pw_salvus = open("%s/cassandra/salvus"%SECRETS).read()
    cmd("""echo "CREATE USER salvus WITH PASSWORD '%s' SUPERUSER;" | cqlsh localhost -u cassandra -p cassandra"""%pw_salvus, verbose=0)
    cmd("""echo "ALTER USER cassandra WITH PASSWORD '%s';" | cqlsh localhost -u cassandra -p cassandra"""%pw_hub, verbose=0)
Exemple #43
0
    def gcs_costs(self):
        # usage based on running "time gsutil du -sch" every once in a while, since it takes
        # quite a while to run.
        smc_db_backup = 50
        smc_projects_backup = 1300
        # This takes about 15 minutes and gives the usage above
        #     time gsutil du -sch gs://smc-projects-bup
        #     time gsutil du -sch gs://smc-db-backup

        usage = (smc_db_backup + smc_projects_backup)
        costs = usage * PRICING['gcs-nearline']
        log("CLOUD STORAGE: %8s/month -- approx. %sGB nearline", money(costs), usage)
        return costs
Exemple #44
0
    def _create_smc_server(self, node, zone='us-central1-c', machine_type='n1-highmem-2',
                          disk_size=100, network='default', devel=False):
        zone = self.expand_zone(zone)
        name = self.instance_name(node=node, prefix='smc', zone=zone, devel=devel)
        disk_name = "%s-cassandra"%name

        log("creating hard disk root filesystem image")
        try:
            cmd(['gcloud', 'compute', '--project', self.project, 'disks', 'create', name,
                 '--zone', zone, '--source-snapshot', self.newest_snapshot('smc'),
                 '--type', 'pd-standard'])
        except Exception, mesg:
            if 'already exists' not in str(mesg):
                raise
Exemple #45
0
    def _create_storage_server(self, node, zone, machine_type,
                               disk_size, network, devel):
        zone = self.expand_zone(zone)
        name = self.instance_name(node=node, prefix='storage', zone=zone, devel=devel)
        disk_name = "%s-projects"%name

        log("creating hard disk root filesystem image")
        try:
            cmd(['gcloud', 'compute', '--project', self.project, 'disks', 'create', name,
                 '--zone', zone, '--source-snapshot', self.newest_snapshot('storage'),
                 '--type', 'pd-standard'])
        except Exception, mesg:
            if 'already exists' not in str(mesg):
                raise
Exemple #46
0
 def costs(self):
     costs = {}
     total_lower = 0
     total_upper = 0
     for t in ['snapshot', 'disk', 'instance', 'network', 'gcs']:
         costs[t] = getattr(self, '%s_costs' % t)()
         if isinstance(costs[t], dict):
             total_lower += costs[t]['lower']
             total_upper += costs[t]['upper']
         else:
             total_lower += costs[t]
             total_upper += costs[t]
     log("TOTAL        : between %s/month and %s/month", money(total_lower),
         money(total_upper))
Exemple #47
0
    def _create_storage_server(self, node, zone, machine_type,
                               disk_size, network, devel):
        zone = self.expand_zone(zone)
        name = self.instance_name(node=node, prefix='storage', zone=zone, devel=devel)
        disk_name = "%s-projects"%name

        log("creating hard disk root filesystem image")
        try:
            cmd(['gcloud', 'compute', '--project', self.project, 'disks', 'create', name,
                 '--zone', zone, '--source-snapshot', self.newest_snapshot('storage'),
                 '--type', 'pd-standard'])
        except Exception, mesg:
            if 'already exists' not in str(mesg):
                raise
Exemple #48
0
    def _create_smc_server(self, node, zone='us-central1-c', machine_type='n1-highmem-2',
                          disk_size=100, network='default', devel=False):
        zone = self.expand_zone(zone)
        name = self.instance_name(node=node, prefix='smc', zone=zone, devel=devel)
        disk_name = "%s-cassandra"%name

        log("creating hard disk root filesystem image")
        try:
            cmd(['gcloud', 'compute', '--project', self.project, 'disks', 'create', name,
                 '--zone', zone, '--source-snapshot', self.newest_snapshot('smc'),
                 '--type', 'pd-standard'])
        except Exception, mesg:
            if 'already exists' not in str(mesg):
                raise
Exemple #49
0
 def disk_costs(self):
     cost = 0
     usage_standard = 0
     usage_ssd = 0
     for x in cmd(['gcloud', 'compute', 'disks', 'list'], verbose=0).splitlines()[1:]:
         v = x.split()
         size = int(v[2])
         typ = v[3]
         if typ == 'pd-ssd':
             usage_ssd += size
         elif typ == 'pd-standard':
             usage_standard += size
         cost += size * PRICING[typ]
     log("DISK         : %8s/month -- storage (standard=%sGB, ssd=%sGB)",
         money(cost), usage_standard, usage_ssd)
     return cost
Exemple #50
0
 def disk_costs(self):
     cost = 0
     usage_standard = 0
     usage_ssd = 0
     for x in cmd(['gcloud', 'compute', 'disks', 'list'], verbose=0).splitlines()[1:]:
         v = x.split()
         size = int(v[2])
         typ = v[3]
         if typ == 'pd-ssd':
             usage_ssd += size
         elif typ == 'pd-standard':
             usage_standard += size
         cost += size * PRICING[typ]
     log("DISK         : %8s/month -- storage (standard=%sGB, ssd=%sGB)",
         money(cost), usage_standard, usage_ssd)
     return cost
Exemple #51
0
 def autostart(self, instance):
     """
     Ensure that each instance in the input is running.
     """
     for x in cmd(['gcloud', 'compute', 'instances', 'list'] + instance,
                  verbose=0).splitlines()[1:]:
         v = x.split()
         if len(v) > 2 and v[-1] != 'RUNNING':
             name = v[0]
             zone = v[1]
             log("Starting %s...", name)
             cmd(' '.join([
                 'gcloud', 'compute', 'instances', 'start', '--zone', zone,
                 name
             ]) + '&',
                 system=True)
Exemple #52
0
def init_compute_server():
    log("starting compute server")
    cmd("compute start")
    log("making log link: ~/logs/compute.log")
    cmd("ln -sf /projects/conf/compute.log %s/logs/compute.log"%os.environ['HOME'])
    log("waiting a few seconds")
    import time; time.sleep(5)
    log("adding compute server to database")
    cmd(r"""echo "require('compute').compute_server(keyspace:'salvus', cb:(e,s)->console.log(e); s.add_server(host:'%s', cb:(e)->console.log('done',e);process.exit(0)))" | coffee """%hostname)
Exemple #53
0
 def instance_costs(self):
     cost_lower = cost_upper = 0
     n_compute = 0
     n_smc = 0
     n_other = 0
     n_preempt = 0
     for x in cmd(['gcloud', 'compute', 'instances', 'list'],
                  verbose=0).splitlines()[1:]:
         v = x.split()
         zone = v[1]
         machine_type = v[2]
         status = v[-1]
         if status != 'RUNNING':
             continue
         if len(v) == 7:
             preempt = (v[3] == 'true')
             n_preempt += 1
         else:
             preempt = False
         if v[0].startswith('compute'):
             n_compute += 1
         elif v[0].startswith('smc'):
             n_smc += 1
         else:
             n_other += 1
         t = machine_type.split('-')
         if len(t) == 3:
             b = '-'.join(t[:2])
             cpus = int(t[2])
         else:
             b = machine_type
             cpus = 1
         if preempt:
             pricing_hour = PRICING[b + '-hour-pre']
             pricing_month = pricing_hour * 24 * 30.5
         else:
             pricing_hour = PRICING[b + '-hour']
             pricing_month = PRICING[b + '-month']
         cost_lower += pricing_month * cpus * PRICING[zone.split('-')[0]]
         cost_upper += pricing_hour * 30.5 * 24 * cpus * PRICING[zone.split(
             '-')[0]]
     log(
         "INSTANCES    : compute=%s, smc=%s, other=%s (preempt=%s): %s/month (or %s/month with sustained use)",
         n_compute, n_smc, n_other, n_preempt, money(cost_upper),
         money(cost_lower))
     return {'lower': cost_lower, 'upper': cost_upper}
Exemple #54
0
 def costs(self):
     costs = {}
     total_lower = 0
     total_upper = 0
     for t in ['snapshot', 'disk', 'instance', 'network', 'gcs']:
         costs[t] = getattr(self, '%s_costs' % t)()
         if isinstance(costs[t], dict):
             total_lower += costs[t]['lower']
             total_upper += costs[t]['upper']
         else:
             total_lower += costs[t]
             total_upper += costs[t]
     log("SALES TAX    : %8s/month -- 9.5%% WA+Seattle sales tax",
         money(total_lower * 0.095))
     log(
         "TOTAL        : %8s/month -- up to as worse as %8s/month without sustained",
         money(total_lower * 1.095), money(total_upper * 1.095))
Exemple #55
0
    def _create_compute_server(self, node, zone='us-central1-c',
                              machine_type='n1-highmem-4', network='default',
                              projects_ssd=False, base_ssd=False,
                              projects_size=150,
                              devel=False,
                              address=None,
                              preemptible=False):
        zone = self.expand_zone(zone)
        name = self.instance_name(node=node, prefix='compute', zone=zone, devel=devel)

        log("creating root filesystem image")
        try:
            opts = ['gcloud', 'compute', '--project', self.project, 'disks', 'create', name,
                 '--zone', zone, '--source-snapshot', self.newest_snapshot('compute')]
            if base_ssd:
                opts.extend(['--type', 'pd-ssd'])
            cmd(opts)
        except Exception as mesg:
            if 'already exists' not in str(mesg):
                raise
            log("%s already exists", name)

        log("creating /dev/sdb persistent disk")
        disk_name = "%s-projects"%name
        try:
            opts = ['gcloud', 'compute', '--project', self.project, 'disks', 'create', disk_name,
                    '--size', projects_size, '--zone', zone]
            if projects_ssd:
                opts.extend(['--type', 'pd-ssd'])
            cmd(opts)
        except Exception as mesg:
            if 'already exists' not in str(mesg):
                raise

        log("creating and starting compute instance")
        opts =['gcloud', 'compute', '--project', self.project, 'instances', 'create', name,
             '--zone', zone, '--machine-type', machine_type, '--network', network]
        if address:
            opts.extend(["--address", address])
        if preemptible:
            opts.append('--preemptible')
        else:
            opts.extend(['--maintenance-policy', 'MIGRATE'])
        opts.extend(['--scopes', 'https://www.googleapis.com/auth/logging.write',
             '--disk', 'name=%s,device-name=%s,mode=rw,boot=yes'%(name, name)])
        opts.extend(['--disk', 'name=%s'%disk_name, 'device-name=%s'%disk_name, 'mode=rw'])
        opts.extend(['--tags', 'compute'])
        #if local_ssd:
        #    opts.append('--local-ssd')
        #else:
        cmd(opts, system=True)

        if devel:
            self.set_boot_auto_delete(name=name, zone=zone)
Exemple #56
0
    def _create_compute_server(self, node, zone='us-central1-c',
                              machine_type='n1-highmem-4', network='default',
                              projects_ssd=False, base_ssd=False,
                               projects_size=150,
                              devel=False):
        zone = self.expand_zone(zone)
        name = self.instance_name(node=node, prefix='compute', zone=zone, devel=devel)

        log("creating root filesystem image")
        try:
            opts = ['gcloud', 'compute', '--project', self.project, 'disks', 'create', name,
                 '--zone', zone, '--source-snapshot', self.newest_snapshot('compute')]
            if base_ssd:
                opts.extend(['--type', 'pd-ssd'])
            cmd(opts)
        except Exception, mesg:
            if 'already exists' not in str(mesg):
                raise
            log("%s already exists", name)
Exemple #57
0
 def instance_costs(self):
     cost_lower = cost_upper = 0
     n_compute = 0
     n_smc = 0
     n_other = 0
     n_preempt = 0
     for x in cmd(['gcloud', 'compute', 'instances', 'list'], verbose=0).splitlines()[1:]:
         v = x.split()
         zone         = v[1]
         machine_type = v[2]
         status       = v[-1]
         if status != 'RUNNING':
             continue
         if len(v) == 7:
             preempt = (v[3] == 'true')
             n_preempt += 1
         else:
             preempt = False
         if v[0].startswith('compute'):
             n_compute += 1
         elif v[0].startswith('smc'):
             n_smc += 1
         else:
             n_other += 1
         t = machine_type.split('-')
         if len(t) == 3:
             b = '-'.join(t[:2])
             cpus = int(t[2])
         else:
             b = machine_type
             cpus = 1
         if preempt:
             pricing_hour  = PRICING[b+'-hour-pre']
             pricing_month = pricing_hour*24*30.5
         else:
             pricing_hour  = PRICING[b+'-hour']
             pricing_month = PRICING[b+'-month']
         cost_lower += pricing_month * cpus * PRICING[zone.split('-')[0]]
         cost_upper += pricing_hour *30.5*24* cpus * PRICING[zone.split('-')[0]]
     log("INSTANCES    : compute=%s, smc=%s, other=%s (preempt=%s): %s/month (or %s/month with sustained use)",
         n_compute, n_smc, n_other, n_preempt, money(cost_upper), money(cost_lower))
     return {'lower':cost_lower, 'upper':cost_upper}
Exemple #58
0
 def create_all_data_snapshots(self):
     log("snapshotting a database node")
     self.create_data_snapshot(node=0, prefix='smc', zone='us-central1-c', devel=False)
     log("snapshotting storage data")
     self.create_data_snapshot(node=0, prefix='storage', zone='us-central1-c', devel=False)
     log("snapshotting live user data")
     for n in ['0', '1', '2', '3', '4']:  # TODO -- automate this!!!!!
         self.create_data_snapshot(node=n, prefix='compute', zone='us-central1-c', devel=False)
Exemple #59
0
def start_cassandra():
    log("start_cassandra...")
    services = admin.Services('conf/deploy_devel/', password='')
    services.start('cassandra')
    cmd("ln -sf %s/data/cassandra-0/logs/system.log %s/logs/cassandra.log"%(SALVUS_ROOT, os.environ['HOME']))
    log("cassandra started")
    log("waiting 30 seconds...")
    import time; time.sleep(30)
Exemple #60
0
def setup_quota():
    log("quota packages")
    cmd("sudo apt-get install -y libatlas3gf-base liblapack-dev  quota quotatool linux-image-extra-virtual cgroup-lite cgmanager-utils cgroup-bin libpam-cgroup cgmanager cgmanager-utils  cgroup-bin smem", system=True)
    log("quota stuff")
    cmd("echo 'LABEL=cloudimg-rootfs /        ext4   defaults,usrquota       0 0' | sudo tee /etc/fstab")
    cmd("sudo mount -o remount /")
    log("initializing quota, which will take a while")
    cmd("sudo quotacheck -fucm /")
    cmd("sudo quotaon /")