def create_data_snapshot(self,
node,
prefix,
zone='us-central1-c',
devel=False):
"""
Snapshot the data disk on the given machine. Typically used for
backing up very important data.
"""
zone = self.expand_zone(zone)
instance_name = self.instance_name(node, prefix, zone, devel=devel)
info = json.loads(
cmd([
'gcloud', 'compute', 'instances', 'describe', instance_name,
'--zone', zone, '--format=json'
],
verbose=0))
for disk in info['disks']:
if disk.get('boot', False):
continue
src = disk['deviceName']
target = 'data-%s-%s' % (src, time.strftime(TIMESTAMP_FORMAT))
log("%s --> %s", src, target)
try:
cmd([
'gcloud', 'compute', 'disks', 'snapshot', '--project',
self.project, src, '--snapshot-names', target, '--zone',
zone
],
system=True)
except Exception, mesg:
log("WARNING: issue making snapshot %s -- %s", target, mesg)
def create_all_boot_snapshots(self):
v = []
v.append(('kubectl', ''))
v.append(('postgres', 0))
#for i in [0,1,2,3,4,5]:
# v.append(('db', i))
#for name in self.dev_instances():
# node = name.split('v')[1]
# v.append(('dev', node))
#for i in [0,1,2]:
# v.append(('web',i))
v.append(('admin',0))
log("snapshotting storage machine boot images")
for i in [0,1,2,3,4,5]:
v.append(('storage', i))
log("snapshotting compute machine boot images")
for i in [0,1,2,3,4,5,6,7,8]:
v.append(('compute', i))
errors = []
log("snapshotting boot images: %s"%v)
for prefix, node in v:
try:
log('snapshotting %s%s'%(prefix, node))
self.create_boot_snapshot(node=node, prefix=prefix, zone='us-central1-c', devel=False)
except Exception, mesg:
errors.append(mesg)
log("WARNING: issue making snapshot -- %s", mesg)
def create_data_snapshot(self, node, prefix, zone='us-central1-c', devel=False):
"""
Snapshot the data disk on the given machine. Typically used for
backing up very important data.
"""
zone = self.expand_zone(zone)
instance_name = self.instance_name(node, prefix, zone, devel=devel)
info = json.loads(cmd(['gcloud', 'compute', 'instances', 'describe',
instance_name, '--zone', zone, '--format=json'], verbose=0))
errors = []
for disk in info['disks']:
# ignore boot disks (would be True)
if disk.get('boot', False):
continue
# ignore read-only disks (like for globally mounted data volumes)
# they would be snapshotted manually
if disk.get('mode', 'READ_WRITE') == 'READ_ONLY':
continue
src = disk['source'].split('/')[-1]
if 'swap' in src: continue
if 'tmp' in src: continue
target = 'data-%s-%s'%(src, time.strftime(TIMESTAMP_FORMAT))
log("%s --> %s", src, target)
try:
cmd(['gcloud', 'compute', 'disks', 'snapshot',
'--project', self.project,
src,
'--snapshot-names', target,
'--zone', zone], system=True)
except Exception, mesg:
log("WARNING: issue making snapshot %s -- %s", target, mesg)
errors.append(mesg)
def network_costs(self):
# These are estimates based on usage during March and April. May be lower in future
# do to moving everything to GCE. Not sure.
us = 700; aus = 150; china = 20
costs = 700 * PRICING['egress'] + 150*PRICING['egress-australia'] + 20*PRICING['egress-china']
log("NETWORK : %8s/month -- approx. %sGB Americas, %sGB EUR, %sGB CHINA", money(costs), us, aus, china)
return costs
def create_data_snapshot(self, node, prefix, zone='us-central1-c', devel=False):
"""
Snapshot the data disk on the given machine. Typically used for
backing up very important data.
"""
zone = self.expand_zone(zone)
instance_name = self.instance_name(node, prefix, zone, devel=devel)
info = json.loads(cmd(['gcloud', 'compute', 'instances', 'describe',
instance_name, '--zone', zone, '--format=json'], verbose=0))
errors = []
for disk in info['disks']:
if disk.get('boot', False):
continue
src = disk['deviceName']
if 'swap' in src: continue
target = 'data-%s-%s'%(src, time.strftime(TIMESTAMP_FORMAT))
log("%s --> %s", src, target)
try:
cmd(['gcloud', 'compute', 'disks', 'snapshot',
'--project', self.project,
src,
'--snapshot-names', target,
'--zone', zone], system=True)
except Exception, mesg:
log("WARNING: issue making snapshot %s -- %s", target, mesg)
errors.append(mesg)
def create_all_boot_snapshots(self):
log("snapshotting storage boot image")
self.create_boot_snapshot(node=0,
prefix='storage',
zone='us-central1-c',
devel=False)
log("snapshotting backup boot image")
self.create_boot_snapshot(node=0,
prefix='backup',
zone='us-central1-c',
devel=False)
log("snapshotting admin boot image")
self.create_boot_snapshot(node='',
prefix='admin',
zone='us-central1-c',
devel=False)
log("snapshotting SMC server boot image")
self.create_boot_snapshot(node=0,
prefix='smc',
zone='us-central1-c',
devel=False)
log("snapshotting compute machine boot image")
self.create_boot_snapshot(node=0,
prefix='compute',
zone='us-central1-c',
devel=False)
def instance_costs(self):
cost = cost_upper = 0
n_compute = 0
n_smc = 0
n_other = 0
for x in cmd(['gcloud', 'compute', 'instances', 'list'], verbose=0).splitlines()[1:]:
v = x.split()
zone = v[1]
machine_type = v[2]
status = v[-1]
if v[0].startswith('compute'):
n_compute += 1
elif v[0].startswith('smc'):
n_smc += 1
else:
n_other += 1
if status == "RUNNING":
t = machine_type.split('-')
if len(t) == 3:
b = '-'.join(t[:2])
cpus = int(t[2])
else:
b = machine_type
cpus = 1
cost += PRICING[b+'-month'] * cpus * PRICING[zone.split('-')[0]]
cost_upper += PRICING[b+'-hour'] *30.5*24* cpus * PRICING[zone.split('-')[0]]
log("INSTANCES : compute=%s, smc=%s, other=%s: %s/month (or %s/month with sustained use)",
n_compute, n_smc, n_other, money(cost_upper), money(cost))
return cost_upper
def create_all_boot_snapshots(self):
v = []
v.append(('kubectl', ''))
for i in [0,1,2,3,4,5]:
v.append(('db', i))
#for name in self.dev_instances():
# node = name.split('v')[1]
# v.append(('dev', node))
for i in [0,1,2]:
v.append(('web',i))
v.append(('admin',0))
log("snapshotting storage machine boot images")
for i in [0,1,2,3,4,5]:
v.append(('storage', i))
log("snapshotting compute machine boot images")
for i in [0,1,2,3,4,5,6,7]:
v.append(('compute', i))
errors = []
log("snapshotting boot images: %s"%v)
for prefix, node in v:
try:
log('snapshotting %s%s'%(prefix, node))
self.create_boot_snapshot(node=node, prefix=prefix, zone='us-central1-c', devel=False)
except Exception, mesg:
errors.append(mesg)
log("WARNING: issue making snapshot -- %s", mesg)
def _create_compute_server(self,
node,
zone='us-central1-c',
machine_type='n1-highmem-4',
network='default',
projects_ssd=False,
base_ssd=False,
projects_size=150,
devel=False,
address=None,
preemptible=False):
zone = self.expand_zone(zone)
name = self.instance_name(node=node,
prefix='compute',
zone=zone,
devel=devel)
log("creating root filesystem image")
try:
opts = [
'gcloud', 'compute', '--project', self.project, 'disks',
'create', name, '--zone', zone, '--source-snapshot',
self.newest_snapshot('compute')
]
if base_ssd:
opts.extend(['--type', 'pd-ssd'])
cmd(opts)
except Exception, mesg:
if 'already exists' not in str(mesg):
raise
log("%s already exists", name)
def instance_costs(self):
cost = cost_upper = 0
n_compute = 0
n_smc = 0
n_other = 0
for x in cmd(['gcloud', 'compute', 'instances', 'list'],
verbose=0).splitlines()[1:]:
v = x.split()
zone = v[1]
machine_type = v[2]
status = v[-1]
if v[0].startswith('compute'):
n_compute += 1
elif v[0].startswith('smc'):
n_smc += 1
else:
n_other += 1
if status == "RUNNING":
t = machine_type.split('-')
if len(t) == 3:
b = '-'.join(t[:2])
cpus = int(t[2])
else:
b = machine_type
cpus = 1
cost += PRICING[b +
'-month'] * cpus * PRICING[zone.split('-')[0]]
cost_upper += PRICING[b +
'-hour'] * 30.5 * 24 * cpus * PRICING[
zone.split('-')[0]]
log(
"INSTANCES : compute=%s, smc=%s, other=%s: %s/month (or %s/month with sustained use)",
n_compute, n_smc, n_other, money(cost_upper), money(cost))
return cost_upper
def create_data_snapshot(self, node, prefix, zone='us-central1-c', devel=False):
"""
Snapshot the data disk on the given machine. Typically used for
backing up very important data.
"""
zone = self.expand_zone(zone)
instance_name = self.instance_name(node, prefix, zone, devel=devel)
info = json.loads(cmd(['gcloud', 'compute', 'instances', 'describe',
instance_name, '--zone', zone, '--format=json'], verbose=0))
errors = []
for disk in info['disks']:
# ignore boot disks (would be True)
if disk.get('boot', False):
continue
# ignore read-only disks (like for globally mounted data volumes)
# they would be snapshotted manually
if disk.get('mode', 'READ_WRITE') == 'READ_ONLY':
continue
src = disk['source'].split('/')[-1]
if 'swap' in src: continue
if 'tmp' in src: continue
target = 'data-%s-%s'%(src, time.strftime(TIMESTAMP_FORMAT))
log("%s --> %s", src, target)
try:
cmd(['gcloud', 'compute', 'disks', 'snapshot',
'--project', self.project,
src,
'--snapshot-names', target,
'--zone', zone], system=True)
except Exception, mesg:
log("WARNING: issue making snapshot %s -- %s", target, mesg)
errors.append(mesg)
def network_costs(self):
# These are estimates based on usage during March and April. May be lower in future
# do to moving everything to GCE. Not sure.
us = 700; aus = 150; china = 20
costs = 700 * PRICING['egress'] + 150*PRICING['egress-australia'] + 20*PRICING['egress-china']
log("NETWORK : %8s/month -- approx. %sGB Americas, %sGB EUR, %sGB CHINA", money(costs), us, aus, china)
return costs
def network_costs(self):
# These are estimates based on usage during March and April. May be lower in future
# do to moving everything to GCE. Not sure.
costs = 800 * PRICING['egress'] + 15 * PRICING[
'egress-australia'] + 15 * PRICING['egress-china']
log("NETWORK : approx. %s/month", money(costs))
return costs
def set_boot_auto_delete(self, name, zone):
log("set boot disk of %s to auto-delete" % name)
cmd([
'gcloud', 'compute', '--project', self.project, 'instances',
'set-disk-auto-delete', name, '--zone', zone, '--disk', name,
'--auto-delete'
])
def costs(self):
costs = {}
total = 0
for t in ['snapshot', 'disk', 'instance', 'network', 'gcs']:
costs[t] = getattr(self, '%s_costs' % t)()
total += costs[t]
log("TOTAL : %s/month", money(total))
return costs
def costs(self):
costs = {}
total = 0
for t in ['snapshot', 'disk', 'instance', 'network', 'gcs']:
costs[t] = getattr(self, '%s_costs'%t)()
total += costs[t]
log("TOTAL : %s/month", money(total))
return costs
def create_all_data_snapshots(self, zone='us-central1-c'):
for i in range(6):
log("snapshotting storage%s storage data"%i)
self.create_data_snapshot(node=i, prefix='storage', zone=zone, devel=False)
log("snapshotting live user data")
for n in self.compute_nodes(zone):
self.create_data_snapshot(node=n, prefix='compute', zone=zone, devel=False)
def instance_costs(self):
cost_lower = cost_upper = 0
n_compute = 0
n_web = 0
n_db = 0
n_other = 0
n_dev = 0
n_admin = 0
n_storage = 0
n_preempt = 0
for x in cmd(['gcloud', 'compute', 'instances', 'list'],
verbose=0).splitlines()[1:]:
v = x.split()
zone = v[1]
machine_type = v[2]
status = v[-1]
if status != 'RUNNING':
continue
if len(v) == 7:
preempt = (v[3] == 'true')
n_preempt += 1
else:
preempt = False
if v[0].startswith('compute'):
n_compute += 1
elif v[0].startswith('web'):
n_web += 1
elif v[0].startswith('db'):
n_db += 1
elif v[0].startswith('dev'):
n_dev += 1
elif v[0].startswith('admin'):
n_admin += 1
elif v[0].startswith('storage'):
n_storage += 1
else:
n_other += 1
t = machine_type.split('-')
if len(t) == 3:
b = '-'.join(t[:2])
cpus = int(t[2])
else:
b = machine_type
cpus = 1
if preempt:
pricing_hour = PRICING[b + '-hour-pre']
pricing_month = pricing_hour * 24 * 30.5
else:
pricing_hour = PRICING[b + '-hour']
pricing_month = PRICING[b + '-month']
cost_lower += pricing_month * cpus * PRICING[zone.split('-')[0]]
cost_upper += pricing_hour * 30.5 * 24 * cpus * PRICING[zone.split(
'-')[0]]
log(
"INSTANCES : %8s/month -- (or %8s/month without sustained!); compute=%s, web=%s, db=%s, dev=%s, admin=%s, storage=%s, other=%s (preempt=%s)",
money(cost_lower), money(cost_upper), n_compute, n_web, n_db,
n_dev, n_admin, n_storage, n_other, n_preempt)
return {'lower': cost_lower, 'upper': cost_upper}
def delete_devel_instances(self):
for x in cmd(['gcloud', 'compute', 'instances', 'list'], verbose=0).splitlines()[1:]:
v = x.split()
name = v[0]
if '-devel-' in name:
zone = v[1]
status = v[-1]
log("deleting devel instance: %s"%name)
cmd(['gcloud', 'compute', 'instances', 'delete', '--zone', zone, name], system=True)
def setup_projects_path():
log("create paths")
cmd("sudo mkdir -p /projects")
cmd("sudo chmod a+x /projects")
cmd("sudo touch /projects/snapshots; sudo chmod a+r /projects/snapshots")
cmd("sudo mkdir -p /projects/conf")
cmd("sudo chown salvus. /projects/conf")
cmd("sudo mkdir -p /projects/sagemathcloud")
cmd("sudo rsync -LrxH --delete /home/salvus/salvus/salvus/local_hub_template/ /projects/sagemathcloud/")
def instance_costs(self):
cost_lower = cost_upper = 0
n_compute = 0
n_web = 0
n_db = 0
n_other = 0
n_dev =0
n_admin =0
n_storage =0
n_preempt = 0
for x in cmd(['gcloud', 'compute', 'instances', 'list'], verbose=0).splitlines()[1:]:
v = x.split()
zone = v[1]
machine_type = v[2]
status = v[-1]
if status != 'RUNNING':
continue
if len(v) == 7:
preempt = (v[3] == 'true')
n_preempt += 1
else:
preempt = False
if v[0].startswith('compute'):
n_compute += 1
elif v[0].startswith('web'):
n_web += 1
elif v[0].startswith('db'):
n_db += 1
elif v[0].startswith('dev'):
n_dev += 1
elif v[0].startswith('admin'):
n_admin += 1
elif v[0].startswith('storage'):
n_storage += 1
else:
n_other += 1
t = machine_type.split('-')
if len(t) == 3:
b = '-'.join(t[:2])
cpus = int(t[2])
else:
b = machine_type
cpus = 1
if b == 'custom':
print("warning -custom machine types not supported; skipping ", x)
continue
if preempt:
pricing_hour = PRICING[b+'-hour-pre']
pricing_month = pricing_hour*24*30.5
else:
pricing_hour = PRICING[b+'-hour']
pricing_month = PRICING[b+'-month']
cost_lower += pricing_month * cpus * PRICING[zone.split('-')[0]]
cost_upper += pricing_hour *30.5*24* cpus * PRICING[zone.split('-')[0]]
log("INSTANCES : %8s/month -- (or %8s/month without sustained!); compute=%s, web=%s, db=%s, dev=%s, admin=%s, storage=%s, other=%s (preempt=%s)",
money(cost_lower), money(cost_upper), n_compute, n_web, n_db, n_dev, n_admin, n_storage, n_other, n_preempt)
return {'lower':cost_lower, 'upper':cost_upper}
def delete_devel_instances(self):
for x in cmd(['gcloud', 'compute', 'instances', 'list'], verbose=0).splitlines()[1:]:
v = x.split()
name = v[0]
if '-devel-' in name:
zone = v[1]
status = v[-1]
log("deleting devel instance: %s"%name)
cmd(['gcloud', 'compute', 'instances', 'delete', '--zone', zone, name], system=True)
def setup_projects_path():
log("create paths")
cmd("sudo mkdir -p /projects")
cmd("sudo chmod a+x /projects")
cmd("sudo touch /projects/snapshots; sudo chmod a+r /projects/snapshots")
cmd("sudo mkdir -p /projects/conf")
cmd("sudo chown salvus. /projects/conf")
cmd("sudo mkdir -p /projects/sagemathcloud")
cmd("sudo rsync -LrxH --delete /home/salvus/salvus/salvus/local_hub_template/ /projects/sagemathcloud/"
)
def start_devel_instances(self):
for x in cmd(['gcloud', 'compute', 'instances', 'list']).splitlines()[1:]:
v = x.split()
name = v[0]
if '-devel-' in name:
zone = v[1]
status = v[-1]
if status == "TERMINATED":
log("starting %s"%name)
cmd(['gcloud', 'compute', 'instances', 'start', '--zone', zone, name])
def start_devel_instances(self):
for x in cmd(['gcloud', 'compute', 'instances', 'list']).splitlines()[1:]:
v = x.split()
name = v[0]
if '-devel-' in name:
zone = v[1]
status = v[-1]
if status == "TERMINATED":
log("starting %s"%name)
cmd(['gcloud', 'compute', 'instances', 'start', '--zone', zone, name])
def delete_secrets():
log("delete any possible sensitive info from the production install")
log("wipe root ssh keys")
cmd("sudo rm -f /root/.ssh/id_rsa /root/.ssh/id_rsa.pub")
log("wipe salvus ssh keys")
cmd("sudo rm -rf /home/salvus/.ssh/id_rsa*")
log("wipe salvus secrets")
cmd("sudo rm -rf /home/salvus/salvus/salvus/data/secrets/")
log("wipe production logs")
cmd("sudo rm -rf /home/salvus/logs/*")
def create_data_secrets():
cmd("mkdir -p %s"%SECRETS)
log("sendgrid fake password (will not work)")
cmd("echo 'will-not-work' > %s/sendgrid_email_password"%SECRETS)
log("generate cassandra passwords")
cmd("mkdir -p %s/cassandra"%SECRETS)
cmd("makepasswd -q > %s/cassandra/hub"%SECRETS)
cmd("makepasswd -q > %s/cassandra/salvus"%SECRETS)
cmd("mkdir -p %s/sagemath.com"%SECRETS)
cmd("yes US | openssl req -x509 -newkey rsa:2048 -keyout key.pem -out cert.pem -nodes -days 10000 && cat key.pem cert.pem > %s/sagemath.com/nopassphrase.pem"%SECRETS)
def delete_secrets():
log("delete any possible sensitive info from the production install")
log("wipe root ssh keys")
cmd("sudo rm -f /root/.ssh/id_rsa /root/.ssh/id_rsa.pub")
log("wipe salvus ssh keys")
cmd("sudo rm -rf /home/salvus/.ssh/id_rsa*")
log("wipe salvus secrets")
cmd("sudo rm -rf /home/salvus/salvus/salvus/data/secrets/")
log("wipe production logs")
cmd("sudo rm -rf /home/salvus/logs/*")
def create_all_data_snapshots(self, zone='us-central1-c'):
# database backup disk
#self.create_data_snapshot(node='-backup', prefix='db', zone=zone, devel=False)
for i in range(6):
log("snapshotting storage%s storage data"%i)
self.create_data_snapshot(node=i, prefix='storage', zone=zone, devel=False)
log("snapshotting live user data")
for n in self.compute_nodes(zone):
self.create_data_snapshot(node=n, prefix='compute', zone=zone, devel=False)
def create_all_boot_snapshots(self):
log("snapshotting storage boot image")
self.create_boot_snapshot(node=0, prefix='storage', zone='us-central1-c', devel=False)
log("snapshotting backup boot image")
self.create_boot_snapshot(node=0, prefix='backup', zone='us-central1-c', devel=False)
log("snapshotting admin boot image")
self.create_boot_snapshot(node='',prefix='admin', zone='us-central1-c', devel=False)
log("snapshotting SMC server boot image")
self.create_boot_snapshot(node=0, prefix='smc', zone='us-central1-c', devel=False)
log("snapshotting compute machine boot image")
self.create_boot_snapshot(node=0, prefix='compute', zone='us-central1-c', devel=False)
def create_data_secrets():
cmd("mkdir -p %s" % SECRETS)
log("sendgrid fake password (will not work)")
cmd("echo 'will-not-work' > %s/sendgrid_email_password" % SECRETS)
log("generate cassandra passwords")
cmd("mkdir -p %s/cassandra" % SECRETS)
cmd("makepasswd -q > %s/cassandra/hub" % SECRETS)
cmd("makepasswd -q > %s/cassandra/salvus" % SECRETS)
cmd("mkdir -p %s/sagemath.com" % SECRETS)
cmd("yes US | openssl req -x509 -newkey rsa:2048 -keyout key.pem -out cert.pem -nodes -days 10000 && cat key.pem cert.pem > %s/sagemath.com/nopassphrase.pem"
% SECRETS)
def create_all_data_snapshots(self, zone='us-central1-c'):
log("snapshotting storage data")
self.create_data_snapshot(node=0,
prefix='storage',
zone=zone,
devel=False)
log("snapshotting live user data")
for n in self.compute_nodes(zone):
self.create_data_snapshot(node=n,
prefix='compute',
zone=zone,
devel=False)
def gcs_costs(self):
# usage based on running "time gsutil du -sch" every once in a while, since it takes
# quite a while to run.
cassandra = 200
database_backup = 200
gb_archive = 650 # delete in a few weeks...
projects_backup = 1500
usage = (database_backup + gb_archive + projects_backup)
costs = usage * PRICING['gcs-nearline']
log("CLOUD STORAGE: approx. %sGB nearline: %s/month", usage, money(costs))
return costs
def create_dev(self, node, zone='us-central1-c', machine_type='n1-standard-1', size=30, preemptible=True, address=''):
zone = self.expand_zone(zone)
name = self.instance_name(node=node, prefix='dev', zone=zone)
log("creating %sGB hard disk root filesystem image based on last smc snapshot", size)
try:
cmd(['gcloud', 'compute', '--project', self.project, 'disks', 'create', name,
'--zone', zone, '--source-snapshot', self.newest_snapshot('smc'),
'--size', size, '--type', 'pd-standard'])
except Exception, mesg:
if 'already exists' not in str(mesg):
raise
def create_dev(self, node, zone='us-central1-c', machine_type='n1-standard-1', size=30, preemptible=True, address=''):
zone = self.expand_zone(zone)
name = self.instance_name(node=node, prefix='dev', zone=zone)
log("creating %sGB hard disk root filesystem image based on last smc snapshot", size)
try:
cmd(['gcloud', 'compute', '--project', self.project, 'disks', 'create', name,
'--zone', zone, '--source-snapshot', self.newest_snapshot('smc'),
'--size', size, '--type', 'pd-standard'])
except Exception, mesg:
if 'already exists' not in str(mesg):
raise
def autostart(self, instance):
"""
Ensure that each instance in the input is running.
"""
for x in cmd(['gcloud', 'compute', 'instances', 'list'], verbose=0).splitlines()[1:]:
v = x.split()
if len(v) > 2 and v[-1] != 'RUNNING':
name = v[0]; zone = v[1]
for x in instance:
if name.startswith(x):
log("Starting %s... at %s", name, time.asctime())
cmd(' '.join(['gcloud', 'compute', 'instances', 'start', '--zone', zone, name]) + '&', system=True)
break
def gcs_costs(self):
# usage based on running "time gsutil du -sch" every once in a while, since it takes
# quite a while to run.
smc_db_backup = 50
smc_projects_backup = 1300
# This takes about 15 minutes and gives the usage above
# time gsutil du -sch gs://smc-projects-bup
# time gsutil du -sch gs://smc-db-backup
usage = (smc_db_backup + smc_projects_backup)
costs = usage * PRICING['gcs-nearline']
log("CLOUD STORAGE: %8s/month -- approx. %sGB nearline", money(costs), usage)
return costs
def gcs_costs(self):
# usage based on running "time gsutil du -sch" every once in a while, since it takes
# quite a while to run.
cassandra = 200
database_backup = 200
gb_archive = 650 # delete in a few weeks...
projects_backup = 1500
usage = (database_backup + gb_archive + projects_backup)
costs = usage * PRICING['gcs-nearline']
log("CLOUD STORAGE: approx. %sGB nearline: %s/month", usage,
money(costs))
return costs
def costs(self):
costs = {}
total_lower = 0
total_upper = 0
for t in ['snapshot', 'disk', 'instance', 'network', 'gcs']:
costs[t] = getattr(self, '%s_costs'%t)()
if isinstance(costs[t], dict):
total_lower += costs[t]['lower']
total_upper += costs[t]['upper']
else:
total_lower += costs[t]
total_upper += costs[t]
log("TOTAL : between %s/month and %s/month", money(total_lower), money(total_upper))
def costs(self):
costs = {}
total_lower = 0
total_upper = 0
for t in ['snapshot', 'disk', 'instance', 'network', 'gcs']:
costs[t] = getattr(self, '%s_costs'%t)()
if isinstance(costs[t], dict):
total_lower += costs[t]['lower']
total_upper += costs[t]['upper']
else:
total_lower += costs[t]
total_upper += costs[t]
log("TOTAL : %8s/month -- up to as worse as %8s/month without sustained", money(total_lower), money(total_upper))
def autostart(self, instance):
"""
Ensure that each instance in the input is running.
"""
for x in cmd(['gcloud', 'compute', 'instances', 'list'], verbose=0).splitlines()[1:]:
v = x.split()
if len(v) > 2 and v[-1] != 'RUNNING':
name = v[0]; zone = v[1]
for x in instance:
if name.startswith(x):
log("Starting %s... at %s", name, time.asctime())
cmd(' '.join(['gcloud', 'compute', 'instances', 'start', '--zone', zone, name]) + '&', system=True)
break
def init_cassandra_users():
pw_hub = open("%s/cassandra/hub"%SECRETS).read()
cmd("""echo "CREATE USER hub WITH PASSWORD '%s' SUPERUSER;" | cqlsh localhost -u cassandra -p cassandra"""%pw_hub, verbose=0)
rc = "%s/.cqlshrc"%os.environ['HOME']
log("writing %s", rc)
open(rc, 'w').write("""
[authentication]
username=hub
password=%s
"""%pw_hub)
pw_salvus = open("%s/cassandra/salvus"%SECRETS).read()
cmd("""echo "CREATE USER salvus WITH PASSWORD '%s' SUPERUSER;" | cqlsh localhost -u cassandra -p cassandra"""%pw_salvus, verbose=0)
cmd("""echo "ALTER USER cassandra WITH PASSWORD '%s';" | cqlsh localhost -u cassandra -p cassandra"""%pw_hub, verbose=0)
def gcs_costs(self):
# usage based on running "time gsutil du -sch" every once in a while, since it takes
# quite a while to run.
smc_db_backup = 50
smc_projects_backup = 1300
# This takes about 15 minutes and gives the usage above
# time gsutil du -sch gs://smc-projects-bup
# time gsutil du -sch gs://smc-db-backup
usage = (smc_db_backup + smc_projects_backup)
costs = usage * PRICING['gcs-nearline']
log("CLOUD STORAGE: %8s/month -- approx. %sGB nearline", money(costs), usage)
return costs
def _create_smc_server(self, node, zone='us-central1-c', machine_type='n1-highmem-2',
disk_size=100, network='default', devel=False):
zone = self.expand_zone(zone)
name = self.instance_name(node=node, prefix='smc', zone=zone, devel=devel)
disk_name = "%s-cassandra"%name
log("creating hard disk root filesystem image")
try:
cmd(['gcloud', 'compute', '--project', self.project, 'disks', 'create', name,
'--zone', zone, '--source-snapshot', self.newest_snapshot('smc'),
'--type', 'pd-standard'])
except Exception, mesg:
if 'already exists' not in str(mesg):
raise
def _create_storage_server(self, node, zone, machine_type,
disk_size, network, devel):
zone = self.expand_zone(zone)
name = self.instance_name(node=node, prefix='storage', zone=zone, devel=devel)
disk_name = "%s-projects"%name
log("creating hard disk root filesystem image")
try:
cmd(['gcloud', 'compute', '--project', self.project, 'disks', 'create', name,
'--zone', zone, '--source-snapshot', self.newest_snapshot('storage'),
'--type', 'pd-standard'])
except Exception, mesg:
if 'already exists' not in str(mesg):
raise
def costs(self):
costs = {}
total_lower = 0
total_upper = 0
for t in ['snapshot', 'disk', 'instance', 'network', 'gcs']:
costs[t] = getattr(self, '%s_costs' % t)()
if isinstance(costs[t], dict):
total_lower += costs[t]['lower']
total_upper += costs[t]['upper']
else:
total_lower += costs[t]
total_upper += costs[t]
log("TOTAL : between %s/month and %s/month", money(total_lower),
money(total_upper))
def _create_storage_server(self, node, zone, machine_type,
disk_size, network, devel):
zone = self.expand_zone(zone)
name = self.instance_name(node=node, prefix='storage', zone=zone, devel=devel)
disk_name = "%s-projects"%name
log("creating hard disk root filesystem image")
try:
cmd(['gcloud', 'compute', '--project', self.project, 'disks', 'create', name,
'--zone', zone, '--source-snapshot', self.newest_snapshot('storage'),
'--type', 'pd-standard'])
except Exception, mesg:
if 'already exists' not in str(mesg):
raise
def _create_smc_server(self, node, zone='us-central1-c', machine_type='n1-highmem-2',
disk_size=100, network='default', devel=False):
zone = self.expand_zone(zone)
name = self.instance_name(node=node, prefix='smc', zone=zone, devel=devel)
disk_name = "%s-cassandra"%name
log("creating hard disk root filesystem image")
try:
cmd(['gcloud', 'compute', '--project', self.project, 'disks', 'create', name,
'--zone', zone, '--source-snapshot', self.newest_snapshot('smc'),
'--type', 'pd-standard'])
except Exception, mesg:
if 'already exists' not in str(mesg):
raise
def disk_costs(self):
cost = 0
usage_standard = 0
usage_ssd = 0
for x in cmd(['gcloud', 'compute', 'disks', 'list'], verbose=0).splitlines()[1:]:
v = x.split()
size = int(v[2])
typ = v[3]
if typ == 'pd-ssd':
usage_ssd += size
elif typ == 'pd-standard':
usage_standard += size
cost += size * PRICING[typ]
log("DISK : %8s/month -- storage (standard=%sGB, ssd=%sGB)",
money(cost), usage_standard, usage_ssd)
return cost
def disk_costs(self):
cost = 0
usage_standard = 0
usage_ssd = 0
for x in cmd(['gcloud', 'compute', 'disks', 'list'], verbose=0).splitlines()[1:]:
v = x.split()
size = int(v[2])
typ = v[3]
if typ == 'pd-ssd':
usage_ssd += size
elif typ == 'pd-standard':
usage_standard += size
cost += size * PRICING[typ]
log("DISK : %8s/month -- storage (standard=%sGB, ssd=%sGB)",
money(cost), usage_standard, usage_ssd)
return cost
def autostart(self, instance):
"""
Ensure that each instance in the input is running.
"""
for x in cmd(['gcloud', 'compute', 'instances', 'list'] + instance,
verbose=0).splitlines()[1:]:
v = x.split()
if len(v) > 2 and v[-1] != 'RUNNING':
name = v[0]
zone = v[1]
log("Starting %s...", name)
cmd(' '.join([
'gcloud', 'compute', 'instances', 'start', '--zone', zone,
name
]) + '&',
system=True)
def init_compute_server():
log("starting compute server")
cmd("compute start")
log("making log link: ~/logs/compute.log")
cmd("ln -sf /projects/conf/compute.log %s/logs/compute.log"%os.environ['HOME'])
log("waiting a few seconds")
import time; time.sleep(5)
log("adding compute server to database")
cmd(r"""echo "require('compute').compute_server(keyspace:'salvus', cb:(e,s)->console.log(e); s.add_server(host:'%s', cb:(e)->console.log('done',e);process.exit(0)))" | coffee """%hostname)
def instance_costs(self):
cost_lower = cost_upper = 0
n_compute = 0
n_smc = 0
n_other = 0
n_preempt = 0
for x in cmd(['gcloud', 'compute', 'instances', 'list'],
verbose=0).splitlines()[1:]:
v = x.split()
zone = v[1]
machine_type = v[2]
status = v[-1]
if status != 'RUNNING':
continue
if len(v) == 7:
preempt = (v[3] == 'true')
n_preempt += 1
else:
preempt = False
if v[0].startswith('compute'):
n_compute += 1
elif v[0].startswith('smc'):
n_smc += 1
else:
n_other += 1
t = machine_type.split('-')
if len(t) == 3:
b = '-'.join(t[:2])
cpus = int(t[2])
else:
b = machine_type
cpus = 1
if preempt:
pricing_hour = PRICING[b + '-hour-pre']
pricing_month = pricing_hour * 24 * 30.5
else:
pricing_hour = PRICING[b + '-hour']
pricing_month = PRICING[b + '-month']
cost_lower += pricing_month * cpus * PRICING[zone.split('-')[0]]
cost_upper += pricing_hour * 30.5 * 24 * cpus * PRICING[zone.split(
'-')[0]]
log(
"INSTANCES : compute=%s, smc=%s, other=%s (preempt=%s): %s/month (or %s/month with sustained use)",
n_compute, n_smc, n_other, n_preempt, money(cost_upper),
money(cost_lower))
return {'lower': cost_lower, 'upper': cost_upper}
def costs(self):
costs = {}
total_lower = 0
total_upper = 0
for t in ['snapshot', 'disk', 'instance', 'network', 'gcs']:
costs[t] = getattr(self, '%s_costs' % t)()
if isinstance(costs[t], dict):
total_lower += costs[t]['lower']
total_upper += costs[t]['upper']
else:
total_lower += costs[t]
total_upper += costs[t]
log("SALES TAX : %8s/month -- 9.5%% WA+Seattle sales tax",
money(total_lower * 0.095))
log(
"TOTAL : %8s/month -- up to as worse as %8s/month without sustained",
money(total_lower * 1.095), money(total_upper * 1.095))
def _create_compute_server(self, node, zone='us-central1-c',
machine_type='n1-highmem-4', network='default',
projects_ssd=False, base_ssd=False,
projects_size=150,
devel=False,
address=None,
preemptible=False):
zone = self.expand_zone(zone)
name = self.instance_name(node=node, prefix='compute', zone=zone, devel=devel)
log("creating root filesystem image")
try:
opts = ['gcloud', 'compute', '--project', self.project, 'disks', 'create', name,
'--zone', zone, '--source-snapshot', self.newest_snapshot('compute')]
if base_ssd:
opts.extend(['--type', 'pd-ssd'])
cmd(opts)
except Exception as mesg:
if 'already exists' not in str(mesg):
raise
log("%s already exists", name)
log("creating /dev/sdb persistent disk")
disk_name = "%s-projects"%name
try:
opts = ['gcloud', 'compute', '--project', self.project, 'disks', 'create', disk_name,
'--size', projects_size, '--zone', zone]
if projects_ssd:
opts.extend(['--type', 'pd-ssd'])
cmd(opts)
except Exception as mesg:
if 'already exists' not in str(mesg):
raise
log("creating and starting compute instance")
opts =['gcloud', 'compute', '--project', self.project, 'instances', 'create', name,
'--zone', zone, '--machine-type', machine_type, '--network', network]
if address:
opts.extend(["--address", address])
if preemptible:
opts.append('--preemptible')
else:
opts.extend(['--maintenance-policy', 'MIGRATE'])
opts.extend(['--scopes', 'https://www.googleapis.com/auth/logging.write',
'--disk', 'name=%s,device-name=%s,mode=rw,boot=yes'%(name, name)])
opts.extend(['--disk', 'name=%s'%disk_name, 'device-name=%s'%disk_name, 'mode=rw'])
opts.extend(['--tags', 'compute'])
#if local_ssd:
# opts.append('--local-ssd')
#else:
cmd(opts, system=True)
if devel:
self.set_boot_auto_delete(name=name, zone=zone)
def _create_compute_server(self, node, zone='us-central1-c',
machine_type='n1-highmem-4', network='default',
projects_ssd=False, base_ssd=False,
projects_size=150,
devel=False):
zone = self.expand_zone(zone)
name = self.instance_name(node=node, prefix='compute', zone=zone, devel=devel)
log("creating root filesystem image")
try:
opts = ['gcloud', 'compute', '--project', self.project, 'disks', 'create', name,
'--zone', zone, '--source-snapshot', self.newest_snapshot('compute')]
if base_ssd:
opts.extend(['--type', 'pd-ssd'])
cmd(opts)
except Exception, mesg:
if 'already exists' not in str(mesg):
raise
log("%s already exists", name)
def instance_costs(self):
cost_lower = cost_upper = 0
n_compute = 0
n_smc = 0
n_other = 0
n_preempt = 0
for x in cmd(['gcloud', 'compute', 'instances', 'list'], verbose=0).splitlines()[1:]:
v = x.split()
zone = v[1]
machine_type = v[2]
status = v[-1]
if status != 'RUNNING':
continue
if len(v) == 7:
preempt = (v[3] == 'true')
n_preempt += 1
else:
preempt = False
if v[0].startswith('compute'):
n_compute += 1
elif v[0].startswith('smc'):
n_smc += 1
else:
n_other += 1
t = machine_type.split('-')
if len(t) == 3:
b = '-'.join(t[:2])
cpus = int(t[2])
else:
b = machine_type
cpus = 1
if preempt:
pricing_hour = PRICING[b+'-hour-pre']
pricing_month = pricing_hour*24*30.5
else:
pricing_hour = PRICING[b+'-hour']
pricing_month = PRICING[b+'-month']
cost_lower += pricing_month * cpus * PRICING[zone.split('-')[0]]
cost_upper += pricing_hour *30.5*24* cpus * PRICING[zone.split('-')[0]]
log("INSTANCES : compute=%s, smc=%s, other=%s (preempt=%s): %s/month (or %s/month with sustained use)",
n_compute, n_smc, n_other, n_preempt, money(cost_upper), money(cost_lower))
return {'lower':cost_lower, 'upper':cost_upper}
def create_all_data_snapshots(self):
log("snapshotting a database node")
self.create_data_snapshot(node=0, prefix='smc', zone='us-central1-c', devel=False)
log("snapshotting storage data")
self.create_data_snapshot(node=0, prefix='storage', zone='us-central1-c', devel=False)
log("snapshotting live user data")
for n in ['0', '1', '2', '3', '4']: # TODO -- automate this!!!!!
self.create_data_snapshot(node=n, prefix='compute', zone='us-central1-c', devel=False)
def start_cassandra():
log("start_cassandra...")
services = admin.Services('conf/deploy_devel/', password='')
services.start('cassandra')
cmd("ln -sf %s/data/cassandra-0/logs/system.log %s/logs/cassandra.log"%(SALVUS_ROOT, os.environ['HOME']))
log("cassandra started")
log("waiting 30 seconds...")
import time; time.sleep(30)
def setup_quota():
log("quota packages")
cmd("sudo apt-get install -y libatlas3gf-base liblapack-dev quota quotatool linux-image-extra-virtual cgroup-lite cgmanager-utils cgroup-bin libpam-cgroup cgmanager cgmanager-utils cgroup-bin smem", system=True)
log("quota stuff")
cmd("echo 'LABEL=cloudimg-rootfs / ext4 defaults,usrquota 0 0' | sudo tee /etc/fstab")
cmd("sudo mount -o remount /")
log("initializing quota, which will take a while")
cmd("sudo quotacheck -fucm /")
cmd("sudo quotaon /")
