def verify_csv_schema_upload(shard_type, date, schema_host, schema_db,
schema_upload_path_raw):
""" Confirm that schema files are uploaded
Args:
shard_type - In this case, a hostname or shard type (generally
one in the same)
date - The date to search for
schema_host - A for to examine to find which tables should exist
schema_db - Which db to inxpect on schema_host
schema_upload_path_raw - A string that can be format'ed in order to create
a S3 key path
Returns True for no problems found, False otherwise.
"""
boto_conn = boto.connect_s3()
bucket = boto_conn.get_bucket(environment_specific.S3_CSV_BUCKET, validate=False)
tables = mysql_lib.get_tables(schema_host,
environment_specific.convert_shard_to_db(schema_db),
skip_views=True)
return_status = True
for table in tables:
path = schema_upload_path_raw.format(table=table,
hostname_prefix=shard_type,
date=date,
db_name=schema_db)
if not bucket.get_key(path):
print 'Expected key {key} is missing'.format(key=path)
return_status = False
return return_status, tables
def verify_csv_schema_upload(shard_type, date, schema_host, schema_db,
schema_upload_path_raw):
""" Confirm that schema files are uploaded
Args:
shard_type - In this case, a hostname or shard type (generally
one in the same)
date - The date to search for
schema_host - A for to examine to find which tables should exist
schema_db - Which db to inxpect on schema_host
schema_upload_path_raw - A string that can be format'ed in order to create
a S3 key path
Returns True for no problems found, False otherwise.
"""
boto_conn = boto.connect_s3()
bucket = boto_conn.get_bucket(environment_specific.S3_CSV_BUCKET,
validate=False)
tables = mysql_lib.get_tables(
schema_host,
environment_specific.convert_shard_to_db(schema_db),
skip_views=True)
return_status = True
for table in tables:
path = schema_upload_path_raw.format(table=table,
hostname_prefix=shard_type,
date=date,
db_name=schema_db)
if not bucket.get_key(path):
print 'Expected key {key} is missing'.format(key=path)
return_status = False
return return_status, tables
def upload_schema(self, db, table, tmp_dir_db):
""" Upload the schema of a table to s3
Args:
db - the db to be backed up
table - the table to be backed up
tmp_dir_db - temporary storage used for all tables in the db
"""
if self.instance.replica_type in environment_specific.SHARDED_DBS_PREFIX_MAP:
if db != environment_specific.convert_shard_to_db(
environment_specific.SHARDED_DBS_PREFIX_MAP[self.instance.replica_type]['example_schema']):
return
s3_path_raw = PATH_DAILY_BACKUP_SHARDED_SCHEMA
else:
s3_path_raw = PATH_DAILY_BACKUP_NONSHARDED_SCHEMA
s3_path = s3_path_raw.format(table=table,
hostname_prefix=self.instance.replica_type,
date=self.datestamp,
db_name=db)
create_stm = mysql_lib.show_create_table(self.instance, db, table)
log.debug('{proc_id}: Uploading schema to {s3_path}'
''.format(s3_path=s3_path,
proc_id=multiprocessing.current_process().name))
boto_conn = boto.connect_s3()
bucket = boto_conn.get_bucket(environment_specific.S3_CSV_BUCKET, validate=False)
key = bucket.new_key(s3_path)
key.set_contents_from_string(create_stm)
def upload_schema(self, db, table, tmp_dir_db):
""" Upload the schema of a table to s3
Args:
db - the db to be backed up
table - the table to be backed up
tmp_dir_db - temporary storage used for all tables in the db
"""
if self.instance.replica_type in environment_specific.SHARDED_DBS_PREFIX_MAP:
if db != environment_specific.convert_shard_to_db(
environment_specific.SHARDED_DBS_PREFIX_MAP[
self.instance.replica_type]['example_schema']):
return
s3_path_raw = PATH_DAILY_BACKUP_SHARDED_SCHEMA
else:
s3_path_raw = PATH_DAILY_BACKUP_NONSHARDED_SCHEMA
s3_path = s3_path_raw.format(
table=table,
hostname_prefix=self.instance.replica_type,
date=self.datestamp,
db_name=db)
create_stm = mysql_lib.show_create_table(self.instance, db, table)
log.debug('{proc_id}: Uploading schema to {s3_path}'
''.format(s3_path=s3_path,
proc_id=multiprocessing.current_process().name))
boto_conn = boto.connect_s3()
bucket = boto_conn.get_bucket(environment_specific.S3_CSV_BUCKET,
validate=False)
key = bucket.new_key(s3_path)
key.set_contents_from_string(create_stm)
def get_missing_uploads(args):
""" Check to see if all backups are present
Args: A tuple which can be expanded to:
table - table name
shard_type - sharddb, etc
shards - a set of shards
Returns: a set of shards which are not backed up
"""
(table, shard_type, date, shards) = args
expected_s3_keys = set()
prefix = None
for shard in shards:
(_, data_path, _) = environment_specific.get_csv_backup_paths(
date, environment_specific.convert_shard_to_db(shard), table,
shard_type)
expected_s3_keys.add(data_path)
if not prefix:
prefix = os.path.dirname(data_path)
boto_conn = boto.connect_s3()
bucket = boto_conn.get_bucket(environment_specific.S3_CSV_BUCKET,
validate=False)
uploaded_keys = set()
for key in bucket.list(prefix=prefix):
uploaded_keys.add(key.name)
missing_uploads = expected_s3_keys.difference(uploaded_keys)
for entry in copy.copy(missing_uploads):
# the list api occassionally has issues, so we will recheck any missing
# entries. If any are actually missing we will quit checking because
# there is definitely work that needs to be done
if bucket.get_key(entry):
print 'List method erronious did not return data for key:{entry}'.format(
entry=entry)
missing_uploads.discard(entry)
else:
return missing_uploads
return missing_uploads
def get_missing_uploads(args):
""" Check to see if all backups are present
Args: A tuple which can be expanded to:
table - table name
shard_type - sharddb, etc
shards - a set of shards
Returns: a set of shards which are not backed up
"""
(table, shard_type, date, shards) = args
expected_s3_keys = set()
prefix = None
for shard in shards:
(_, data_path, _) = environment_specific.get_csv_backup_paths(
date, environment_specific.convert_shard_to_db(shard),
table, shard_type)
expected_s3_keys.add(data_path)
if not prefix:
prefix = os.path.dirname(data_path)
boto_conn = boto.connect_s3()
bucket = boto_conn.get_bucket(environment_specific.S3_CSV_BUCKET, validate=False)
uploaded_keys = set()
for key in bucket.list(prefix=prefix):
uploaded_keys.add(key.name)
missing_uploads = expected_s3_keys.difference(uploaded_keys)
for entry in copy.copy(missing_uploads):
# the list api occassionally has issues, so we will recheck any missing
# entries. If any are actually missing we will quit checking because
# there is definitely work that needs to be done
if bucket.get_key(entry):
print 'List method erronious did not return data for key:{entry}'.format(entry=entry)
missing_uploads.discard(entry)
else:
return missing_uploads
return missing_uploads
def verify_sharded_csv_backup(shard_type, date, instance=None):
""" Verify that a sharded data set has been backed up to hive
Args:
shard_type - i.e. 'sharddb', etc
date - The date to search for
instance - Restrict the search to problem on a single instnace
Returns True for no problems found, False otherwise.
"""
zk = host_utils.MysqlZookeeper()
example_shard = environment_specific.SHARDED_DBS_PREFIX_MAP[shard_type][
'example_shard']
schema_host = zk.shard_to_instance(example_shard,
repl_type=host_utils.REPLICA_ROLE_SLAVE)
tables = mysql_backup_csv.mysql_backup_csv(
schema_host).get_tables_to_backup(
environment_specific.convert_shard_to_db(example_shard))
success = verify_csv_schema_upload(
shard_type, date, schema_host,
environment_specific.convert_shard_to_db(example_shard), tables)
if instance:
host_shard_map = zk.get_host_shard_map()
(replica_set,
replica_type) = zk.get_replica_set_from_instance(instance)
master = zk.get_mysql_instance_from_replica_set(
replica_set, host_utils.REPLICA_ROLE_MASTER)
shards = host_shard_map[master.__str__()]
else:
shards = zk.get_shards_by_shard_type(shard_type)
pool = multiprocessing.Pool(processes=CSV_CHECK_PROCESSES)
pool_args = list()
if not tables:
raise Exception('No tables will be checked for backups')
if not shards:
raise Exception('No shards will be checked for backups')
for table in tables:
pool_args.append((table, shard_type, date, shards))
results = pool.map(get_missing_uploads, pool_args)
missing_uploads = set()
for result in results:
missing_uploads.update(result)
if missing_uploads or not success:
if len(missing_uploads) < MISSING_BACKUP_VERBOSE_LIMIT:
print('Shard type {shard_type} is missing uploads:'
''.format(shard_type=shard_type))
pprint.pprint(missing_uploads)
else:
print('Shard type {shard_type} is missing {num} uploads'
''.format(num=len(missing_uploads), shard_type=shard_type))
return False
else:
if instance:
print 'Instance {instance} is backed up'.format(instance=instance)
else:
# we have checked all shards, all are good, create success files
boto_conn = boto.connect_s3()
bucket = boto_conn.get_bucket(environment_specific.S3_CSV_BUCKET,
validate=False)
for table in tables:
(_, _,
success_path) = environment_specific.get_csv_backup_paths(
date,
environment_specific.convert_shard_to_db(example_shard),
table, shard_type)
if not bucket.get_key(success_path):
print 'Creating success key {key}'.format(key=success_path)
key = bucket.new_key(success_path)
key.set_contents_from_string('')
print 'Shard type {shard_type} is backed up'.format(
shard_type=shard_type)
return True
def verify_sharded_csv_backup(shard_type, date, instance=None):
""" Verify that a sharded data set has been backed up to hive
Args:
shard_type - i.e. 'sharddb', etc
date - The date to search for
instance - Restrict the search to problem on a single instnace
Returns True for no problems found, False otherwise.
"""
if instance:
# if we are running the check on a specific instance, we will add in
# some additional logic to not cause false alarms during the backup
# process.
if (date == (datetime.datetime.utcnow().date() -
datetime.timedelta(days=1)).strftime("%Y-%m-%d")):
# For todays date, we give CSV_STARTUP minutes before checking anything.
if datetime.datetime.utcnow().time() < CSV_STARTUP:
print 'Backup startup time has not yet passed'
return True
if datetime.datetime.utcnow().time() < CSV_COMPLETION_TIME:
# For todays date, until after CSV_COMPLETION_TIME it is good enough
# to check if backups are running. If they are running, everything
# is ok. If they are not running, we will do all the normal checks.
if csv_backups_running(instance):
print 'Backup running on {i}'.format(i=instance)
return True
schema_db = environment_specific.SHARDED_DBS_PREFIX_MAP[shard_type][
'example_schema']
zk = host_utils.MysqlZookeeper()
schema_host = zk.shard_to_instance(schema_db,
repl_type=host_utils.REPLICA_ROLE_SLAVE)
(success, tables) = \
verify_csv_schema_upload(shard_type, date, schema_host, schema_db,
mysql_backup_csv.PATH_DAILY_BACKUP_SHARDED_SCHEMA)
if not success:
# problem with schema, don't bother verifying data
return False
if instance:
host_shard_map = zk.get_host_shard_map()
(replica_set,
replica_type) = zk.get_replica_set_from_instance(instance)
master = zk.get_mysql_instance_from_replica_set(
replica_set, host_utils.REPLICA_ROLE_MASTER)
shards = host_shard_map[master.__str__()]
else:
shards = zk.get_shards_by_shard_type(shard_type)
missing_uploads = set()
for table in tables:
expected_s3_keys = set()
prefix = None
for shard in shards:
key = mysql_backup_csv.PATH_DAILY_BACKUP.format(
table=table,
hostname_prefix=shard_type,
date=date,
db_name=environment_specific.convert_shard_to_db(shard))
expected_s3_keys.add(key)
if not prefix:
prefix = os.path.dirname(key)
boto_conn = boto.connect_s3()
bucket = boto_conn.get_bucket(environment_specific.S3_CSV_BUCKET,
validate=False)
uploaded_keys = set()
for key in bucket.list(prefix=prefix):
uploaded_keys.add(key.name)
missing = expected_s3_keys.difference(uploaded_keys)
if missing and table not in environment_specific.IGNORABLE_MISSING_TABLES:
missing_uploads.update(missing)
if missing_uploads:
if len(missing_uploads) < MISSING_BACKUP_VERBOSE_LIMIT:
print 'Missing uploads: {uploads}'.format(uploads=missing_uploads)
else:
print 'Missing {num} uploads'.format(num=len(missing_uploads))
return False
else:
if not instance:
# we have checked all shards, all are good, create success files
for table in tables:
key_name = mysql_backup_csv.PATH_DAILY_SUCCESS.format(
table=table, hostname_prefix=shard_type, date=date)
if bucket.get_key(key_name):
print 'Key already exists {key}'.format(key=key)
else:
print 'Creating success key {key}'.format(key=key)
key = bucket.new_key(key_name)
key.set_contents_from_string('')
return True
def main():
parser = argparse.ArgumentParser()
parser.add_argument('db',
help='What server, shard or replica set to connect to '
'(ie sharddb021b[:3306], db00003, pbdata03862, '
'follower_zendata001002)')
parser.add_argument('-p',
'--privileges',
help=''.join(('Default is ', DEFAULT_ROLE)),
default=DEFAULT_ROLE,
choices=environment_specific.CLI_ROLES.keys())
parser.add_argument('-l',
'--longquery',
default=False,
action='store_true',
help='For standard read or write access, use this '
'flag if you expect the query to take more than '
'10 seconds.')
parser.add_argument('-e',
'--execute',
help='An optional SQL command to run.',
default=False)
args = parser.parse_args()
zk = host_utils.MysqlZookeeper()
host = None
db = ''
role_modifier = 'default'
long_query = ''
if args.longquery:
role_modifier = 'long'
long_query = '(long queries enabled)'
# check if db exists in dns, if so the supplied argument will be considered
# a hostname, otherwise a replica set.
try:
socket.gethostbyname(args.db)
host = host_utils.HostAddr(args.db)
log.info('{db} appears to be a hostname'.format(db=args.db))
except:
log.info('{db} appears not to be a hostname'.format(db=args.db))
# Maybe it is a replica set
if not host:
config = zk.get_all_mysql_config()
if args.db in config:
master = config[args.db]['master']
log.info('{db} appears to be a replica set'.format(db=args.db))
host = host_utils.HostAddr(''.join((master['host'],
':',
str(master['port']))))
else:
log.info('{db} appears not to be a replica set'.format(db=args.db))
# Perhaps a shard?
if not host:
shard_map = zk.get_host_shard_map()
for master in shard_map:
if args.db in shard_map[master]:
log.info('{db} appears to be a shard'.format(db=args.db))
host = host_utils.HostAddr(master)
db = environment_specific.convert_shard_to_db(args.db)
break
if not host:
log.info('{db} appears not to be a shard'.format(db=args.db))
if not host:
raise Exception('Could not determine what host to connect to')
log.info('Will connect to {host} with {privileges} '
'privileges {lq}'.format(host=host,
privileges=args.privileges,
lq=long_query))
(username, password) = mysql_lib.get_mysql_user_for_role(
environment_specific.CLI_ROLES[args.privileges][role_modifier])
if args.execute:
execute_escaped = string.replace(args.execute, '"', '\\"')
cmd = MYSQL_CLI_EX.format(host=host.hostname,
port=host.port,
db=db,
user=username,
password=password,
execute=execute_escaped)
else:
cmd = MYSQL_CLI.format(host=host.hostname,
port=host.port,
db=db,
user=username,
password=password)
log.info(cmd)
proc = subprocess.Popen(cmd, shell=True)
proc.wait()
def verify_sharded_csv_backup(shard_type, date, instance=None):
""" Verify that a sharded data set has been backed up to hive
Args:
shard_type - i.e. 'sharddb', etc
date - The date to search for
instance - Restrict the search to problem on a single instnace
Returns True for no problems found, False otherwise.
"""
if instance:
# if we are running the check on a specific instance, we will add in
# some additional logic to not cause false alarms during the backup
# process.
if (date == (datetime.datetime.utcnow().date() - datetime.timedelta(days=1)).strftime("%Y-%m-%d")):
# For todays date, we give CSV_STARTUP minutes before checking anything.
if datetime.datetime.utcnow().time() < CSV_STARTUP:
print 'Backup startup time has not yet passed'
return True
if datetime.datetime.utcnow().time() < CSV_COMPLETION_TIME:
# For todays date, until after CSV_COMPLETION_TIME it is good enough
# to check if backups are running. If they are running, everything
# is ok. If they are not running, we will do all the normal checks.
if csv_backups_running(instance):
print 'Backup running on {i}'.format(i=instance)
return True
schema_db = environment_specific.SHARDED_DBS_PREFIX_MAP[shard_type]['example_schema']
zk = host_utils.MysqlZookeeper()
schema_host = zk.shard_to_instance(schema_db, repl_type=host_utils.REPLICA_ROLE_SLAVE)
(success, tables) = \
verify_csv_schema_upload(shard_type, date, schema_host, schema_db,
mysql_backup_csv.PATH_DAILY_BACKUP_SHARDED_SCHEMA)
if not success:
# problem with schema, don't bother verifying data
return False
if instance:
host_shard_map = zk.get_host_shard_map()
(replica_set, replica_type) = zk.get_replica_set_from_instance(instance)
master = zk.get_mysql_instance_from_replica_set(replica_set, host_utils.REPLICA_ROLE_MASTER)
shards = host_shard_map[master.__str__()]
else:
shards = zk.get_shards_by_shard_type(shard_type)
missing_uploads = set()
for table in tables:
expected_s3_keys = set()
prefix = None
for shard in shards:
key = mysql_backup_csv.PATH_DAILY_BACKUP.format(table=table,
hostname_prefix=shard_type,
date=date,
db_name=environment_specific.convert_shard_to_db(shard))
expected_s3_keys.add(key)
if not prefix:
prefix = os.path.dirname(key)
boto_conn = boto.connect_s3()
bucket = boto_conn.get_bucket(environment_specific.S3_CSV_BUCKET, validate=False)
uploaded_keys = set()
for key in bucket.list(prefix=prefix):
uploaded_keys.add(key.name)
missing = expected_s3_keys.difference(uploaded_keys)
if missing and table not in environment_specific.IGNORABLE_MISSING_TABLES:
missing_uploads.update(missing)
if missing_uploads:
if len(missing_uploads) < MISSING_BACKUP_VERBOSE_LIMIT:
print 'Missing uploads: {uploads}'.format(uploads=missing_uploads)
else:
print 'Missing {num} uploads'.format(num=len(missing_uploads))
return False
else:
if not instance:
# we have checked all shards, all are good, create success files
for table in tables:
key_name = mysql_backup_csv.PATH_DAILY_SUCCESS.format(table=table,
hostname_prefix=shard_type,
date=date)
if bucket.get_key(key_name):
print 'Key already exists {key}'.format(key=key)
else:
print 'Creating success key {key}'.format(key=key)
key = bucket.new_key(key_name)
key.set_contents_from_string('')
return True
def main():
parser = argparse.ArgumentParser()
parser.add_argument('db',
help='What server, shard or replica set to connect to '
'(ie sharddb021b[:3306], db00003, pbdata03862, '
'follower_zendata001002)')
parser.add_argument('-p',
'--privileges',
help=''.join(('Default is ', DEFAULT_ROLE)),
default=DEFAULT_ROLE,
choices=environment_specific.CLI_ROLES.keys())
parser.add_argument('-l',
'--longquery',
default=False,
action='store_true',
help='For standard read or write access, use this '
'flag if you expect the query to take more than '
'10 seconds.')
parser.add_argument('-e',
'--execute',
help='An optional SQL command to run.',
default=False)
args = parser.parse_args()
zk = host_utils.MysqlZookeeper()
host = None
db = ''
role_modifier = 'default'
long_query = ''
if args.longquery:
role_modifier = 'long'
long_query = '(long queries enabled)'
# check if db exists in dns, if so the supplied argument will be considered
# a hostname, otherwise a replica set.
try:
socket.gethostbyname(args.db)
host = host_utils.HostAddr(args.db)
log.info('{db} appears to be a hostname'.format(db=args.db))
except:
log.info('{db} appears not to be a hostname'.format(db=args.db))
# Maybe it is a replica set
if not host:
config = zk.get_all_mysql_config()
if args.db in config:
master = config[args.db]['master']
log.info('{db} appears to be a replica set'.format(db=args.db))
host = host_utils.HostAddr(''.join((master['host'],
':',
str(master['port']))))
else:
log.info('{db} appears not to be a replica set'.format(db=args.db))
# Perhaps a shard?
if not host:
shard_map = zk.get_host_shard_map()
for master in shard_map:
if args.db in shard_map[master]:
log.info('{db} appears to be a shard'.format(db=args.db))
host = host_utils.HostAddr(master)
db = environment_specific.convert_shard_to_db(args.db)
break
if not host:
log.info('{db} appears not to be a shard'.format(db=args.db))
if not host:
raise Exception('Could not determine what host to connect to')
log.info('Will connect to {host} with {privileges} '
'privileges {lq}'.format(host=host,
privileges=args.privileges,
lq=long_query))
(username, password) = mysql_lib.get_mysql_user_for_role(
environment_specific.CLI_ROLES[args.privileges][role_modifier])
# we may or may not know what replica set we're connecting to at
# this point.
sql_safe = ''
try:
replica_set, _ = zk.get_replica_set_from_instance(host)
except Exception as e:
if 'is not in zk' in e.message:
log.warning('SERVER IS NOT IN ZK!!!')
replica_set = None
else:
raise
try:
# do we need a prompt?
if replica_set in environment_specific.EXTRA_PARANOID_REPLICA_SETS:
if args.privileges in ['read-write', 'admin']:
resp = raw_input("You've asked for {priv} access to replica "
"set {rs}. Are you sure? (Y/N): ".format(
priv=args.privileges,
rs=replica_set))
if not resp or resp[0] not in ['Y', 'y']:
raise Exception('Connection aborted by user!')
# should we enable safe-updates?
if replica_set in environment_specific.PARANOID_REPLICA_SETS:
if args.privileges in ['read-write', 'admin']:
sql_safe = '--init-command="SET SESSION SQL_SAFE_UPDATES=ON"'
except Exception as e:
log.error("Unable to continue: {}".format(e))
return
if args.execute:
execute_escaped = string.replace(args.execute, '"', '\\"')
cmd = MYSQL_CLI_EX.format(host=host.hostname,
port=host.port,
db=db,
user=username,
password=password,
sql_safe=sql_safe,
execute=execute_escaped)
else:
cmd = MYSQL_CLI.format(host=host.hostname,
port=host.port,
db=db,
user=username,
password=password,
sql_safe=sql_safe)
log.info(cmd)
proc = subprocess.Popen(cmd, shell=True)
proc.wait()
def verify_sharded_csv_backup(shard_type, date, instance=None):
""" Verify that a sharded data set has been backed up to hive
Args:
shard_type - i.e. 'sharddb', etc
date - The date to search for
instance - Restrict the search to problem on a single instnace
Returns True for no problems found, False otherwise.
"""
zk = host_utils.MysqlZookeeper()
example_shard = environment_specific.SHARDED_DBS_PREFIX_MAP[shard_type]['example_shard']
schema_host = zk.shard_to_instance(example_shard, repl_type=host_utils.REPLICA_ROLE_SLAVE)
tables = mysql_backup_csv.mysql_backup_csv(schema_host).get_tables_to_backup(environment_specific.convert_shard_to_db(example_shard))
success = verify_csv_schema_upload(shard_type, date, schema_host,
environment_specific.convert_shard_to_db(example_shard), tables)
if instance:
host_shard_map = zk.get_host_shard_map()
(replica_set, replica_type) = zk.get_replica_set_from_instance(instance)
master = zk.get_mysql_instance_from_replica_set(replica_set, host_utils.REPLICA_ROLE_MASTER)
shards = host_shard_map[master.__str__()]
else:
shards = zk.get_shards_by_shard_type(shard_type)
pool = multiprocessing.Pool(processes=CSV_CHECK_PROCESSES)
pool_args = list()
if not tables:
raise Exception('No tables will be checked for backups')
if not shards:
raise Exception('No shards will be checked for backups')
for table in tables:
pool_args.append((table, shard_type, date, shards))
results = pool.map(get_missing_uploads, pool_args)
missing_uploads = set()
for result in results:
missing_uploads.update(result)
if missing_uploads or not success:
if len(missing_uploads) < MISSING_BACKUP_VERBOSE_LIMIT:
print ('Shard type {shard_type} is missing uploads:'
''.format(shard_type=shard_type))
pprint.pprint(missing_uploads)
else:
print ('Shard type {shard_type} is missing {num} uploads'
''.format(num=len(missing_uploads),
shard_type=shard_type))
return False
else:
if instance:
print 'Instance {instance} is backed up'.format(instance=instance)
else:
# we have checked all shards, all are good, create success files
boto_conn = boto.connect_s3()
bucket = boto_conn.get_bucket(environment_specific.S3_CSV_BUCKET, validate=False)
for table in tables:
(_, _, success_path) = environment_specific.get_csv_backup_paths(date,
environment_specific.convert_shard_to_db(example_shard),
table, shard_type)
if not bucket.get_key(success_path):
print 'Creating success key {key}'.format(key=success_path)
key = bucket.new_key(success_path)
key.set_contents_from_string('')
print 'Shard type {shard_type} is backed up'.format(shard_type=shard_type)
return True
def main():
parser = argparse.ArgumentParser()
parser.add_argument('db',
help='What server, shard or replica set to connect to '
'(ie sharddb021b[:3306], db00003, pbdata03862, '
'follower_zendata001002)')
parser.add_argument('-p',
'--privileges',
help=''.join(('Default is ', DEFAULT_ROLE)),
default=DEFAULT_ROLE,
choices=environment_specific.CLI_ROLES.keys())
parser.add_argument('-l',
'--longquery',
default=False,
action='store_true',
help='For standard read or write access, use this '
'flag if you expect the query to take more than '
'10 seconds.')
parser.add_argument('-e',
'--execute',
help='An optional SQL command to run.',
default=False)
args = parser.parse_args()
zk = host_utils.MysqlZookeeper()
host = None
db = ''
role_modifier = 'default'
long_query = ''
if args.longquery:
role_modifier = 'long'
long_query = '(long queries enabled)'
# check if db exists in dns, if so the supplied argument will be considered
# a hostname, otherwise a replica set.
try:
socket.gethostbyname(args.db)
host = host_utils.HostAddr(args.db)
log.info('{db} appears to be a hostname'.format(db=args.db))
except:
log.info('{db} appears not to be a hostname'.format(db=args.db))
# Maybe it is a replica set
if not host:
config = zk.get_all_mysql_config()
if args.db in config:
master = config[args.db]['master']
log.info('{db} appears to be a replica set'.format(db=args.db))
host = host_utils.HostAddr(''.join(
(master['host'], ':', str(master['port']))))
else:
log.info('{db} appears not to be a replica set'.format(db=args.db))
# Perhaps a shard?
if not host:
shard_map = zk.get_host_shard_map()
for master in shard_map:
if args.db in shard_map[master]:
log.info('{db} appears to be a shard'.format(db=args.db))
host = host_utils.HostAddr(master)
db = environment_specific.convert_shard_to_db(args.db)
break
if not host:
log.info('{db} appears not to be a shard'.format(db=args.db))
if not host:
raise Exception('Could not determine what host to connect to')
log.info('Will connect to {host} with {privileges} '
'privileges {lq}'.format(host=host,
privileges=args.privileges,
lq=long_query))
(username, password) = mysql_lib.get_mysql_user_for_role(
environment_specific.CLI_ROLES[args.privileges][role_modifier])
if args.execute:
execute_escaped = string.replace(args.execute, '"', '\\"')
cmd = MYSQL_CLI_EX.format(host=host.hostname,
port=host.port,
db=db,
user=username,
password=password,
execute=execute_escaped)
else:
cmd = MYSQL_CLI.format(host=host.hostname,
port=host.port,
db=db,
user=username,
password=password)
log.info(cmd)
proc = subprocess.Popen(cmd, shell=True)
proc.wait()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('db',
help='What server, shard or replica set to connect to '
'(ie sharddb021b[:3306], db00003, pbdata03862, '
'follower_zendata001002)')
parser.add_argument('-p',
'--privileges',
help=''.join(('Default is ', DEFAULT_ROLE)),
default=DEFAULT_ROLE,
choices=environment_specific.CLI_ROLES.keys())
parser.add_argument('-l',
'--longquery',
default=False,
action='store_true',
help='For standard read or write access, use this '
'flag if you expect the query to take more than '
'10 seconds.')
parser.add_argument('-e',
'--execute',
help='An optional SQL command to run.',
default=False)
args = parser.parse_args()
zk = host_utils.MysqlZookeeper()
host = None
db = ''
role_modifier = 'default'
long_query = ''
if args.longquery:
role_modifier = 'long'
long_query = '(long queries enabled)'
# check if db exists in dns, if so the supplied argument will be considered
# a hostname, otherwise a replica set.
try:
socket.gethostbyname(args.db)
host = host_utils.HostAddr(args.db)
log.info('{db} appears to be a hostname'.format(db=args.db))
except:
log.info('{db} appears not to be a hostname'.format(db=args.db))
# Maybe it is a replica set
if not host:
config = zk.get_all_mysql_config()
if args.db in config:
master = config[args.db]['master']
log.info('{db} appears to be a replica set'.format(db=args.db))
host = host_utils.HostAddr(''.join(
(master['host'], ':', str(master['port']))))
else:
log.info('{db} appears not to be a replica set'.format(db=args.db))
# Perhaps a shard?
if not host:
shard_map = zk.get_host_shard_map()
for master in shard_map:
if args.db in shard_map[master]:
log.info('{db} appears to be a shard'.format(db=args.db))
host = host_utils.HostAddr(master)
db = environment_specific.convert_shard_to_db(args.db)
break
if not host:
log.info('{db} appears not to be a shard'.format(db=args.db))
if not host:
raise Exception('Could not determine what host to connect to')
log.info('Will connect to {host} with {privileges} '
'privileges {lq}'.format(host=host,
privileges=args.privileges,
lq=long_query))
(username, password) = mysql_lib.get_mysql_user_for_role(
environment_specific.CLI_ROLES[args.privileges][role_modifier])
# we may or may not know what replica set we're connecting to at
# this point.
sql_safe = ''
try:
replica_set, _ = zk.get_replica_set_from_instance(host)
except Exception as e:
if 'is not in zk' in e.message:
log.warning('SERVER IS NOT IN ZK!!!')
replica_set = None
else:
raise
try:
# do we need a prompt?
if replica_set in environment_specific.EXTRA_PARANOID_REPLICA_SETS:
if args.privileges in ['read-write', 'admin']:
resp = raw_input("You've asked for {priv} access to replica "
"set {rs}. Are you sure? (Y/N): ".format(
priv=args.privileges, rs=replica_set))
if not resp or resp[0] not in ['Y', 'y']:
raise Exception('Connection aborted by user!')
# should we enable safe-updates?
if replica_set in environment_specific.PARANOID_REPLICA_SETS:
if args.privileges in ['read-write', 'admin']:
sql_safe = '--init-command="SET SESSION SQL_SAFE_UPDATES=ON"'
except Exception as e:
log.error("Unable to continue: {}".format(e))
return
if args.execute:
execute_escaped = string.replace(args.execute, '"', '\\"')
cmd = MYSQL_CLI_EX.format(host=host.hostname,
port=host.port,
db=db,
user=username,
password=password,
sql_safe=sql_safe,
execute=execute_escaped)
else:
cmd = MYSQL_CLI.format(host=host.hostname,
port=host.port,
db=db,
user=username,
password=password,
sql_safe=sql_safe)
log.info(cmd)
proc = subprocess.Popen(cmd, shell=True)
proc.wait()
