def verify_csv_schema_upload(instance, db, tables, date, dev_bucket=False):
""" Confirm that schema files are uploaded
Args:
instance - A hostaddr object
db - which db to inspect
tables - A set of tables to look for
date - The date to search for
dev_bucket - Are we checking the dev bucket or not?
Returns:
True for no problems found, False otherwise.
"""
return_status = True
missing = set()
boto_conn = boto.connect_s3()
bucket_name = environment_specific.S3_CSV_BUCKET_DEV if dev_bucket \
else environment_specific.S3_CSV_BUCKET
bucket = boto_conn.get_bucket(bucket_name, validate=False)
for table in tables:
(path, _, _) = backup.get_csv_backup_paths(instance, db, table, date,
0)
if not bucket.get_key(path):
missing.add(path)
return_status = False
if missing:
raise Exception('Expected schema files missing: {}'.format(missing))
return return_status
def get_sharded_db_missing_uploads(args):
""" Check to see if all backups are present
Args: A tuple which can be expanded to:
table_tuple - a tuple of (db.table, partition_name, partition_num)
shard_type - sharddb, etc
shards - a set of shards
dev_bucket - check the dev bucket instead of the prod bucket?
Returns: the table name that was checked and a set of shards which
are not backed up for the table in question.
"""
(table_tuple, date, shards, dev_bucket) = args
zk = host_utils.MysqlZookeeper()
expected_s3_keys = set()
prefix = None
table_name = table_tuple[0].split('.')[1]
for shard in shards:
(replica_set, db) = zk.map_shard_to_replica_and_db(shard)
instance = zk.get_mysql_instance_from_replica_set(
replica_set, repl_type=host_utils.REPLICA_ROLE_SLAVE)
(_, data_path,
_) = backup.get_csv_backup_paths(instance, db, table_name, date,
table_tuple[2])
expected_s3_keys.add(data_path)
if not prefix:
prefix = os.path.dirname(data_path)
boto_conn = boto.connect_s3()
bucket_name = environment_specific.S3_CSV_BUCKET_DEV if dev_bucket \
else environment_specific.S3_CSV_BUCKET
bucket = boto_conn.get_bucket(bucket_name, validate=False)
uploaded_keys = set()
for key in bucket.list(prefix=prefix):
if key.size > 0:
uploaded_keys.add(key.name)
elif key.name.split('/')[-1][0] != '_':
# If we have a zero-length file that doesn't start with
# an underscore, it shouldn't be here.
key.delete()
missing_uploads = expected_s3_keys.difference(uploaded_keys)
for entry in copy.copy(missing_uploads):
# The list API occassionally has issues, so we will recheck any missing
# entries. If any are actually missing we will quit checking because
# there is definitely work that needs to be done
k = bucket.get_key(entry)
if k and k.size > 0:
print 'List method did not return data for key:{}'.format(entry)
missing_uploads.discard(entry)
else:
return ({'table': table_name, 'missing_uploads': missing_uploads})
return ({'table': table_name, 'missing_uploads': missing_uploads})
def upload_schema(self, db, table, tmp_dir_db):
""" Upload the schema of a table to s3
Args:
db - the db to be backed up
table - the table to be backed up
tmp_dir_db - temporary storage used for all tables in the db
"""
(schema_path, _,
_) = backup.get_csv_backup_paths(self.instance, db, table,
self.datestamp)
create_stm = mysql_lib.show_create_table(self.instance, db, table)
log.debug('{proc_id}: Uploading schema to {schema_path}'
''.format(schema_path=schema_path,
proc_id=multiprocessing.current_process().name))
boto_conn = boto.connect_s3()
bucket = boto_conn.get_bucket(self.upload_bucket, validate=False)
key = bucket.new_key(schema_path)
key.set_contents_from_string(create_stm)
def already_backed_up(self, table_tuple):
""" Check to see if a particular partition has already been uploaded
to s3
Args:
table_tuple - (table, partition name, part number)
Returns:
bool - True if the partition has already been backed up,
False otherwise
"""
boto_conn = boto.connect_s3()
bucket = boto_conn.get_bucket(self.upload_bucket, validate=False)
(_, data_path,
_) = backup.get_csv_backup_paths(self.instance,
*table_tuple[0].split('.'),
date=self.datestamp,
partition_number=table_tuple[2])
if not bucket.get_key(data_path):
return False
return True
def verify_flexsharded_csv_backup(shard_type, date, dev_bucket=False):
""" Verify that a flexsharded data set has been backed up to hive
Args:
shard_type - i.e. 'commercefeeddb', etc
date - The date to search for
dev_bucket - Look in the dev bucket?
Returns:
True for no problems found, False otherwise.
"""
success = True
replica_sets = set()
zk = host_utils.MysqlZookeeper()
# Figure out what replica sets to check based on a prefix
for replica_set in zk.get_all_mysql_replica_sets():
if replica_set.startswith(
environment_specific.FLEXSHARD_DBS[shard_type]['zk_prefix']):
replica_sets.add(replica_set)
# Example schema host
schema_host = zk.get_mysql_instance_from_replica_set(
environment_specific.FLEXSHARD_DBS[shard_type]
['example_shard_replica_set'],
repl_type=host_utils.REPLICA_ROLE_SLAVE)
boto_conn = boto.connect_s3()
bucket_name = environment_specific.S3_CSV_BUCKET_DEV if dev_bucket \
else environment_specific.S3_CSV_BUCKET
bucket = boto_conn.get_bucket(bucket_name, validate=False)
missing_uploads = set()
for db in mysql_lib.get_dbs(schema_host):
table_list = [
'{}.{}'.format(db, x)
for x in mysql_lib.get_tables(schema_host, db, True)
]
table_tuples = backup.filter_tables_to_csv_backup(
schema_host, table_list)
for t in table_tuples:
try:
verify_csv_schema_upload(schema_host,
db, [t[0].split('.')[1]],
date=date,
dev_bucket=dev_bucket)
except:
continue
table_missing_uploads = set()
for replica_set in replica_sets:
chk_instance = zk.get_mysql_instance_from_replica_set(
replica_set)
(_, data_path, success_path) = backup.get_csv_backup_paths(
chk_instance,
db,
t[0].split('.')[1],
date=date,
partition_number=t[2])
k = bucket.get_key(data_path)
if k is None:
table_missing_uploads.add(data_path)
success = False
elif k.size == 0:
# we should not have zero-length files, because even if
# we send zero bytes to lzop, there's a 55-byte header.
# so, if this actually happened, it probably means that
# something is wrong. delete the key and add it to the
# missing_uploads list so that we'll try again.
k.delete()
table_missing_uploads.add(data_path)
success = False
if not table_missing_uploads and not bucket.get_key(success_path):
print 'Creating success key {b}/{k}'.format(b=bucket_name,
k=success_path)
key = bucket.new_key(success_path)
key.set_contents_from_string(' ')
missing_uploads.update(table_missing_uploads)
if missing_uploads:
if len(missing_uploads) < MISSING_BACKUP_VERBOSE_LIMIT:
print('Shard type {} is missing uploads:'.format(shard_type))
pprint.pprint(missing_uploads)
else:
print('Shard type {shard_type} is missing {num} uploads'
''.format(num=len(missing_uploads), shard_type=shard_type))
if not missing_uploads and success:
print 'Shard type {} is backed up'.format(shard_type)
return success
def verify_unsharded_csv_backups(instance, date, dev_bucket=False):
""" Verify csv backups for an instance which is not part of a sharded
system
Args:
instance - The instance to inspect for backups being done
date - The date to search for
dev_bucket - Use the dev bucket?
Returns:
True for no problems found, False otherwise.
"""
return_status = True
boto_conn = boto.connect_s3()
bucket_name = environment_specific.S3_CSV_BUCKET_DEV if dev_bucket \
else environment_specific.S3_CSV_BUCKET
bucket = boto_conn.get_bucket(bucket_name, validate=False)
missing_uploads = set()
for db in mysql_lib.get_dbs(instance):
table_list = [
'{}.{}'.format(db, x)
for x in mysql_lib.get_tables(instance, db, True)
]
table_tuples = backup.filter_tables_to_csv_backup(instance, table_list)
try:
verify_csv_schema_upload(
instance, db, [x[0].split('.')[1] for x in table_tuples], date,
dev_bucket)
except Exception as e:
print e
return_status = False
continue
table_names = [x[0] for x in table_tuples]
expected_partitions = dict(
(x, table_names.count(x)) for x in table_names)
found_partitions = dict()
for t in table_tuples:
(_, data_path, success_path) = \
backup.get_csv_backup_paths(instance, *t[0].split('.'), date=date,
partition_number=t[2])
k = bucket.get_key(data_path)
if k is None:
missing_uploads.add(data_path)
elif k.size == 0:
# we should not have zero-length files, because even if
# we send zero bytes to lzop, there's a 55-byte header.
# so, if this actually happened, it probably means that
# something is wrong. delete the key and add it to the
# missing_uploads list so that we'll try again.
k.delete()
missing_uploads.add(data_path)
else:
found_partitions[t[0]] = 1 + found_partitions.get(t[0], 0)
# We still need to create a success file for the data
# team for this table, even if something else is AWOL
# later in the backup.
s_key = bucket.get_key(success_path)
if s_key:
if found_partitions.get(t[0], 0) < expected_partitions[t[0]]:
print(
'Success key {b}/{k} exists but it should '
'not - deleting it!').format(b=bucket_name,
k=success_path)
s_key.delete()
elif found_partitions.get(t[0],
0) == expected_partitions[t[0]]:
print 'Success key {b}/{k} exists!'.format(b=bucket_name,
k=success_path)
elif found_partitions.get(t[0], 0) == expected_partitions[t[0]]:
print 'Creating success key {b}/{k}'.format(b=bucket_name,
k=success_path)
key = bucket.new_key(success_path)
key.set_contents_from_string(' ')
if missing_uploads:
if len(missing_uploads) < MISSING_BACKUP_VERBOSE_LIMIT:
print 'Missing uploads: {}'.format(missing_uploads)
else:
print 'Missing {} uploads'.format(len(missing_uploads))
return_status = False
if return_status:
log_csv_backup_success(instance, date, dev_bucket)
return return_status
def verify_sharded_csv_backup_by_shard_type(shard_type,
date,
dev_bucket=False):
""" Verify that a sharded data set has been backed up to hive
Args:
shard_type - i.e. 'sharddb', etc
date - The date to search for
dev_bucket - Look in the dev bucket
Returns:
True for no problems found, False otherwise.
"""
zk = host_utils.MysqlZookeeper()
(replica_set,
db) = zk.get_example_db_and_replica_set_for_shard_type(shard_type)
schema_host = zk.get_mysql_instance_from_replica_set(
replica_set, repl_type=host_utils.REPLICA_ROLE_SLAVE)
table_list = [
'{}.{}'.format(db, x)
for x in mysql_lib.get_tables(schema_host, db, True)
]
table_tuples = backup.filter_tables_to_csv_backup(schema_host, table_list)
if not table_tuples:
raise Exception('No tables will be checked for backups')
verify_csv_schema_upload(schema_host, db,
[x[0].split('.')[1] for x in table_tuples], date,
dev_bucket)
shards = zk.get_shards_by_shard_type(shard_type)
if not shards:
raise Exception('No shards will be checked for backups')
(finished_uploads, missing_uploads) = verify_sharded_csv_backup_by_shards(
shards, table_tuples, date, dev_bucket)
if finished_uploads:
boto_conn = boto.connect_s3()
bucket_name = environment_specific.S3_CSV_BUCKET_DEV \
if dev_bucket else environment_specific.S3_CSV_BUCKET
bucket = boto_conn.get_bucket(bucket_name, validate=False)
for tbl in finished_uploads:
(_, _, success_path) = backup.get_csv_backup_paths(
schema_host, db, tbl, date)
if not bucket.get_key(success_path):
print 'Creating success key {b}/{k}'.format(b=bucket_name,
k=success_path)
key = bucket.new_key(success_path)
key.set_contents_from_string(' ')
if missing_uploads:
if len(missing_uploads) < MISSING_BACKUP_VERBOSE_LIMIT:
print('Shard type {} is missing uploads:'.format(shard_type))
pprint.pprint(missing_uploads)
else:
print('Shard type {shard_type} is missing {num} uploads'
''.format(num=len(missing_uploads), shard_type=shard_type))
return False
else:
# we have checked all shards, all are good, create success files
# that might not already be present. theoretically, everything here
# will get picked up by the finished_uploads stanza earlier, but
# we have this here as a failsafe.
boto_conn = boto.connect_s3()
bucket_name = environment_specific.S3_CSV_BUCKET_DEV \
if dev_bucket else environment_specific.S3_CSV_BUCKET
bucket = boto_conn.get_bucket(bucket_name, validate=False)
for t in table_tuples:
(_, _,
success_path) = backup.get_csv_backup_paths(schema_host,
*t[0].split('.'),
date=date,
partition_number=t[2])
if not bucket.get_key(success_path):
print 'Creating success key {b}/{k}'.format(b=bucket_name,
k=success_path)
key = bucket.new_key(success_path)
key.set_contents_from_string(' ')
print 'Shard type {} is backed up'.format(shard_type)
return True
def mysql_backup_one_partition(self, table_tuple, tmp_dir_db, conn):
""" Back up a single partition of a single table
Args:
table_tuple - the table_tuple (db, partition name, partition number)
to be backed up
tmp_dir_db - temporary storage used for all tables in the db
conn - a connection the the mysql instance
"""
proc_id = multiprocessing.current_process().name
(_, data_path,
_) = backup.get_csv_backup_paths(self.instance,
*table_tuple[0].split('.'),
date=self.datestamp,
partition_number=table_tuple[2])
log.debug('{proc_id}: {tbl} partition {p} dump to {path} started'
''.format(proc_id=proc_id,
tbl=table_tuple[0],
p=table_tuple[2],
path=data_path))
self.upload_schema(*table_tuple[0].split('.'), tmp_dir_db=tmp_dir_db)
fifo = os.path.join(
tmp_dir_db, '{tbl}{part}'.format(tbl=table_tuple[0].split('.')[1],
part=table_tuple[2]))
procs = dict()
try:
# giant try so we can try to clean things up in case of errors
self.create_fifo(fifo)
# Start creating processes
procs['cat'] = subprocess.Popen(['cat', fifo],
stdout=subprocess.PIPE)
procs['nullescape'] = subprocess.Popen(['nullescape'],
stdin=procs['cat'].stdout,
stdout=subprocess.PIPE)
procs['lzop'] = subprocess.Popen(['lzop'],
stdin=procs['nullescape'].stdout,
stdout=subprocess.PIPE)
# Start dump query
return_value = set()
query_thread = threading.Thread(target=self.run_dump_query,
args=(table_tuple, fifo, conn,
procs['cat'], return_value))
query_thread.daemon = True
query_thread.start()
# And run the upload
safe_uploader.safe_upload(precursor_procs=procs,
stdin=procs['lzop'].stdout,
bucket=self.upload_bucket,
key=data_path,
check_func=self.check_dump_success,
check_arg=return_value)
os.remove(fifo)
log.debug('{proc_id}: {tbl} partition {p} clean up complete'
''.format(proc_id=proc_id,
tbl=table_tuple[0],
p=table_tuple[2]))
except:
log.debug('{}: in exception handling for failed table '
'upload'.format(proc_id))
if os.path.exists(fifo):
self.cleanup_fifo(fifo)
raise