def repair_partitions(context, crawler_name, query_parser):
print "Updating indexes for '{}'".format(crawler_name)
#run the index repair on external tables if it is not currently running.
context[recovery.STATE_CONTEXT_QUERY] = query_parser
query_id = recovery.db_key(crawler_name, "msck_query_id")
if query_id in context:
id = context[query_id]
query = context[c.KEY_ATHENA_QUERY].client.get_query_execution(id)
if query['Status']['State'] == "SUCCEEDED":
del context[query_id]
repair = RepairTable(context)
repair.execute(query_id)
def __is_table_data_not_valid(context, crawler_name, query_parser):
query_id = recovery.db_key(crawler_name, "count_uid")
state_context = __create_state_context(context, crawler_name, query_parser)
validate = ValidateTableData(state_context)
result = None
while True:
print validate, 'is EXECUTING'
result = validate.execute(query_id)
if result != None:
context[c.KEY_DB].delete_item(query_id)
break
time.sleep(10)
if result is None:
print "Validation query failed. Validating table exists."
if not __is_valid_table(context, crawler_name, query_parser):
#No table found, which is a valid case.
print "Table not found"
return False
else:
print "Corrupt table found"
return True
return False
def __is_recovery_in_progress(context, crawler_name):
return recovery.db_key(crawler_name, recovery.STATE_CONTEXT_DDL) in context