def test_consul_backup(self):
'test making backup of Consul key/value store'
from utils import get_consul_session
from utils import get_records_from_consul
from utils import save_records
import json
# Get session
session = get_consul_session('localhost', '8500')
# Did we get a proper Consul session object back?
expected = 'Consul'
class_name = session.__class__.__name__
self.assertEqual(class_name, expected)
# Assuming we successfully connected to the Consul cluster,
# we should be able to ask for "peers"
peers = session.status.peers()
# which should be a list
self.assertTrue(isinstance(peers, list))
# of length 3
self.assertEqual(len(peers), 3)
# Now add some data to the key/value store
for key, value in test_data.items():
session.kv[key] = value
# Now retrieve the records
records = get_records_from_consul(session)
# Did we get the expected data back?
expected_records = '[["key1", 0, "value1"], ["key2", 0, "value2"], ["key3", 0, "value3"]]'
self.assertEqual(json.dumps(records), expected_records)
# Now test file creation
current_dir = os.path.dirname(os.path.realpath(__file__))
backup_file = '%s/test_backup.json' % current_dir
save_records(records, backup_file)
# Check that the file was created
self.assertTrue(os.path.exists(backup_file))
# Now check if its contents are what we expect
backup = open(backup_file).read().strip()
self.assertEqual(backup, expected_records)
# Delete the backup file
os.remove(backup_file)
def test_consul_backup(self):
'test making backup of Consul key/value store'
from utils import get_consul_session
from utils import get_records_from_consul
from utils import save_records
import json
# Get session
session = get_consul_session('localhost', '8500')
# Did we get a proper Consul session object back?
expected = 'Consul'
class_name = session.__class__.__name__
self.assertEqual(class_name, expected)
# Assuming we successfully connected to the Consul cluster,
# we should be able to ask for "peers"
peers = session.status.peers()
# which should be a list
self.assertTrue(isinstance(peers, list))
# of length 3
self.assertEqual(len(peers), 3)
# Now add some data to the key/value store
for key,value in test_data.items():
session.kv[key] = value
# Now retrieve the records
records = get_records_from_consul(session)
# Did we get the expected data back?
expected_records = '[["key1", 0, "value1"], ["key2", 0, "value2"], ["key3", 0, "value3"]]'
self.assertEqual(json.dumps(records), expected_records)
# Now test file creation
current_dir = os.path.dirname(os.path.realpath(__file__))
backup_file = '%s/test_backup.json' % current_dir
save_records(records, backup_file)
# Check that the file was created
self.assertTrue(os.path.exists(backup_file))
# Now check if its contents are what we expect
backup = open(backup_file).read().strip()
self.assertEqual(backup, expected_records)
# Delete the backup file
os.remove(backup_file)
if action == 'backup':
# Get today's date for the time stamp
now = str(datetime.datetime.now().date())
# Construct the name of the backup file
fname = os.environ.get('BACKUP_FILE','adsabs_consul_kv')
version = os.environ.get('VERSION',now)
backup_file = '%s/%s.%s.json' % (tmp_dir,fname,version)
# Get the records from the Consul key/value store
try:
records = get_records_from_consul(session)
except Exception, e:
logging.error('Unable to retrieve records from Consul store: %s'%e)
sys.exit(2)
# Write the records to the backup file
try:
save_records(records, backup_file)
except Exception, e:
logging.error('Unable to write to backup file: %s (%s)'%(backup_file,e))
sys.exit(2)
logging.info('Backup was written to: %s'%backup_file)
# Now copy the backup to S3
s3 = get_s3_resource()
try:
s3_upload_file(s3, backup_file, backup_folder)
except Exception, e:
logging.error('Unable to move backup to S3: %s'%e)
# Finally, remove the local copy
logging.info('Removing local copy of backup file: %s' % backup_file)
os.remove(backup_file)
elif action == 'restore':
# Construct the name of the backup file to retrieve
sys.exit(2)
# Now take the appropriate action depending on the value of the ACTION variable
if action == 'backup':
# Get today's date for the time stamp
now = str(datetime.datetime.now().date())
# Construct the name of the backup file
backup_file = '%s/adsabs_consul_kv.%s.json' % (tmp_dir,now)
# Get the records from the Consul key/value store
try:
records = get_records_from_consul(session)
except Exception, e:
logging.error('Unable to retrieve records from Consul store: %s'%e)
sys.exit(2)
# Write the records to the backup file
try:
save_records(records, backup_file)
except Exception, e:
logging.error('Unable to write to backup file: %s (%s)'%(backup_file,e))
sys.exit(2)
logging.info('Backup was written to: %s'%backup_file)
# Now copy the backup to S3
s3 = get_s3_resource()
try:
s3_upload_file(s3, backup_file, backup_folder)
except Exception, e:
logging.error('Unable to move backup to S3: %s'%e)
# Finally, remove the local copy
logging.info('Removing local copy of backup file: %s' % backup_file)
os.remove(backup_file)
elif action == 'restore':
# Construct the name of the backup file to retrieve