def on_insert_code(items):
"""
Deploy code onto servers as the items are created.
If a new code item 'is_current', PATCH 'is_current' code with the same name
and type to no longer be current.
:param items: List of dicts for items to be created.
"""
log.debug('code | Insert | items - %s', items)
for item in items:
log.debug('code | POST | On Insert callback | %s', item)
# Check to see if we have a current profile and core.
code_query = 'where={{"meta.name":"{0}","meta.version":"{1}","meta.code_type":"{2}"}}'.format(item['meta']['name'], item['meta']['version'], item['meta']['code_type'])
code = utilities.get_eve('code', code_query)
log.debug('code | POST | On Insert callback | Code query result | %s', code)
if not code['_meta']['total'] == 0:
log.error('code | POST | On Insert callback | %s named %s-%s already exists', item['meta']['code_type'], item['meta']['name'], item['meta']['version'])
abort(409, 'Error: A {0} named {1}-{2} already exists.'.format(item['meta']['code_type'], item['meta']['name'], item['meta']['version']))
if item.get('meta') and item['meta'].get('is_current') and item['meta']['is_current'] is True:
query = 'where={{"meta.name":"{0}","meta.code_type":"{1}","meta.is_current": true}}'.format(item['meta']['name'], item['meta']['code_type'])
code_get = utilities.get_eve('code', query)
log.debug('code | Insert | current code - %s', code_get)
if code_get['_meta']['total'] != 0:
for code in code_get['_items']:
request_payload = {'meta.is_current': False}
utilities.patch_eve('code', code['_id'], request_payload)
log.debug('code | Insert | Ready to deploy item - %s', item)
tasks.code_deploy.delay(item)
def _diff_f5():
"""
Copy f5 configuration file to local sever, parse txt and create or update
site items.
"""
f5_config_dir = "{0}/atlas/fabfile".format(path)
f5_config_file = "{0}/{1}".format(f5_config_dir, f5_config_files[environment])
# If an older config file exists, copy it to a backup folder.
if os.path.isfile(f5_config_file):
local(
"mv {0} /data/code/inventory/fabfile/backup/{1}.{2}".format(
f5_config_file, f5_config_files[environment], str(time()).split(".")[0]
)
)
# Copy config file from the f5 server to the Atlas server.
local(
"scp {0}:/config/{1} {2}/".format(
serverdefs[environment]["f5_servers"][0], f5_config_files[environment], f5_config_dir
)
)
# Open file from f5
with open(f5_config_file, "r") as ifile:
data = ifile.read()
# Use regex to parse out path values
p = re.compile('"(.+/?)" := "(\w+(-\w+)?)",')
sites = p.findall(data)
# Iterate through sites found in f5 data
for site in sites:
f5only = False
if site[0] in f5exceptions:
f5only = True
# Get path without leading slash
path = site[0][1:]
pool = site[1]
# Set a type value based on pool
if pool == "WWWLegacy":
type = "legacy"
elif pool == "poola-homepage" or pool == "poolb-homepage":
type = "homepage"
elif pool == "poolb-express":
type = "express"
else:
type = "custom"
site_query = 'where={{"path":"{0}"}}'.format(path)
sites = utilities.get_eve("sites", site_query)
if not sites or len(sites["_items"]) == 0:
payload = {"name": path, "path": path, "pool": pool, "status": "launched", "type": type, "f5only": f5only}
utilities.post_eve("sites", payload)
print ("Created site record based on f5.\n{0}".format(payload))
elif pool != data["_items"][0]["pool"]:
site = data["_items"][0]
payload = {"pool": pool, "status": "launched", "type": type}
utilities.patch_eve("sites", site["_id"], payload)
print "Updated site based on f5.\n{0}".format(payload)
def delete_all_available_sites():
"""
Get a list of available sites and delete them
"""
site_query = 'where={"status":"available"}'
sites = utilities.get_eve("sites", site_query)
payload = {"status": "delete"}
for site in sites:
utilities.patch_eve("sites", site["_id"], payload)
def delete_stale_pending_sites():
site_query = 'where={"status":"pending"}'
sites = utilities.get_eve("sites", site_query)
# Loop through and remove sites that are more than 30 minutes old.
for site in sites["_items"]:
# Parse date string into structured time.
# See https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior for mask format.
date_created = time.strptime(site["_created"], "%Y-%m-%d %H:%M:%S %Z")
# Get time now, Convert date_created to seconds from epoch and
# calculate the age of the site.
seconds_since_creation = time.time() - time.mktime(date_created)
# 30 min * 60 sec = 1800 seconds
if seconds_since_creation > 1800:
payload = {"status": "delete"}
utilities.patch_eve("sites", site["_id"], payload)
def site_provision(site):
"""
Provision a new instance with the given parameters.
:param site: A single site.
:return:
"""
logger.debug("Site provision - {0}".format(site))
# 'db_key' needs to be added here and not in Eve so that the encryption
# works properly.
site["db_key"] = utilities.encrypt_string(utilities.mysql_password())
fab_task = execute(fabfile.site_provision, site=site)
logger.debug(fab_task)
logger.debug(fab_task.values)
patch_payload = {"status": "available", "db_key": site["db_key"]}
patch = utilities.patch_eve("sites", site["_id"], patch_payload)
logger.debug("Site has been provisioned\n{0}".format(patch))
slack_title = "{0}/{1}".format(base_urls[environment], site["sid"])
slack_link = "{0}/{1}".format(base_urls[environment], site["sid"])
attachment_text = "{0}/sites/{1}".format(api_server, site["_id"])
if False not in fab_task.values():
slack_message = "Site provision - Success"
slack_color = "good"
utilities.post_to_slack(
message=slack_message,
title=slack_title,
link=slack_link,
attachment_text=attachment_text,
level=slack_color,
)
def on_update_code(updates, original):
"""
Update code on the servers as the item is updated.
:param updates:
:param original:
"""
log.debug('code | on update | updates - %s | original - %s', updates, original)
# If this 'is_current' PATCH code with the same name and code_type.
if updates.get('meta') and updates['meta'].get('is_current') and updates['meta']['is_current'] is True:
# If the name and code_type are not changing, we need to load them from the original.
name = updates['meta']['name'] if updates['meta'].get('name') else original['meta']['name']
code_type = updates['meta']['code_type'] if updates['meta'].get('code_type') else original['meta']['code_type']
query = 'where={{"meta.name":"{0}","meta.code_type":"{1}","meta.is_current": true,"_id":{{"$ne":"{2}"}}}}'.format(
name, code_type, original['_id'])
code_get = utilities.get_eve('code', query)
log.debug('code | on update | Current code - %s', code_get)
for code in code_get['_items']:
request_payload = {'meta.is_current': False}
utilities.patch_eve('code', code['_id'], request_payload)
# We need the whole record so that we can manipulate code in the right place.
# Copy 'original' to a new dict, then update it with values from 'updates' to create an item to
# deploy. Need to do the same process for meta first, otherwise the update will fully overwrite.
if updates.get('meta'):
meta = original['meta'].copy()
meta.update(updates['meta'])
updated_item = original.copy()
updated_item.update(updates)
if updates.get('meta'):
updated_item['meta'] = meta
if updates.has_key('meta') and (updates['meta'].has_key('name') or updates['meta'].has_key('version') or updates['meta'].has_key('code_type')):
update_code = True
elif updates.has_key('commit_hash') or updates.has_key('git_url'):
update_code = True
else:
update_code = False
if update_code:
log.debug('code | on update | Ready to hand to Celery')
# chord two tasks
chord(tasks.code_update.s(updated_item, original), tasks.clear_php_cache.si())()
def backup_restore(backup_record, original_instance, package_list):
"""
Restore database and files to a new instance.
"""
log.info('Instance | Restore Backup | %s | %s', backup_record, original_instance)
start_time = time()
file_date = datetime.strptime(backup_record['backup_date'], "%Y-%m-%d %H:%M:%S %Z")
pretty_filename = '{0}_{1}'.format(
original_instance['sid'], file_date.strftime("%Y-%m-%d-%H-%M-%S"))
pretty_database_filename = '{0}.sql'.format(pretty_filename)
database_path = '{0}/backups/{1}'.format(BACKUP_PATH, pretty_database_filename)
pretty_files_filename = '{0}.tar.gz'.format(pretty_filename)
files_path = '{0}/backups/{1}'.format(BACKUP_PATH, pretty_files_filename)
# Grab available instance and add packages if needed
available_instances = utilities.get_eve('sites', 'where={"status":"available"}')
log.debug('Instance | Restore Backup | Avaiable Instances - %s', available_instances)
new_instance = next(iter(available_instances['_items']), None)
# TODO: Don't switch if the code is the same
if new_instance is not None:
payload = {'status': 'installing'}
if package_list:
packages = {'code': {'package': package_list}}
payload.update(packages)
utilities.patch_eve('sites', new_instance['_id'], payload)
else:
exit('No available instances.')
# Wait for code and status to update.
attempts = 18 # Tries every 10 seconds to a max of 18 (or 3 minutes).
while attempts:
try:
new_instance_refresh = utilities.get_single_eve('sites', new_instance['_id'])
if new_instance_refresh['status'] != 'installed':
log.info('Instance | Restore Backup | New instance is not ready | %s', new_instance['_id'])
raise ValueError('Status has not yet updated.')
break
except ValueError, e:
# If the status is not updated and we have attempts left,
# remove an attempt and wait 10 seconds.
attempts -= 1
if attempts is not 0:
sleep(10)
else:
exit(str(e))
def site_update(site, updates, original):
"""
Update an instance with the given parameters.
:param site: A complete site item, including new values.
:param updates: A partial site item, including only changed keys.
:param original: Complete original site item.
:return:
"""
logger.debug("Site update - {0}\n{1}\n\n{2}".format(site["_id"], site, updates))
if updates.get("code"):
logger.debug("Found code changes.")
if "package" in updates["code"]:
logger.debug("Found package changes.")
execute(fabfile.site_package_update, site=site)
execute(fabfile.update_database, site=site)
if updates["code"].get("core") != original["code"].get("core"):
logger.debug("Found core change.")
execute(fabfile.site_core_update, site=site)
execute(fabfile.update_database, site=site)
if updates["code"].get("profile") != original["code"].get("profile"):
logger.debug("Found profile change.")
execute(fabfile.site_profile_update, site=site, original=original, updates=updates)
execute(fabfile.update_database, site=site)
if updates.get("status"):
logger.debug("Found status change.")
if updates["status"] in ["installing", "launching", "take_down", "restore"]:
if updates["status"] == "installing":
logger.debug("Status changed to installing")
patch_payload = '{"status": "installed"}'
elif updates["status"] == "launching":
logger.debug("Status changed to launching")
execute(fabfile.site_launch, site=site)
patch_payload = '{"status": "launched"}'
elif updates["status"] == "take_down":
logger.debug("Status changed to take_down")
execute(fabfile.site_take_down, site=site)
patch_payload = '{"status": "down"}'
elif updates["status"] == "restore":
logger.debug("Status changed to restore")
execute(fabfile.site_restore, site=site)
execute(fabfile.update_database, site=site)
patch_payload = '{"status": "installed"}'
patch = utilities.patch_eve("sites", site["_id"], patch_payload)
logger.debug(patch)
slack_title = "{0}/{1}".format(base_urls[environment], site["sid"])
slack_link = "{0}/{1}".format(base_urls[environment], site["sid"])
attachment_text = "{0}/sites/{1}".format(api_server, site["_id"])
slack_message = "Site Update - Success"
slack_color = "good"
utilities.post_to_slack(
message=slack_message, title=slack_title, link=slack_link, attachment_text=attachment_text, level=slack_color
)
def _launch_site(site, gsa_collection=False):
"""
Create symlinks with new site name.
"""
print ("Launch subtask")
code_directory = "{0}/{1}".format(sites_code_root, site["sid"])
code_directory_current = "{0}/current".format(code_directory)
if site["pool"] in ["poolb-express", "poolb-homepage"]:
if site["pool"] == "poolb-express":
web_directory = "{0}/{1}".format(sites_web_root, site["type"])
web_directory_path = "{0}/{1}".format(web_directory, site["path"])
with cd(web_directory):
# If the path is nested like 'lab/atlas', make the 'lab' directory
if "/" in site["path"]:
lead_path = "/".join(site["path"].split("/")[:-1])
_create_directory_structure(lead_path)
# Create a new symlink using site's updated path
if not exists(web_directory_path):
_update_symlink(code_directory_current, site["path"])
# enter new site directory
with cd(web_directory_path):
clear_apc()
if gsa_collection:
# Set the collection name
run("drush vset --yes google_appliance_collection {0}".format(gsa_collection))
# Clear caches at the end of the launch process to show
# correct pathologic rendered URLS.
drush_cache_clear(site["sid"])
# Assign it to an update group.
update_group = randint(0, 10)
if site["pool"] == "poolb-homepage":
web_directory = "{0}/{1}".format(sites_web_root, "homepage")
with cd(sites_web_root):
_update_symlink(code_directory_current, web_directory)
# enter new site directory
with cd(web_directory):
clear_apc()
drush_cache_clear(site["sid"])
# Assign site to update group 12.
update_group = 12
payload = {"status": "launched", "update_group": update_group}
utilities.patch_eve("sites", site["_id"], payload)
def on_updated_code(updates, original):
"""
Find instances that use this code asset and re-add them.
:param updates:
:param original:
"""
log.debug('code | on updated | updates - %s | original - %s', updates, original)
# First get the code_type from either the update or original, then convert package types for
# querying instance objects.
if updates.get('meta') and updates['meta'].get('code_type'):
code_type = updates['meta']['code_type']
else:
code_type = original['meta']['code_type']
if code_type in ['module', 'theme', 'library']:
code_type = 'package'
if updates.has_key('meta'):
if updates['meta']['name'] != original['meta']['name'] or updates['meta']['version'] != original['meta']['version'] or updates['meta']['code_type'] != original['meta']['code_type']:
update_sites = True
log.debug('code | on updated | Found meta data changes | %s', updates['meta'])
else:
log.debug('code | on updated | Found no meta changes that require an update')
update_sites = False
elif updates.has_key('commit_hash') or updates.has_key('git_url'):
update_sites = True
log.debug('code | on updated | Found git data changes')
else:
log.debug('code | on updated | Found no changes')
update_sites = False
if update_sites:
log.info('Code | on updated | Preparing to update instances')
query = 'where={{"code.{0}":"{1}"}}'.format(code_type, original['_id'])
sites_get = utilities.get_eve('sites', query)
if sites_get['_meta']['total'] is not 0:
for site in sites_get['_items']:
log.debug('code | on updated | site - %s', site)
code_id_string = site['code'][code_type]
payload = {'code': {code_type: code_id_string}}
log.debug('code | on updated | payload - %s', payload)
utilities.patch_eve('sites', site['_id'], payload)
def take_down_installed_35_day_old_sites():
if environment != "prod":
site_query = 'where={"status":"installed"}'
sites = utilities.get_eve("sites", site_query)
# Loop through and remove sites that are more than 35 days old.
for site in sites["_items"]:
# Parse date string into structured time.
# See https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior
# for mask format.
date_created = time.strptime(site["_created"], "%Y-%m-%d %H:%M:%S %Z")
# Get time now, Convert date_created to seconds from epoch and
# calculate the age of the site.
seconds_since_creation = time.time() - time.mktime(date_created)
print("{0} is {1} seconds old".format(site["sid"], seconds_since_creation))
# 35 days * 24 hrs * 60 min * 60 sec = 302400 seconds
if seconds_since_creation > 3024000:
# Patch the status to 'take_down'.
payload = {"status": "take_down"}
utilities.patch_eve("sites", site["_id"], payload)
def on_insert_code_callback(items):
"""
Deploy code onto servers as the items are created.
If a new code item 'is_current', PATCH 'is_current' code with the same name
and type to no longer be current.
:param items: List of dicts for items to be created.
"""
app.logger.debug(items)
for item in items:
if item.get('meta') and item['meta'].get('is_current') and item['meta']['is_current'] == True:
# Need a lowercase string when querying boolean values. Python
# stores it as 'True'.
query = 'where={{"meta.name":"{0}","meta.code_type":"{1}","meta.is_current": {2}}}'.format(item['meta']['name'], item['meta']['code_type'], str(item['meta']['is_current']).lower())
code_get = utilities.get_eve('code', query)
app.logger.debug(code_get)
for code in code_get['_items']:
request_payload = {'meta.is_current': False}
utilities.patch_eve('code', code['_id'], request_payload)
app.logger.debug('Ready to send to Celery\n{0}'.format(item))
tasks.code_deploy.delay(item)
def on_update_code_callback(updates, original):
"""
Update code on the servers as the item is updated.
:param updates:
:param original:
"""
app.logger.debug(updates)
app.logger.debug(original)
# If this 'is_current' PATCH code with the same name and code_type.
if updates.get('meta') and updates['meta'].get('is_current') and updates['meta']['is_current'] == True:
# If the name and code_type are not changing, we need to load them from
# the original.
name = updates['meta']['name'] if updates['meta'].get('name') else original['meta']['name']
code_type = updates['meta']['code_type'] if updates['meta'].get('code_type') else original['meta']['code_type']
query = 'where={{"meta.name":"{0}","meta.code_type":"{1}","meta.is_current": {2}}}'.format(name, code_type, str(updates['meta']['is_current']).lower())
code_get = utilities.get_eve('code', query)
# TODO: Filter out the site we are updating.
app.logger.debug(code_get)
for code in code_get['_items']:
request_payload = {'meta.is_current': False}
utilities.patch_eve('code', code['_id'], request_payload)
# We need the whole record so that we can manipulate code in the right
# place.
# Copy 'original' to a new dict, then update it with values from 'updates'
# to create an item to deploy. Need to do the same process for meta first,
# otherwise the update will fully overwrite.
meta = original['meta'].copy()
meta.update(updates['meta'])
updated_item = original.copy()
updated_item.update(updates)
updated_item['meta'] = meta
app.logger.debug('Ready to hand to Celery\n{0}\n{1}'.format(updated_item, original))
tasks.code_update.delay(updated_item, original)
def backup_create(site, backup_type):
"""
Backup the database and files for an site.
"""
log.debug('Backup | Create | site - %s', site)
# Create the stub for the backup
post_payload = {
'site': site['_id'],
'site_version': site['_version'],
'backup_type': backup_type,
'state': 'pending'
}
post_url = '{0}/backup'.format(API_URLS[ENVIRONMENT])
post = requests.post(
post_url,
data=post_payload,
auth=(SERVICE_ACCOUNT_USERNAME, SERVICE_ACCOUNT_PASSWORD),
verify=SSL_VERIFICATION,
)
if post.ok:
log.info('Backup | Create | POST - OK | %s | %s', post.content, post.headers)
else:
log.error('Backup | Create | POST - Error | %s', json.dumps(post.text))
backup_item = post.json()
log.info('Backup | Create | POST | Backup item - %s', backup_item)
# Setup dates and times.
start_time = time()
date = datetime.now()
date_time_string = date.strftime("%Y-%m-%d-%H-%M-%S")
datetime_string = date.strftime("%Y-%m-%d %H:%M:%S GMT")
# Instance paths
web_directory = '{0}/{1}'.format(WEB_ROOT, site['sid'])
database_result_file = '{0}_{1}.sql'.format(site['sid'], date_time_string)
database_result_file_path = '{0}/backups/{1}'.format(BACKUP_PATH, database_result_file)
if NFS_MOUNT_FILES_DIR:
files_dir = '{0}/{1}/files'.format(NFS_MOUNT_LOCATION[ENVIRONMENT], site['sid'])
else:
files_dir = '{0}/{1}/sites/default/files'.format(WEB_ROOT, site['sid'])
files_result_file = '{0}_{1}.tar.gz'.format(site['sid'], date_time_string)
files_result_file_path = '{0}/backups/{1}'.format(BACKUP_PATH, files_result_file)
# Start the actual process.
with cd(web_directory):
run('drush sql-dump --structure-tables-list=cache,cache_*,sessions,watchdog,history --result-file={0}'.format(
database_result_file_path))
with cd(files_dir):
log.debug('File dir | %s', files_dir)
run('tar --exclude "imagecache" --exclude "css" --exclude "js" --exclude "backup_migrate" --exclude "styles" --exclude "xmlsitemap" --exclude "honeypot" -czf {0} *'.format(
files_result_file_path))
backup_time = time() - start_time
# File size with thousand seperator, converted to MB.
db_size = '{:,.0f}'.format(os.path.getsize(database_result_file_path)/float(1 << 20))+" MB"
file_size = '{:,.0f}'.format(os.path.getsize(files_result_file_path)/float(1 << 20))+" MB"
patch_payload = {
'site': site['_id'],
'site_version': site['_version'],
'backup_date': datetime_string,
'backup_type': backup_type,
'files': files_result_file,
'database': database_result_file,
'state': 'complete'
}
log.debug('Backup | Create | Ready to update record | Payload - %s', patch_payload)
utilities.patch_eve('backup', backup_item['_id'], patch_payload)
log.info('Operational statistic | Backup Create | SID - %s | Time - %s | DB size - %s | File size - %s',
site['sid'], backup_time, db_size, file_size)
