def on_insert_code(items): """ Deploy code onto servers as the items are created. If a new code item 'is_current', PATCH 'is_current' code with the same name and type to no longer be current. :param items: List of dicts for items to be created. """ log.debug('code | Insert | items - %s', items) for item in items: log.debug('code | POST | On Insert callback | %s', item) # Check to see if we have a current profile and core. code_query = 'where={{"meta.name":"{0}","meta.version":"{1}","meta.code_type":"{2}"}}'.format(item['meta']['name'], item['meta']['version'], item['meta']['code_type']) code = utilities.get_eve('code', code_query) log.debug('code | POST | On Insert callback | Code query result | %s', code) if not code['_meta']['total'] == 0: log.error('code | POST | On Insert callback | %s named %s-%s already exists', item['meta']['code_type'], item['meta']['name'], item['meta']['version']) abort(409, 'Error: A {0} named {1}-{2} already exists.'.format(item['meta']['code_type'], item['meta']['name'], item['meta']['version'])) if item.get('meta') and item['meta'].get('is_current') and item['meta']['is_current'] is True: query = 'where={{"meta.name":"{0}","meta.code_type":"{1}","meta.is_current": true}}'.format(item['meta']['name'], item['meta']['code_type']) code_get = utilities.get_eve('code', query) log.debug('code | Insert | current code - %s', code_get) if code_get['_meta']['total'] != 0: for code in code_get['_items']: request_payload = {'meta.is_current': False} utilities.patch_eve('code', code['_id'], request_payload) log.debug('code | Insert | Ready to deploy item - %s', item) tasks.code_deploy.delay(item)
def _diff_f5(): """ Copy f5 configuration file to local sever, parse txt and create or update site items. """ f5_config_dir = "{0}/atlas/fabfile".format(path) f5_config_file = "{0}/{1}".format(f5_config_dir, f5_config_files[environment]) # If an older config file exists, copy it to a backup folder. if os.path.isfile(f5_config_file): local( "mv {0} /data/code/inventory/fabfile/backup/{1}.{2}".format( f5_config_file, f5_config_files[environment], str(time()).split(".")[0] ) ) # Copy config file from the f5 server to the Atlas server. local( "scp {0}:/config/{1} {2}/".format( serverdefs[environment]["f5_servers"][0], f5_config_files[environment], f5_config_dir ) ) # Open file from f5 with open(f5_config_file, "r") as ifile: data = ifile.read() # Use regex to parse out path values p = re.compile('"(.+/?)" := "(\w+(-\w+)?)",') sites = p.findall(data) # Iterate through sites found in f5 data for site in sites: f5only = False if site[0] in f5exceptions: f5only = True # Get path without leading slash path = site[0][1:] pool = site[1] # Set a type value based on pool if pool == "WWWLegacy": type = "legacy" elif pool == "poola-homepage" or pool == "poolb-homepage": type = "homepage" elif pool == "poolb-express": type = "express" else: type = "custom" site_query = 'where={{"path":"{0}"}}'.format(path) sites = utilities.get_eve("sites", site_query) if not sites or len(sites["_items"]) == 0: payload = {"name": path, "path": path, "pool": pool, "status": "launched", "type": type, "f5only": f5only} utilities.post_eve("sites", payload) print ("Created site record based on f5.\n{0}".format(payload)) elif pool != data["_items"][0]["pool"]: site = data["_items"][0] payload = {"pool": pool, "status": "launched", "type": type} utilities.patch_eve("sites", site["_id"], payload) print "Updated site based on f5.\n{0}".format(payload)
def delete_all_available_sites(): """ Get a list of available sites and delete them """ site_query = 'where={"status":"available"}' sites = utilities.get_eve("sites", site_query) payload = {"status": "delete"} for site in sites: utilities.patch_eve("sites", site["_id"], payload)
def delete_stale_pending_sites(): site_query = 'where={"status":"pending"}' sites = utilities.get_eve("sites", site_query) # Loop through and remove sites that are more than 30 minutes old. for site in sites["_items"]: # Parse date string into structured time. # See https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior for mask format. date_created = time.strptime(site["_created"], "%Y-%m-%d %H:%M:%S %Z") # Get time now, Convert date_created to seconds from epoch and # calculate the age of the site. seconds_since_creation = time.time() - time.mktime(date_created) # 30 min * 60 sec = 1800 seconds if seconds_since_creation > 1800: payload = {"status": "delete"} utilities.patch_eve("sites", site["_id"], payload)
def site_provision(site): """ Provision a new instance with the given parameters. :param site: A single site. :return: """ logger.debug("Site provision - {0}".format(site)) # 'db_key' needs to be added here and not in Eve so that the encryption # works properly. site["db_key"] = utilities.encrypt_string(utilities.mysql_password()) fab_task = execute(fabfile.site_provision, site=site) logger.debug(fab_task) logger.debug(fab_task.values) patch_payload = {"status": "available", "db_key": site["db_key"]} patch = utilities.patch_eve("sites", site["_id"], patch_payload) logger.debug("Site has been provisioned\n{0}".format(patch)) slack_title = "{0}/{1}".format(base_urls[environment], site["sid"]) slack_link = "{0}/{1}".format(base_urls[environment], site["sid"]) attachment_text = "{0}/sites/{1}".format(api_server, site["_id"]) if False not in fab_task.values(): slack_message = "Site provision - Success" slack_color = "good" utilities.post_to_slack( message=slack_message, title=slack_title, link=slack_link, attachment_text=attachment_text, level=slack_color, )
def on_update_code(updates, original): """ Update code on the servers as the item is updated. :param updates: :param original: """ log.debug('code | on update | updates - %s | original - %s', updates, original) # If this 'is_current' PATCH code with the same name and code_type. if updates.get('meta') and updates['meta'].get('is_current') and updates['meta']['is_current'] is True: # If the name and code_type are not changing, we need to load them from the original. name = updates['meta']['name'] if updates['meta'].get('name') else original['meta']['name'] code_type = updates['meta']['code_type'] if updates['meta'].get('code_type') else original['meta']['code_type'] query = 'where={{"meta.name":"{0}","meta.code_type":"{1}","meta.is_current": true,"_id":{{"$ne":"{2}"}}}}'.format( name, code_type, original['_id']) code_get = utilities.get_eve('code', query) log.debug('code | on update | Current code - %s', code_get) for code in code_get['_items']: request_payload = {'meta.is_current': False} utilities.patch_eve('code', code['_id'], request_payload) # We need the whole record so that we can manipulate code in the right place. # Copy 'original' to a new dict, then update it with values from 'updates' to create an item to # deploy. Need to do the same process for meta first, otherwise the update will fully overwrite. if updates.get('meta'): meta = original['meta'].copy() meta.update(updates['meta']) updated_item = original.copy() updated_item.update(updates) if updates.get('meta'): updated_item['meta'] = meta if updates.has_key('meta') and (updates['meta'].has_key('name') or updates['meta'].has_key('version') or updates['meta'].has_key('code_type')): update_code = True elif updates.has_key('commit_hash') or updates.has_key('git_url'): update_code = True else: update_code = False if update_code: log.debug('code | on update | Ready to hand to Celery') # chord two tasks chord(tasks.code_update.s(updated_item, original), tasks.clear_php_cache.si())()
def backup_restore(backup_record, original_instance, package_list): """ Restore database and files to a new instance. """ log.info('Instance | Restore Backup | %s | %s', backup_record, original_instance) start_time = time() file_date = datetime.strptime(backup_record['backup_date'], "%Y-%m-%d %H:%M:%S %Z") pretty_filename = '{0}_{1}'.format( original_instance['sid'], file_date.strftime("%Y-%m-%d-%H-%M-%S")) pretty_database_filename = '{0}.sql'.format(pretty_filename) database_path = '{0}/backups/{1}'.format(BACKUP_PATH, pretty_database_filename) pretty_files_filename = '{0}.tar.gz'.format(pretty_filename) files_path = '{0}/backups/{1}'.format(BACKUP_PATH, pretty_files_filename) # Grab available instance and add packages if needed available_instances = utilities.get_eve('sites', 'where={"status":"available"}') log.debug('Instance | Restore Backup | Avaiable Instances - %s', available_instances) new_instance = next(iter(available_instances['_items']), None) # TODO: Don't switch if the code is the same if new_instance is not None: payload = {'status': 'installing'} if package_list: packages = {'code': {'package': package_list}} payload.update(packages) utilities.patch_eve('sites', new_instance['_id'], payload) else: exit('No available instances.') # Wait for code and status to update. attempts = 18 # Tries every 10 seconds to a max of 18 (or 3 minutes). while attempts: try: new_instance_refresh = utilities.get_single_eve('sites', new_instance['_id']) if new_instance_refresh['status'] != 'installed': log.info('Instance | Restore Backup | New instance is not ready | %s', new_instance['_id']) raise ValueError('Status has not yet updated.') break except ValueError, e: # If the status is not updated and we have attempts left, # remove an attempt and wait 10 seconds. attempts -= 1 if attempts is not 0: sleep(10) else: exit(str(e))
def site_update(site, updates, original): """ Update an instance with the given parameters. :param site: A complete site item, including new values. :param updates: A partial site item, including only changed keys. :param original: Complete original site item. :return: """ logger.debug("Site update - {0}\n{1}\n\n{2}".format(site["_id"], site, updates)) if updates.get("code"): logger.debug("Found code changes.") if "package" in updates["code"]: logger.debug("Found package changes.") execute(fabfile.site_package_update, site=site) execute(fabfile.update_database, site=site) if updates["code"].get("core") != original["code"].get("core"): logger.debug("Found core change.") execute(fabfile.site_core_update, site=site) execute(fabfile.update_database, site=site) if updates["code"].get("profile") != original["code"].get("profile"): logger.debug("Found profile change.") execute(fabfile.site_profile_update, site=site, original=original, updates=updates) execute(fabfile.update_database, site=site) if updates.get("status"): logger.debug("Found status change.") if updates["status"] in ["installing", "launching", "take_down", "restore"]: if updates["status"] == "installing": logger.debug("Status changed to installing") patch_payload = '{"status": "installed"}' elif updates["status"] == "launching": logger.debug("Status changed to launching") execute(fabfile.site_launch, site=site) patch_payload = '{"status": "launched"}' elif updates["status"] == "take_down": logger.debug("Status changed to take_down") execute(fabfile.site_take_down, site=site) patch_payload = '{"status": "down"}' elif updates["status"] == "restore": logger.debug("Status changed to restore") execute(fabfile.site_restore, site=site) execute(fabfile.update_database, site=site) patch_payload = '{"status": "installed"}' patch = utilities.patch_eve("sites", site["_id"], patch_payload) logger.debug(patch) slack_title = "{0}/{1}".format(base_urls[environment], site["sid"]) slack_link = "{0}/{1}".format(base_urls[environment], site["sid"]) attachment_text = "{0}/sites/{1}".format(api_server, site["_id"]) slack_message = "Site Update - Success" slack_color = "good" utilities.post_to_slack( message=slack_message, title=slack_title, link=slack_link, attachment_text=attachment_text, level=slack_color )
def _launch_site(site, gsa_collection=False): """ Create symlinks with new site name. """ print ("Launch subtask") code_directory = "{0}/{1}".format(sites_code_root, site["sid"]) code_directory_current = "{0}/current".format(code_directory) if site["pool"] in ["poolb-express", "poolb-homepage"]: if site["pool"] == "poolb-express": web_directory = "{0}/{1}".format(sites_web_root, site["type"]) web_directory_path = "{0}/{1}".format(web_directory, site["path"]) with cd(web_directory): # If the path is nested like 'lab/atlas', make the 'lab' directory if "/" in site["path"]: lead_path = "/".join(site["path"].split("/")[:-1]) _create_directory_structure(lead_path) # Create a new symlink using site's updated path if not exists(web_directory_path): _update_symlink(code_directory_current, site["path"]) # enter new site directory with cd(web_directory_path): clear_apc() if gsa_collection: # Set the collection name run("drush vset --yes google_appliance_collection {0}".format(gsa_collection)) # Clear caches at the end of the launch process to show # correct pathologic rendered URLS. drush_cache_clear(site["sid"]) # Assign it to an update group. update_group = randint(0, 10) if site["pool"] == "poolb-homepage": web_directory = "{0}/{1}".format(sites_web_root, "homepage") with cd(sites_web_root): _update_symlink(code_directory_current, web_directory) # enter new site directory with cd(web_directory): clear_apc() drush_cache_clear(site["sid"]) # Assign site to update group 12. update_group = 12 payload = {"status": "launched", "update_group": update_group} utilities.patch_eve("sites", site["_id"], payload)
def on_updated_code(updates, original): """ Find instances that use this code asset and re-add them. :param updates: :param original: """ log.debug('code | on updated | updates - %s | original - %s', updates, original) # First get the code_type from either the update or original, then convert package types for # querying instance objects. if updates.get('meta') and updates['meta'].get('code_type'): code_type = updates['meta']['code_type'] else: code_type = original['meta']['code_type'] if code_type in ['module', 'theme', 'library']: code_type = 'package' if updates.has_key('meta'): if updates['meta']['name'] != original['meta']['name'] or updates['meta']['version'] != original['meta']['version'] or updates['meta']['code_type'] != original['meta']['code_type']: update_sites = True log.debug('code | on updated | Found meta data changes | %s', updates['meta']) else: log.debug('code | on updated | Found no meta changes that require an update') update_sites = False elif updates.has_key('commit_hash') or updates.has_key('git_url'): update_sites = True log.debug('code | on updated | Found git data changes') else: log.debug('code | on updated | Found no changes') update_sites = False if update_sites: log.info('Code | on updated | Preparing to update instances') query = 'where={{"code.{0}":"{1}"}}'.format(code_type, original['_id']) sites_get = utilities.get_eve('sites', query) if sites_get['_meta']['total'] is not 0: for site in sites_get['_items']: log.debug('code | on updated | site - %s', site) code_id_string = site['code'][code_type] payload = {'code': {code_type: code_id_string}} log.debug('code | on updated | payload - %s', payload) utilities.patch_eve('sites', site['_id'], payload)
def take_down_installed_35_day_old_sites(): if environment != "prod": site_query = 'where={"status":"installed"}' sites = utilities.get_eve("sites", site_query) # Loop through and remove sites that are more than 35 days old. for site in sites["_items"]: # Parse date string into structured time. # See https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior # for mask format. date_created = time.strptime(site["_created"], "%Y-%m-%d %H:%M:%S %Z") # Get time now, Convert date_created to seconds from epoch and # calculate the age of the site. seconds_since_creation = time.time() - time.mktime(date_created) print("{0} is {1} seconds old".format(site["sid"], seconds_since_creation)) # 35 days * 24 hrs * 60 min * 60 sec = 302400 seconds if seconds_since_creation > 3024000: # Patch the status to 'take_down'. payload = {"status": "take_down"} utilities.patch_eve("sites", site["_id"], payload)
def on_insert_code_callback(items): """ Deploy code onto servers as the items are created. If a new code item 'is_current', PATCH 'is_current' code with the same name and type to no longer be current. :param items: List of dicts for items to be created. """ app.logger.debug(items) for item in items: if item.get('meta') and item['meta'].get('is_current') and item['meta']['is_current'] == True: # Need a lowercase string when querying boolean values. Python # stores it as 'True'. query = 'where={{"meta.name":"{0}","meta.code_type":"{1}","meta.is_current": {2}}}'.format(item['meta']['name'], item['meta']['code_type'], str(item['meta']['is_current']).lower()) code_get = utilities.get_eve('code', query) app.logger.debug(code_get) for code in code_get['_items']: request_payload = {'meta.is_current': False} utilities.patch_eve('code', code['_id'], request_payload) app.logger.debug('Ready to send to Celery\n{0}'.format(item)) tasks.code_deploy.delay(item)
def on_update_code_callback(updates, original): """ Update code on the servers as the item is updated. :param updates: :param original: """ app.logger.debug(updates) app.logger.debug(original) # If this 'is_current' PATCH code with the same name and code_type. if updates.get('meta') and updates['meta'].get('is_current') and updates['meta']['is_current'] == True: # If the name and code_type are not changing, we need to load them from # the original. name = updates['meta']['name'] if updates['meta'].get('name') else original['meta']['name'] code_type = updates['meta']['code_type'] if updates['meta'].get('code_type') else original['meta']['code_type'] query = 'where={{"meta.name":"{0}","meta.code_type":"{1}","meta.is_current": {2}}}'.format(name, code_type, str(updates['meta']['is_current']).lower()) code_get = utilities.get_eve('code', query) # TODO: Filter out the site we are updating. app.logger.debug(code_get) for code in code_get['_items']: request_payload = {'meta.is_current': False} utilities.patch_eve('code', code['_id'], request_payload) # We need the whole record so that we can manipulate code in the right # place. # Copy 'original' to a new dict, then update it with values from 'updates' # to create an item to deploy. Need to do the same process for meta first, # otherwise the update will fully overwrite. meta = original['meta'].copy() meta.update(updates['meta']) updated_item = original.copy() updated_item.update(updates) updated_item['meta'] = meta app.logger.debug('Ready to hand to Celery\n{0}\n{1}'.format(updated_item, original)) tasks.code_update.delay(updated_item, original)
def backup_create(site, backup_type): """ Backup the database and files for an site. """ log.debug('Backup | Create | site - %s', site) # Create the stub for the backup post_payload = { 'site': site['_id'], 'site_version': site['_version'], 'backup_type': backup_type, 'state': 'pending' } post_url = '{0}/backup'.format(API_URLS[ENVIRONMENT]) post = requests.post( post_url, data=post_payload, auth=(SERVICE_ACCOUNT_USERNAME, SERVICE_ACCOUNT_PASSWORD), verify=SSL_VERIFICATION, ) if post.ok: log.info('Backup | Create | POST - OK | %s | %s', post.content, post.headers) else: log.error('Backup | Create | POST - Error | %s', json.dumps(post.text)) backup_item = post.json() log.info('Backup | Create | POST | Backup item - %s', backup_item) # Setup dates and times. start_time = time() date = datetime.now() date_time_string = date.strftime("%Y-%m-%d-%H-%M-%S") datetime_string = date.strftime("%Y-%m-%d %H:%M:%S GMT") # Instance paths web_directory = '{0}/{1}'.format(WEB_ROOT, site['sid']) database_result_file = '{0}_{1}.sql'.format(site['sid'], date_time_string) database_result_file_path = '{0}/backups/{1}'.format(BACKUP_PATH, database_result_file) if NFS_MOUNT_FILES_DIR: files_dir = '{0}/{1}/files'.format(NFS_MOUNT_LOCATION[ENVIRONMENT], site['sid']) else: files_dir = '{0}/{1}/sites/default/files'.format(WEB_ROOT, site['sid']) files_result_file = '{0}_{1}.tar.gz'.format(site['sid'], date_time_string) files_result_file_path = '{0}/backups/{1}'.format(BACKUP_PATH, files_result_file) # Start the actual process. with cd(web_directory): run('drush sql-dump --structure-tables-list=cache,cache_*,sessions,watchdog,history --result-file={0}'.format( database_result_file_path)) with cd(files_dir): log.debug('File dir | %s', files_dir) run('tar --exclude "imagecache" --exclude "css" --exclude "js" --exclude "backup_migrate" --exclude "styles" --exclude "xmlsitemap" --exclude "honeypot" -czf {0} *'.format( files_result_file_path)) backup_time = time() - start_time # File size with thousand seperator, converted to MB. db_size = '{:,.0f}'.format(os.path.getsize(database_result_file_path)/float(1 << 20))+" MB" file_size = '{:,.0f}'.format(os.path.getsize(files_result_file_path)/float(1 << 20))+" MB" patch_payload = { 'site': site['_id'], 'site_version': site['_version'], 'backup_date': datetime_string, 'backup_type': backup_type, 'files': files_result_file, 'database': database_result_file, 'state': 'complete' } log.debug('Backup | Create | Ready to update record | Payload - %s', patch_payload) utilities.patch_eve('backup', backup_item['_id'], patch_payload) log.info('Operational statistic | Backup Create | SID - %s | Time - %s | DB size - %s | File size - %s', site['sid'], backup_time, db_size, file_size)