Exemplo n.º 1
0
def on_insert_code(items):
    """
    Deploy code onto servers as the items are created.

    If a new code item 'is_current', PATCH 'is_current' code with the same name
    and type to no longer be current.

    :param items: List of dicts for items to be created.
    """
    log.debug('code | Insert | items - %s', items)
    for item in items:
        log.debug('code | POST | On Insert callback | %s', item)
        # Check to see if we have a current profile and core.
        code_query = 'where={{"meta.name":"{0}","meta.version":"{1}","meta.code_type":"{2}"}}'.format(item['meta']['name'], item['meta']['version'], item['meta']['code_type'])
        code = utilities.get_eve('code', code_query)
        log.debug('code | POST | On Insert callback | Code query result | %s', code)
        if not code['_meta']['total'] == 0:
            log.error('code | POST | On Insert callback | %s named %s-%s already exists', item['meta']['code_type'], item['meta']['name'], item['meta']['version'])
            abort(409, 'Error: A {0} named {1}-{2} already exists.'.format(item['meta']['code_type'], item['meta']['name'], item['meta']['version']))

        if item.get('meta') and item['meta'].get('is_current') and item['meta']['is_current'] is True:
            query = 'where={{"meta.name":"{0}","meta.code_type":"{1}","meta.is_current": true}}'.format(item['meta']['name'], item['meta']['code_type'])
            code_get = utilities.get_eve('code', query)
            log.debug('code | Insert | current code - %s', code_get)
            if code_get['_meta']['total'] != 0:
                for code in code_get['_items']:
                    request_payload = {'meta.is_current': False}
                    utilities.patch_eve('code', code['_id'], request_payload)
        log.debug('code | Insert | Ready to deploy item - %s', item)
        tasks.code_deploy.delay(item)
Exemplo n.º 2
0
def _diff_f5():
    """
    Copy f5 configuration file to local sever, parse txt and create or update
    site items.

    """
    f5_config_dir = "{0}/atlas/fabfile".format(path)
    f5_config_file = "{0}/{1}".format(f5_config_dir, f5_config_files[environment])
    # If an older config file exists, copy it to a backup folder.
    if os.path.isfile(f5_config_file):
        local(
            "mv {0} /data/code/inventory/fabfile/backup/{1}.{2}".format(
                f5_config_file, f5_config_files[environment], str(time()).split(".")[0]
            )
        )
    # Copy config file from the f5 server to the Atlas server.
    local(
        "scp {0}:/config/{1} {2}/".format(
            serverdefs[environment]["f5_servers"][0], f5_config_files[environment], f5_config_dir
        )
    )

    # Open file from f5
    with open(f5_config_file, "r") as ifile:
        data = ifile.read()
    # Use regex to parse out path values
    p = re.compile('"(.+/?)" := "(\w+(-\w+)?)",')
    sites = p.findall(data)
    # Iterate through sites found in f5 data
    for site in sites:
        f5only = False
        if site[0] in f5exceptions:
            f5only = True
        # Get path without leading slash
        path = site[0][1:]
        pool = site[1]
        # Set a type value based on pool
        if pool == "WWWLegacy":
            type = "legacy"
        elif pool == "poola-homepage" or pool == "poolb-homepage":
            type = "homepage"
        elif pool == "poolb-express":
            type = "express"
        else:
            type = "custom"

        site_query = 'where={{"path":"{0}"}}'.format(path)
        sites = utilities.get_eve("sites", site_query)

        if not sites or len(sites["_items"]) == 0:
            payload = {"name": path, "path": path, "pool": pool, "status": "launched", "type": type, "f5only": f5only}
            utilities.post_eve("sites", payload)
            print ("Created site record based on f5.\n{0}".format(payload))
        elif pool != data["_items"][0]["pool"]:
            site = data["_items"][0]
            payload = {"pool": pool, "status": "launched", "type": type}
            utilities.patch_eve("sites", site["_id"], payload)
            print "Updated site based on f5.\n{0}".format(payload)
Exemplo n.º 3
0
def delete_all_available_sites():
    """
    Get a list of available sites and delete them
    """
    site_query = 'where={"status":"available"}'
    sites = utilities.get_eve("sites", site_query)
    payload = {"status": "delete"}
    for site in sites:
        utilities.patch_eve("sites", site["_id"], payload)
Exemplo n.º 4
0
def delete_stale_pending_sites():
    site_query = 'where={"status":"pending"}'
    sites = utilities.get_eve("sites", site_query)
    # Loop through and remove sites that are more than 30 minutes old.
    for site in sites["_items"]:
        # Parse date string into structured time.
        # See https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior for mask format.
        date_created = time.strptime(site["_created"], "%Y-%m-%d %H:%M:%S %Z")
        # Get time now, Convert date_created to seconds from epoch and
        # calculate the age of the site.
        seconds_since_creation = time.time() - time.mktime(date_created)
        # 30 min * 60 sec = 1800 seconds
        if seconds_since_creation > 1800:
            payload = {"status": "delete"}
            utilities.patch_eve("sites", site["_id"], payload)
Exemplo n.º 5
0
def site_provision(site):
    """
    Provision a new instance with the given parameters.

    :param site: A single site.
    :return:
    """
    logger.debug("Site provision - {0}".format(site))
    # 'db_key' needs to be added here and not in Eve so that the encryption
    # works properly.
    site["db_key"] = utilities.encrypt_string(utilities.mysql_password())
    fab_task = execute(fabfile.site_provision, site=site)
    logger.debug(fab_task)
    logger.debug(fab_task.values)

    patch_payload = {"status": "available", "db_key": site["db_key"]}
    patch = utilities.patch_eve("sites", site["_id"], patch_payload)

    logger.debug("Site has been provisioned\n{0}".format(patch))

    slack_title = "{0}/{1}".format(base_urls[environment], site["sid"])
    slack_link = "{0}/{1}".format(base_urls[environment], site["sid"])
    attachment_text = "{0}/sites/{1}".format(api_server, site["_id"])
    if False not in fab_task.values():
        slack_message = "Site provision - Success"
        slack_color = "good"
        utilities.post_to_slack(
            message=slack_message,
            title=slack_title,
            link=slack_link,
            attachment_text=attachment_text,
            level=slack_color,
        )
Exemplo n.º 6
0
def on_update_code(updates, original):
    """
    Update code on the servers as the item is updated.

    :param updates:
    :param original:
    """
    log.debug('code | on update | updates - %s | original - %s', updates, original)
    # If this 'is_current' PATCH code with the same name and code_type.
    if updates.get('meta') and updates['meta'].get('is_current') and updates['meta']['is_current'] is True:
        # If the name and code_type are not changing, we need to load them from the original.
        name = updates['meta']['name'] if updates['meta'].get('name') else original['meta']['name']
        code_type = updates['meta']['code_type'] if updates['meta'].get('code_type') else original['meta']['code_type']

        query = 'where={{"meta.name":"{0}","meta.code_type":"{1}","meta.is_current": true,"_id":{{"$ne":"{2}"}}}}'.format(
            name, code_type, original['_id'])
        code_get = utilities.get_eve('code', query)
        log.debug('code | on update | Current code - %s', code_get)

        for code in code_get['_items']:
            request_payload = {'meta.is_current': False}
            utilities.patch_eve('code', code['_id'], request_payload)

    # We need the whole record so that we can manipulate code in the right place.
    # Copy 'original' to a new dict, then update it with values from 'updates' to create an item to
    # deploy. Need to do the same process for meta first, otherwise the update will fully overwrite.
    if updates.get('meta'):
        meta = original['meta'].copy()
        meta.update(updates['meta'])
    updated_item = original.copy()
    updated_item.update(updates)
    if updates.get('meta'):
        updated_item['meta'] = meta

    if updates.has_key('meta') and (updates['meta'].has_key('name') or updates['meta'].has_key('version') or updates['meta'].has_key('code_type')):
        update_code = True
    elif updates.has_key('commit_hash') or updates.has_key('git_url'):
        update_code = True
    else:
        update_code = False

    if update_code:
        log.debug('code | on update | Ready to hand to Celery')
        # chord two tasks
        chord(tasks.code_update.s(updated_item, original), tasks.clear_php_cache.si())()
Exemplo n.º 7
0
def backup_restore(backup_record, original_instance, package_list):
    """
    Restore database and files to a new instance.
    """
    log.info('Instance | Restore Backup | %s | %s', backup_record, original_instance)
    start_time = time()
    file_date = datetime.strptime(backup_record['backup_date'], "%Y-%m-%d %H:%M:%S %Z")
    pretty_filename = '{0}_{1}'.format(
        original_instance['sid'], file_date.strftime("%Y-%m-%d-%H-%M-%S"))
    pretty_database_filename = '{0}.sql'.format(pretty_filename)
    database_path = '{0}/backups/{1}'.format(BACKUP_PATH, pretty_database_filename)
    pretty_files_filename = '{0}.tar.gz'.format(pretty_filename)
    files_path = '{0}/backups/{1}'.format(BACKUP_PATH, pretty_files_filename)

    # Grab available instance and add packages if needed
    available_instances = utilities.get_eve('sites', 'where={"status":"available"}')
    log.debug('Instance | Restore Backup | Avaiable Instances - %s', available_instances)
    new_instance = next(iter(available_instances['_items']), None)
    # TODO: Don't switch if the code is the same
    if new_instance is not None:
        payload = {'status': 'installing'}
        if package_list:
            packages = {'code': {'package': package_list}}
            payload.update(packages)
        utilities.patch_eve('sites', new_instance['_id'], payload)
    else:
        exit('No available instances.')

    # Wait for code and status to update.
    attempts = 18  # Tries every 10 seconds to a max of 18 (or 3 minutes).
    while attempts:
        try:
            new_instance_refresh = utilities.get_single_eve('sites', new_instance['_id'])
            if new_instance_refresh['status'] != 'installed':
                log.info('Instance | Restore Backup | New instance is not ready | %s', new_instance['_id'])
                raise ValueError('Status has not yet updated.')
            break
        except ValueError, e:
            # If the status is not updated and we have attempts left,
            # remove an attempt and wait 10 seconds.
            attempts -= 1
            if attempts is not 0:
                sleep(10)
            else:
                exit(str(e))
Exemplo n.º 8
0
def site_update(site, updates, original):
    """
    Update an instance with the given parameters.

    :param site: A complete site item, including new values.
    :param updates: A partial site item, including only changed keys.
    :param original: Complete original site item.
    :return:
    """
    logger.debug("Site update - {0}\n{1}\n\n{2}".format(site["_id"], site, updates))

    if updates.get("code"):
        logger.debug("Found code changes.")
        if "package" in updates["code"]:
            logger.debug("Found package changes.")
            execute(fabfile.site_package_update, site=site)
            execute(fabfile.update_database, site=site)
        if updates["code"].get("core") != original["code"].get("core"):
            logger.debug("Found core change.")
            execute(fabfile.site_core_update, site=site)
            execute(fabfile.update_database, site=site)
        if updates["code"].get("profile") != original["code"].get("profile"):
            logger.debug("Found profile change.")
            execute(fabfile.site_profile_update, site=site, original=original, updates=updates)
            execute(fabfile.update_database, site=site)

    if updates.get("status"):
        logger.debug("Found status change.")
        if updates["status"] in ["installing", "launching", "take_down", "restore"]:
            if updates["status"] == "installing":
                logger.debug("Status changed to installing")
                patch_payload = '{"status": "installed"}'
            elif updates["status"] == "launching":
                logger.debug("Status changed to launching")
                execute(fabfile.site_launch, site=site)
                patch_payload = '{"status": "launched"}'
            elif updates["status"] == "take_down":
                logger.debug("Status changed to take_down")
                execute(fabfile.site_take_down, site=site)
                patch_payload = '{"status": "down"}'
            elif updates["status"] == "restore":
                logger.debug("Status changed to restore")
                execute(fabfile.site_restore, site=site)
                execute(fabfile.update_database, site=site)
                patch_payload = '{"status": "installed"}'

            patch = utilities.patch_eve("sites", site["_id"], patch_payload)
            logger.debug(patch)

    slack_title = "{0}/{1}".format(base_urls[environment], site["sid"])
    slack_link = "{0}/{1}".format(base_urls[environment], site["sid"])
    attachment_text = "{0}/sites/{1}".format(api_server, site["_id"])
    slack_message = "Site Update - Success"
    slack_color = "good"
    utilities.post_to_slack(
        message=slack_message, title=slack_title, link=slack_link, attachment_text=attachment_text, level=slack_color
    )
Exemplo n.º 9
0
def _launch_site(site, gsa_collection=False):
    """
    Create symlinks with new site name.
    """
    print ("Launch subtask")
    code_directory = "{0}/{1}".format(sites_code_root, site["sid"])
    code_directory_current = "{0}/current".format(code_directory)

    if site["pool"] in ["poolb-express", "poolb-homepage"]:
        if site["pool"] == "poolb-express":
            web_directory = "{0}/{1}".format(sites_web_root, site["type"])
            web_directory_path = "{0}/{1}".format(web_directory, site["path"])
            with cd(web_directory):
                # If the path is nested like 'lab/atlas', make the 'lab' directory
                if "/" in site["path"]:
                    lead_path = "/".join(site["path"].split("/")[:-1])
                    _create_directory_structure(lead_path)

                # Create a new symlink using site's updated path
                if not exists(web_directory_path):
                    _update_symlink(code_directory_current, site["path"])
                # enter new site directory
                with cd(web_directory_path):
                    clear_apc()
                    if gsa_collection:
                        # Set the collection name
                        run("drush vset --yes google_appliance_collection {0}".format(gsa_collection))
                    # Clear caches at the end of the launch process to show
                    # correct pathologic rendered URLS.
                    drush_cache_clear(site["sid"])
            # Assign it to an update group.
            update_group = randint(0, 10)
        if site["pool"] == "poolb-homepage":
            web_directory = "{0}/{1}".format(sites_web_root, "homepage")
            with cd(sites_web_root):
                _update_symlink(code_directory_current, web_directory)
                # enter new site directory
            with cd(web_directory):
                clear_apc()
                drush_cache_clear(site["sid"])
            # Assign site to update group 12.
            update_group = 12
        payload = {"status": "launched", "update_group": update_group}
        utilities.patch_eve("sites", site["_id"], payload)
Exemplo n.º 10
0
def on_updated_code(updates, original):
    """
    Find instances that use this code asset and re-add them.

    :param updates:
    :param original:
    """
    log.debug('code | on updated | updates - %s | original - %s', updates, original)
    # First get the code_type from either the update or original, then convert package types for
    # querying instance objects.
    if updates.get('meta') and updates['meta'].get('code_type'):
        code_type = updates['meta']['code_type']
    else:
        code_type = original['meta']['code_type']
    if code_type in ['module', 'theme', 'library']:
        code_type = 'package'

    if updates.has_key('meta'):
        if updates['meta']['name'] != original['meta']['name'] or updates['meta']['version'] != original['meta']['version'] or updates['meta']['code_type'] != original['meta']['code_type']:
            update_sites = True
            log.debug('code | on updated | Found meta data changes | %s', updates['meta'])
        else:
            log.debug('code | on updated | Found no meta changes that require an update')
            update_sites = False
    elif updates.has_key('commit_hash') or updates.has_key('git_url'):
        update_sites = True
        log.debug('code | on updated | Found git data changes')
    else:
        log.debug('code | on updated | Found no changes')
        update_sites = False

    if update_sites:
        log.info('Code | on updated | Preparing to update instances')
        query = 'where={{"code.{0}":"{1}"}}'.format(code_type, original['_id'])
        sites_get = utilities.get_eve('sites', query)

        if sites_get['_meta']['total'] is not 0:
            for site in sites_get['_items']:
                log.debug('code | on updated | site - %s', site)
                code_id_string = site['code'][code_type]
                payload = {'code': {code_type: code_id_string}}
                log.debug('code | on updated | payload - %s', payload)
                utilities.patch_eve('sites', site['_id'], payload)
Exemplo n.º 11
0
def take_down_installed_35_day_old_sites():
    if environment != "prod":
        site_query = 'where={"status":"installed"}'
        sites = utilities.get_eve("sites", site_query)
        # Loop through and remove sites that are more than 35 days old.
        for site in sites["_items"]:
            # Parse date string into structured time.
            # See https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior
            # for mask format.
            date_created = time.strptime(site["_created"], "%Y-%m-%d %H:%M:%S %Z")
            # Get time now, Convert date_created to seconds from epoch and
            # calculate the age of the site.
            seconds_since_creation = time.time() - time.mktime(date_created)
            print("{0} is {1} seconds old".format(site["sid"], seconds_since_creation))
            # 35 days * 24 hrs * 60 min * 60 sec = 302400 seconds
            if seconds_since_creation > 3024000:
                # Patch the status to 'take_down'.
                payload = {"status": "take_down"}
                utilities.patch_eve("sites", site["_id"], payload)
Exemplo n.º 12
0
def on_insert_code_callback(items):
    """
    Deploy code onto servers as the items are created.

    If a new code item 'is_current', PATCH 'is_current' code with the same name
    and type to no longer be current.

    :param items: List of dicts for items to be created.
    """
    app.logger.debug(items)
    for item in items:
        if item.get('meta') and item['meta'].get('is_current') and item['meta']['is_current'] == True:
            # Need a lowercase string when querying boolean values. Python
            # stores it as 'True'.
            query = 'where={{"meta.name":"{0}","meta.code_type":"{1}","meta.is_current": {2}}}'.format(item['meta']['name'], item['meta']['code_type'], str(item['meta']['is_current']).lower())
            code_get = utilities.get_eve('code', query)
            app.logger.debug(code_get)
            for code in code_get['_items']:
                request_payload = {'meta.is_current': False}
                utilities.patch_eve('code', code['_id'], request_payload)
        app.logger.debug('Ready to send to Celery\n{0}'.format(item))
        tasks.code_deploy.delay(item)
Exemplo n.º 13
0
def on_update_code_callback(updates, original):
    """
    Update code on the servers as the item is updated.

    :param updates:
    :param original:
    """
    app.logger.debug(updates)
    app.logger.debug(original)
    # If this 'is_current' PATCH code with the same name and code_type.
    if updates.get('meta') and updates['meta'].get('is_current') and updates['meta']['is_current'] == True:
        # If the name and code_type are not changing, we need to load them from
        # the original.
        name = updates['meta']['name'] if updates['meta'].get('name') else original['meta']['name']
        code_type = updates['meta']['code_type'] if updates['meta'].get('code_type') else original['meta']['code_type']

        query = 'where={{"meta.name":"{0}","meta.code_type":"{1}","meta.is_current": {2}}}'.format(name, code_type, str(updates['meta']['is_current']).lower())
        code_get = utilities.get_eve('code', query)
        # TODO: Filter out the site we are updating.
        app.logger.debug(code_get)

        for code in code_get['_items']:
            request_payload = {'meta.is_current': False}
            utilities.patch_eve('code', code['_id'], request_payload)

    # We need the whole record so that we can manipulate code in the right
    # place.
    # Copy 'original' to a new dict, then update it with values from 'updates'
    # to create an item to deploy. Need to do the same process for meta first,
    # otherwise the update will fully overwrite.
    meta = original['meta'].copy()
    meta.update(updates['meta'])
    updated_item = original.copy()
    updated_item.update(updates)
    updated_item['meta'] = meta

    app.logger.debug('Ready to hand to Celery\n{0}\n{1}'.format(updated_item, original))
    tasks.code_update.delay(updated_item, original)
Exemplo n.º 14
0
def backup_create(site, backup_type):
    """
    Backup the database and files for an site.
    """
    log.debug('Backup | Create | site - %s', site)

    # Create the stub for the backup
    post_payload = {
        'site': site['_id'],
        'site_version': site['_version'],
        'backup_type': backup_type,
        'state': 'pending'
    }
    post_url = '{0}/backup'.format(API_URLS[ENVIRONMENT])
    post = requests.post(
        post_url,
        data=post_payload,
        auth=(SERVICE_ACCOUNT_USERNAME, SERVICE_ACCOUNT_PASSWORD),
        verify=SSL_VERIFICATION,
    )
    if post.ok:
        log.info('Backup | Create | POST - OK | %s | %s', post.content, post.headers)
    else:
        log.error('Backup | Create | POST - Error | %s', json.dumps(post.text))

    backup_item = post.json()
    log.info('Backup | Create | POST | Backup item - %s', backup_item)
    # Setup dates and times.
    start_time = time()
    date = datetime.now()
    date_time_string = date.strftime("%Y-%m-%d-%H-%M-%S")
    datetime_string = date.strftime("%Y-%m-%d %H:%M:%S GMT")

    # Instance paths
    web_directory = '{0}/{1}'.format(WEB_ROOT, site['sid'])
    database_result_file = '{0}_{1}.sql'.format(site['sid'], date_time_string)
    database_result_file_path = '{0}/backups/{1}'.format(BACKUP_PATH, database_result_file)
    if NFS_MOUNT_FILES_DIR:
        files_dir = '{0}/{1}/files'.format(NFS_MOUNT_LOCATION[ENVIRONMENT], site['sid'])
    else:
        files_dir = '{0}/{1}/sites/default/files'.format(WEB_ROOT, site['sid'])
    files_result_file = '{0}_{1}.tar.gz'.format(site['sid'], date_time_string)
    files_result_file_path = '{0}/backups/{1}'.format(BACKUP_PATH, files_result_file)

    # Start the actual process.
    with cd(web_directory):
        run('drush sql-dump --structure-tables-list=cache,cache_*,sessions,watchdog,history --result-file={0}'.format(
            database_result_file_path))
    with cd(files_dir):
        log.debug('File dir | %s', files_dir)
        run('tar --exclude "imagecache" --exclude "css" --exclude "js" --exclude "backup_migrate" --exclude "styles" --exclude "xmlsitemap" --exclude "honeypot" -czf {0} *'.format(
            files_result_file_path))

    backup_time = time() - start_time

    # File size with thousand seperator, converted to MB.
    db_size = '{:,.0f}'.format(os.path.getsize(database_result_file_path)/float(1 << 20))+" MB"
    file_size = '{:,.0f}'.format(os.path.getsize(files_result_file_path)/float(1 << 20))+" MB"

    patch_payload = {
        'site': site['_id'],
        'site_version': site['_version'],
        'backup_date': datetime_string,
        'backup_type': backup_type,
        'files': files_result_file,
        'database': database_result_file,
        'state': 'complete'
    }

    log.debug('Backup | Create | Ready to update record | Payload - %s', patch_payload)
    utilities.patch_eve('backup', backup_item['_id'], patch_payload)

    log.info('Operational statistic | Backup Create | SID - %s | Time - %s | DB size - %s | File size - %s',
             site['sid'], backup_time, db_size, file_size)