def site_profile_update(site, original, updates):
print ("Site Profile Update\n{0}".format(site))
code_directory_sid = "{0}/{1}/{1}".format(sites_code_root, site["sid"])
old_profile = utilities.get_single_eve("code", original["code"]["profile"])
new_profile = utilities.get_single_eve("code", site["code"]["profile"])
new_profile_full_string = _get_code_name_version(site["code"]["profile"])
with cd(code_directory_sid + "/profiles"):
run(
"rm {0}; ln -s {1}/profiles/{2}/{3} {2}".format(
old_profile["meta"]["name"], code_root, new_profile["meta"]["name"], new_profile_full_string
)
)
print ("Rebuild registry.")
run("drush rr")
def pre_delete_code_callback(request, lookup):
"""
Make sure no sites are using the code.
:param request: flask.request object
:param lookup:
"""
code = utilities.get_single_eve('code', lookup['_id'])
app.logger.debug(code)
if code['meta']['code_type'] in ['module', 'theme', 'library']:
code_type = 'package'
else:
code_type = code['meta']['code_type']
app.logger.debug(code_type)
site_query = 'where={{"code.{0}":"{1}"}}'.format(code_type, code['_id'])
sites = utilities.get_eve('sites', site_query)
app.logger.debug(sites)
if not sites['_meta']['total'] == 0:
for site in sites['_items']:
# Create a list of sites that use this code item.
# If 'sid' is a key in the site dict use it, otherwise use '_id'.
site_list = []
if site.get('sid'):
site_list.append(site['sid'])
else:
site_list.append(site['_id'])
site_list_full = ', '.join(site_list)
app.logger.error('Code item is in use by one or more sites:\n{0}'.format(site_list_full))
abort(409, 'A conflict happened while processing the request. Code item is in use by one or more sites.')
def switch_packages(instance):
"""Switch Package symlinks, if no package symlinks are present add them.
Arguments:
instance {dict} -- full instance record
"""
log.info('Instance | Switch package | Instance - %s', instance['sid'])
log.debug('Instance | Switch package | Instance - %s', instance)
instance_code_path_sid = '{0}/{1}/{1}'.format(INSTANCE_ROOT,
instance['sid'])
# Get rid of old symlinks
# List sites/all/{modules|themes|libraries} and remove all symlinks
for package_type_path in ['modules', 'themes', 'libraries']:
package_path = instance_code_path_sid + '/sites/all/' + package_type_path
log.debug('Instance | Switch Packages | listdir - %s',
os.listdir(package_path))
for item in os.listdir(package_path):
# Get full path of item
path = package_path + '/' + item
log.debug('Instance | Switch Packages | Item to unlink - %s', path)
if os.path.islink(path):
os.remove(path)
if 'package' in instance['code']:
for item in instance['code']['package']:
package = utilities.get_single_eve('code', item)
package_path = utilities.code_path(package)
package_type_path = utilities.code_type_directory_name(
package['meta']['code_type'])
destination_path = instance_code_path_sid + '/sites/all/' + \
package_type_path + '/' + package['meta']['name']
# Add new relative symlink
if not os.access(destination_path, os.F_OK):
utilities.relative_symlink(package_path, destination_path)
def pre_delete_code(request, lookup):
"""
Make sure no sites are using the code.
:param request: flask.request object
:param lookup:
"""
code = utilities.get_single_eve('code', lookup['_id'])
log.debug('code | Delete | code - %s', code)
# Check for sites using this piece of code.
if code['meta']['code_type'] in ['module', 'theme', 'library']:
code_type = 'package'
else:
code_type = code['meta']['code_type']
log.debug('code | Delete | code - %s | code_type - %s', code['_id'], code_type)
site_query = 'where={{"code.{0}":"{1}"}}'.format(code_type, code['_id'])
sites = utilities.get_eve('sites', site_query)
log.debug('code | Delete | code - %s | sites result - %s', code['_id'], sites)
if not sites['_meta']['total'] == 0:
site_list = []
for site in sites['_items']:
# Create a list of sites that use this code item.
# If 'sid' is a key in the site dict use it, otherwise use '_id'.
if site.get('sid'):
site_list.append(site['sid'])
else:
site_list.append(site['_id'])
site_list_full = ', '.join(site_list)
log.error('code | Delete | code - %s | Code item is in use by one or more sites - %s',
code['_id'], site_list_full)
abort(409, 'A conflict happened while processing the request. Code item is in use by one or more sites.')
def correct_file_permissions(site_id):
"""
Correct file permissions for an instance's NFS files.
:param machine_name: id of instance to fix
"""
app.logger.debug('Site | Correct file permissions | Site ID - %s', site_id)
instance = utilities.get_single_eve('sites', site_id)
tasks.correct_file_permissions.delay(instance)
return make_response('Fixing permissions for NFS mounted files.')
def update_settings_file(site):
print ("Update Settings Files\n{0}".format(site))
code_directory = "{0}/{1}".format(sites_code_root, site["sid"])
code_directory_current = "{0}/current".format(code_directory)
profile = utilities.get_single_eve("code", site["code"]["profile"])
profile_name = profile["meta"]["name"]
_create_settings_files(site, profile_name)
_push_settings_files(site, code_directory_current)
def _get_code_name_version(code_id):
"""
Get the label and version for a code item.
:param code_id: string '_id' for a code item
:return: string 'label'-'version'
"""
code = utilities.get_single_eve("code", code_id)
code_name = code["meta"]["name"]
code_version = code["meta"]["version"]
return "{0}-{1}".format(code_name, code_version)
def create_backup(site_id):
"""
Create a backup of an instance.
:param machine_name: id of instance to restore
"""
app.logger.debug('Backup | Create | Site ID - %s', site_id)
site = utilities.get_single_eve('sites', site_id)
app.logger.debug('Backup | Create | Site Response - %s', site)
tasks.backup_create.delay(site=site, backup_type='on_demand')
response = make_response('Backup started')
return response
def restore_backup(backup_id):
"""
Restore a backup to a new instance.
:param machine_name: id of backup to restore
"""
app.logger.debug('Backup | Restore | %s', backup_id)
backup_record = utilities.get_single_eve('backup', backup_id)
original_instance = utilities.get_single_eve('sites',
backup_record['site'],
backup_record['site_version'])
# If packages are still active, add them; if not, find a current version
# and add it; if none, error
try:
package_list = utilities.package_import(original_instance)
except Exception as error:
abort(409, error)
tasks.backup_restore.delay(backup_record, original_instance, package_list)
response = make_response('Restore started')
return response
def site_provision(site, install=True):
"""
Responds to POSTs to provision a site to the right places on the server.
:param site: The flask.request object, JSON encoded
:param install: Boolean. Indicates if the install command will run.
:return:
"""
print ("Site Provision - Install: {0}\n{1}".format(install, site))
code_directory = "{0}/{1}".format(sites_code_root, site["sid"])
code_directory_sid = "{0}/{1}".format(code_directory, site["sid"])
code_directory_current = "{0}/current".format(code_directory)
web_directory = "{0}/{1}/{2}".format(sites_web_root, site["type"], site["sid"])
profile = utilities.get_single_eve("code", site["code"]["profile"])
profile_name = profile["meta"]["name"]
_create_database(site)
_create_settings_files(site, profile_name)
_create_directory_structure(code_directory)
with cd(code_directory):
core = _get_code_name_version(site["code"]["core"])
run("drush dslm-new {0} {1}".format(site["sid"], core))
_update_symlink(code_directory_sid, code_directory_current)
with cd(code_directory_current):
profile = _get_code_name_version(site["code"]["profile"])
run("drush dslm-add-profile {0}".format(profile))
if nfs_mount_files_dir:
nfs_dir = nfs_mount_location[environment]
nfs_files_dir = "{0}/sitefiles/{1}/files".format(nfs_dir, site["sid"])
nfs_tmp_dir = "{0}/sitefiles/{1}/tmp".format(nfs_dir, site["sid"])
_create_directory_structure(nfs_files_dir)
_create_directory_structure(nfs_tmp_dir)
# Replace default files dir with this one
site_files_dir = code_directory_current + "/sites/default/files"
_replace_files_directory(nfs_files_dir, site_files_dir)
_push_settings_files(site, code_directory_current)
_update_symlink(code_directory_current, web_directory)
correct_file_directory_permissions(site)
if install:
_install_site(profile_name, code_directory_current)
correct_file_directory_permissions(site)
def site_install(site):
"""Run Drupal install
"""
code_directory = '{0}/{1}'.format(INSTANCE_ROOT, site['sid'])
code_directory_current = '{0}/current'.format(code_directory)
profile = utilities.get_single_eve('code', site['code']['profile'])
profile_name = profile['meta']['name']
try:
with cd(code_directory_current):
run('drush site-install -y {0}'.format(profile_name))
except FabricException as error:
log.error('Site | Install | Instance install failed | Error - %s', error)
return error
def backup_restore(backup_record, original_instance, package_list):
"""
Restore database and files to a new instance.
"""
log.info('Instance | Restore Backup | %s | %s', backup_record, original_instance)
start_time = time()
file_date = datetime.strptime(backup_record['backup_date'], "%Y-%m-%d %H:%M:%S %Z")
pretty_filename = '{0}_{1}'.format(
original_instance['sid'], file_date.strftime("%Y-%m-%d-%H-%M-%S"))
pretty_database_filename = '{0}.sql'.format(pretty_filename)
database_path = '{0}/backups/{1}'.format(BACKUP_PATH, pretty_database_filename)
pretty_files_filename = '{0}.tar.gz'.format(pretty_filename)
files_path = '{0}/backups/{1}'.format(BACKUP_PATH, pretty_files_filename)
# Grab available instance and add packages if needed
available_instances = utilities.get_eve('sites', 'where={"status":"available"}')
log.debug('Instance | Restore Backup | Avaiable Instances - %s', available_instances)
new_instance = next(iter(available_instances['_items']), None)
# TODO: Don't switch if the code is the same
if new_instance is not None:
payload = {'status': 'installing'}
if package_list:
packages = {'code': {'package': package_list}}
payload.update(packages)
utilities.patch_eve('sites', new_instance['_id'], payload)
else:
exit('No available instances.')
# Wait for code and status to update.
attempts = 18 # Tries every 10 seconds to a max of 18 (or 3 minutes).
while attempts:
try:
new_instance_refresh = utilities.get_single_eve('sites', new_instance['_id'])
if new_instance_refresh['status'] != 'installed':
log.info('Instance | Restore Backup | New instance is not ready | %s', new_instance['_id'])
raise ValueError('Status has not yet updated.')
break
except ValueError, e:
# If the status is not updated and we have attempts left,
# remove an attempt and wait 10 seconds.
attempts -= 1
if attempts is not 0:
sleep(10)
else:
exit(str(e))
def pre_delete_sites(request, lookup):
"""
Remove site from servers right before the item is removed.
:param request: flask.request object
:param lookup:
"""
log.debug('Instances | Pre Delete | lookup - %s', lookup)
instance = utilities.get_single_eve('sites', lookup['_id'])
log.debug('Instances | Pre Delete | instance - %s', instance)
# Check if instance is launched.
if not instance['status'] in ['launched', 'launching']:
log.debug('Instances | Pre Delete | instance - %s | Instance state - %s | Okay to delete', instance, instance['status'])
tasks.site_remove.delay(instance)
else:
log.error('Instances | Delete | instance - %s | Instance is launched or launching',
instance['_id'])
abort(409, 'Instance is launched or launching. To delete, take instance down first.')
def switch_profile(instance):
"""Switch non-core Profile symlinks, if no appropriate symlinks are present add them.
Arguments:
instance {dict} -- full instance record
"""
log.info('Instance | Switch profile | Instance - %s', instance['sid'])
log.debug('Instance | Switch profile | Instance - %s', instance)
# Lookup the profile we want to use.
profile = utilities.get_single_eve('code', instance['code']['profile'])
# Setup variables
profile_path = utilities.code_path(profile)
instance_code_path_sid = '{0}/{1}/{1}'.format(INSTANCE_ROOT,
instance['sid'])
destination_path = instance_code_path_sid + '/profiles/' + profile['meta'][
'name']
# Remove old symlink
if os.path.islink(destination_path):
os.remove(destination_path)
# Add new relative symlink
if not os.access(destination_path, os.F_OK):
utilities.relative_symlink(profile_path, destination_path)
def backup_delete(item):
"""Remove backup files from servers
Arguments:
item {string} -- Backup item to remove
"""
log.debug('Backup | Delete | Item - %s', item)
log.info('Backup | Delete | Item - %s ', item['_id'])
instance = utilities.get_single_eve('sites', item['site'],
item['site_version'])
pretty_filename = '{0}_{1}'.format(
instance['sid'], item['backup_date'].strftime("%Y-%m-%d-%H-%M-%S"))
pretty_database_filename = '{0}.sql'.format(pretty_filename)
database_path = '{0}/backups/{1}'.format(BACKUP_PATH,
pretty_database_filename)
pretty_files_filename = '{0}.tar.gz'.format(pretty_filename)
files_path = '{0}/backups/{1}'.format(BACKUP_PATH, pretty_files_filename)
os.remove(files_path)
os.remove(database_path)
log.info('Backup | Delete | Complete | Item - %s', item['_id'])
def switch_settings_files(instance):
"""Create settings.php from template and render the resulting file onto the server.
Arguments:
instance {dict} -- full instance record
"""
log.info('Instance | Settings file | Instance ID - %s', instance['_id'])
# If the settings file exists, change permissions to allow us to update the template.
file_destination = "{0}/{1}/{1}/sites/default/settings.php".format(
INSTANCE_ROOT, instance['sid'])
# Check to see if file exists and is writable.
utilities.file_accessable_and_writable(file_destination)
# Setup variables
if instance['settings'].get('siteimprove_site'):
siteimprove_site = instance['settings']['siteimprove_site']
else:
siteimprove_site = None
if instance['settings'].get('siteimprove_group'):
siteimprove_group = instance['settings']['siteimprove_group']
else:
siteimprove_group = None
profile = utilities.get_single_eve('code', instance['code']['profile'])
if ('cse_creator' in instance['settings']) and ('cse_id'
in instance['settings']):
google_cse_csx = instance['settings']['cse_creator'] + ':' + instance[
'settings']['cse_id']
else:
google_cse_csx = None
if NFS_MOUNT_FILES_DIR:
tmp_path = '{0}/{1}/tmp'.format(NFS_MOUNT_LOCATION[ENVIRONMENT],
instance['sid'])
else:
tmp_path = '/tmp'
if 'google_tag_client_container_id' in instance['settings']:
google_tag_client_container_id = instance['settings'][
'google_tag_client_container_id']
else:
google_tag_client_container_id = None
domain = BASE_URLS[ENVIRONMENT].split('://')[1]
settings_variables = {
'profile': profile['meta']['name'],
'sid': instance['sid'],
'atlas_id': instance['_id'],
'atlas_url': API_URLS[ENVIRONMENT] + '/',
'atlas_logging_url': ATLAS_LOGGING_URLS[ENVIRONMENT],
'atlas_username': SERVICE_ACCOUNT_USERNAME,
'atlas_password': SERVICE_ACCOUNT_PASSWORD,
'path': instance['path'],
'status': instance['status'],
'atlas_statistics_id': instance['statistics'],
'siteimprove_site': siteimprove_site,
'siteimprove_group': siteimprove_group,
'google_cse_csx': google_cse_csx,
'google_tag_client_container_id': google_tag_client_container_id,
'reverse_proxies': SERVERDEFS[ENVIRONMENT]['varnish_servers'],
'varnish_control': VARNISH_CONTROL_TERMINALS[ENVIRONMENT],
'varnish_control_key': VARNISH_CONTROL_KEY,
'pw': utilities.decrypt_string(instance['db_key']),
'page_cache_maximum_age':
instance['settings']['page_cache_maximum_age'],
'database_servers': SERVERDEFS[ENVIRONMENT]['database_servers'],
'environment': ENVIRONMENT,
'tmp_path': tmp_path,
'saml_pw': SAML_AUTH,
'smtp_client_hostname': BASE_URLS[ENVIRONMENT],
'smtp_password': SMTP_PASSWORD,
'base_url': BASE_URLS[ENVIRONMENT],
'domain': domain,
'servicenow_key': SERVICENOW_KEY
}
log.info(
'Instance | Settings file | Render settings file | Instance ID - %s',
instance['_id'])
# Create a template environment with the default settings and a loader that looks up the
# templates in the templates folder inside the Atlas python package.
# We don't do autoescaping, because there is no PHP support.
jinja_env = Environment(loader=PackageLoader('atlas', 'templates'))
template = jinja_env.get_template('settings.php')
render = template.render(settings_variables)
# Remove the existing file.
if os.access(file_destination, os.F_OK):
os.remove(file_destination)
# Write the render to a file.
with open(file_destination, "wb") as open_file:
open_file.write(render)
# Set file permissions
# Octet mode, Python 3 compatible
os.chmod(file_destination, 0o444)
def import_backup():
"""
Import a backup to a new instance on the current version of core, profile, and any packages
that are present. If a current version of a package is not available, the import will abort.
"""
backup_request = request.get_json()
app.logger.debug('Backup | Import | %s', backup_request)
# Get the backup and then the site records.
# TODO Get the list of env from the config files.
# TODO Verify import is from different env, recommend restore if it is the same env.
if not (backup_request.get('env') and backup_request.get('id')):
abort(409, 'Error: Missing env (local, dev, test, prod) and id.')
elif not backup_request.get('env'):
abort(409, 'Error: Missing env (local, dev, test, prod).')
elif not backup_request.get('id'):
abort(409, 'Error: Missing id.')
elif backup_request['env'] not in ['local', 'dev', 'test', 'prod']:
abort(409, 'Error: Invalid env choose from [local, dev, test, prod]')
backup_record = utilities.get_single_eve('backup',
backup_request['id'],
env=backup_request['env'])
app.logger.debug('Backup | Import | Backup record - %s', backup_record)
remote_site_record = utilities.get_single_eve(
'sites',
backup_record['site'],
backup_record['site_version'],
env=backup_request['env'])
app.logger.debug('Backup | Import | Site record - %s', remote_site_record)
# Get a list of packages to include
try:
package_list = utilities.package_import_cross_env(
remote_site_record, env=backup_request['env'])
except Exception as error:
abort(500, error)
app.logger.info('Backup | Import | Package list - %s', package_list)
# Try to get the p1 record.
local_p1_instance_record = utilities.get_single_eve(
'sites', remote_site_record['sid'])
app.logger.debug('Backup | Import | Local instance record - %s',
local_p1_instance_record)
# Try to get the path record if the site is launched.
local_path_instance_record = False
if remote_site_record['path'] != remote_site_record['sid']:
query_string = 'where={{"path":"{0}"}}'.format(
remote_site_record['path'])
local_path_instance_records = utilities.get_eve('sites', query_string)
app.logger.info('Backup | Import | Local path instance record - %s',
local_path_instance_records)
if local_path_instance_records['_meta']['total'] == 1:
local_path_instance_record = True
if local_p1_instance_record['_error'] and local_p1_instance_record[
'_error']['code'] == 404 and not local_path_instance_record:
# Create an instance with the same sid
payload = {
"status": remote_site_record['status'],
"sid": remote_site_record['sid'],
"path": remote_site_record['path']
}
response_string = 'the same'
else:
app.logger.info('Backup | Import | Instance sid or path exists')
payload = {"status": "installed"}
response_string = 'a new'
# Add package list to payload if it exists
if package_list:
payload['code'] = {"package": package_list}
# Set install
payload['install'] = False
new_instance = utilities.post_eve('sites', payload)
app.logger.debug('Backup | Import | New instance record - %s',
new_instance)
env = backup_request['env']
backup_id = backup_request['id']
target_instance = new_instance['_id']
tasks.import_backup.apply_async([env, backup_id, target_instance],
countdown=30)
return make_response(
'Attempting to import backup to {0} sid'.format(response_string))
def switch_core(instance):
"""Switch Drupal core symlinks, if no core symlinks are present add them.
Arguments:
instance {dict} -- full instance record
"""
# Lookup the core we want to use.
core = utilities.get_single_eve('code', instance['code']['core'])
# Setup variables
core_path = utilities.code_path(core)
instance_code_path_sid = '{0}/{1}/{1}'.format(INSTANCE_ROOT,
instance['sid'])
# Get a list of files in the Core source directory
core_files = os.listdir(core_path)
# Get a list of files in the Instance target directory
instance_files = os.listdir(instance_code_path_sid)
# Remove any existing symlinks to a core.
for instance_file in instance_files:
full_path = instance_code_path_sid + '/' + instance_file
# Check if path is a symlink.
if os.path.islink(full_path):
# Get the target of the symlink.
symlink_target = os.readlink(full_path)
# Get the name of the directory that contains the symlink target
code_dir = os.path.dirname(symlink_target)
# Check to see if the directory is a Drupal core, if so remove the symlink.
regex = '((drupal)\-([\d\.x]+\-*[dev|alph|beta|rc|pl]*[\d]*))$i'
if re.match(regex, code_dir):
os.remove(full_path)
# Iterate through the source files and symlink when applicable.
for core_file in core_files:
if core_file in ['sites', 'profiles']:
continue
if utilities.ignore_code_file(core_file):
continue
source_path = core_path + '/' + core_file
destination_path = instance_code_path_sid + '/' + core_file
# Remove existing symlink and add new one.
if os.path.islink(destination_path):
os.remove(destination_path)
# F_OK to test the existence of path
if not os.access(destination_path, os.F_OK):
utilities.relative_symlink(source_path, destination_path)
# Create Instance specific directory structure
directories_to_create = [
'sites', 'sites/all', 'sites/all/modules', 'sites/all/libraries',
'sites/all/themes', 'sites/default', 'sites/default/files', 'profiles'
]
for directory in directories_to_create:
target_dir = instance_code_path_sid + '/' + directory
# If the directoey does not already exist, create it.
if not os.access(target_dir, os.F_OK):
os.mkdir(target_dir)
# Copy over default settings file so that this instance is like a default install.
source_path = core_path + '/sites/default/default.settings.php'
destination_path = instance_code_path_sid + '/sites/default/default.settings.php'
if os.access(destination_path, os.F_OK):
os.remove(destination_path)
copyfile(source_path, destination_path)
# Include links to the profiles that we are not using so that the site doesn't white screen if
# the deployed profile gets disabled.
core_profiles = os.listdir(core_path + '/profiles')
for core_profile in core_profiles:
source_path = core_path + '/profiles/' + core_profile
destination_path = instance_code_path_sid + '/profiles/' + core_profile
if os.path.islink(destination_path):
os.remove(destination_path)
# F_OK to test the existence of path
if not os.access(destination_path, os.F_OK):
utilities.relative_symlink(source_path, destination_path)
