class Reports(Plugin):
""" Class for print reports. """
def __init__(self):
# load the Plugin _plugins property
self.settings = Settings()
tool = self.settings.get('reportingTool').title()
self._plugin = Plugin.load_plugin(tool)
class_ = getattr(self._plugin, tool)
self._instance = class_()
def json(self, report):
import json
return json.dumps(report)
def yaml(self, report):
import yaml
return yaml.dump(report, default_flow_style=False)
def text(self, report, text=""):
""" Format the report dictionary into a string. """
return self.yaml(report)
def send(self, report):
""" Deliver the report. """
report_format = self.settings.get('reportingFormat')
class_ = getattr(self, report_format)
report_text = class_(report)
return self._instance.send_message(report_text)
class Repolist(Repotool):
""" Return git repository list manually maintained in the settings file. """
def __init__(self):
current_dir = os.path.dirname(os.path.realpath(__file__))
settings_file = current_dir + '/settings/default.yaml'
self.settings = Settings()
self.settings.add(settings_file)
def git_repos(self):
"""Return a list of repos from the Settings.
The expected format is a dictionary whose keys are the folder names and
values are the ssh connection strings to the repo.
Example:
{
'drupal': 'http://git.drupal.org/project/drupal.git'
}
"""
repo_dict = self.settings.get('repoDict')
if (not repo_dict) or (not isinstance(repo_dict, dict)):
return {}
else:
return repo_dict
def __init__(self, site_name, ssh, working_dir):
self.settings = Settings()
self._site_name = site_name
self.site_dir = os.path.join(working_dir, self._site_name)
self.ssh = ssh
self.utilities = Utils()
self.si_files = copy.copy(self.settings.get('drushSiFiles'))
def __init__(self):
# load the Plugin _plugins property
self.settings = Settings()
tool = self.settings.get('gitRepoName').title()
self._plugin = Plugin.load_plugin(tool)
class_ = getattr(self._plugin, tool)
self._instance = class_()
def __init__(self):
current_dir = os.path.dirname(os.path.realpath(__file__))
settings_file = current_dir + '/settings/default.yaml'
self.settings = Settings()
self.settings.add(settings_file)
self._pm_label = self.settings.get('pmName')
base = self.settings.get('attaskBaseURL')
api = self.settings.get('attaskAPIVersion')
api_base = urljoin(base, 'attask/api/')
self._attask_api_url = urljoin(api_base, api)
class Sendmail(Report):
""" sendmail report plugin. """
def __init__(self):
current_dir = os.path.dirname(os.path.realpath(__file__))
settings_file = current_dir + '/settings/default.yaml'
self.settings = Settings()
self.settings.add(settings_file)
def send_message(self, report_text):
""" Send the report via email using sendmail."""
today = str(datetime.date.today())
msg = MIMEText(report_text)
msg["From"] = self.settings.get('reportSender')
msg["To"] = self.settings.get('reportRecipient')
msg["Subject"] = "Drupdates report {0}.".format(today)
mail = Popen(["sendmail", "-t"], stdin=PIPE)
mail.communicate(msg.as_string())
def __init__(self):
current_dir = os.path.dirname(os.path.realpath(__file__))
settings_file = current_dir + "/settings/default.yaml"
self.settings = Settings()
self.settings.add(settings_file)
base = self.settings.get("jiraBaseURL")
api = self.settings.get("jiraAPIVersion")
if not api.endswith("/"):
api += "/"
jira_api_base = urljoin(base, "rest/api/")
self._jira_api_url = urljoin(jira_api_base, api)
def __init__(self):
self.settings = Settings()
self.install()
self.utilities = Utils()
self.working_dirs = self.settings.get('workingDir')
self.single_site = ''
self.alias_file = None
if isinstance(self.working_dirs, str):
self.working_dirs = [self.working_dirs]
# by design, SingleSite setting only works with single working directory
if len(self.working_dirs) == 1:
self.single_site = self.settings.get('singleSite')
def __init__(self, site_name, ssh, working_dir):
self.settings = Settings()
self.working_branch = self.settings.get('workingBranch')
self._site_name = site_name
self.working_dir = working_dir
self.site_dir = os.path.join(working_dir, self._site_name)
self.ssh = ssh
self.utilities = Utils()
self.site_web_root = None
self._commit_hash = None
self.repo_status = None
self.sub_sites = Drush.get_sub_site_aliases(self._site_name)
class Jira(Pmtool):
""" Plugin to wotk with JIRA. """
def __init__(self):
current_dir = os.path.dirname(os.path.realpath(__file__))
settings_file = current_dir + "/settings/default.yaml"
self.settings = Settings()
self.settings.add(settings_file)
base = self.settings.get("jiraBaseURL")
api = self.settings.get("jiraAPIVersion")
if not api.endswith("/"):
api += "/"
jira_api_base = urljoin(base, "rest/api/")
self._jira_api_url = urljoin(jira_api_base, api)
def submit_deploy_ticket(self, site, environments, description, target_date):
""" Submit a deployment ticket to JIRA/Agile Ready. """
issue_uri = self.settings.get("jiraIssueURL")
issue_url = urljoin(self._jira_api_url, issue_uri)
jira_user = self.settings.get("jiraUser")
jira_pword = self.settings.get("jiraPword")
message = {}
for environment in environments:
request = self.build_reqest(site, environment, description, target_date)
headers = {"content-type": "application/json"}
response = Utils.api_call(
issue_url, "Jira", "post", data=request, auth=(jira_user, jira_pword), headers=headers
)
if not response == False:
url = response["key"]
message[environment] = "The {0} deploy ticket is {1}".format(environment, url)
else:
message[environment] = "JIRA ticket submission failed for {0}".format(environment)
return message
def build_reqest(self, site, environment, description, target_date):
""" Build the JSON request to be submitted to JIRA. """
summary = "{0} Deployment for {1} w.e. {2}".format(environment, site, target_date)
request = {}
request["fields"] = {}
request["fields"]["project"] = {"key": self.settings.get("jiraProjectID")}
request["fields"]["summary"] = summary
request["fields"]["description"] = description
request["fields"]["issuetype"] = {"name": self.settings.get("jiraIssueType")}
return json.dumps(request)
class Pmtools(Plugin):
""" Submit requests to Project Management tools. """
def __init__(self):
self.settings = Settings()
tool = self.settings.get('pmName').title()
self._plugin = Plugin.load_plugin(tool)
class_ = getattr(self._plugin, tool)
self._instance = class_()
def target_date(self):
""" Get the date string for the following Friday. """
value = self.settings.get('targetDate')
if not value:
today = datetime.date.today()
# If today is a Friday, we skip to next Friday
if datetime.datetime.today().weekday() == 4:
friday = str(today + datetime.timedelta((3-today.weekday()) % 7 + 1))
else:
friday = str(today + datetime.timedelta((4-today.weekday()) % 7))
return friday
else:
return value
@staticmethod
def description(site, git_hash):
""" Collect data for ticket's descriotion field. """
description_list = []
description_list.append("Git Hash = <" + git_hash + ">")
description_list.append("Post deployment steps:")
description_list.append("drush @" + site +" updb -y")
return '\n'.join(description_list)
def deploy_ticket(self, site, commit_hash):
""" Submit ticket requesting deployment(s). """
description = Pmtools.description(site, commit_hash)
environments = self.settings.get('deploymentTickets')
target = self.target_date()
return self._instance.submit_deploy_ticket(site, environments, description, target)
class Repos(Plugin):
""" Build Git repository list. """
def __init__(self):
# load the Plugin _plugins property
self.settings = Settings()
tool = self.settings.get('gitRepoName').title()
self._plugin = Plugin.load_plugin(tool)
class_ = getattr(self._plugin, tool)
self._instance = class_()
def get(self):
""" Get repository list from plugin. """
return self._instance.git_repos()
class Stash(Repotool):
""" Return git repository list from Stash project. """
def __init__(self):
current_dir = os.path.dirname(os.path.realpath(__file__))
settings_file = current_dir + '/settings/default.yaml'
self.settings = Settings()
self.settings.add(settings_file)
def git_repos(self):
""" Get list of Stash repos from a specific Project.
Note: this request will only bring back 9,999 repos, which should
suffice, if it doesn't updaqte the stashLimit setting.
"""
stash_url = self.settings.get('stashURL')
git_repo_name = self.settings.get('gitRepoName')
stash_user = self.settings.get('stashUser')
stash_pword = self.settings.get('stashPword')
stash_cert_verify = self.settings.get('stashCertVerify')
stash_limit = self.settings.get('stashLimit')
stash_params = {'limit' : stash_limit}
response = Utils.api_call(stash_url, git_repo_name, 'get',
auth=(stash_user, stash_pword),
verify=stash_cert_verify,
params=stash_params)
if not response == False:
repos = Stash.parse_repos(response['values'])
return repos
else:
return {}
@staticmethod
def parse_repos(raw):
""" Parse repo list returned form Stash."""
repos = {}
for repo in raw:
for link in repo['links']['clone']:
if link['name'] == 'ssh':
repos[repo['slug']] = link['href']
return repos
class Slack(Report):
""" Slack report plugin. """
def __init__(self):
current_dir = os.path.dirname(os.path.realpath(__file__))
settings_file = current_dir + '/settings/default.yaml'
self.settings = Settings()
self.settings.add(settings_file)
def send_message(self, report_text):
""" Post the report to a Slack channel or DM a specific user."""
url = self.settings.get('slackURL')
user = self.settings.get('slackUser')
payload = {}
payload['text'] = report_text
payload['new-bot-name'] = user
direct = self.settings.get('slackRecipient')
channel = self.settings.get('slackChannel')
if direct:
payload['channel'] = '@' + direct
elif channel:
payload['channel'] = '#' + direct
Utils.api_call(url, 'Slack', 'post', data=json.dumps(payload))
def __init__(self):
current_dir = os.path.dirname(os.path.realpath(__file__))
settings_file = current_dir + '/settings/default.yaml'
self.settings = Settings()
self.settings.add(settings_file)
class Attask(Pmtool):
""" Plugin to wotk with AtTask. """
def __init__(self):
current_dir = os.path.dirname(os.path.realpath(__file__))
settings_file = current_dir + '/settings/default.yaml'
self.settings = Settings()
self.settings.add(settings_file)
self._pm_label = self.settings.get('pmName')
base = self.settings.get('attaskBaseURL')
api = self.settings.get('attaskAPIVersion')
api_base = urljoin(base, 'attask/api/')
self._attask_api_url = urljoin(api_base, api)
def get_session_id(self):
""" Get a session ID from AtTask. """
attask_pword = self.settings.get('attaskPword')
attask_user = self.settings.get('attaskUser')
at_params = {'username': attask_user, 'password': attask_pword}
login_url = urljoin(self._attask_api_url, self.settings.get('attaskLoginUrl'))
response = Utils.api_call(login_url, self._pm_label, 'post', params=at_params)
if response == False:
return False
else:
return response['data']['sessionID']
def submit_deploy_ticket(self, site, environments, description, target_date):
""" Submit a Deployment request to AtTask.
site -- The site the ticket is for
environments -- The name(s) of the environments to deploy to
description -- The description text to go in the task
targetDate -- The date to put in the label fo the ticket
"""
sessparam = {}
session_id = self.get_session_id()
# Make sure you can get a Session ID
if session_id:
sessparam['sessionID'] = session_id
else:
return False
# Set-up AtTask request
attask_project_id = self.settings.get('attaskProjectID')
dev_ops_team_id = self.settings.get('devOpsTeamID')
attask_base_url = self.settings.get('attaskBaseURL')
attask_assignee_type = self.settings.get('attaskAssigneeType')
task_url = urljoin(self._attask_api_url, self.settings.get('attaskTaskURL'))
message = {}
for environment in environments:
title = environment + ' Deployment for ' + site +' w.e. ' + target_date
at_params = {'name': title, 'projectID': attask_project_id,
attask_assignee_type: dev_ops_team_id, 'description': description}
response = Utils.api_call(task_url, self._pm_label, 'post',
params=at_params, headers=sessparam)
if not response == False:
data = response['data']
msg = "The {0} deploy ticket is".format(environment)
msg += " <{0}task/view?ID={1}>".format(attask_base_url, data['ID'])
message[environment] = msg
else:
msg = "The {0} deploy ticket did not submit to".format(environment)
msg += "{0} properly".format(self._pm_label)
message[environment] = msg
return message
class Siteupdate(object):
""" Update the modules and/or core in a completely built Drupal site. """
def __init__(self, site_name, ssh, working_dir):
self.settings = Settings()
self.working_branch = self.settings.get('workingBranch')
self._site_name = site_name
self.working_dir = working_dir
self.site_dir = os.path.join(working_dir, self._site_name)
self.ssh = ssh
self.utilities = Utils()
self.site_web_root = None
self._commit_hash = None
self.repo_status = None
self.sub_sites = Drush.get_sub_site_aliases(self._site_name)
@property
def commit_hash(self):
""" commit_hash getter. """
return self._commit_hash
@commit_hash.setter
def commit_hash(self, value):
""" commit_hash setter. """
self._commit_hash = value
def update(self):
""" Set-up to and run Drush update(s) (i.e. up or ups). """
report = {}
self.utilities.sys_commands(self, 'preUpdateCmds')
self.repo_status = Drush.call(['st'], self._site_name, True)
try:
updates = self.run_updates()
except DrupdatesError as updates_error:
raise DrupdatesUpdateError(20, updates_error.msg)
# If no updates move to the next repo
if not updates:
self.commit_hash = ""
report['status'] = "Did not have any updates to apply"
return report
report['status'] = "The following updates were applied"
report['updates'] = updates
report['commit'] = "The commit hash is {0}".format(self.commit_hash)
self.utilities.sys_commands(self, 'postUpdateCmds')
if self.settings.get('submitDeployTicket') and self.commit_hash:
report[self._site_name] = {}
pm_name = self.settings.get('pmName').title()
try:
report[self._site_name][pm_name] = Pmtools().deploy_ticket(self._site_name,
self.commit_hash)
except DrupdatesError as api_error:
report[self._site_name][pm_name] = api_error.msg
return report
def run_updates(self):
""" Run the site updates.
The updates are done either by downloading the updates, updating the
make file or both.
- First, run drush pm-updatestatus to get a list of eligible updates for
the site/sub-sites.
- Second, build the report to return to Updates().
- Third, apply the updates.
"""
updates = {}
try:
sites = self.get_sites_to_update()
except DrupdatesError as update_status_error:
raise DrupdatesUpdateError(20, update_status_error)
if not sites['count']:
return updates
else:
sites.pop('count')
# Note: call Drush.call() without site alias as alias comes after dd argument.
drush_dd = Drush.call(['dd', '@drupdates.' + self._site_name])
self.site_web_root = drush_dd[0]
# Create seperate commits for each project (ie module/theme)
one_commit_per_project = self.settings.get('oneCommitPerProject')
# Iterate through the site/sub-sites and perform updates, update files etc...
sites_copy = copy.copy(sites)
for site, data in sites.items():
if 'modules' not in data:
sites_copy.pop(site)
continue
modules = copy.copy(data['modules'])
x = 0
for project, descriptions in data['modules'].items():
if self.settings.get('useMakeFile'):
self.update_make_file(project, descriptions['current'], descriptions['candidate'])
if one_commit_per_project:
if x:
build = Sitebuild(self._site_name, self.ssh, self.working_dir)
build.build()
self._update_code(site, [project])
modules.pop(project)
updates = self._build_commit_message(sites_copy, site, project)
self._cleanup_and_commit(updates)
x += 1
if self.settings.get('buildSource') == 'make' and self.settings.get('useMakeFile'):
self.utilities.make_site(self._site_name, self.site_dir)
elif len(modules):
self._update_code(site, modules.keys())
if not one_commit_per_project:
updates = self._build_commit_message(sites_copy)
self._cleanup_and_commit(updates)
return updates
def get_sites_to_update(self):
""" Build dictionary of sites/sub-sites and modules needing updated. """
ups_cmds = self.settings.get('upsCmds')
updates_ret = {}
count = 0
sites = {}
sites[self._site_name] = {}
for alias, data in self.sub_sites.items():
sites[alias] = {}
for site in sites:
try:
updates_ret = Drush.call(ups_cmds, site, True)
except DrupdatesError as updates_error:
parse_error = updates_error.msg.split('\n')
if parse_error[2][0:14] == "Drush message:":
# If there are not updates to apply.
continue
else:
raise updates_error
else:
# Parse the results of drush pm-updatestatus
count += len(updates_ret)
modules = {}
for module, update in updates_ret.items():
modules[module] = {}
api = update['api_version']
modules[module]['current'] = update['existing_version'].replace(api + '-', '')
modules[module]['candidate'] = update['candidate_version'].replace(api + '-', '')
msg = "Update {0} from {1} to {2}"
modules[module]['report_txt'] = msg.format(module.title(),
modules[module]['current'],
modules[module]['candidate'])
sites[site]['modules'] = modules
sites['count'] = count
return sites
def update_make_file(self, module, current, candidate):
""" Update the make file.
Keyword arguments:
module -- the drupal module or core (required)
current -- the current version
candidate -- the version to update two
"""
make_file = self.utilities.find_make_file(self._site_name, self.site_dir)
make_format = self.settings.get('makeFormat')
if make_format == 'make':
openfile = open(make_file)
makef = openfile.read()
openfile.close()
current_str = 'projects[{0}][version] = \"{1}\"'.format(module, current)
candidate_str = 'projects[{0}][version] = \"{1}\"'.format(module, candidate)
newdata = makef.replace(current_str, candidate_str)
openfile = open(make_file, 'w')
openfile.write(newdata)
openfile.close()
elif make_format == 'yaml':
make = open(make_file)
makef = yaml.load(make)
make.close()
makef['projects'][module]['version'] = str(candidate)
openfile = open(make_file, 'w')
yaml.dump(makef, openfile, default_flow_style=False)
def _update_code(self, site, modules):
""" Run drush make or pm-update to make te actual code updates.
Keyword arguments:
site -- site alias of the site to update.
modules -- list containing modules to update.
"""
up_cmds = copy.copy(self.settings.get('upCmds'))
up_cmds += modules
try:
Drush.call(up_cmds, site)
except DrupdatesError as updates_error:
raise updates_error
def _build_commit_message(self, sites, site = '', module = ''):
""" Build a commit message for one project update or multiple.
Keyword arguments:
sites -- dictionary containing meta data about update for each site.
site -- if only one site needs updated.
module -- if only one module needs updated.
"""
msg = {}
if module and site:
msg[site] = [sites[site]['modules'][module]['report_txt']]
else:
for site, data in sites.items():
msg[site] = []
for module, status in data['modules'].items():
msg[site].append(status['report_txt'])
return msg
def _cleanup_and_commit(self, updates):
""" Clean-up webroot and commit changes.
Keyword arguments:
updates -- list of update message to put in commit message.
"""
self._clean_up_web_root()
self._git_apply_changes(updates)
def _git_apply_changes(self, updates):
""" add/remove changed files.
Keyword arguments:
updates -- list of update message to put in commit message.
notes:
- Will ignore file mode changes and anything in the commonIgnore setting.
"""
os.chdir(self.site_dir)
repo = Repo(self.site_dir)
for ignore_file in self.settings.get('commonIgnore'):
try:
repo.git.checkout(os.path.join(self.site_web_root, ignore_file))
except git.exc.GitCommandError:
pass
if self.repo_status['modules'] and self.settings.get('ignoreCustomModules'):
custom_module_dir = os.path.join(self.site_web_root,
self.repo_status['modules'], 'custom')
try:
repo.git.checkout(custom_module_dir)
except git.exc.GitCommandError:
pass
# Instruct Git to ignore file mode changes.
cwriter = repo.config_writer('global')
cwriter.set_value('core', 'fileMode', 'false')
cwriter.release()
# Add new/changed files to Git's index
try:
repo.git.add('--all')
except git.exc.GitCommandError as git_add_error:
raise DrupdatesUpdateError(20, git_add_error)
# Remove deleted files from Git's index.
deleted = repo.git.ls_files('--deleted')
for filepath in deleted.split():
repo.git.rm(filepath)
# Commit all the changes.
if self.settings.get('useFeatureBranch'):
if self.settings.get('featureBranchName'):
branch_name = self.settings.get('featureBranchName')
else:
ts = time.time()
stamp = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
branch_name = "drupdates-{0}".format(stamp)
repo.git.checkout(self.working_branch, b=branch_name)
else:
branch_name = self.settings.get('workingBranch')
repo.git.checkout(self.working_branch)
msg = ''
for site, update in updates.items():
msg += "\n{0} \n {1}".format(site, '\n'.join(update))
commit_author = Actor(self.settings.get('commitAuthorName'), self.settings.get('commitAuthorEmail'))
repo.index.commit(message=msg, author=commit_author)
# Save the commit hash for the Drupdates report to use.
heads = repo.heads
branch = heads[branch_name]
self.commit_hash = branch.commit
# Push the changes to the origin repo.
repo.git.push(self._site_name, branch_name)
def _clean_up_web_root(self):
""" Clean-up artifacts from drush pm-update/core-quick-drupal. """
use_make_file = self.settings.get('useMakeFile')
if self.settings.get('buildSource') == 'make' and use_make_file:
# Remove web root folder if repo only ships a make file.
shutil.rmtree(self.site_web_root)
else:
rebuilt = self._rebuild_web_root()
if not rebuilt:
report['status'] = "The webroot re-build failed."
if use_make_file:
make_err = " Ensure the make file format is correct "
make_err += "and Drush make didn't fail on a bad patch."
report['status'] += make_err
return report
# Remove <webroot>/drush folder
drush_path = os.path.join(self.site_web_root, 'drush')
if os.path.isdir(drush_path):
self.utilities.remove_dir(drush_path)
try:
# Remove all SQLite files
os.remove(self.repo_status['db-name'])
for alias, data in self.sub_sites.items():
db_file = data['databases']['default']['default']['database']
if os.path.isfile(db_file):
os.remove(db_file)
except OSError:
pass
def _rebuild_web_root(self):
""" Rebuild the web root folder completely after running pm-update.
Drush pm-update of Drupal Core deletes the .git folder therefore need to
move the updated folder to a temp dir and re-build the webroot folder.
"""
temp_dir = tempfile.mkdtemp(self._site_name)
shutil.move(self.site_web_root, temp_dir)
add_dir = self.settings.get('webrootDir')
if add_dir:
repo = Repo(self.site_dir)
repo.git.checkout(add_dir)
else:
repo = Repo.init(self.site_dir)
try:
remote = git.Remote.create(repo, self._site_name, self.ssh)
except git.exc.GitCommandError as error:
if not error.status == 128:
msg = "Could not establish a remote for the {0} repo".format(self._site_name)
print(msg)
remote.fetch(self.working_branch)
try:
repo.git.checkout('FETCH_HEAD', b=self.working_branch)
except git.exc.GitCommandError as error:
repo.git.checkout(self.working_branch)
add_dir = self._site_name
if 'modules' in self.repo_status:
module_dir = self.repo_status['modules']
shutil.rmtree(os.path.join(self.site_web_root, module_dir))
if 'themes' in self.repo_status:
theme_dir = self.repo_status['themes']
shutil.rmtree(os.path.join(self.site_web_root, theme_dir))
self.utilities.rm_common(self.site_web_root, os.path.join(temp_dir, add_dir))
try:
Utils.copytree(os.path.join(temp_dir, add_dir),
self.site_web_root,
symlinks=True)
except OSError as copy_error:
raise DrupdatesUpdateError(20, copy_error)
except IOError as error:
msg = "Can't copy updates from: \n"
msg += "{0} temp dir to {1}\n".format(temp_dir, self.site_web_root)
msg += "Error: {0}".format(error.strerror)
raise DrupdatesUpdateError(20, msg)
shutil.rmtree(temp_dir)
return True
class Utils(object):
""" Class of utilities used throughout the module. """
def __init__(self):
self.settings = Settings()
@staticmethod
def detect_home_dir(directory):
""" If dir is relative to home dir rewrite as OS agnostic path. """
parts = directory.split('/')
if parts[0] == '~' or parts[0].upper() == '$HOME':
del parts[0]
directory = os.path.join(expanduser('~'), '/'.join(parts))
return directory
@staticmethod
def check_dir(directory):
""" Ensure the directory is writable. """
directory = Utils.detect_home_dir(directory)
if not os.path.isdir(directory):
try:
os.makedirs(directory)
except OSError as error:
msg = 'Unable to create non-existant directory {0} \n'.format(directory)
msg += 'Error: {0}\n'.format(error.strerror)
msg += 'Moving to next working directory, if applicable'
raise DrupdatesError(20, msg)
filepath = os.path.join(directory, "text.txt")
try:
open(filepath, "w")
except IOError:
msg = 'Unable to write to directory {0} \n'.format(directory)
raise DrupdatesError(20, msg)
os.remove(filepath)
return directory
@staticmethod
def remove_dir(directory):
""" Try and remove the directory. """
if os.path.isdir(directory):
try:
shutil.rmtree(directory)
except OSError as error:
msg = "Can't remove site dir {0}\n Error: {1}".format(directory, error.strerror)
raise DrupdatesError(20, msg)
return True
def find_make_file(self, site_name, directory):
""" Find the make file and test to ensure it exists. """
make_format = self.settings.get('makeFormat')
make_folder = self.settings.get('makeFolder')
file_name = self.settings.get('makeFileName')
make_file = site_name + '.make'
if file_name:
make_file_short = file_name
else:
make_file_short = site_name
if make_format == 'yaml':
make_file += '.yaml'
make_file_short += '.yaml'
if make_folder:
directory = os.path.join(directory, make_folder)
file_name = os.path.join(directory, make_file)
file_name_short = os.path.join(directory, make_file_short)
if os.path.isfile(file_name):
return file_name
if os.path.isfile(file_name_short):
return file_name_short
return False
def make_site(self, site_name, site_dir):
""" Build a webroot based on a make file. """
web_root = self.settings.get('webrootDir')
folder = os.path.join(site_dir, web_root)
make_file = self.find_make_file(site_name, site_dir)
Utils.remove_dir(folder)
if make_file and web_root:
# Run drush make
# Get the repo webroot
make_opts = self.settings.get('makeOpts')
make_cmds = ['make', make_file, folder]
make_cmds += make_opts
make = Drush.call(make_cmds)
return make
@staticmethod
def api_call(uri, name, method='get', **kwargs):
""" Perform and API call, expecting a JSON response.
Largely a wrapper around the request module
Keyword arguments:
uri -- the uri of the Restful Web Service (required)
name -- the human readable label for the service being called (required)
method -- HTTP method to use (default = 'get')
kwargs -- dictionary of arguments passed directly to requests module method
"""
# Ensure uri is valid
if not bool(urlparse(uri).netloc):
msg = ("Error: {0} is not a valid url").format(uri)
raise DrupdatesAPIError(20, msg)
func = getattr(requests, method)
args = {}
args['timeout'] = (10, 10)
for key, value in kwargs.items():
args[key] = value
try:
response = func(uri, **args)
except requests.exceptions.Timeout:
msg = "The api call to {0} timed out".format(uri)
raise DrupdatesAPIError(20, msg)
except requests.exceptions.TooManyRedirects:
msg = "The api call to {0} appears incorrect, returned: too many re-directs".format(uri)
raise DrupdatesAPIError(20, msg)
except requests.exceptions.RequestException as error:
msg = "The api call to {0} failed\n Error {1}".format(uri, error)
raise DrupdatesAPIError(20, msg)
try:
response_dictionary = response.json()
except ValueError:
return response
#If API call errors out print the error and quit the script
if response.status_code not in [200, 201]:
if 'errors' in response_dictionary:
errors = response_dictionary.pop('errors')
first_error = errors.pop()
elif 'error' in response_dictionary:
first_error = response_dictionary.pop('error')
else:
first_error['message'] = "No error message provided by response"
msg = "{0} returned an error, exiting the script.\n".format(name)
msg += "Status Code: {0} \n".format(response.status_code)
msg += "Error: {0}".format(first_error['message'])
raise DrupdatesAPIError(20, msg)
else:
return response_dictionary
def sys_commands(self, obj, phase=''):
""" Run a system command based on the subprocess.popen method.
For example: maybe you want a symbolic link, on a unix box,
from /opt/drupal to /var/www/drupal you would add the command(s)
to the appropriate phase setting in you yaml settings files.
Note: the format of the setting is a multi-dimensional list
Example (from Sitebuild.build():
postBuildCmds:
value:
-
- ln
- -s
- /var/www/drupal
- /opt/drupal
Note: You can refer to an attribute in the calling class, assuming they are
set, by prefixing them with "att_" in the settings yaml above,
ex. att_site_dir would pass the Sitebuild.site_dir attribute
Keyword arguments:
phase -- the phase the script is at when sysCommands is called (default "")
object -- the object the call to sysCommand is housed within
"""
commands = self.settings.get(phase)
if commands and isinstance(commands, list):
for command in commands:
if isinstance(command, list):
# Find list items that match the string after "att_",
# these are names names of attribute in the calling class
for key, item in enumerate(command):
if item[:4] == 'att_':
attribute = item[4:]
try:
command[key] = getattr(obj, attribute)
except AttributeError:
continue
try:
popen = subprocess.Popen(command, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except OSError as error:
msg = "Cannot run {0} the command doesn't exist,\n".format(command.pop(0))
msg += "Error: {1}".format(error.strerror)
print(msg)
results = popen.communicate()
if results[1]:
print("Running {0}, \n Error: {1}".format(command, results[1]))
else:
continue
def rm_common(self, dir_delete, dir_compare):
""" Delete files in dir_delete that are in dir_compare.
Iterate over the sites directory and delete any files/folders not in the
commonIgnore setting.
keyword arguments:
dir_delete -- The directory to have it's file/folders deleted.
dir_compare -- The directory to compare dirDelete with.
"""
ignore = self.settings.get('commonIgnore')
if isinstance(ignore, str):
ignore = [ignore]
dcmp = dircmp(dir_delete, dir_compare, ignore)
for file_name in dcmp.common_files:
os.remove(dir_delete + '/' + file_name)
for directory in dcmp.common_dirs:
shutil.rmtree(dir_delete + '/' + directory)
def write_debug_file(self):
""" Write debug file for this run.
Write file containing your system settings to be used to record python
and Drupdates state at the time Drupdates was run.
"""
base_dir = self.settings.get('baseDir')
directory = Utils.check_dir(base_dir)
debug_file_name = os.path.join(directory, 'drupdates.debug')
debug_file = open(debug_file_name, 'w')
debug_file.write("Python Version:\n")
python_version = "{0}\n\n".format(sys.version)
debug_file.write(python_version)
# Get version data for system dependancies
dependancies = ['sqlite3', 'drush', 'git', 'php']
for dependancy in dependancies:
commands = [dependancy, '--version']
popen = subprocess.Popen(commands,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
results = popen.communicate()
if popen.returncode != 0:
stdout = "Check returned error."
else:
stdout = results[0]
debug_file.write("{0} Version:\n".format(dependancy.title()))
debug_file.write("{0}\n".format(stdout.decode()))
installed_packages = pip.get_installed_distributions()
if len(installed_packages):
debug_file.write("Installed Packages:\n\n")
for i in installed_packages:
package = "{0}\n".format(str(i))
debug_file.write(package)
settings = self.settings.list()
debug_file.write("\nDrupdates Settings:\n\n")
for name, setting in settings.items():
line = "{0} : {1}\n".format(name, str(setting['value']))
debug_file.write(line)
def load_dir_settings(self, dir):
""" Add custom settings for the a given directory. """
settings_file = os.path.join(dir, '.drupdates/settings.yaml')
if os.path.isfile(settings_file):
self.settings.add(settings_file, True)
@staticmethod
def copytree(src, dst, symlinks = False, ignore = None):
""" Recursively copy a directory tree from src to dst.
Taken from http://stackoverflow.com/a/22331852/1120125.
Needed because distutils.dir_util.copy_tree will only copy a given
directory one time. Which is annoying!
"""
if not os.path.exists(dst):
os.makedirs(dst)
shutil.copystat(src, dst)
lst = os.listdir(src)
if ignore:
excl = ignore(src, lst)
lst = [x for x in lst if x not in excl]
for item in lst:
s = os.path.join(src, item)
d = os.path.join(dst, item)
if symlinks and os.path.islink(s):
if os.path.lexists(d):
os.remove(d)
os.symlink(os.readlink(s), d)
try:
st = os.lstat(s)
mode = stat.S_IMODE(st.st_mode)
os.lchmod(d, mode)
except:
pass # lchmod not available
elif os.path.isdir(s):
Utils.copytree(s, d, symlinks, ignore)
else:
shutil.copy2(s, d)
def __init__(self):
self.settings = Settings()
class Updates(object):
""" Run through the working directories and sites updating them. """
def __init__(self):
self.settings = Settings()
self.install()
self.utilities = Utils()
self.working_dirs = self.settings.get('workingDir')
self.single_site = ''
self.alias_file = None
if isinstance(self.working_dirs, str):
self.working_dirs = [self.working_dirs]
# by design, SingleSite setting only works with single working directory
if len(self.working_dirs) == 1:
self.single_site = self.settings.get('singleSite')
def install(self):
""" Basic Installation of Drupdates. """
base_dir = self.settings.get('baseDir')
backup_dir = self.settings.get('backupDir')
dirs = [backup_dir, base_dir]
for directory in dirs:
Utils.check_dir(directory)
current_dir = os.path.dirname(os.path.realpath(__file__))
src = os.path.join(current_dir, "templates/settings.template")
settings_file = os.path.join(Utils.check_dir(base_dir), 'settings.yaml')
instructions_url = "http://drupdates.readthedocs.org/en/latest/setup/"
if not os.path.isfile(settings_file):
shutil.copy(src, settings_file)
msg = "The Settings file {0} was created and needs updated.\n".format(settings_file)
msg += "See {0} for instructions".format(instructions_url)
print(msg)
sys.exit(1)
current_settings = open(settings_file, 'r')
settings = yaml.load(current_settings)
if 'repoDict' in settings and 'example' in settings['repoDict']['value']:
msg = "The default Settings file, {0}, needs updated. \n ".format(settings_file)
msg += "See {0} for instructions".format(instructions_url)
print(msg)
sys.exit(1)
def run_updates(self):
""" Drupdates main function. """
if self.settings.get('debug'):
self.utilities.write_debug_file()
report = {}
for current_working_dir in self.working_dirs:
try:
current_working_dir = Utils.check_dir(current_working_dir)
self.utilities.load_dir_settings(current_working_dir)
update = self.update_sites(current_working_dir)
report[current_working_dir] = update
except DrupdatesError as update_error:
report[current_working_dir] = update_error.msg
if update_error.level >= 30:
break
else:
continue
try:
reporting = Reports()
except DrupdatesError as reports_error:
print("Reporting error: \n {0}".format(reports_error.msg))
sys.exit(1)
reporting.send(report)
def update_sites(self, working_dir):
""" Run updates for a working directory's sites. """
report = {}
self.aliases(working_dir)
blacklist = self.settings.get('blacklist')
sites = Repos().get()
if self.single_site:
sites = {self.single_site : sites[self.single_site]}
for site_name, ssh in sites.items():
if self.settings.get('verbose'):
msg = "Drupdates is working on the site: {0} ...".format(site_name)
print(msg)
report[site_name] = {}
if site_name in blacklist:
continue
self.utilities.load_dir_settings(working_dir)
for phase in self.settings.get("drupdatesPhases"):
mod = __import__('drupdates.' + phase['name'].lower(), fromlist=[phase])
class_ = getattr(mod, phase['name'])
instance = class_(site_name, ssh, working_dir)
result = ''
try:
call = getattr(instance, phase['method'])
result = call()
except DrupdatesError as error:
result = error.msg
if error.level < 30:
break
if error.level >= 30:
msg = "Drupdates: fatal error\n Drupdates returned: {0}".format(result)
raise DrupdatesError(error.level, msg)
finally:
report[site_name][phase['name']] = result
self.settings.reset()
self.delete_files()
return report
def aliases(self, working_dir):
""" Build a Drush alias file in $HOME/.drush, with alises to be used later.
Notes:
The file name is controlled by the drushAliasFile settings
All of the aliases will be prefixed with "drupdates" if the default file name
is retained
"""
alias_file_name = self.settings.get('drushAliasFile')
drush_folder = os.path.join(expanduser('~'), '.drush')
self.alias_file = os.path.join(drush_folder, alias_file_name)
if not os.path.isdir(drush_folder):
try:
os.makedirs(drush_folder)
except OSError as error:
msg = "Could not create ~/.drush folder \n Error: {0}".format(error.strerror)
raise DrupdatesError(30, msg)
current_dir = os.path.dirname(os.path.realpath(__file__))
# Symlink the Drush aliases file
src = os.path.join(current_dir, "templates/aliases.template")
doc = open(src)
template = Template(doc.read())
doc.close()
try:
filepath = open(self.alias_file, 'w')
except OSError as error:
msg = "Could not create {0} file\n Error: {1}".format(self.alias_file, error.strerror)
raise DrupdatesError(30, msg)
webroot_dir = self.settings.get('webrootDir')
filepath.write(template.safe_substitute(path=working_dir,
webroot=webroot_dir))
filepath.close()
def delete_files(self):
""" Clean up files used by Drupdates. """
if os.path.isfile(self.alias_file):
try:
os.remove(self.alias_file)
except OSError as error:
msg = "Clean-up error, couldn't remove {0}\n".format(self.alias_file)
msg += "Error: {1}".format(error.strerror)
print(msg)
return True
def __init__(self):
self.settings = Settings()
tool = self.settings.get('pmName').title()
self._plugin = Plugin.load_plugin(tool)
class_ = getattr(self._plugin, tool)
self._instance = class_()
class Sitebuild(object):
""" Build out the repository folder. """
def __init__(self, site_name, ssh, working_dir):
self.settings = Settings()
self._site_name = site_name
self.site_dir = os.path.join(working_dir, self._site_name)
self.ssh = ssh
self.utilities = Utils()
self.si_files = copy.copy(self.settings.get('drushSiFiles'))
def build(self):
""" Core build method. """
working_branch = self.settings.get('workingBranch')
try:
Utils.remove_dir(self.site_dir)
except DrupdatesError as remove_error:
raise DrupdatesBuildError(20, remove_error.msg)
self.utilities.sys_commands(self, 'preBuildCmds')
repository = Repo.init(self.site_dir)
remote = git.Remote.create(repository, self._site_name, self.ssh)
try:
remote.fetch(working_branch, depth=1)
except git.exc.GitCommandError as error:
msg = "{0}: Could not checkout {1}. \n".format(self._site_name, working_branch)
msg += "Error: {0}".format(error)
raise DrupdatesBuildError(20, msg)
git_repo = repository.git
git_repo.checkout('FETCH_HEAD', b=working_branch)
self.utilities.load_dir_settings(self.site_dir)
self.standup_site()
try:
repo_status = Drush.call(['st'], self._site_name, True)
except DrupdatesError as st_error:
raise DrupdatesBuildError(20, st_error.msg)
finally:
self.file_cleanup()
if not 'bootstrap' in repo_status:
msg = "{0} failed to Stand-up properly after running drush qd".format(self._site_name)
raise DrupdatesBuildError(20, msg)
self.utilities.sys_commands(self, 'postBuildCmds')
return "Site build for {0} successful".format(self._site_name)
def standup_site(self):
""" Using the drush core-quick-drupal (qd) command stand-up a Drupal site.
This will:
- Perform site install with sqlite.
- If needed, build webroot from a make file.
- Install any sub sites (ie multi-sites)
- Ensure that all the files in the web root are writable.
"""
qd_settings = self.settings.get('qdCmds')
qd_cmds = copy.copy(qd_settings)
backup_dir = Utils.check_dir(self.settings.get('backupDir'))
qd_cmds += ['--backup-dir=' + backup_dir]
try:
qd_cmds.remove('--no-backup')
except ValueError:
pass
if self.settings.get('useMakeFile'):
make_file = self.utilities.find_make_file(self._site_name, self.site_dir)
if make_file:
qd_cmds += ['--makefile=' + make_file]
else:
msg = "Can't find make file in {0} for {1}".format(self.site_dir, self._site_name)
raise DrupdatesBuildError(20, msg)
if self.settings.get('buildSource') == 'make':
qd_cmds.remove('--use-existing')
try:
Drush.call(qd_cmds, self._site_name)
sub_sites = Drush.get_sub_site_aliases(self._site_name)
for alias, data in sub_sites.items():
Drush.call(qd_cmds, alias)
# Add sub site settings.php to list of file_cleanup() files.
sub_site_st = Drush.call(['st'], alias, True)
self.si_files.append(sub_site_st['site'] + '/settings.php')
self.si_files.append(sub_site_st['files'] + '/.htaccess')
self.si_files.append(sub_site_st['site'])
except DrupdatesError as standup_error:
raise standup_error
def file_cleanup(self):
""" Drush sets the folder permissions for some file to be 0444, convert to 0777. """
drush_dd = Drush.call(['dd', '@drupdates.' + self._site_name])
site_webroot = drush_dd[0]
for name in self.si_files:
complete_name = os.path.join(site_webroot, name)
if os.path.isfile(complete_name) or os.path.isdir(complete_name):
try:
os.chmod(complete_name, 0o777)
except OSError:
msg = "Couldn't change file permission for {0}".format(complete_name)
raise DrupdatesBuildError(20, msg)
