class Reports(Plugin): """ Class for print reports. """ def __init__(self): # load the Plugin _plugins property self.settings = Settings() tool = self.settings.get('reportingTool').title() self._plugin = Plugin.load_plugin(tool) class_ = getattr(self._plugin, tool) self._instance = class_() def json(self, report): import json return json.dumps(report) def yaml(self, report): import yaml return yaml.dump(report, default_flow_style=False) def text(self, report, text=""): """ Format the report dictionary into a string. """ return self.yaml(report) def send(self, report): """ Deliver the report. """ report_format = self.settings.get('reportingFormat') class_ = getattr(self, report_format) report_text = class_(report) return self._instance.send_message(report_text)
class Repolist(Repotool): """ Return git repository list manually maintained in the settings file. """ def __init__(self): current_dir = os.path.dirname(os.path.realpath(__file__)) settings_file = current_dir + '/settings/default.yaml' self.settings = Settings() self.settings.add(settings_file) def git_repos(self): """Return a list of repos from the Settings. The expected format is a dictionary whose keys are the folder names and values are the ssh connection strings to the repo. Example: { 'drupal': 'http://git.drupal.org/project/drupal.git' } """ repo_dict = self.settings.get('repoDict') if (not repo_dict) or (not isinstance(repo_dict, dict)): return {} else: return repo_dict
def __init__(self, site_name, ssh, working_dir): self.settings = Settings() self._site_name = site_name self.site_dir = os.path.join(working_dir, self._site_name) self.ssh = ssh self.utilities = Utils() self.si_files = copy.copy(self.settings.get('drushSiFiles'))
def __init__(self): # load the Plugin _plugins property self.settings = Settings() tool = self.settings.get('gitRepoName').title() self._plugin = Plugin.load_plugin(tool) class_ = getattr(self._plugin, tool) self._instance = class_()
def __init__(self): current_dir = os.path.dirname(os.path.realpath(__file__)) settings_file = current_dir + '/settings/default.yaml' self.settings = Settings() self.settings.add(settings_file) self._pm_label = self.settings.get('pmName') base = self.settings.get('attaskBaseURL') api = self.settings.get('attaskAPIVersion') api_base = urljoin(base, 'attask/api/') self._attask_api_url = urljoin(api_base, api)
class Sendmail(Report): """ sendmail report plugin. """ def __init__(self): current_dir = os.path.dirname(os.path.realpath(__file__)) settings_file = current_dir + '/settings/default.yaml' self.settings = Settings() self.settings.add(settings_file) def send_message(self, report_text): """ Send the report via email using sendmail.""" today = str(datetime.date.today()) msg = MIMEText(report_text) msg["From"] = self.settings.get('reportSender') msg["To"] = self.settings.get('reportRecipient') msg["Subject"] = "Drupdates report {0}.".format(today) mail = Popen(["sendmail", "-t"], stdin=PIPE) mail.communicate(msg.as_string())
def __init__(self): current_dir = os.path.dirname(os.path.realpath(__file__)) settings_file = current_dir + "/settings/default.yaml" self.settings = Settings() self.settings.add(settings_file) base = self.settings.get("jiraBaseURL") api = self.settings.get("jiraAPIVersion") if not api.endswith("/"): api += "/" jira_api_base = urljoin(base, "rest/api/") self._jira_api_url = urljoin(jira_api_base, api)
def __init__(self): self.settings = Settings() self.install() self.utilities = Utils() self.working_dirs = self.settings.get('workingDir') self.single_site = '' self.alias_file = None if isinstance(self.working_dirs, str): self.working_dirs = [self.working_dirs] # by design, SingleSite setting only works with single working directory if len(self.working_dirs) == 1: self.single_site = self.settings.get('singleSite')
def __init__(self, site_name, ssh, working_dir): self.settings = Settings() self.working_branch = self.settings.get('workingBranch') self._site_name = site_name self.working_dir = working_dir self.site_dir = os.path.join(working_dir, self._site_name) self.ssh = ssh self.utilities = Utils() self.site_web_root = None self._commit_hash = None self.repo_status = None self.sub_sites = Drush.get_sub_site_aliases(self._site_name)
class Jira(Pmtool): """ Plugin to wotk with JIRA. """ def __init__(self): current_dir = os.path.dirname(os.path.realpath(__file__)) settings_file = current_dir + "/settings/default.yaml" self.settings = Settings() self.settings.add(settings_file) base = self.settings.get("jiraBaseURL") api = self.settings.get("jiraAPIVersion") if not api.endswith("/"): api += "/" jira_api_base = urljoin(base, "rest/api/") self._jira_api_url = urljoin(jira_api_base, api) def submit_deploy_ticket(self, site, environments, description, target_date): """ Submit a deployment ticket to JIRA/Agile Ready. """ issue_uri = self.settings.get("jiraIssueURL") issue_url = urljoin(self._jira_api_url, issue_uri) jira_user = self.settings.get("jiraUser") jira_pword = self.settings.get("jiraPword") message = {} for environment in environments: request = self.build_reqest(site, environment, description, target_date) headers = {"content-type": "application/json"} response = Utils.api_call( issue_url, "Jira", "post", data=request, auth=(jira_user, jira_pword), headers=headers ) if not response == False: url = response["key"] message[environment] = "The {0} deploy ticket is {1}".format(environment, url) else: message[environment] = "JIRA ticket submission failed for {0}".format(environment) return message def build_reqest(self, site, environment, description, target_date): """ Build the JSON request to be submitted to JIRA. """ summary = "{0} Deployment for {1} w.e. {2}".format(environment, site, target_date) request = {} request["fields"] = {} request["fields"]["project"] = {"key": self.settings.get("jiraProjectID")} request["fields"]["summary"] = summary request["fields"]["description"] = description request["fields"]["issuetype"] = {"name": self.settings.get("jiraIssueType")} return json.dumps(request)
class Pmtools(Plugin): """ Submit requests to Project Management tools. """ def __init__(self): self.settings = Settings() tool = self.settings.get('pmName').title() self._plugin = Plugin.load_plugin(tool) class_ = getattr(self._plugin, tool) self._instance = class_() def target_date(self): """ Get the date string for the following Friday. """ value = self.settings.get('targetDate') if not value: today = datetime.date.today() # If today is a Friday, we skip to next Friday if datetime.datetime.today().weekday() == 4: friday = str(today + datetime.timedelta((3-today.weekday()) % 7 + 1)) else: friday = str(today + datetime.timedelta((4-today.weekday()) % 7)) return friday else: return value @staticmethod def description(site, git_hash): """ Collect data for ticket's descriotion field. """ description_list = [] description_list.append("Git Hash = <" + git_hash + ">") description_list.append("Post deployment steps:") description_list.append("drush @" + site +" updb -y") return '\n'.join(description_list) def deploy_ticket(self, site, commit_hash): """ Submit ticket requesting deployment(s). """ description = Pmtools.description(site, commit_hash) environments = self.settings.get('deploymentTickets') target = self.target_date() return self._instance.submit_deploy_ticket(site, environments, description, target)
class Repos(Plugin): """ Build Git repository list. """ def __init__(self): # load the Plugin _plugins property self.settings = Settings() tool = self.settings.get('gitRepoName').title() self._plugin = Plugin.load_plugin(tool) class_ = getattr(self._plugin, tool) self._instance = class_() def get(self): """ Get repository list from plugin. """ return self._instance.git_repos()
class Stash(Repotool): """ Return git repository list from Stash project. """ def __init__(self): current_dir = os.path.dirname(os.path.realpath(__file__)) settings_file = current_dir + '/settings/default.yaml' self.settings = Settings() self.settings.add(settings_file) def git_repos(self): """ Get list of Stash repos from a specific Project. Note: this request will only bring back 9,999 repos, which should suffice, if it doesn't updaqte the stashLimit setting. """ stash_url = self.settings.get('stashURL') git_repo_name = self.settings.get('gitRepoName') stash_user = self.settings.get('stashUser') stash_pword = self.settings.get('stashPword') stash_cert_verify = self.settings.get('stashCertVerify') stash_limit = self.settings.get('stashLimit') stash_params = {'limit' : stash_limit} response = Utils.api_call(stash_url, git_repo_name, 'get', auth=(stash_user, stash_pword), verify=stash_cert_verify, params=stash_params) if not response == False: repos = Stash.parse_repos(response['values']) return repos else: return {} @staticmethod def parse_repos(raw): """ Parse repo list returned form Stash.""" repos = {} for repo in raw: for link in repo['links']['clone']: if link['name'] == 'ssh': repos[repo['slug']] = link['href'] return repos
class Slack(Report): """ Slack report plugin. """ def __init__(self): current_dir = os.path.dirname(os.path.realpath(__file__)) settings_file = current_dir + '/settings/default.yaml' self.settings = Settings() self.settings.add(settings_file) def send_message(self, report_text): """ Post the report to a Slack channel or DM a specific user.""" url = self.settings.get('slackURL') user = self.settings.get('slackUser') payload = {} payload['text'] = report_text payload['new-bot-name'] = user direct = self.settings.get('slackRecipient') channel = self.settings.get('slackChannel') if direct: payload['channel'] = '@' + direct elif channel: payload['channel'] = '#' + direct Utils.api_call(url, 'Slack', 'post', data=json.dumps(payload))
def __init__(self): current_dir = os.path.dirname(os.path.realpath(__file__)) settings_file = current_dir + '/settings/default.yaml' self.settings = Settings() self.settings.add(settings_file)
class Attask(Pmtool): """ Plugin to wotk with AtTask. """ def __init__(self): current_dir = os.path.dirname(os.path.realpath(__file__)) settings_file = current_dir + '/settings/default.yaml' self.settings = Settings() self.settings.add(settings_file) self._pm_label = self.settings.get('pmName') base = self.settings.get('attaskBaseURL') api = self.settings.get('attaskAPIVersion') api_base = urljoin(base, 'attask/api/') self._attask_api_url = urljoin(api_base, api) def get_session_id(self): """ Get a session ID from AtTask. """ attask_pword = self.settings.get('attaskPword') attask_user = self.settings.get('attaskUser') at_params = {'username': attask_user, 'password': attask_pword} login_url = urljoin(self._attask_api_url, self.settings.get('attaskLoginUrl')) response = Utils.api_call(login_url, self._pm_label, 'post', params=at_params) if response == False: return False else: return response['data']['sessionID'] def submit_deploy_ticket(self, site, environments, description, target_date): """ Submit a Deployment request to AtTask. site -- The site the ticket is for environments -- The name(s) of the environments to deploy to description -- The description text to go in the task targetDate -- The date to put in the label fo the ticket """ sessparam = {} session_id = self.get_session_id() # Make sure you can get a Session ID if session_id: sessparam['sessionID'] = session_id else: return False # Set-up AtTask request attask_project_id = self.settings.get('attaskProjectID') dev_ops_team_id = self.settings.get('devOpsTeamID') attask_base_url = self.settings.get('attaskBaseURL') attask_assignee_type = self.settings.get('attaskAssigneeType') task_url = urljoin(self._attask_api_url, self.settings.get('attaskTaskURL')) message = {} for environment in environments: title = environment + ' Deployment for ' + site +' w.e. ' + target_date at_params = {'name': title, 'projectID': attask_project_id, attask_assignee_type: dev_ops_team_id, 'description': description} response = Utils.api_call(task_url, self._pm_label, 'post', params=at_params, headers=sessparam) if not response == False: data = response['data'] msg = "The {0} deploy ticket is".format(environment) msg += " <{0}task/view?ID={1}>".format(attask_base_url, data['ID']) message[environment] = msg else: msg = "The {0} deploy ticket did not submit to".format(environment) msg += "{0} properly".format(self._pm_label) message[environment] = msg return message
class Siteupdate(object): """ Update the modules and/or core in a completely built Drupal site. """ def __init__(self, site_name, ssh, working_dir): self.settings = Settings() self.working_branch = self.settings.get('workingBranch') self._site_name = site_name self.working_dir = working_dir self.site_dir = os.path.join(working_dir, self._site_name) self.ssh = ssh self.utilities = Utils() self.site_web_root = None self._commit_hash = None self.repo_status = None self.sub_sites = Drush.get_sub_site_aliases(self._site_name) @property def commit_hash(self): """ commit_hash getter. """ return self._commit_hash @commit_hash.setter def commit_hash(self, value): """ commit_hash setter. """ self._commit_hash = value def update(self): """ Set-up to and run Drush update(s) (i.e. up or ups). """ report = {} self.utilities.sys_commands(self, 'preUpdateCmds') self.repo_status = Drush.call(['st'], self._site_name, True) try: updates = self.run_updates() except DrupdatesError as updates_error: raise DrupdatesUpdateError(20, updates_error.msg) # If no updates move to the next repo if not updates: self.commit_hash = "" report['status'] = "Did not have any updates to apply" return report report['status'] = "The following updates were applied" report['updates'] = updates report['commit'] = "The commit hash is {0}".format(self.commit_hash) self.utilities.sys_commands(self, 'postUpdateCmds') if self.settings.get('submitDeployTicket') and self.commit_hash: report[self._site_name] = {} pm_name = self.settings.get('pmName').title() try: report[self._site_name][pm_name] = Pmtools().deploy_ticket(self._site_name, self.commit_hash) except DrupdatesError as api_error: report[self._site_name][pm_name] = api_error.msg return report def run_updates(self): """ Run the site updates. The updates are done either by downloading the updates, updating the make file or both. - First, run drush pm-updatestatus to get a list of eligible updates for the site/sub-sites. - Second, build the report to return to Updates(). - Third, apply the updates. """ updates = {} try: sites = self.get_sites_to_update() except DrupdatesError as update_status_error: raise DrupdatesUpdateError(20, update_status_error) if not sites['count']: return updates else: sites.pop('count') # Note: call Drush.call() without site alias as alias comes after dd argument. drush_dd = Drush.call(['dd', '@drupdates.' + self._site_name]) self.site_web_root = drush_dd[0] # Create seperate commits for each project (ie module/theme) one_commit_per_project = self.settings.get('oneCommitPerProject') # Iterate through the site/sub-sites and perform updates, update files etc... sites_copy = copy.copy(sites) for site, data in sites.items(): if 'modules' not in data: sites_copy.pop(site) continue modules = copy.copy(data['modules']) x = 0 for project, descriptions in data['modules'].items(): if self.settings.get('useMakeFile'): self.update_make_file(project, descriptions['current'], descriptions['candidate']) if one_commit_per_project: if x: build = Sitebuild(self._site_name, self.ssh, self.working_dir) build.build() self._update_code(site, [project]) modules.pop(project) updates = self._build_commit_message(sites_copy, site, project) self._cleanup_and_commit(updates) x += 1 if self.settings.get('buildSource') == 'make' and self.settings.get('useMakeFile'): self.utilities.make_site(self._site_name, self.site_dir) elif len(modules): self._update_code(site, modules.keys()) if not one_commit_per_project: updates = self._build_commit_message(sites_copy) self._cleanup_and_commit(updates) return updates def get_sites_to_update(self): """ Build dictionary of sites/sub-sites and modules needing updated. """ ups_cmds = self.settings.get('upsCmds') updates_ret = {} count = 0 sites = {} sites[self._site_name] = {} for alias, data in self.sub_sites.items(): sites[alias] = {} for site in sites: try: updates_ret = Drush.call(ups_cmds, site, True) except DrupdatesError as updates_error: parse_error = updates_error.msg.split('\n') if parse_error[2][0:14] == "Drush message:": # If there are not updates to apply. continue else: raise updates_error else: # Parse the results of drush pm-updatestatus count += len(updates_ret) modules = {} for module, update in updates_ret.items(): modules[module] = {} api = update['api_version'] modules[module]['current'] = update['existing_version'].replace(api + '-', '') modules[module]['candidate'] = update['candidate_version'].replace(api + '-', '') msg = "Update {0} from {1} to {2}" modules[module]['report_txt'] = msg.format(module.title(), modules[module]['current'], modules[module]['candidate']) sites[site]['modules'] = modules sites['count'] = count return sites def update_make_file(self, module, current, candidate): """ Update the make file. Keyword arguments: module -- the drupal module or core (required) current -- the current version candidate -- the version to update two """ make_file = self.utilities.find_make_file(self._site_name, self.site_dir) make_format = self.settings.get('makeFormat') if make_format == 'make': openfile = open(make_file) makef = openfile.read() openfile.close() current_str = 'projects[{0}][version] = \"{1}\"'.format(module, current) candidate_str = 'projects[{0}][version] = \"{1}\"'.format(module, candidate) newdata = makef.replace(current_str, candidate_str) openfile = open(make_file, 'w') openfile.write(newdata) openfile.close() elif make_format == 'yaml': make = open(make_file) makef = yaml.load(make) make.close() makef['projects'][module]['version'] = str(candidate) openfile = open(make_file, 'w') yaml.dump(makef, openfile, default_flow_style=False) def _update_code(self, site, modules): """ Run drush make or pm-update to make te actual code updates. Keyword arguments: site -- site alias of the site to update. modules -- list containing modules to update. """ up_cmds = copy.copy(self.settings.get('upCmds')) up_cmds += modules try: Drush.call(up_cmds, site) except DrupdatesError as updates_error: raise updates_error def _build_commit_message(self, sites, site = '', module = ''): """ Build a commit message for one project update or multiple. Keyword arguments: sites -- dictionary containing meta data about update for each site. site -- if only one site needs updated. module -- if only one module needs updated. """ msg = {} if module and site: msg[site] = [sites[site]['modules'][module]['report_txt']] else: for site, data in sites.items(): msg[site] = [] for module, status in data['modules'].items(): msg[site].append(status['report_txt']) return msg def _cleanup_and_commit(self, updates): """ Clean-up webroot and commit changes. Keyword arguments: updates -- list of update message to put in commit message. """ self._clean_up_web_root() self._git_apply_changes(updates) def _git_apply_changes(self, updates): """ add/remove changed files. Keyword arguments: updates -- list of update message to put in commit message. notes: - Will ignore file mode changes and anything in the commonIgnore setting. """ os.chdir(self.site_dir) repo = Repo(self.site_dir) for ignore_file in self.settings.get('commonIgnore'): try: repo.git.checkout(os.path.join(self.site_web_root, ignore_file)) except git.exc.GitCommandError: pass if self.repo_status['modules'] and self.settings.get('ignoreCustomModules'): custom_module_dir = os.path.join(self.site_web_root, self.repo_status['modules'], 'custom') try: repo.git.checkout(custom_module_dir) except git.exc.GitCommandError: pass # Instruct Git to ignore file mode changes. cwriter = repo.config_writer('global') cwriter.set_value('core', 'fileMode', 'false') cwriter.release() # Add new/changed files to Git's index try: repo.git.add('--all') except git.exc.GitCommandError as git_add_error: raise DrupdatesUpdateError(20, git_add_error) # Remove deleted files from Git's index. deleted = repo.git.ls_files('--deleted') for filepath in deleted.split(): repo.git.rm(filepath) # Commit all the changes. if self.settings.get('useFeatureBranch'): if self.settings.get('featureBranchName'): branch_name = self.settings.get('featureBranchName') else: ts = time.time() stamp = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S') branch_name = "drupdates-{0}".format(stamp) repo.git.checkout(self.working_branch, b=branch_name) else: branch_name = self.settings.get('workingBranch') repo.git.checkout(self.working_branch) msg = '' for site, update in updates.items(): msg += "\n{0} \n {1}".format(site, '\n'.join(update)) commit_author = Actor(self.settings.get('commitAuthorName'), self.settings.get('commitAuthorEmail')) repo.index.commit(message=msg, author=commit_author) # Save the commit hash for the Drupdates report to use. heads = repo.heads branch = heads[branch_name] self.commit_hash = branch.commit # Push the changes to the origin repo. repo.git.push(self._site_name, branch_name) def _clean_up_web_root(self): """ Clean-up artifacts from drush pm-update/core-quick-drupal. """ use_make_file = self.settings.get('useMakeFile') if self.settings.get('buildSource') == 'make' and use_make_file: # Remove web root folder if repo only ships a make file. shutil.rmtree(self.site_web_root) else: rebuilt = self._rebuild_web_root() if not rebuilt: report['status'] = "The webroot re-build failed." if use_make_file: make_err = " Ensure the make file format is correct " make_err += "and Drush make didn't fail on a bad patch." report['status'] += make_err return report # Remove <webroot>/drush folder drush_path = os.path.join(self.site_web_root, 'drush') if os.path.isdir(drush_path): self.utilities.remove_dir(drush_path) try: # Remove all SQLite files os.remove(self.repo_status['db-name']) for alias, data in self.sub_sites.items(): db_file = data['databases']['default']['default']['database'] if os.path.isfile(db_file): os.remove(db_file) except OSError: pass def _rebuild_web_root(self): """ Rebuild the web root folder completely after running pm-update. Drush pm-update of Drupal Core deletes the .git folder therefore need to move the updated folder to a temp dir and re-build the webroot folder. """ temp_dir = tempfile.mkdtemp(self._site_name) shutil.move(self.site_web_root, temp_dir) add_dir = self.settings.get('webrootDir') if add_dir: repo = Repo(self.site_dir) repo.git.checkout(add_dir) else: repo = Repo.init(self.site_dir) try: remote = git.Remote.create(repo, self._site_name, self.ssh) except git.exc.GitCommandError as error: if not error.status == 128: msg = "Could not establish a remote for the {0} repo".format(self._site_name) print(msg) remote.fetch(self.working_branch) try: repo.git.checkout('FETCH_HEAD', b=self.working_branch) except git.exc.GitCommandError as error: repo.git.checkout(self.working_branch) add_dir = self._site_name if 'modules' in self.repo_status: module_dir = self.repo_status['modules'] shutil.rmtree(os.path.join(self.site_web_root, module_dir)) if 'themes' in self.repo_status: theme_dir = self.repo_status['themes'] shutil.rmtree(os.path.join(self.site_web_root, theme_dir)) self.utilities.rm_common(self.site_web_root, os.path.join(temp_dir, add_dir)) try: Utils.copytree(os.path.join(temp_dir, add_dir), self.site_web_root, symlinks=True) except OSError as copy_error: raise DrupdatesUpdateError(20, copy_error) except IOError as error: msg = "Can't copy updates from: \n" msg += "{0} temp dir to {1}\n".format(temp_dir, self.site_web_root) msg += "Error: {0}".format(error.strerror) raise DrupdatesUpdateError(20, msg) shutil.rmtree(temp_dir) return True
class Utils(object): """ Class of utilities used throughout the module. """ def __init__(self): self.settings = Settings() @staticmethod def detect_home_dir(directory): """ If dir is relative to home dir rewrite as OS agnostic path. """ parts = directory.split('/') if parts[0] == '~' or parts[0].upper() == '$HOME': del parts[0] directory = os.path.join(expanduser('~'), '/'.join(parts)) return directory @staticmethod def check_dir(directory): """ Ensure the directory is writable. """ directory = Utils.detect_home_dir(directory) if not os.path.isdir(directory): try: os.makedirs(directory) except OSError as error: msg = 'Unable to create non-existant directory {0} \n'.format(directory) msg += 'Error: {0}\n'.format(error.strerror) msg += 'Moving to next working directory, if applicable' raise DrupdatesError(20, msg) filepath = os.path.join(directory, "text.txt") try: open(filepath, "w") except IOError: msg = 'Unable to write to directory {0} \n'.format(directory) raise DrupdatesError(20, msg) os.remove(filepath) return directory @staticmethod def remove_dir(directory): """ Try and remove the directory. """ if os.path.isdir(directory): try: shutil.rmtree(directory) except OSError as error: msg = "Can't remove site dir {0}\n Error: {1}".format(directory, error.strerror) raise DrupdatesError(20, msg) return True def find_make_file(self, site_name, directory): """ Find the make file and test to ensure it exists. """ make_format = self.settings.get('makeFormat') make_folder = self.settings.get('makeFolder') file_name = self.settings.get('makeFileName') make_file = site_name + '.make' if file_name: make_file_short = file_name else: make_file_short = site_name if make_format == 'yaml': make_file += '.yaml' make_file_short += '.yaml' if make_folder: directory = os.path.join(directory, make_folder) file_name = os.path.join(directory, make_file) file_name_short = os.path.join(directory, make_file_short) if os.path.isfile(file_name): return file_name if os.path.isfile(file_name_short): return file_name_short return False def make_site(self, site_name, site_dir): """ Build a webroot based on a make file. """ web_root = self.settings.get('webrootDir') folder = os.path.join(site_dir, web_root) make_file = self.find_make_file(site_name, site_dir) Utils.remove_dir(folder) if make_file and web_root: # Run drush make # Get the repo webroot make_opts = self.settings.get('makeOpts') make_cmds = ['make', make_file, folder] make_cmds += make_opts make = Drush.call(make_cmds) return make @staticmethod def api_call(uri, name, method='get', **kwargs): """ Perform and API call, expecting a JSON response. Largely a wrapper around the request module Keyword arguments: uri -- the uri of the Restful Web Service (required) name -- the human readable label for the service being called (required) method -- HTTP method to use (default = 'get') kwargs -- dictionary of arguments passed directly to requests module method """ # Ensure uri is valid if not bool(urlparse(uri).netloc): msg = ("Error: {0} is not a valid url").format(uri) raise DrupdatesAPIError(20, msg) func = getattr(requests, method) args = {} args['timeout'] = (10, 10) for key, value in kwargs.items(): args[key] = value try: response = func(uri, **args) except requests.exceptions.Timeout: msg = "The api call to {0} timed out".format(uri) raise DrupdatesAPIError(20, msg) except requests.exceptions.TooManyRedirects: msg = "The api call to {0} appears incorrect, returned: too many re-directs".format(uri) raise DrupdatesAPIError(20, msg) except requests.exceptions.RequestException as error: msg = "The api call to {0} failed\n Error {1}".format(uri, error) raise DrupdatesAPIError(20, msg) try: response_dictionary = response.json() except ValueError: return response #If API call errors out print the error and quit the script if response.status_code not in [200, 201]: if 'errors' in response_dictionary: errors = response_dictionary.pop('errors') first_error = errors.pop() elif 'error' in response_dictionary: first_error = response_dictionary.pop('error') else: first_error['message'] = "No error message provided by response" msg = "{0} returned an error, exiting the script.\n".format(name) msg += "Status Code: {0} \n".format(response.status_code) msg += "Error: {0}".format(first_error['message']) raise DrupdatesAPIError(20, msg) else: return response_dictionary def sys_commands(self, obj, phase=''): """ Run a system command based on the subprocess.popen method. For example: maybe you want a symbolic link, on a unix box, from /opt/drupal to /var/www/drupal you would add the command(s) to the appropriate phase setting in you yaml settings files. Note: the format of the setting is a multi-dimensional list Example (from Sitebuild.build(): postBuildCmds: value: - - ln - -s - /var/www/drupal - /opt/drupal Note: You can refer to an attribute in the calling class, assuming they are set, by prefixing them with "att_" in the settings yaml above, ex. att_site_dir would pass the Sitebuild.site_dir attribute Keyword arguments: phase -- the phase the script is at when sysCommands is called (default "") object -- the object the call to sysCommand is housed within """ commands = self.settings.get(phase) if commands and isinstance(commands, list): for command in commands: if isinstance(command, list): # Find list items that match the string after "att_", # these are names names of attribute in the calling class for key, item in enumerate(command): if item[:4] == 'att_': attribute = item[4:] try: command[key] = getattr(obj, attribute) except AttributeError: continue try: popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) except OSError as error: msg = "Cannot run {0} the command doesn't exist,\n".format(command.pop(0)) msg += "Error: {1}".format(error.strerror) print(msg) results = popen.communicate() if results[1]: print("Running {0}, \n Error: {1}".format(command, results[1])) else: continue def rm_common(self, dir_delete, dir_compare): """ Delete files in dir_delete that are in dir_compare. Iterate over the sites directory and delete any files/folders not in the commonIgnore setting. keyword arguments: dir_delete -- The directory to have it's file/folders deleted. dir_compare -- The directory to compare dirDelete with. """ ignore = self.settings.get('commonIgnore') if isinstance(ignore, str): ignore = [ignore] dcmp = dircmp(dir_delete, dir_compare, ignore) for file_name in dcmp.common_files: os.remove(dir_delete + '/' + file_name) for directory in dcmp.common_dirs: shutil.rmtree(dir_delete + '/' + directory) def write_debug_file(self): """ Write debug file for this run. Write file containing your system settings to be used to record python and Drupdates state at the time Drupdates was run. """ base_dir = self.settings.get('baseDir') directory = Utils.check_dir(base_dir) debug_file_name = os.path.join(directory, 'drupdates.debug') debug_file = open(debug_file_name, 'w') debug_file.write("Python Version:\n") python_version = "{0}\n\n".format(sys.version) debug_file.write(python_version) # Get version data for system dependancies dependancies = ['sqlite3', 'drush', 'git', 'php'] for dependancy in dependancies: commands = [dependancy, '--version'] popen = subprocess.Popen(commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE) results = popen.communicate() if popen.returncode != 0: stdout = "Check returned error." else: stdout = results[0] debug_file.write("{0} Version:\n".format(dependancy.title())) debug_file.write("{0}\n".format(stdout.decode())) installed_packages = pip.get_installed_distributions() if len(installed_packages): debug_file.write("Installed Packages:\n\n") for i in installed_packages: package = "{0}\n".format(str(i)) debug_file.write(package) settings = self.settings.list() debug_file.write("\nDrupdates Settings:\n\n") for name, setting in settings.items(): line = "{0} : {1}\n".format(name, str(setting['value'])) debug_file.write(line) def load_dir_settings(self, dir): """ Add custom settings for the a given directory. """ settings_file = os.path.join(dir, '.drupdates/settings.yaml') if os.path.isfile(settings_file): self.settings.add(settings_file, True) @staticmethod def copytree(src, dst, symlinks = False, ignore = None): """ Recursively copy a directory tree from src to dst. Taken from http://stackoverflow.com/a/22331852/1120125. Needed because distutils.dir_util.copy_tree will only copy a given directory one time. Which is annoying! """ if not os.path.exists(dst): os.makedirs(dst) shutil.copystat(src, dst) lst = os.listdir(src) if ignore: excl = ignore(src, lst) lst = [x for x in lst if x not in excl] for item in lst: s = os.path.join(src, item) d = os.path.join(dst, item) if symlinks and os.path.islink(s): if os.path.lexists(d): os.remove(d) os.symlink(os.readlink(s), d) try: st = os.lstat(s) mode = stat.S_IMODE(st.st_mode) os.lchmod(d, mode) except: pass # lchmod not available elif os.path.isdir(s): Utils.copytree(s, d, symlinks, ignore) else: shutil.copy2(s, d)
def __init__(self): self.settings = Settings()
class Updates(object): """ Run through the working directories and sites updating them. """ def __init__(self): self.settings = Settings() self.install() self.utilities = Utils() self.working_dirs = self.settings.get('workingDir') self.single_site = '' self.alias_file = None if isinstance(self.working_dirs, str): self.working_dirs = [self.working_dirs] # by design, SingleSite setting only works with single working directory if len(self.working_dirs) == 1: self.single_site = self.settings.get('singleSite') def install(self): """ Basic Installation of Drupdates. """ base_dir = self.settings.get('baseDir') backup_dir = self.settings.get('backupDir') dirs = [backup_dir, base_dir] for directory in dirs: Utils.check_dir(directory) current_dir = os.path.dirname(os.path.realpath(__file__)) src = os.path.join(current_dir, "templates/settings.template") settings_file = os.path.join(Utils.check_dir(base_dir), 'settings.yaml') instructions_url = "http://drupdates.readthedocs.org/en/latest/setup/" if not os.path.isfile(settings_file): shutil.copy(src, settings_file) msg = "The Settings file {0} was created and needs updated.\n".format(settings_file) msg += "See {0} for instructions".format(instructions_url) print(msg) sys.exit(1) current_settings = open(settings_file, 'r') settings = yaml.load(current_settings) if 'repoDict' in settings and 'example' in settings['repoDict']['value']: msg = "The default Settings file, {0}, needs updated. \n ".format(settings_file) msg += "See {0} for instructions".format(instructions_url) print(msg) sys.exit(1) def run_updates(self): """ Drupdates main function. """ if self.settings.get('debug'): self.utilities.write_debug_file() report = {} for current_working_dir in self.working_dirs: try: current_working_dir = Utils.check_dir(current_working_dir) self.utilities.load_dir_settings(current_working_dir) update = self.update_sites(current_working_dir) report[current_working_dir] = update except DrupdatesError as update_error: report[current_working_dir] = update_error.msg if update_error.level >= 30: break else: continue try: reporting = Reports() except DrupdatesError as reports_error: print("Reporting error: \n {0}".format(reports_error.msg)) sys.exit(1) reporting.send(report) def update_sites(self, working_dir): """ Run updates for a working directory's sites. """ report = {} self.aliases(working_dir) blacklist = self.settings.get('blacklist') sites = Repos().get() if self.single_site: sites = {self.single_site : sites[self.single_site]} for site_name, ssh in sites.items(): if self.settings.get('verbose'): msg = "Drupdates is working on the site: {0} ...".format(site_name) print(msg) report[site_name] = {} if site_name in blacklist: continue self.utilities.load_dir_settings(working_dir) for phase in self.settings.get("drupdatesPhases"): mod = __import__('drupdates.' + phase['name'].lower(), fromlist=[phase]) class_ = getattr(mod, phase['name']) instance = class_(site_name, ssh, working_dir) result = '' try: call = getattr(instance, phase['method']) result = call() except DrupdatesError as error: result = error.msg if error.level < 30: break if error.level >= 30: msg = "Drupdates: fatal error\n Drupdates returned: {0}".format(result) raise DrupdatesError(error.level, msg) finally: report[site_name][phase['name']] = result self.settings.reset() self.delete_files() return report def aliases(self, working_dir): """ Build a Drush alias file in $HOME/.drush, with alises to be used later. Notes: The file name is controlled by the drushAliasFile settings All of the aliases will be prefixed with "drupdates" if the default file name is retained """ alias_file_name = self.settings.get('drushAliasFile') drush_folder = os.path.join(expanduser('~'), '.drush') self.alias_file = os.path.join(drush_folder, alias_file_name) if not os.path.isdir(drush_folder): try: os.makedirs(drush_folder) except OSError as error: msg = "Could not create ~/.drush folder \n Error: {0}".format(error.strerror) raise DrupdatesError(30, msg) current_dir = os.path.dirname(os.path.realpath(__file__)) # Symlink the Drush aliases file src = os.path.join(current_dir, "templates/aliases.template") doc = open(src) template = Template(doc.read()) doc.close() try: filepath = open(self.alias_file, 'w') except OSError as error: msg = "Could not create {0} file\n Error: {1}".format(self.alias_file, error.strerror) raise DrupdatesError(30, msg) webroot_dir = self.settings.get('webrootDir') filepath.write(template.safe_substitute(path=working_dir, webroot=webroot_dir)) filepath.close() def delete_files(self): """ Clean up files used by Drupdates. """ if os.path.isfile(self.alias_file): try: os.remove(self.alias_file) except OSError as error: msg = "Clean-up error, couldn't remove {0}\n".format(self.alias_file) msg += "Error: {1}".format(error.strerror) print(msg) return True
def __init__(self): self.settings = Settings() tool = self.settings.get('pmName').title() self._plugin = Plugin.load_plugin(tool) class_ = getattr(self._plugin, tool) self._instance = class_()
class Sitebuild(object): """ Build out the repository folder. """ def __init__(self, site_name, ssh, working_dir): self.settings = Settings() self._site_name = site_name self.site_dir = os.path.join(working_dir, self._site_name) self.ssh = ssh self.utilities = Utils() self.si_files = copy.copy(self.settings.get('drushSiFiles')) def build(self): """ Core build method. """ working_branch = self.settings.get('workingBranch') try: Utils.remove_dir(self.site_dir) except DrupdatesError as remove_error: raise DrupdatesBuildError(20, remove_error.msg) self.utilities.sys_commands(self, 'preBuildCmds') repository = Repo.init(self.site_dir) remote = git.Remote.create(repository, self._site_name, self.ssh) try: remote.fetch(working_branch, depth=1) except git.exc.GitCommandError as error: msg = "{0}: Could not checkout {1}. \n".format(self._site_name, working_branch) msg += "Error: {0}".format(error) raise DrupdatesBuildError(20, msg) git_repo = repository.git git_repo.checkout('FETCH_HEAD', b=working_branch) self.utilities.load_dir_settings(self.site_dir) self.standup_site() try: repo_status = Drush.call(['st'], self._site_name, True) except DrupdatesError as st_error: raise DrupdatesBuildError(20, st_error.msg) finally: self.file_cleanup() if not 'bootstrap' in repo_status: msg = "{0} failed to Stand-up properly after running drush qd".format(self._site_name) raise DrupdatesBuildError(20, msg) self.utilities.sys_commands(self, 'postBuildCmds') return "Site build for {0} successful".format(self._site_name) def standup_site(self): """ Using the drush core-quick-drupal (qd) command stand-up a Drupal site. This will: - Perform site install with sqlite. - If needed, build webroot from a make file. - Install any sub sites (ie multi-sites) - Ensure that all the files in the web root are writable. """ qd_settings = self.settings.get('qdCmds') qd_cmds = copy.copy(qd_settings) backup_dir = Utils.check_dir(self.settings.get('backupDir')) qd_cmds += ['--backup-dir=' + backup_dir] try: qd_cmds.remove('--no-backup') except ValueError: pass if self.settings.get('useMakeFile'): make_file = self.utilities.find_make_file(self._site_name, self.site_dir) if make_file: qd_cmds += ['--makefile=' + make_file] else: msg = "Can't find make file in {0} for {1}".format(self.site_dir, self._site_name) raise DrupdatesBuildError(20, msg) if self.settings.get('buildSource') == 'make': qd_cmds.remove('--use-existing') try: Drush.call(qd_cmds, self._site_name) sub_sites = Drush.get_sub_site_aliases(self._site_name) for alias, data in sub_sites.items(): Drush.call(qd_cmds, alias) # Add sub site settings.php to list of file_cleanup() files. sub_site_st = Drush.call(['st'], alias, True) self.si_files.append(sub_site_st['site'] + '/settings.php') self.si_files.append(sub_site_st['files'] + '/.htaccess') self.si_files.append(sub_site_st['site']) except DrupdatesError as standup_error: raise standup_error def file_cleanup(self): """ Drush sets the folder permissions for some file to be 0444, convert to 0777. """ drush_dd = Drush.call(['dd', '@drupdates.' + self._site_name]) site_webroot = drush_dd[0] for name in self.si_files: complete_name = os.path.join(site_webroot, name) if os.path.isfile(complete_name) or os.path.isdir(complete_name): try: os.chmod(complete_name, 0o777) except OSError: msg = "Couldn't change file permission for {0}".format(complete_name) raise DrupdatesBuildError(20, msg)