def log_tasks(self, revision, branch='^/trunk'): args = ('log',) if revision: args += ('-r', '%s:HEAD' % revision) else: args += ('-l', '100') args += (branch,) cerr("Running svn client") stdout, stderr, return_code = self.svn(args) cerr("Collecting tasks") if return_code != 0: raise SVNError(stderr) tasks = dict() last_rev = None for line in stdout.splitlines(): m = re.match(r'^r([\d]+) \|', line) if m: last_rev = int(m.group(1)) continue m = re.match(r'^%s-([\d]+) (.*)' % self.project_key, line) if not m: continue task = int(m.group(1)) message = m.group(2) tasks.setdefault(task, []) tasks[task].append((last_rev, message)) return tasks
def log_tasks(self, revision, branch='^/trunk'): args = ('log', ) if revision: args += ('-r', '%s:HEAD' % revision) else: args += ('-l', '100') args += (branch, ) cerr("Running svn client") stdout, stderr, return_code = self.svn(args) cerr("Collecting tasks") if return_code != 0: raise SVNError(stderr) tasks = dict() last_rev = None for line in stdout.splitlines(): m = re.match(r'^r([\d]+) \|', line) if m: last_rev = int(m.group(1)) continue m = re.match(r'^%s-([\d]+) (.*)' % self.project_key, line) if not m: continue task = int(m.group(1)) message = m.group(2) tasks.setdefault(task, []) tasks[task].append((last_rev, message)) return tasks
def upload(self, source, destination, quiet=False, interactive=False): args = ('/usr/bin/env', 'curl', '-T', source, destination) cerr("Upload command: %s" % ' '.join(args[1:])) if interactive and not query_yes_no('upload file?', default='yes'): cerr('Aborted') return return self.execute(args, quiet)
def create_stable(self, stable, task, branch=None, interactive=False): stable_path = os.path.join(self.project_root, self.stable_dir) if not self.check_dir_exists(stable_path): self.makedir(stable_path, task, interactive=interactive) stable_path = os.path.join(stable_path, stable) if self.check_dir_exists(stable_path): cerr("Stable already exists") return if branch: svn_base = re.sub('trunk/?$', '', self.project_root) branch = re.sub('^\^/?', '', branch) branch = os.path.join(svn_base, branch) cerr('Source overriden from command line: %s' % branch) source = branch.replace('^', self.project_root) else: source = self.compute_stable_source(stable) if not source.endswith('trunk'): if not self.check_dir_exists(source): raise SVNError("Source for stable does not exists") msg = '%s creating stable %s' % (task, stable) self.svn_copy(source, stable_path, task, message=msg, interactive=interactive)
def check_version(self, version): """ Проверяет, что мы можем собирать указанную версию релиза. Например, мы не можем собирать мажор 3.0.0 пока ещё не закрыт мажор 2.0.0 или когда уже началась сборка мажора 4.0.0 :param version: версия для проверки """ cerr("Checking version %s before release" % version) # не можем собрать тег, если текущая версия уже зарелизена if self.find_tags(self.release_tag(version)): raise GitError("Cannot add features to %s version because it has " "already released" % version) prev_version = self.previous_version(version) # Не можем создать релиз, если ещё не зарелизена окончательно предыдущая # версия, за исключением случаев, если это вообще первая версия if (prev_version != self.big_bang_version and not self.find_tags(self.release_tag(prev_version))): raise GitError("Cannot create %s release because previous " "%s release does not exist" % (version, prev_version)) next_version = self.next_version(version) # Если для следующей версии (той, что использует тот же стейбл) # была уже хоть одна сборка - не можем создать релиз if self.find_tags(self.rc_tag(next_version, "*")): raise GitError("Cannot create %s release because %s release " "already started" % (version, next_version)) cerr("Checking complete")
def check_version(self, version): """ Проверяет, что мы можем собирать указанную версию релиза. Например, мы не можем собирать мажор 3.0.0 пока ещё не закрыт мажор 2.0.0 или когда уже началась сборка мажора 4.0.0 :param version: версия для проверки """ cerr("Checking version %s before release" % version) # не можем собрать тег, если текущая версия уже зарелизена if self.find_tags(self.release_tag(version)): raise GitError("Cannot add features to %s version because it has " "already released" % version) prev_version = self.previous_version(version) # Не можем создать релиз, если ещё не зарелизена окончательно предыдущая # версия, за исключением случаев, если это вообще первая версия if prev_version != self.big_bang_version and not self.find_tags(self.release_tag(prev_version)): raise GitError( "Cannot create %s release because previous " "%s release does not exist" % (version, prev_version) ) next_version = self.next_version(version) # Если для следующей версии (той, что использует тот же стейбл) # была уже хоть одна сборка - не можем создать релиз if self.find_tags(self.rc_tag(next_version, "*")): raise GitError("Cannot create %s release because %s release " "already started" % (version, next_version)) cerr("Checking complete")
def check_collected_tasks(self, collected, tasks, not_found_msg='These tasks not found in SVN log:', found_msg='collected tasks:'): not_found = set(tasks) - set(collected) if not_found: cerr(not_found_msg, ','.join(not_found)) cerr(found_msg, ','.join(collected)) raise SVNError('not all tasks collected')
def check_for_conflicts(self): cerr('Checking merge result') args = ('st', ) stdout, stderr, return_code = self.svn(args) for line in stdout.splitlines(): if 'C' in line[:8]: raise SVNError('Conflict found') if return_code != 0: raise SVNError(stderr)
def check_for_conflicts(self): cerr('Checking merge result') args = ('st',) stdout, stderr, return_code = self.svn(args) for line in stdout.splitlines(): if 'C' in line[:8]: raise SVNError('Conflict found') if return_code != 0: raise SVNError(stderr)
def svn_copy(self, source, destination, task, message=None, interactive=False): message = message or '%s copy %s to %s' % (task, source, destination) args = ('cp', source, destination, '-m', message) if interactive: self.confirm_execution(args) cerr("Copying %s to %s" % (source, destination)) stdout, stderr, return_code = self.svn(args) if return_code != 0: raise SVNError(stderr)
def git(self, args, quiet=False): if not isinstance(args, tuple): args = tuple(args) args = (('/usr/bin/env', 'git') + args) stdout, stderr, returncode = self.execute(args, quiet) if returncode != 0: cerr(stdout) cerr(stderr) raise GitError() return stdout
def check_collected_tasks( self, collected, tasks, not_found_msg='These tasks not found in SVN log:', found_msg='collected tasks:'): not_found = set(tasks) - set(collected) if not_found: cerr(not_found_msg, ','.join(not_found)) cerr(found_msg, ','.join(collected)) raise SVNError('not all tasks collected')
def git(self, args, quiet=False): if not isinstance(args, tuple): args = tuple(args) args = ("/usr/bin/env", "git") + args stdout, stderr, returncode = self.execute(args, quiet) if returncode != 0: cerr(stdout) cerr(stderr) raise GitError() return stdout
def delete(self, path_or_url, interactive=False, message=''): cerr('Deleting path: {0}'.format(path_or_url)) args = ['delete'] if message is not None: args.extend(['-m', message]) if not interactive: args.append('--non-interactive') args.append(path_or_url) stdout, stderr, return_code = self.svn(args) if return_code != 0: raise SVNError(stderr)
def svn_commit(self, interactive): args = ('ci', '-F', self.commit_message_filename) if interactive: cerr("Commit message:") cerr("-" * 40) for line in open(self.commit_message_filename, 'r').readlines(): cerr(line) cerr("-" * 40) self.confirm_execution(args) cerr("Committing merge to SVN") stdout, stderr, return_code = self.svn(args) if return_code != 0: raise SVNError(stderr)
def release(self, task_key, release, interactive=False): cerr("Checking for tags dir existance") released_tags = os.path.join(self.project_root, self.tags_dir, release) if not self.check_dir_exists(released_tags): cerr("Creating tags dir") self.makedir(released_tags, task_key, interactive=interactive) last_tag = self.get_last_tag(released_tags) new_tag = '%02d' % (last_tag + 1) tag = os.path.join(released_tags, new_tag) stable = self.compute_stable_path(release) msg = '%s create tag %s-%s' % (task_key, release, new_tag) self.svn_copy(stable, tag, task_key, message=msg, interactive=interactive) return new_tag
def svn_commit(self, interactive): args = ('ci', '-F', self.commit_message_filename) if interactive: cerr("Commit message:") cerr("-" * 40) with open(self.commit_message_filename, 'r') as f: for line in f.readlines(): cerr(line) cerr("-" * 40) self.confirm_execution(args) cerr("Committing merge to SVN") stdout, stderr, return_code = self.svn(args) if return_code != 0: raise SVNError(stderr)
def build(self, release, interactive=False, build_cmd=None, terminate=False, build=None, cleanup=True): released_tags = os.path.join(self.project_root, self.tags_dir, release) tag = build or '%02d' % self.get_last_tag(released_tags) remote = os.path.join(released_tags, str(tag)) package_name = '%s-%s-%s' % (self.project_key, release, tag) local_path = os.path.join(self.temp_dir, package_name) if os.path.exists(local_path): if not interactive or query_yes_no('remove %s?' % local_path, default='yes'): shutil.rmtree(local_path) else: cerr('Aborted') sys.exit(0) self.export(remote, local_path) if build_cmd: os.environ['PACKAGE'] = package_name os.chdir(local_path) cerr("Build cmd: %s" % build_cmd) cerr("Package name: %s" % package_name) if interactive and not query_yes_no('execute?', default='yes'): cerr('Aborted') return args = ('/usr/bin/env', 'sh', '-c', build_cmd) stdout, stderr, ret = self.execute(args) cout(stdout) if ret: cerr(stderr) sys.exit(ret) if terminate: if cleanup: shutil.rmtree(local_path) return archive_name = os.path.join(self.temp_dir, '%s.tgz' % package_name) self.tar(archive_name, self.temp_dir, package_name, quiet=True) dest = os.path.join(self.repo_url, self.project_key) if not dest.endswith('/'): dest += '/' self.upload(archive_name, dest, interactive=interactive) if cleanup: cout("cleanup") shutil.rmtree(local_path) os.unlink(archive_name)
def merge_tasks(self, task_key, tasks, branch='trunk', interactive=False, dry_run=False): if not tasks: raise ValueError('No tasks requested') self.revert_working_copy() self.svn_update() with open(self.commit_message_filename, 'w+') as commit_msg_file: commit_msg_file.write('%s merge tasks %s\n' % (task_key, ','.join(tasks))) source = os.path.join(self.project_root, branch) commit_msg_file.write('Request revisions from %s:\n' % source) logged = self.log_tasks(None, branch=source) collected = [ '%s-%s' % (self.project_key, t) for t in logged.keys() ] self.check_collected_tasks(collected, tasks) cerr('Merging with svn merge --non-interactive -c $REV') revisions = [] for t, revs in logged.items(): for r, msg in revs: revisions.append((r, t, msg)) revisions = sorted(revisions, key=lambda item: item[0]) for r, t, msg in revisions: jira_task = '%s-%s' % (self.project_key, t) if jira_task not in tasks: continue msg = self.remove_smart_commits(msg) commit_msg_file.write('r%s %s %s\n' % (r, jira_task, msg)) self.merge(source, revision=r) commit_msg_file.flush() commit_msg_file.seek(0) merged = [] for line in commit_msg_file.readlines(): m = re.match(r'^r[\d]+ ([A-Z]+-[\d]+)', line) if not m: continue merged.append(m.group(1)) self.check_collected_tasks(merged, tasks, not_found_msg='These tasks not merged:', found_msg='Merged tasks:') self.check_for_conflicts() if not dry_run: self.svn_commit(interactive)
def compute_stable_source(self, stable): parts = stable.split('.') if parts[1] == 'x': source = os.path.join(self.project_root, 'trunk') cerr("Major stable %s assumed created from %s" % (stable, source)) elif parts[2] == 'x': released = '%s.%s.0' % (parts[0], parts[1]) build = self.get_last_tag( os.path.join(self.project_root, self.tags_dir, released)) if not build: raise SVNError("Minor release tag for %s doesn't exist" % released) source = os.path.join(self.project_root, self.tags_dir, released, '%02d' % build) cerr("Minor stable %s assumed created from %s" % (stable, source)) else: raise ValueError("Don't know how to make stable %s" % stable) return source
def compute_stable_source(self, stable): parts = stable.split('.') if parts[1] == 'x': source = os.path.join(self.project_root, 'trunk') cerr("Major stable %s assumed created from %s" % (stable, source)) elif parts[2] == 'x': released = '%s.%s.0' % (parts[0], parts[1]) build = self.get_last_tag(os.path.join(self.project_root, self.tags_dir, released)) if not build: raise SVNError( "Minor release tag for %s doesn't exist" % released) source = os.path.join(self.project_root, self.tags_dir, released, '%02d' % build) cerr("Minor stable %s assumed created from %s" % (stable, source)) else: raise ValueError("Don't know how to make stable %s" % stable) return source
def merge_tasks(self, task_key, tasks, branch='trunk', interactive=False, dry_run=False): if not tasks: raise ValueError('No tasks requested') self.revert_working_copy() self.svn_update() with open(self.commit_message_filename, 'w+') as commit_msg_file: commit_msg_file.write( '%s merge tasks %s\n' % (task_key, ','.join(tasks))) source = os.path.join(self.project_root, branch) commit_msg_file.write('Request revisions from %s:\n' % source) logged = self.log_tasks(None, branch=source) collected = ['%s-%s' % (self.project_key, t) for t in logged.keys()] self.check_collected_tasks(collected, tasks) cerr('Merging with svn merge --non-interactive -c $REV') revisions = [] for t, revs in logged.items(): for r, msg in revs: revisions.append((r, t, msg)) revisions = sorted(revisions, key=lambda item: item[0]) for r, t, msg in revisions: jira_task = '%s-%s' % (self.project_key, t) if jira_task not in tasks: continue msg = self.remove_smart_commits(msg) commit_msg_file.write('r%s %s %s\n' % (r, jira_task, msg)) args = ('merge', '--non-interactive', '-c', 'r%s' % r, source) self.svn(args) commit_msg_file.flush() commit_msg_file.seek(0) merged = [] for line in commit_msg_file.readlines(): m = re.match(r'^r[\d]+ ([A-Z]+-[\d]+)', line) if not m: continue merged.append(m.group(1)) self.check_collected_tasks(merged, tasks, not_found_msg='These tasks not merged:', found_msg='Merged tasks:') self.check_for_conflicts() if not dry_run: self.svn_commit(interactive)
def build(self, release, interactive=False, build_cmd=None, terminate=False, build=None, cleanup=True): tag = build or "%02d" % self.get_last_tag(release) package_name = "%s-%s-%s" % (self.project_key, release, tag) local_path = os.path.join(self.temp_dir, package_name) if os.path.exists(local_path): if not interactive or query_yes_no("remove %s?" % local_path, default="yes"): shutil.rmtree(local_path) else: cerr("Aborted") sys.exit(0) self.clone(local_path) self.checkout(self.rc_tag(release, build)) if build_cmd: os.environ["PACKAGE"] = package_name os.chdir(local_path) cerr("Build cmd: %s" % build_cmd) cerr("Package name: %s" % package_name) if interactive and not query_yes_no("execute?", default="yes"): cerr("Aborted") return args = ("/usr/bin/env", "sh", "-c", build_cmd) stdout, stderr, ret = self.execute(args) cout(stdout) if ret: cerr(stderr) sys.exit(ret) if terminate: if cleanup: shutil.rmtree(local_path) return archive_name = os.path.join(self.temp_dir, "%s.tgz" % package_name) self.tar(archive_name, self.temp_dir, package_name, quiet=True) dest = os.path.join(self.repo_url, self.project_key) if not dest.endswith("/"): dest += "/" self.upload(archive_name, dest, interactive=interactive) if cleanup: cout("cleanup") shutil.rmtree(local_path) os.unlink(archive_name)
def get_or_create_stable(self, version, task, interactive=False): """ Проверяет наличие или создает ветку, в которую будем собирать изменения :return: Название ветки стейбла """ branch = self.get_stable_branch(version) if not self.git(("branch", "--list", branch)): if self.git(("branch", "-r", "--list", self.remote(branch))): # если на сервере уже есть ветка, то будем использовать ей start_point = self.remote(branch) cerr("Checkout release branch %s for version %s" % (branch, version)) else: # иначе создадим ветку из релиза предыдущей версии start_point = self.release_tag(self.base_version(version)) cerr("Create release branch %s for version %s" % (branch, version)) # создаем локальную ветку и связываем её с удаленной. сам по себе # чекаут здесь не так уже важен self.git(("checkout", "-b", branch, start_point)) return branch
from pprint import pprint from bamboo.tasks import Tasks from bamboo.helpers import get_stable, cerr project_key = os.environ['BUILD_KEY'].split('-')[0] print "Project key:", project_key jira = Tasks(configfile='/data/bamboo.cfg') with open('collected-tasks.txt', 'r') as f: all_stables = dict() version_int_tasks = dict() for line in f.readlines(): task_key = line.strip() versions = jira.get_versions(task_key) stables = {v: get_stable(v) for v in versions} if len(set(stables.values()))!=len(versions): cerr('Incorrect versions and stables for %s' % task_key) cerr('Versions: %s' % ', '.join('%s: %s' % (k, v) for k, v in stables.items())) sys.exit(-1) for v, s in stables.items(): all_stables.setdefault(s, {}) all_stables[s].setdefault(v, []) all_stables[s][v].append(task_key) version_int_tasks[v] = None print "Merging plan:" pprint(all_stables) plan = open('merge-plan.json', 'w') json.dump(all_stables, plan) plan.close() for v in version_int_tasks.keys(): tasks = jira.search_tasks( project_key,
def revert_working_copy(self): cerr('Cleaning working copy') args = ('revert', '-R', '.') stdout, stderr, return_code = self.svn(args) if return_code != 0: raise SVNError(stderr)
def svn_update(self): cerr('Updating from SVN') args = ('up', ) stdout, stderr, return_code = self.svn(args) if return_code != 0: raise SVNError(stderr)
def confirm_execution(self, args): cerr('SVN command:') cerr('svn ' + ' '.join('"%s"' % a if ' ' in a else a for a in args)) if not query_yes_no('commit?', default='yes'): cerr('Aborted') sys.exit(0)
def check_dir_exists(self, path): args = ('info', path) cerr("Checking existance of %s" % path) stdout, stderr, return_code = self.svn(args, quiet=True) if return_code == 0: return True
def svn_update(self): cerr('Updating from SVN') args = ('up',) stdout, stderr, return_code = self.svn(args) if return_code != 0: raise SVNError(stderr)
def execute_make(self, make_args): cerr(' '.join(make_args)) cerr('=' * 40) os.execv('/usr/bin/env', (self.make,) + make_args)
# # скрипт, переводящий задачи на интеграцию в статус Integrating и на # пользователя, из-под которого осуществляется сборка # # На входе integration-tasks.json # import json from bamboo.tasks import Tasks from bamboo.helpers import cerr jira = Tasks(configfile='/data/bamboo.cfg') f = open('integration-tasks.json', 'r') integration_tasks = json.load(f) f.close() for task_key in integration_tasks.values(): info = dict(jira.task_info(task_key)) if info['status'] != 'Integrating': cerr('Taking task %s' % task_key) transitions = jira.get_transitions(task_key) for trans in transitions: name = trans['to']['name'].lower().replace(' ', '-') print name if name == 'integrating': print "Transition to %s" % name jira.transition(task_key, trans['id']) elif info['assignee'] != 'bamboo': cerr('Assigning %s to me' % task_key) jira.assign(task_key, 'bamboo')