def _KeepDirInSync(file1, file2): isfile = os.path.isfile isdir = os.path.isdir join = os.path.join names1 = set(os.listdir(file1)) names2 = set(os.listdir(file2)) both = names1.intersection(names2) if names1 != names2: Print( 'Warning: the directory of %s structure is different from %s. Only files in both are kept in sync.' % (names1, names2)) for name in both: full1 = join(file1, name) full2 = join(file2, name) if isfile(full1) and isfile(full2): _KeepInSyncThread._instance.files_to_keep_in_sync_queue.put( _KeepInSyncStruct(full1, full2)) elif isdir(full1) and isdir(full2): _KeepDirInSync(full1, full2) else: Print('Expected %s and %s to be both files or dirs.' % (full1, full2))
def Run(params, on_output=None): args = params.args config = params.config arg0 = args[0] if arg0 == 'st': args[0] = 'status' if len(args) == 1: args.insert(1, '-s') elif arg0 == 'co': args[0] = 'checkout' elif len(args) == 1: if arg0 == 'mu-branch': args[0] = 'rev-parse' args.insert(1, '--abbrev-ref') args.insert(2, 'HEAD') if not config.repos: msg = 'No repository registered. Use mu register repo_name to register repository.' Print(msg) return Status(msg, True, config) commands = [] for repo in config.repos: if not os.path.exists(repo): Print('%s does not exist' % (repo, )) else: commands.append(ParallelCmd(repo, [config.git] + args)) ExecuteInParallel(commands, on_output, serial=config.serial) return Status('Finished', True)
def Run(params): args = params.args if len(args) != 2 or args[1].count('=') != 1: msg = 'Syntax for set-var is "mu set-var key=value"' Print(msg) return Status(msg, True, params.config) var, value = args[1].split('=') var = var.strip().lower() if var == 'serial': if IsFalse(value): params.config.serial = False else: params.config.serial = True elif var == 'git': params.config.git = value else: msg = 'Variable to set: "%s" not recognized.' % (var, ) Print(msg) return Status(msg, False, params.config) with open(params.config_file, 'w') as f: f.write(str(params.config)) msg = 'Variable %s set to %s' % (var, value) Print(msg) return Status(msg, True, params.config)
def Run(params): base_branch = params.args[1] repos_and_local_branches = GetReposAndLocalBranches( params, patterns=['*%s*' % base_branch]) # Now, do things the other way, show a connection from the branch to the repos which have it! branch_to_repos = ConvertRepoToBranchesToBranchToRepos( repos_and_local_branches) if len(params.config.repos) == 1: params.config.serial = True if base_branch in branch_to_repos or not branch_to_repos: # Ok, the default one matches, just go on with it... from .action_default import Run return Run(params) if len(branch_to_repos) == 1: # The default one does not match but we have a single match, let's use it! branch, _repo = iteritems(branch_to_repos).next() params.args[1] = branch from .action_default import Run # @Reimport return Run(params) # Print it for the user Print( 'Found more than one branch that matches ${START_COLOR}%s${RESET_COLOR}:\n' % params.args[1]) PrintBranchToRepos(branch_to_repos, params) Print( '\n${START_COLOR}ERROR${RESET_COLOR}: unable to decide branch to work on.', __color__='RED')
def Run(params): args = params.args config_file = params.config_file config = params.config if len(args) < 2: msg = 'Repository (dir name|--all) to track not passed' Print(msg) return Status(msg, False) repos = config.repos msgs = [] args = args[1:] remove_all = '--all' in args if remove_all and args != ['--all']: msg = 'If --all is given, no other parameter should be passed.' Print(msg) return Status(msg, False) new_args = [] for arg in args: if arg.endswith('\\') or arg.endswith('/'): arg = arg[:-1] new_args.append(arg) args = new_args group_repos = config.groups.get(config.current_group, None) if remove_all: repos[:] = [] msg = 'Removed all repositories.' if group_repos: group_repos[:] = [] msg += ' (removed from group "%s")' % config.current_group Print(msg) msgs.append(msg) else: for repo in args: if repo in repos: msg = 'Repository: %s unregistered' % (repo, ) repos.remove(repo) else: msg = 'Repository: %s skipped' % (repo, ) if group_repos is not None: if repo in group_repos: group_repos.remove(repo) msg += ' (removed from group "%s")' % config.current_group Print(msg) msgs.append(msg) with open(config_file, 'w') as f: f.write(str(config)) return Status('\n'.join(msgs), True, config)
def check_structure(): prev = os.path.join('.mu.diff.git.tmp', 'REPO', 'test_diff_command_git_repo_dir', 'folder1', 'out.txt') curr = os.path.join('.mu.diff.git.tmp', 'WORKING', 'test_diff_command_git_repo_dir', 'folder1', 'out.txt') assert os.path.exists(prev) assert os.path.exists(curr) Print('prev', open(prev, 'r').read()) Print('curr', open(curr, 'r').read())
def Fix(root, filename): path = os.path.join(root, filename) if not os.path.exists(path): Print('Skip removed file:', path) return contents = open(path, 'rb').read() if b'\r' in contents: Print('Fixing:', path) contents = contents.replace(b'\r\n', b'\n').replace(b'\r', b'\n') open(path, 'wb').write(contents)
def Run(params): config = params.config if not config.repos: msg = 'No repository registered. Use mu register repo_name to register repository.' Print(msg) return Status(msg, True, config) else: repo_str = '\n'.join(sorted(config.repos)) Print('Tracked Repositories:\n') Print(repo_str) return Status(repo_str, True, config)
def Run(params): args = params.args config_file = params.config_file config = params.config if len(args) < 2: msg = 'Repository (dir name|--all) to track not passed' Print(msg) return Status(msg, False) repos = config.repos msgs = [] args = args[1:] join = os.path.join isdir = os.path.isdir if '--all' in args: if len(args) > 1: Print( 'If --all is passed in mu register, no other parameter should be passed.' ) return args = [repo for repo in os.listdir('.') if isdir(join(repo, '.git'))] new_args = [] for arg in args: if arg.endswith('\\') or arg.endswith('/'): arg = arg[:-1] new_args.append(arg) args = new_args group_repos = config.groups.get(config.current_group, None) for repo in args: if repo in repos: msg = 'Repository: %s skipped, already registered' % (repo, ) else: repos.append(repo) msg = 'Repository: %s registered' % (repo, ) if group_repos is not None: if repo not in group_repos: group_repos.append(repo) msg += ' (added to group "%s")' % config.current_group else: msg += ' (already in group "%s")' % config.current_group Print(msg) msgs.append(msg) with open(config_file, 'w') as f: f.write(str(config)) return Status('\n'.join(msgs), True, config)
def OnOutput(output): #@DuplicatedSignature stdout = output.stdout if stdout.strip(): filename = '__diff__.parent.' + output.repo + '.patch' Print('Writing parent diff : ', filename) with open(filename, 'w') as f: f.write(stdout)
def Fix(root, filename): path = os.path.join(root, filename) contents = open(path, 'rb').read() if '\r' in contents: Print('Fixing:', path) contents = contents.replace('\r\n', '\n').replace('\r', '\n') open(path, 'wb').write(contents)
def Run(params): repos_and_curr_branch = GetReposAndCurrBranch(params) commands = [] for repo, branch in repos_and_curr_branch: #We want to update origin/master and not FETCH_HEAD #See: http://stackoverflow.com/questions/11051761/why-git-fetch-specifying-branch-does-not-match-fetch-without-specifying-branch/ commands.append( ParallelCmd(repo, [ params.config.git, 'fetch', 'origin', '%s:refs/remotes/origin/%s' % (branch, branch) ])) repos = [] def on_output(output): if not output.stdout.strip() and not output.stderr.strip(): repos.append(output.repo) else: Print(output) ExecuteInParallel(commands, on_output=on_output) if repos: Print( CreateJoinedReposMsg('Repositories fetched with no changes:', repos)) return repos_and_curr_branch
def Run(params): #Update them from .action_up import Run repos_and_curr_branch = Run(params) if not repos_and_curr_branch: Print('No tracked repos!') return branch_to_repos = {} #Check if all on the same branch for repo, branch in repos_and_curr_branch: curr = branch_to_repos.setdefault(branch, []) curr.append(repo) if len(branch_to_repos) > 1: msg = '\n${START_COLOR}Warning: found repos in different branches${RESET_COLOR}:\n %s\nProceed?(y/n)' % ( '\n '.join([ str('Branch: ${START_COLOR}%s${RESET_COLOR} (%s)' % (key, ', '.join(val))) for (key, val) in branch_to_repos.items() ])) ret = '' while ret not in ('y', 'n'): Print(msg) ret = raw_input().strip().lower() if ret != 'y': return #Diff it from .action_diff import Run #@Reimport initial_args = params.args[:] initial_repos = params.config.repos[:] for branch, repos in iteritems(branch_to_repos): params.args = initial_args + ['origin/' + branch] params.config.repos = repos if len(branch_to_repos) > 1: Print('\nOutput for branch: ${START_COLOR}%s${RESET_COLOR} (%s)' % (branch, ', '.join(repos))) Run(params) params.args = initial_args params.config.repos = initial_repos
def _handle_decoded(self, msgs): if not isinstance(msgs, (tuple, list)): msgs = [msgs] from mu_repo.print_ import Print for msg in msgs: Print(msg) event.set()
def OnOutput(output): stdout = output.stdout.strip() if stdout: repos_and_curr_branch.append((output.repo, stdout)) else: if verbose: Print( 'Unable to update (could not get current branch for: %s)' % (output.repo, ))
def NotifyErrorListeners(): import StringIO import traceback cstr = StringIO.StringIO() traceback.print_exc(file=cstr) error = cstr.getvalue() for listener in on_errors_listeners: listener(error) Print(error)
def NotifyErrorListeners(): from mu_repo.backwards_stringio import StringIO import traceback cstr = StringIO() traceback.print_exc(file=cstr) error = cstr.getvalue() for listener in on_errors_listeners: listener(error) Print(error)
def PrintBranchToRepos(branch_to_repos, params): for branch, repos in sorted(iteritems(branch_to_repos)): if len(repos) == 1: msg = '${START_COLOR}%s${RESET_COLOR}' % (branch, ) elif len(repos) == len(set(params.config.repos)): msg = '${START_COLOR}%s${RESET_COLOR} (all repos)' % (branch, ) else: msg = '${START_COLOR}%s${RESET_COLOR} (%s)' % (branch, ', '.join( sorted(repos))) Print(msg)
def OnOutput(output): stdout = output.stdout if stdout.strip(): diffed_repos.append(output.repo) filename = '__diff__.' + output.repo + '.patch' Print('Writing diff --cached: ', filename) with open(filename, 'w') as f: f.write(stdout) else: empty_diff_repos.add(output.repo)
def Run(params): ''' This action will grab the latest version of mu-repo from the repository. ''' import mu_repo import os.path repo_dir = os.path.dirname(os.path.dirname(mu_repo.__file__)) if not os.path.exists(os.path.join(repo_dir, '.git')): Print( 'Can only automatically update mu-repo if it was properly gotten from a git repository.' ) return config = params.config git = config.git or 'git' ExecuteCommand([git, 'pull', '--rebase'], repo=repo_dir)
def KeepInSync(file1, file2): with _lock: if _KeepInSyncThread._instance is None: _KeepInSyncThread._instance = _KeepInSyncThread() _KeepInSyncThread._instance.start() isfile = os.path.isfile isdir = os.path.isdir if isfile(file1) and isfile(file2): _KeepInSyncThread._instance.files_to_keep_in_sync_queue.put( _KeepInSyncStruct(file1, file2)) elif isdir(file1) and isdir(file2): _KeepDirInSync(file1, file2) else: Print('Expected %s and %s to be both files or dirs.' % (file1, file2))
def run(self, serial=False): repo = self.repo cmd = self.cmd msg = ' '.join([START_COLOR, '\n', repo, ':'] + cmd + [RESET_COLOR]) shell = UseShellOnSubprocess() if serial: #Print directly to stdout/stderr without buffering. if not IsQuietMode(): Print(msg) p = None try: p = subprocess.Popen(cmd, cwd=repo, shell=shell) except: PrintError('Error executing: ' + ' '.join(cmd) + ' on: ' + repo) if p is not None: p.wait() else: try: p = subprocess.Popen(cmd, cwd=repo, stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=shell) except: import os PrintError('Error executing: ' + ' '.join(cmd) + ' on: ' + repo + ' cwd: ' + os.path.abspath('.')) self.output_queue.put( Output(repo, 'Error executing: %s on repo: %s' % (cmd, repo), '', '')) return self.stdout_thread = self._CreateReaderThread(p, 'stdout') self.stderr_thread = self._CreateReaderThread(p, 'stderr') p.wait() self.stdout_thread.join(2) #finish in at most 2 seconds self.stderr_thread.join(2) #finish in at most 2 seconds stdout = AsStr(self.stdout_thread.GetFullOutput()) stderr = AsStr(self.stderr_thread.GetFullOutput()) self._HandleOutput(msg, stdout, stderr)
def OnOutput(output): branch_name = as_dict.get(output.repo, 'UNKNOWN_BRANCH') if not output.stdout: empty_repos_and_branches.append((output.repo, branch_name)) else: status = [ START_COLOR, output.repo, ' ', branch_name, ':', RESET_COLOR, '\n', Indent(output.stdout), '\n', ] Print(''.join(status))
def Run(params): ''' Note: this action always runs in parallel. ''' repos_and_curr_branch = GetReposAndCurrBranch(params, verbose=False) as_dict = dict(repos_and_curr_branch) commands = [] for repo, _branch in repos_and_curr_branch: commands.append( ParallelCmd(repo, [params.config.git] + ['status', '-s'])) empty_repos_and_branches = [] def OnOutput(output): branch_name = as_dict.get(output.repo, 'UNKNOWN_BRANCH') if not output.stdout: empty_repos_and_branches.append((output.repo, branch_name)) else: status = [ START_COLOR, output.repo, ' ', branch_name, ':', RESET_COLOR, '\n', Indent(output.stdout), '\n', ] Print(''.join(status)) ExecuteInParallel(commands, on_output=OnOutput) if empty_repos_and_branches: branch_to_repos = {} for repo, branch in empty_repos_and_branches: branch_to_repos.setdefault(branch, []).append(repo) for branch, repos in branch_to_repos.iteritems(): Print( "${START_COLOR}Unchanged:${RESET_COLOR} %s\nat branch: ${START_COLOR}%s${RESET_COLOR}\n" % (CreateJoinedReposMsg('', repos), branch)) return repos_and_curr_branch
def _Clone(remote, repo, params, other_cmd_line_args): created_dir = os.path.join('.', repo) if os.path.exists(os.path.join(created_dir, '.git')): # If it already exists, bail out! Print( 'Skipping clone of: ${START_COLOR}%s${RESET_COLOR} because it already exists.' % (repo, )) return True remote_path = remote if not remote_path.endswith('/') and not remote_path.endswith('\\'): remote_path += '/' git = params.config.git ExecuteCommand([git, 'clone', '%s%s' % (remote_path, repo)] + other_cmd_line_args, '.') if os.path.exists(os.path.join(created_dir, '.git')): return True return False
def OnOutput(output): stdout = output.stdout.strip() if stdout: branches = set() for line in stdout.splitlines(): branch = line.strip() if branch.startswith('*'): branch = branch[1:].strip() if not patterns: branches.add(branch) else: for pat in patterns: if fnmatch(branch, pat): branches.add(branch) repos_and_curr_branch.append((output.repo, branches)) else: Print('Unable to execute git branch for: %s' % (output.repo, ))
def run(self): while True: action = self.output_queue.get(True) try: if action is self.FINISH_PROCESSING_ITEM: return if isinstance(action, Output): if self.on_output is not None: self.on_output(action) #else: # Note: in this case, the output will be printed by the # ExecuteThreadsHandlingOutputQueue method (along with the stderr). # Print(action) else: Print(action) #Progress message. except: PrintError() finally: self.output_queue.task_done()
def GetReposAndCurrBranch(params, verbose=True): ''' :param params: Params The parameters used to get the repos and current branch (mostly using config). :return: list(tuple(str, str)) A list with the repository and current branch for that repository. ''' repos_and_curr_branch = [] def OnOutput(output): stdout = output.stdout.strip() if stdout: repos_and_curr_branch.append((output.repo, stdout)) else: if verbose: Print( 'Unable to update (could not get current branch for: %s)' % (output.repo, )) from .action_default import Run #@Reimport from mu_repo import Params old_serial = params.config.serial params.config.serial = False #Cannot be serial as we want to get the output Run(Params(params.config, ['rev-parse', '--abbrev-ref', 'HEAD'], params.config_file), on_output=OnOutput) if verbose: branch_to_repos = {} for repo, branch in repos_and_curr_branch: branch_to_repos.setdefault(branch, []).append(repo) for branch, repos in branch_to_repos.iteritems(): Print( "Will handle ${START_COLOR}origin %s${RESET_COLOR} for: %s\n" % (branch, ', '.join(sorted(repos)))) #Restore serial for next command. params.config.serial = old_serial return repos_and_curr_branch
def ExecuteCommand(cmd, repo, return_stdout=False, verbose=True): ''' Execute command letting stderr go to the default sys.stderr. Will block until the command finishes. @param cmd: list(str) The command to be executed. @param repo: str The repository (working dir) where the command should be executed. @param return_stdout: bool If True, will grab stdout and return it, otherwise will let it go to sys.stderr. @param verbose: bool If True will print the command being executed. @return: the redirected stdout (if return_stdout is True) or None otherwise ''' if verbose: msg = ' '.join([START_COLOR, '\n', repo, ':'] + cmd + [RESET_COLOR]) Print(msg) try: shell = UseShellOnSubprocess() if return_stdout: p = subprocess.Popen(cmd, cwd=repo, stdout=subprocess.PIPE, shell=shell) else: p = subprocess.Popen(cmd, cwd=repo, shell=shell) except: PrintError('Error executing: ' + ' '.join(cmd) + ' on: ' + repo) raise if not return_stdout: p.wait() else: stdout, _stderr = p.communicate() return stdout
def Sync(self): try: st1 = os.stat(self.file1) st2 = os.stat(self.file2) except: #If any of those fails, we either don't have access or the file was removed. return changed_file_1 = self.file1_time != st1.st_mtime or self.file1_size != st1.st_size changed_file_2 = self.file2_time != st2.st_mtime or self.file2_size != st2.st_size if changed_file_1 or changed_file_2: if changed_file_1 and changed_file_2: Print( 'Unable to synchronize: both files: %s and %s are changed.' % (self.file1, self.file2)) return if changed_file_1: shutil.copyfile(self.file1, self.file2) else: shutil.copyfile(self.file2, self.file1) self._CollectStats()