def _KeepDirInSync(file1, file2):
isfile = os.path.isfile
isdir = os.path.isdir
join = os.path.join
names1 = set(os.listdir(file1))
names2 = set(os.listdir(file2))
both = names1.intersection(names2)
if names1 != names2:
Print(
'Warning: the directory of %s structure is different from %s. Only files in both are kept in sync.'
% (names1, names2))
for name in both:
full1 = join(file1, name)
full2 = join(file2, name)
if isfile(full1) and isfile(full2):
_KeepInSyncThread._instance.files_to_keep_in_sync_queue.put(
_KeepInSyncStruct(full1, full2))
elif isdir(full1) and isdir(full2):
_KeepDirInSync(full1, full2)
else:
Print('Expected %s and %s to be both files or dirs.' %
(full1, full2))
def Run(params, on_output=None):
args = params.args
config = params.config
arg0 = args[0]
if arg0 == 'st':
args[0] = 'status'
if len(args) == 1:
args.insert(1, '-s')
elif arg0 == 'co':
args[0] = 'checkout'
elif len(args) == 1:
if arg0 == 'mu-branch':
args[0] = 'rev-parse'
args.insert(1, '--abbrev-ref')
args.insert(2, 'HEAD')
if not config.repos:
msg = 'No repository registered. Use mu register repo_name to register repository.'
Print(msg)
return Status(msg, True, config)
commands = []
for repo in config.repos:
if not os.path.exists(repo):
Print('%s does not exist' % (repo, ))
else:
commands.append(ParallelCmd(repo, [config.git] + args))
ExecuteInParallel(commands, on_output, serial=config.serial)
return Status('Finished', True)
def Run(params):
args = params.args
if len(args) != 2 or args[1].count('=') != 1:
msg = 'Syntax for set-var is "mu set-var key=value"'
Print(msg)
return Status(msg, True, params.config)
var, value = args[1].split('=')
var = var.strip().lower()
if var == 'serial':
if IsFalse(value):
params.config.serial = False
else:
params.config.serial = True
elif var == 'git':
params.config.git = value
else:
msg = 'Variable to set: "%s" not recognized.' % (var, )
Print(msg)
return Status(msg, False, params.config)
with open(params.config_file, 'w') as f:
f.write(str(params.config))
msg = 'Variable %s set to %s' % (var, value)
Print(msg)
return Status(msg, True, params.config)
def Run(params):
base_branch = params.args[1]
repos_and_local_branches = GetReposAndLocalBranches(
params, patterns=['*%s*' % base_branch])
# Now, do things the other way, show a connection from the branch to the repos which have it!
branch_to_repos = ConvertRepoToBranchesToBranchToRepos(
repos_and_local_branches)
if len(params.config.repos) == 1:
params.config.serial = True
if base_branch in branch_to_repos or not branch_to_repos:
# Ok, the default one matches, just go on with it...
from .action_default import Run
return Run(params)
if len(branch_to_repos) == 1:
# The default one does not match but we have a single match, let's use it!
branch, _repo = iteritems(branch_to_repos).next()
params.args[1] = branch
from .action_default import Run # @Reimport
return Run(params)
# Print it for the user
Print(
'Found more than one branch that matches ${START_COLOR}%s${RESET_COLOR}:\n'
% params.args[1])
PrintBranchToRepos(branch_to_repos, params)
Print(
'\n${START_COLOR}ERROR${RESET_COLOR}: unable to decide branch to work on.',
__color__='RED')
def Run(params):
args = params.args
config_file = params.config_file
config = params.config
if len(args) < 2:
msg = 'Repository (dir name|--all) to track not passed'
Print(msg)
return Status(msg, False)
repos = config.repos
msgs = []
args = args[1:]
remove_all = '--all' in args
if remove_all and args != ['--all']:
msg = 'If --all is given, no other parameter should be passed.'
Print(msg)
return Status(msg, False)
new_args = []
for arg in args:
if arg.endswith('\\') or arg.endswith('/'):
arg = arg[:-1]
new_args.append(arg)
args = new_args
group_repos = config.groups.get(config.current_group, None)
if remove_all:
repos[:] = []
msg = 'Removed all repositories.'
if group_repos:
group_repos[:] = []
msg += ' (removed from group "%s")' % config.current_group
Print(msg)
msgs.append(msg)
else:
for repo in args:
if repo in repos:
msg = 'Repository: %s unregistered' % (repo, )
repos.remove(repo)
else:
msg = 'Repository: %s skipped' % (repo, )
if group_repos is not None:
if repo in group_repos:
group_repos.remove(repo)
msg += ' (removed from group "%s")' % config.current_group
Print(msg)
msgs.append(msg)
with open(config_file, 'w') as f:
f.write(str(config))
return Status('\n'.join(msgs), True, config)
def check_structure():
prev = os.path.join('.mu.diff.git.tmp', 'REPO',
'test_diff_command_git_repo_dir', 'folder1',
'out.txt')
curr = os.path.join('.mu.diff.git.tmp', 'WORKING',
'test_diff_command_git_repo_dir', 'folder1',
'out.txt')
assert os.path.exists(prev)
assert os.path.exists(curr)
Print('prev', open(prev, 'r').read())
Print('curr', open(curr, 'r').read())
def Fix(root, filename):
path = os.path.join(root, filename)
if not os.path.exists(path):
Print('Skip removed file:', path)
return
contents = open(path, 'rb').read()
if b'\r' in contents:
Print('Fixing:', path)
contents = contents.replace(b'\r\n', b'\n').replace(b'\r', b'\n')
open(path, 'wb').write(contents)
def Run(params):
config = params.config
if not config.repos:
msg = 'No repository registered. Use mu register repo_name to register repository.'
Print(msg)
return Status(msg, True, config)
else:
repo_str = '\n'.join(sorted(config.repos))
Print('Tracked Repositories:\n')
Print(repo_str)
return Status(repo_str, True, config)
def Run(params):
args = params.args
config_file = params.config_file
config = params.config
if len(args) < 2:
msg = 'Repository (dir name|--all) to track not passed'
Print(msg)
return Status(msg, False)
repos = config.repos
msgs = []
args = args[1:]
join = os.path.join
isdir = os.path.isdir
if '--all' in args:
if len(args) > 1:
Print(
'If --all is passed in mu register, no other parameter should be passed.'
)
return
args = [repo for repo in os.listdir('.') if isdir(join(repo, '.git'))]
new_args = []
for arg in args:
if arg.endswith('\\') or arg.endswith('/'):
arg = arg[:-1]
new_args.append(arg)
args = new_args
group_repos = config.groups.get(config.current_group, None)
for repo in args:
if repo in repos:
msg = 'Repository: %s skipped, already registered' % (repo, )
else:
repos.append(repo)
msg = 'Repository: %s registered' % (repo, )
if group_repos is not None:
if repo not in group_repos:
group_repos.append(repo)
msg += ' (added to group "%s")' % config.current_group
else:
msg += ' (already in group "%s")' % config.current_group
Print(msg)
msgs.append(msg)
with open(config_file, 'w') as f:
f.write(str(config))
return Status('\n'.join(msgs), True, config)
def OnOutput(output): #@DuplicatedSignature
stdout = output.stdout
if stdout.strip():
filename = '__diff__.parent.' + output.repo + '.patch'
Print('Writing parent diff : ', filename)
with open(filename, 'w') as f:
f.write(stdout)
def Fix(root, filename):
path = os.path.join(root, filename)
contents = open(path, 'rb').read()
if '\r' in contents:
Print('Fixing:', path)
contents = contents.replace('\r\n', '\n').replace('\r', '\n')
open(path, 'wb').write(contents)
def Run(params):
repos_and_curr_branch = GetReposAndCurrBranch(params)
commands = []
for repo, branch in repos_and_curr_branch:
#We want to update origin/master and not FETCH_HEAD
#See: http://stackoverflow.com/questions/11051761/why-git-fetch-specifying-branch-does-not-match-fetch-without-specifying-branch/
commands.append(
ParallelCmd(repo, [
params.config.git, 'fetch', 'origin',
'%s:refs/remotes/origin/%s' % (branch, branch)
]))
repos = []
def on_output(output):
if not output.stdout.strip() and not output.stderr.strip():
repos.append(output.repo)
else:
Print(output)
ExecuteInParallel(commands, on_output=on_output)
if repos:
Print(
CreateJoinedReposMsg('Repositories fetched with no changes:',
repos))
return repos_and_curr_branch
def Run(params):
#Update them
from .action_up import Run
repos_and_curr_branch = Run(params)
if not repos_and_curr_branch:
Print('No tracked repos!')
return
branch_to_repos = {}
#Check if all on the same branch
for repo, branch in repos_and_curr_branch:
curr = branch_to_repos.setdefault(branch, [])
curr.append(repo)
if len(branch_to_repos) > 1:
msg = '\n${START_COLOR}Warning: found repos in different branches${RESET_COLOR}:\n %s\nProceed?(y/n)' % (
'\n '.join([
str('Branch: ${START_COLOR}%s${RESET_COLOR} (%s)' %
(key, ', '.join(val)))
for (key, val) in branch_to_repos.items()
]))
ret = ''
while ret not in ('y', 'n'):
Print(msg)
ret = raw_input().strip().lower()
if ret != 'y':
return
#Diff it
from .action_diff import Run #@Reimport
initial_args = params.args[:]
initial_repos = params.config.repos[:]
for branch, repos in iteritems(branch_to_repos):
params.args = initial_args + ['origin/' + branch]
params.config.repos = repos
if len(branch_to_repos) > 1:
Print('\nOutput for branch: ${START_COLOR}%s${RESET_COLOR} (%s)' %
(branch, ', '.join(repos)))
Run(params)
params.args = initial_args
params.config.repos = initial_repos
def _handle_decoded(self, msgs):
if not isinstance(msgs, (tuple, list)):
msgs = [msgs]
from mu_repo.print_ import Print
for msg in msgs:
Print(msg)
event.set()
def OnOutput(output):
stdout = output.stdout.strip()
if stdout:
repos_and_curr_branch.append((output.repo, stdout))
else:
if verbose:
Print(
'Unable to update (could not get current branch for: %s)' %
(output.repo, ))
def NotifyErrorListeners():
import StringIO
import traceback
cstr = StringIO.StringIO()
traceback.print_exc(file=cstr)
error = cstr.getvalue()
for listener in on_errors_listeners:
listener(error)
Print(error)
def NotifyErrorListeners():
from mu_repo.backwards_stringio import StringIO
import traceback
cstr = StringIO()
traceback.print_exc(file=cstr)
error = cstr.getvalue()
for listener in on_errors_listeners:
listener(error)
Print(error)
def PrintBranchToRepos(branch_to_repos, params):
for branch, repos in sorted(iteritems(branch_to_repos)):
if len(repos) == 1:
msg = '${START_COLOR}%s${RESET_COLOR}' % (branch, )
elif len(repos) == len(set(params.config.repos)):
msg = '${START_COLOR}%s${RESET_COLOR} (all repos)' % (branch, )
else:
msg = '${START_COLOR}%s${RESET_COLOR} (%s)' % (branch, ', '.join(
sorted(repos)))
Print(msg)
def OnOutput(output):
stdout = output.stdout
if stdout.strip():
diffed_repos.append(output.repo)
filename = '__diff__.' + output.repo + '.patch'
Print('Writing diff --cached: ', filename)
with open(filename, 'w') as f:
f.write(stdout)
else:
empty_diff_repos.add(output.repo)
def Run(params):
'''
This action will grab the latest version of mu-repo from the repository.
'''
import mu_repo
import os.path
repo_dir = os.path.dirname(os.path.dirname(mu_repo.__file__))
if not os.path.exists(os.path.join(repo_dir, '.git')):
Print(
'Can only automatically update mu-repo if it was properly gotten from a git repository.'
)
return
config = params.config
git = config.git or 'git'
ExecuteCommand([git, 'pull', '--rebase'], repo=repo_dir)
def KeepInSync(file1, file2):
with _lock:
if _KeepInSyncThread._instance is None:
_KeepInSyncThread._instance = _KeepInSyncThread()
_KeepInSyncThread._instance.start()
isfile = os.path.isfile
isdir = os.path.isdir
if isfile(file1) and isfile(file2):
_KeepInSyncThread._instance.files_to_keep_in_sync_queue.put(
_KeepInSyncStruct(file1, file2))
elif isdir(file1) and isdir(file2):
_KeepDirInSync(file1, file2)
else:
Print('Expected %s and %s to be both files or dirs.' % (file1, file2))
def run(self, serial=False):
repo = self.repo
cmd = self.cmd
msg = ' '.join([START_COLOR, '\n', repo, ':'] + cmd + [RESET_COLOR])
shell = UseShellOnSubprocess()
if serial:
#Print directly to stdout/stderr without buffering.
if not IsQuietMode():
Print(msg)
p = None
try:
p = subprocess.Popen(cmd, cwd=repo, shell=shell)
except:
PrintError('Error executing: ' + ' '.join(cmd) + ' on: ' +
repo)
if p is not None:
p.wait()
else:
try:
p = subprocess.Popen(cmd,
cwd=repo,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
shell=shell)
except:
import os
PrintError('Error executing: ' + ' '.join(cmd) + ' on: ' +
repo + ' cwd: ' + os.path.abspath('.'))
self.output_queue.put(
Output(repo,
'Error executing: %s on repo: %s' % (cmd, repo), '',
''))
return
self.stdout_thread = self._CreateReaderThread(p, 'stdout')
self.stderr_thread = self._CreateReaderThread(p, 'stderr')
p.wait()
self.stdout_thread.join(2) #finish in at most 2 seconds
self.stderr_thread.join(2) #finish in at most 2 seconds
stdout = AsStr(self.stdout_thread.GetFullOutput())
stderr = AsStr(self.stderr_thread.GetFullOutput())
self._HandleOutput(msg, stdout, stderr)
def OnOutput(output):
branch_name = as_dict.get(output.repo, 'UNKNOWN_BRANCH')
if not output.stdout:
empty_repos_and_branches.append((output.repo, branch_name))
else:
status = [
START_COLOR,
output.repo,
' ',
branch_name,
':',
RESET_COLOR,
'\n',
Indent(output.stdout),
'\n',
]
Print(''.join(status))
def Run(params):
'''
Note: this action always runs in parallel.
'''
repos_and_curr_branch = GetReposAndCurrBranch(params, verbose=False)
as_dict = dict(repos_and_curr_branch)
commands = []
for repo, _branch in repos_and_curr_branch:
commands.append(
ParallelCmd(repo, [params.config.git] + ['status', '-s']))
empty_repos_and_branches = []
def OnOutput(output):
branch_name = as_dict.get(output.repo, 'UNKNOWN_BRANCH')
if not output.stdout:
empty_repos_and_branches.append((output.repo, branch_name))
else:
status = [
START_COLOR,
output.repo,
' ',
branch_name,
':',
RESET_COLOR,
'\n',
Indent(output.stdout),
'\n',
]
Print(''.join(status))
ExecuteInParallel(commands, on_output=OnOutput)
if empty_repos_and_branches:
branch_to_repos = {}
for repo, branch in empty_repos_and_branches:
branch_to_repos.setdefault(branch, []).append(repo)
for branch, repos in branch_to_repos.iteritems():
Print(
"${START_COLOR}Unchanged:${RESET_COLOR} %s\nat branch: ${START_COLOR}%s${RESET_COLOR}\n"
% (CreateJoinedReposMsg('', repos), branch))
return repos_and_curr_branch
def _Clone(remote, repo, params, other_cmd_line_args):
created_dir = os.path.join('.', repo)
if os.path.exists(os.path.join(created_dir, '.git')):
# If it already exists, bail out!
Print(
'Skipping clone of: ${START_COLOR}%s${RESET_COLOR} because it already exists.'
% (repo, ))
return True
remote_path = remote
if not remote_path.endswith('/') and not remote_path.endswith('\\'):
remote_path += '/'
git = params.config.git
ExecuteCommand([git, 'clone', '%s%s' % (remote_path, repo)] +
other_cmd_line_args, '.')
if os.path.exists(os.path.join(created_dir, '.git')):
return True
return False
def OnOutput(output):
stdout = output.stdout.strip()
if stdout:
branches = set()
for line in stdout.splitlines():
branch = line.strip()
if branch.startswith('*'):
branch = branch[1:].strip()
if not patterns:
branches.add(branch)
else:
for pat in patterns:
if fnmatch(branch, pat):
branches.add(branch)
repos_and_curr_branch.append((output.repo, branches))
else:
Print('Unable to execute git branch for: %s' % (output.repo, ))
def run(self):
while True:
action = self.output_queue.get(True)
try:
if action is self.FINISH_PROCESSING_ITEM:
return
if isinstance(action, Output):
if self.on_output is not None:
self.on_output(action)
#else:
# Note: in this case, the output will be printed by the
# ExecuteThreadsHandlingOutputQueue method (along with the stderr).
# Print(action)
else:
Print(action) #Progress message.
except:
PrintError()
finally:
self.output_queue.task_done()
def GetReposAndCurrBranch(params, verbose=True):
'''
:param params: Params
The parameters used to get the repos and current branch (mostly using config).
:return: list(tuple(str, str))
A list with the repository and current branch for that repository.
'''
repos_and_curr_branch = []
def OnOutput(output):
stdout = output.stdout.strip()
if stdout:
repos_and_curr_branch.append((output.repo, stdout))
else:
if verbose:
Print(
'Unable to update (could not get current branch for: %s)' %
(output.repo, ))
from .action_default import Run #@Reimport
from mu_repo import Params
old_serial = params.config.serial
params.config.serial = False #Cannot be serial as we want to get the output
Run(Params(params.config, ['rev-parse', '--abbrev-ref', 'HEAD'],
params.config_file),
on_output=OnOutput)
if verbose:
branch_to_repos = {}
for repo, branch in repos_and_curr_branch:
branch_to_repos.setdefault(branch, []).append(repo)
for branch, repos in branch_to_repos.iteritems():
Print(
"Will handle ${START_COLOR}origin %s${RESET_COLOR} for: %s\n" %
(branch, ', '.join(sorted(repos))))
#Restore serial for next command.
params.config.serial = old_serial
return repos_and_curr_branch
def ExecuteCommand(cmd, repo, return_stdout=False, verbose=True):
'''
Execute command letting stderr go to the default sys.stderr.
Will block until the command finishes.
@param cmd: list(str)
The command to be executed.
@param repo: str
The repository (working dir) where the command should be executed.
@param return_stdout: bool
If True, will grab stdout and return it, otherwise will let it go to sys.stderr.
@param verbose: bool
If True will print the command being executed.
@return: the redirected stdout (if return_stdout is True) or None otherwise
'''
if verbose:
msg = ' '.join([START_COLOR, '\n', repo, ':'] + cmd + [RESET_COLOR])
Print(msg)
try:
shell = UseShellOnSubprocess()
if return_stdout:
p = subprocess.Popen(cmd,
cwd=repo,
stdout=subprocess.PIPE,
shell=shell)
else:
p = subprocess.Popen(cmd, cwd=repo, shell=shell)
except:
PrintError('Error executing: ' + ' '.join(cmd) + ' on: ' + repo)
raise
if not return_stdout:
p.wait()
else:
stdout, _stderr = p.communicate()
return stdout
def Sync(self):
try:
st1 = os.stat(self.file1)
st2 = os.stat(self.file2)
except:
#If any of those fails, we either don't have access or the file was removed.
return
changed_file_1 = self.file1_time != st1.st_mtime or self.file1_size != st1.st_size
changed_file_2 = self.file2_time != st2.st_mtime or self.file2_size != st2.st_size
if changed_file_1 or changed_file_2:
if changed_file_1 and changed_file_2:
Print(
'Unable to synchronize: both files: %s and %s are changed.'
% (self.file1, self.file2))
return
if changed_file_1:
shutil.copyfile(self.file1, self.file2)
else:
shutil.copyfile(self.file2, self.file1)
self._CollectStats()
