def set_precious_objects(fullpath, enabled=True):
# It's better to just set it blindly without checking first,
# as this results in one fewer shell-out.
logger.debug('Setting preciousObjects for %s', fullpath)
if enabled:
poval = 'true'
else:
poval = 'false'
args = ['config', 'extensions.preciousObjects', poval]
grokmirror.run_git_command(fullpath, args)
def pull_repo(toplevel, gitdir, threadid='X'):
fullpath = os.path.join(toplevel, gitdir.lstrip('/'))
args = ['remote', 'update', '--prune']
retcode, output, error = grokmirror.run_git_command(fullpath, args)
success = False
if retcode == 0:
success = True
if error:
# Put things we recognize into debug
debug = []
warn = []
for line in error.split('\n'):
if line.find('From ') == 0:
debug.append(line)
elif line.find('-> ') > 0:
debug.append(line)
else:
warn.append(line)
if debug:
logger.debug('[Thread-%s] Stderr: %s', threadid, '\n'.join(debug))
if warn:
logger.warning('[Thread-%s] Stderr: %s', threadid, '\n'.join(warn))
return success
def run_git_fsck(fullpath, config, conn_only=False):
args = ['fsck', '--no-dangling', '--no-reflogs']
if conn_only:
args.append('--connectivity-only')
logger.info(' fsck : running with --connectivity-only')
else:
logger.info(' fsck : running full checks')
retcode, output, error = grokmirror.run_git_command(fullpath, args)
if output or error:
# Put things we recognize as fairly benign into debug
debug = []
warn = []
for line in output.split('\n') + error.split('\n'):
if not len(line.strip()):
continue
ignored = False
for estring in config['ignore_errors']:
if line.find(estring) != -1:
ignored = True
debug.append(line)
break
if not ignored:
warn.append(line)
if debug:
logger.debug('Stderr: %s', '\n'.join(debug))
if warn:
logger.critical('%s has critical errors:', fullpath)
for entry in warn:
logger.critical("\t%s", entry)
check_reclone_error(fullpath, config, warn)
def run_git_fsck(fullpath, config, conn_only=False):
args = ['fsck', '--no-progress', '--no-dangling', '--no-reflogs']
if conn_only:
args.append('--connectivity-only')
logger.info(' fsck : running with --connectivity-only')
else:
logger.info(' fsck : running full checks')
retcode, output, error = grokmirror.run_git_command(fullpath, args)
if output or error:
# Put things we recognize as fairly benign into debug
debug = []
warn = []
for line in output.split('\n') + error.split('\n'):
if not len(line.strip()):
continue
ignored = False
for estring in config['ignore_errors']:
if line.find(estring) != -1:
ignored = True
debug.append(line)
break
if not ignored:
warn.append(line)
if debug:
logger.debug('Stderr: %s', '\n'.join(debug))
if warn:
logger.critical('%s has critical errors:', fullpath)
for entry in warn:
logger.critical("\t%s", entry)
check_reclone_error(fullpath, config, warn)
def pull_repo(toplevel, gitdir, threadid='X'):
fullpath = os.path.join(toplevel, gitdir.lstrip('/'))
args = ['remote', 'update', '--prune']
retcode, output, error = grokmirror.run_git_command(fullpath, args)
success = False
if retcode == 0:
success = True
if error:
# Put things we recognize into debug
debug = []
warn = []
for line in error.split('\n'):
if line.find('From ') == 0:
debug.append(line)
elif line.find('-> ') > 0:
debug.append(line)
else:
warn.append(line)
if debug:
logger.debug('[Thread-%s] Stderr: %s', threadid, '\n'.join(debug))
if warn:
logger.warning('[Thread-%s] Stderr: %s', threadid, '\n'.join(warn))
return success
def run_git_commit_graph(fullpath):
# Does our version of git support commit-graph?
if not grokmirror.git_newer_than('2.18.0'):
logger.debug('Git version too old, not generating commit-graph')
logger.info(' graph : generating commit-graph --reachable')
args = ['commit-graph', 'write', '--reachable']
retcode, output, error = grokmirror.run_git_command(fullpath, args)
if retcode == 0:
return True
return False
def get_repo_obj_info(fullpath):
args = ['count-objects', '-v']
retcode, output, error = grokmirror.run_git_command(fullpath, args)
obj_info = {}
if output:
for line in output.split('\n'):
key, value = line.split(':')
obj_info[key] = value.strip()
return obj_info
def get_repo_obj_info(fullpath):
args = ['count-objects', '-v']
retcode, output, error = grokmirror.run_git_command(fullpath, args)
obj_info = {}
if output:
for line in output.split('\n'):
key, value = line.split(':')
obj_info[key] = value.strip()
return obj_info
def git_rev_parse_all(gitdir):
args = ['rev-parse', '--all']
retcode, output, error = grokmirror.run_git_command(gitdir, args)
if error:
# Put things we recognize into debug
debug = []
warn = []
for line in error.split('\n'):
warn.append(line)
if debug:
logger.debug('Stderr: %s', '\n'.join(debug))
if warn:
logger.warning('Stderr: %s', '\n'.join(warn))
return output
def git_remote_update(args, fullpath):
retcode, output, error = grokmirror.run_git_command(fullpath, args)
if error:
# Put things we recognize into debug
debug = []
warn = []
for line in error.split('\n'):
if line.find('From ') == 0:
debug.append(line)
elif line.find('-> ') > 0:
debug.append(line)
else:
warn.append(line)
if debug:
logger.debug('Stderr: %s', '\n'.join(debug))
if warn:
logger.warning('Stderr: %s', '\n'.join(warn))
def run_git_prune(fullpath, config):
prune_ok = True
if 'prune' not in config.keys() or config['prune'] != 'yes':
return prune_ok
# Are any other repos using us in their objects/info/alternates?
gitdir = '/' + os.path.relpath(fullpath, config['toplevel']).lstrip('/')
if grokmirror.is_alt_repo(config['toplevel'], gitdir):
logger.info(' prune : skipped, is alternate to other repos')
return prune_ok
# We set expire to yesterday in order to avoid race conditions
# in repositories that are actively being accessed at the time of
# running the prune job.
args = ['prune', '--expire=yesterday']
logger.info(' prune : pruning')
retcode, output, error = grokmirror.run_git_command(fullpath, args)
if error:
# Put things we recognize as fairly benign into debug
debug = []
warn = []
for line in error.split('\n'):
ignored = False
for estring in config['ignore_errors']:
if line.find(estring) != -1:
ignored = True
debug.append(line)
break
if not ignored:
warn.append(line)
if debug:
logger.debug('Stderr: %s', '\n'.join(debug))
if warn:
logger.critical('Pruning %s returned critical errors:', fullpath)
prune_ok = False
for entry in warn:
logger.critical("\t%s", entry)
check_reclone_error(fullpath, config, warn)
return prune_ok
def run_git_prune(fullpath, config):
prune_ok = True
if 'prune' not in config or config['prune'] != 'yes':
return prune_ok
# Are any other repos using us in their objects/info/alternates?
gitdir = '/' + os.path.relpath(fullpath, config['toplevel']).lstrip('/')
if grokmirror.is_alt_repo(config['toplevel'], gitdir):
logger.info(' prune : skipped, is alternate to other repos')
return prune_ok
# We set expire to yesterday in order to avoid race conditions
# in repositories that are actively being accessed at the time of
# running the prune job.
args = ['prune', '--expire=yesterday']
logger.info(' prune : pruning')
retcode, output, error = grokmirror.run_git_command(fullpath, args)
if error:
# Put things we recognize as fairly benign into debug
debug = []
warn = []
for line in error.split('\n'):
ignored = False
for estring in config['ignore_errors']:
if line.find(estring) != -1:
ignored = True
debug.append(line)
break
if not ignored:
warn.append(line)
if debug:
logger.debug('Stderr: %s', '\n'.join(debug))
if warn:
logger.critical('Pruning %s returned critical errors:', fullpath)
prune_ok = False
for entry in warn:
logger.critical("\t%s", entry)
check_reclone_error(fullpath, config, warn)
return prune_ok
def clone_repo(toplevel, gitdir, site, reference=None):
source = os.path.join(site, gitdir.lstrip('/'))
dest = os.path.join(toplevel, gitdir.lstrip('/'))
args = ['clone', '--mirror']
if reference is not None:
reference = os.path.join(toplevel, reference.lstrip('/'))
args.append('--reference')
args.append(reference)
args.append(source)
args.append(dest)
logger.info('Cloning %s into %s', source, dest)
if reference is not None:
logger.info('With reference to %s', reference)
retcode, output, error = grokmirror.run_git_command(None, args)
success = False
if retcode == 0:
success = True
if error:
# Put things we recognize into debug
debug = []
warn = []
for line in error.split('\n'):
if line.find('cloned an empty repository') > 0:
debug.append(line)
if line.find('into bare repository') > 0:
debug.append(line)
else:
warn.append(line)
if debug:
logger.debug('Stderr: %s', '\n'.join(debug))
if warn:
logger.warning('Stderr: %s', '\n'.join(warn))
return success
def clone_repo(toplevel, gitdir, site, reference=None):
source = os.path.join(site, gitdir.lstrip('/'))
dest = os.path.join(toplevel, gitdir.lstrip('/'))
args = ['clone', '--mirror']
if reference is not None:
reference = os.path.join(toplevel, reference.lstrip('/'))
args.append('--reference')
args.append(reference)
args.append(source)
args.append(dest)
logger.info('Cloning %s into %s', source, dest)
if reference is not None:
logger.info('With reference to %s', reference)
retcode, output, error = grokmirror.run_git_command(None, args)
success = False
if retcode == 0:
success = True
if error:
# Put things we recognize into debug
debug = []
warn = []
for line in error.split('\n'):
if line.find('cloned an empty repository') > 0:
debug.append(line)
if line.find('into bare repository') > 0:
debug.append(line)
else:
warn.append(line)
if debug:
logger.debug('Stderr: %s', '\n'.join(debug))
if warn:
logger.warning('Stderr: %s', '\n'.join(warn))
return success
def run_git_repack(fullpath, config, level=1):
# Returns false if we hit any errors on the way
repack_ok = True
is_precious = check_precious_objects(fullpath)
# Figure out what our repack flags should be.
repack_flags = list()
if 'extra_repack_flags' in config and len(config['extra_repack_flags']):
repack_flags += config['extra_repack_flags'].split()
full_repack_flags = ['-f', '-b', '--pack-kept-objects']
if 'extra_repack_flags_full' in config and len(config['extra_repack_flags_full']):
full_repack_flags += config['extra_repack_flags_full'].split()
# Are any other repos using us in their objects/info/alternates?
gitdir = '/' + os.path.relpath(fullpath, config['toplevel']).lstrip('/')
if grokmirror.is_alt_repo(config['toplevel'], gitdir):
# we are a "mother repo"
if not is_precious and ('precious' in config and config['precious'] == 'yes'):
is_precious = True
set_precious_objects(fullpath)
# are we using alternates ourselves? Multiple levels of alternates are
# a bad idea in general due high possibility of corruption.
if os.path.exists(os.path.join(fullpath, 'objects', 'info', 'alternates')):
logger.warning('warning : has alternates and is used by others for alternates')
logger.warning(' : this can cause grandchild corruption')
repack_flags.append('-A')
repack_flags.append('-l')
else:
repack_flags.append('-a')
if not is_precious:
repack_flags.append('-k')
if level > 1:
logger.info(' repack : performing a full repack for optimal deltas')
repack_flags += full_repack_flags
elif os.path.exists(os.path.join(fullpath, 'objects', 'info', 'alternates')):
# we are a "child repo"
repack_flags.append('-l')
if level > 1:
repack_flags.append('-A')
else:
# we have no relationships with other repos
if level > 1:
logger.info(' repack : performing a full repack for optimal deltas')
repack_flags.append('-a')
if not is_precious:
repack_flags.append('-k')
repack_flags += full_repack_flags
if not is_precious:
repack_flags.append('-d')
args = ['repack'] + repack_flags
logger.info(' repack : repacking with "%s"', ' '.join(repack_flags))
# We always tack on -q
repack_flags.append('-q')
retcode, output, error = grokmirror.run_git_command(fullpath, args)
# With newer versions of git, repack may return warnings that are safe to ignore
# so use the same strategy to weed out things we aren't interested in seeing
if error:
# Put things we recognize as fairly benign into debug
debug = []
warn = []
for line in error.split('\n'):
ignored = False
for estring in config['ignore_errors']:
if line.find(estring) != -1:
ignored = True
debug.append(line)
break
if not ignored:
warn.append(line)
if debug:
logger.debug('Stderr: %s', '\n'.join(debug))
if warn:
logger.critical('Repacking %s returned critical errors:', fullpath)
repack_ok = False
for entry in warn:
logger.critical("\t%s", entry)
check_reclone_error(fullpath, config, warn)
if not repack_ok:
# No need to repack refs if repo is broken
return False
# only repack refs on full repacks
if level < 2:
# do we still have loose objects after repacking?
obj_info = get_repo_obj_info(fullpath)
if obj_info['count'] != '0':
return run_git_prune(fullpath, config)
return repack_ok
# repacking refs requires a separate command, so run it now
args = ['pack-refs', '--all']
logger.info(' repack : repacking refs')
retcode, output, error = grokmirror.run_git_command(fullpath, args)
# pack-refs shouldn't return anything, but use the same ignore_errors block
# to weed out any future potential benign warnings
if error:
# Put things we recognize as fairly benign into debug
debug = []
warn = []
for line in error.split('\n'):
ignored = False
for estring in config['ignore_errors']:
if line.find(estring) != -1:
ignored = True
debug.append(line)
break
if not ignored:
warn.append(line)
if debug:
logger.debug('Stderr: %s', '\n'.join(debug))
if warn:
logger.critical('Repacking refs %s returned critical errors:',
fullpath)
repack_ok = False
for entry in warn:
logger.critical("\t%s", entry)
check_reclone_error(fullpath, config, warn)
if repack_ok and 'prune' in config and config['prune'] == 'yes':
# run prune now
return run_git_prune(fullpath, config)
return repack_ok
def check_precious_objects(fullpath):
args = ['config', '--get', 'extensions.preciousObjects']
retcode, output, error = grokmirror.run_git_command(fullpath, args)
if output.strip().lower() == 'true':
return True
return False
def set_precious_objects(fullpath):
# It's better to just set it blindly without checking first,
# as this results in one fewer shell-out.
logger.debug('Setting preciousObjects for %s', fullpath)
args = ['config', 'extensions.preciousObjects', 'true']
grokmirror.run_git_command(fullpath, args)
def check_precious_objects(fullpath):
args = ['config', '--get', 'extensions.preciousObjects']
retcode, output, error = grokmirror.run_git_command(fullpath, args)
if output.strip().lower() == 'true':
return True
return False
def run_git_repack(fullpath, config, level=1):
# Returns false if we hit any errors on the way
repack_ok = True
is_precious = False
set_precious = False
gen_commitgraph = True
# Figure out what our repack flags should be.
repack_flags = list()
if 'extra_repack_flags' in config and len(config['extra_repack_flags']):
repack_flags += config['extra_repack_flags'].split()
full_repack_flags = ['-f', '-b', '--pack-kept-objects']
if 'extra_repack_flags_full' in config and len(
config['extra_repack_flags_full']):
full_repack_flags += config['extra_repack_flags_full'].split()
# Are any other repos using us in their objects/info/alternates?
gitdir = '/' + os.path.relpath(fullpath, config['toplevel']).lstrip('/')
if grokmirror.is_alt_repo(config['toplevel'], gitdir):
# we are a "mother repo"
# Force preciousObjects if precious is "always"
if config['precious'] == 'always':
is_precious = True
set_precious_objects(fullpath, enabled=True)
else:
# Turn precious off during repacks
set_precious_objects(fullpath, enabled=False)
# Turn it back on after we're done
set_precious = True
# are we using alternates ourselves? Multiple levels of alternates are
# a bad idea in general due high possibility of corruption.
if os.path.exists(
os.path.join(fullpath, 'objects', 'info', 'alternates')):
logger.warning(
'warning : has alternates and is used by others for alternates'
)
logger.warning(' : this can cause grandchild corruption')
repack_flags.append('-A')
repack_flags.append('-l')
else:
repack_flags.append('-a')
if not is_precious:
repack_flags.append('-k')
if level > 1:
logger.info(
' repack : performing a full repack for optimal deltas')
repack_flags += full_repack_flags
elif os.path.exists(os.path.join(fullpath, 'objects', 'info',
'alternates')):
# we are a "child repo"
gen_commitgraph = False
repack_flags.append('-l')
if level > 1:
repack_flags.append('-A')
else:
# we have no relationships with other repos
if level > 1:
logger.info(
' repack : performing a full repack for optimal deltas')
repack_flags.append('-a')
if not is_precious:
repack_flags.append('-k')
repack_flags += full_repack_flags
if not is_precious:
repack_flags.append('-d')
args = ['repack'] + repack_flags
logger.info(' repack : repacking with "%s"', ' '.join(repack_flags))
# We always tack on -q
repack_flags.append('-q')
retcode, output, error = grokmirror.run_git_command(fullpath, args)
if set_precious:
set_precious_objects(fullpath, enabled=True)
# With newer versions of git, repack may return warnings that are safe to ignore
# so use the same strategy to weed out things we aren't interested in seeing
if error:
# Put things we recognize as fairly benign into debug
debug = []
warn = []
for line in error.split('\n'):
ignored = False
for estring in config['ignore_errors']:
if line.find(estring) != -1:
ignored = True
debug.append(line)
break
if not ignored:
warn.append(line)
if debug:
logger.debug('Stderr: %s', '\n'.join(debug))
if warn:
logger.critical('Repacking %s returned critical errors:', fullpath)
repack_ok = False
for entry in warn:
logger.critical("\t%s", entry)
check_reclone_error(fullpath, config, warn)
if not repack_ok:
# No need to repack refs if repo is broken
return False
if gen_commitgraph and config['commitgraph'] == 'yes':
run_git_commit_graph(fullpath)
# only repack refs on full repacks
if level < 2:
# do we still have loose objects after repacking?
obj_info = get_repo_obj_info(fullpath)
if obj_info['count'] != '0':
return run_git_prune(fullpath, config)
return repack_ok
# repacking refs requires a separate command, so run it now
args = ['pack-refs', '--all']
logger.info(' repack : repacking refs')
retcode, output, error = grokmirror.run_git_command(fullpath, args)
# pack-refs shouldn't return anything, but use the same ignore_errors block
# to weed out any future potential benign warnings
if error:
# Put things we recognize as fairly benign into debug
debug = []
warn = []
for line in error.split('\n'):
ignored = False
for estring in config['ignore_errors']:
if line.find(estring) != -1:
ignored = True
debug.append(line)
break
if not ignored:
warn.append(line)
if debug:
logger.debug('Stderr: %s', '\n'.join(debug))
if warn:
logger.critical('Repacking refs %s returned critical errors:',
fullpath)
repack_ok = False
for entry in warn:
logger.critical("\t%s", entry)
check_reclone_error(fullpath, config, warn)
if repack_ok and 'prune' in config and config['prune'] == 'yes':
# run prune now
return run_git_prune(fullpath, config)
return repack_ok
