def set_precious_objects(fullpath, enabled=True): # It's better to just set it blindly without checking first, # as this results in one fewer shell-out. logger.debug('Setting preciousObjects for %s', fullpath) if enabled: poval = 'true' else: poval = 'false' args = ['config', 'extensions.preciousObjects', poval] grokmirror.run_git_command(fullpath, args)
def pull_repo(toplevel, gitdir, threadid='X'): fullpath = os.path.join(toplevel, gitdir.lstrip('/')) args = ['remote', 'update', '--prune'] retcode, output, error = grokmirror.run_git_command(fullpath, args) success = False if retcode == 0: success = True if error: # Put things we recognize into debug debug = [] warn = [] for line in error.split('\n'): if line.find('From ') == 0: debug.append(line) elif line.find('-> ') > 0: debug.append(line) else: warn.append(line) if debug: logger.debug('[Thread-%s] Stderr: %s', threadid, '\n'.join(debug)) if warn: logger.warning('[Thread-%s] Stderr: %s', threadid, '\n'.join(warn)) return success
def run_git_fsck(fullpath, config, conn_only=False): args = ['fsck', '--no-dangling', '--no-reflogs'] if conn_only: args.append('--connectivity-only') logger.info(' fsck : running with --connectivity-only') else: logger.info(' fsck : running full checks') retcode, output, error = grokmirror.run_git_command(fullpath, args) if output or error: # Put things we recognize as fairly benign into debug debug = [] warn = [] for line in output.split('\n') + error.split('\n'): if not len(line.strip()): continue ignored = False for estring in config['ignore_errors']: if line.find(estring) != -1: ignored = True debug.append(line) break if not ignored: warn.append(line) if debug: logger.debug('Stderr: %s', '\n'.join(debug)) if warn: logger.critical('%s has critical errors:', fullpath) for entry in warn: logger.critical("\t%s", entry) check_reclone_error(fullpath, config, warn)
def run_git_fsck(fullpath, config, conn_only=False): args = ['fsck', '--no-progress', '--no-dangling', '--no-reflogs'] if conn_only: args.append('--connectivity-only') logger.info(' fsck : running with --connectivity-only') else: logger.info(' fsck : running full checks') retcode, output, error = grokmirror.run_git_command(fullpath, args) if output or error: # Put things we recognize as fairly benign into debug debug = [] warn = [] for line in output.split('\n') + error.split('\n'): if not len(line.strip()): continue ignored = False for estring in config['ignore_errors']: if line.find(estring) != -1: ignored = True debug.append(line) break if not ignored: warn.append(line) if debug: logger.debug('Stderr: %s', '\n'.join(debug)) if warn: logger.critical('%s has critical errors:', fullpath) for entry in warn: logger.critical("\t%s", entry) check_reclone_error(fullpath, config, warn)
def run_git_commit_graph(fullpath): # Does our version of git support commit-graph? if not grokmirror.git_newer_than('2.18.0'): logger.debug('Git version too old, not generating commit-graph') logger.info(' graph : generating commit-graph --reachable') args = ['commit-graph', 'write', '--reachable'] retcode, output, error = grokmirror.run_git_command(fullpath, args) if retcode == 0: return True return False
def get_repo_obj_info(fullpath): args = ['count-objects', '-v'] retcode, output, error = grokmirror.run_git_command(fullpath, args) obj_info = {} if output: for line in output.split('\n'): key, value = line.split(':') obj_info[key] = value.strip() return obj_info
def git_rev_parse_all(gitdir): args = ['rev-parse', '--all'] retcode, output, error = grokmirror.run_git_command(gitdir, args) if error: # Put things we recognize into debug debug = [] warn = [] for line in error.split('\n'): warn.append(line) if debug: logger.debug('Stderr: %s', '\n'.join(debug)) if warn: logger.warning('Stderr: %s', '\n'.join(warn)) return output
def git_remote_update(args, fullpath): retcode, output, error = grokmirror.run_git_command(fullpath, args) if error: # Put things we recognize into debug debug = [] warn = [] for line in error.split('\n'): if line.find('From ') == 0: debug.append(line) elif line.find('-> ') > 0: debug.append(line) else: warn.append(line) if debug: logger.debug('Stderr: %s', '\n'.join(debug)) if warn: logger.warning('Stderr: %s', '\n'.join(warn))
def run_git_prune(fullpath, config): prune_ok = True if 'prune' not in config.keys() or config['prune'] != 'yes': return prune_ok # Are any other repos using us in their objects/info/alternates? gitdir = '/' + os.path.relpath(fullpath, config['toplevel']).lstrip('/') if grokmirror.is_alt_repo(config['toplevel'], gitdir): logger.info(' prune : skipped, is alternate to other repos') return prune_ok # We set expire to yesterday in order to avoid race conditions # in repositories that are actively being accessed at the time of # running the prune job. args = ['prune', '--expire=yesterday'] logger.info(' prune : pruning') retcode, output, error = grokmirror.run_git_command(fullpath, args) if error: # Put things we recognize as fairly benign into debug debug = [] warn = [] for line in error.split('\n'): ignored = False for estring in config['ignore_errors']: if line.find(estring) != -1: ignored = True debug.append(line) break if not ignored: warn.append(line) if debug: logger.debug('Stderr: %s', '\n'.join(debug)) if warn: logger.critical('Pruning %s returned critical errors:', fullpath) prune_ok = False for entry in warn: logger.critical("\t%s", entry) check_reclone_error(fullpath, config, warn) return prune_ok
def run_git_prune(fullpath, config): prune_ok = True if 'prune' not in config or config['prune'] != 'yes': return prune_ok # Are any other repos using us in their objects/info/alternates? gitdir = '/' + os.path.relpath(fullpath, config['toplevel']).lstrip('/') if grokmirror.is_alt_repo(config['toplevel'], gitdir): logger.info(' prune : skipped, is alternate to other repos') return prune_ok # We set expire to yesterday in order to avoid race conditions # in repositories that are actively being accessed at the time of # running the prune job. args = ['prune', '--expire=yesterday'] logger.info(' prune : pruning') retcode, output, error = grokmirror.run_git_command(fullpath, args) if error: # Put things we recognize as fairly benign into debug debug = [] warn = [] for line in error.split('\n'): ignored = False for estring in config['ignore_errors']: if line.find(estring) != -1: ignored = True debug.append(line) break if not ignored: warn.append(line) if debug: logger.debug('Stderr: %s', '\n'.join(debug)) if warn: logger.critical('Pruning %s returned critical errors:', fullpath) prune_ok = False for entry in warn: logger.critical("\t%s", entry) check_reclone_error(fullpath, config, warn) return prune_ok
def clone_repo(toplevel, gitdir, site, reference=None): source = os.path.join(site, gitdir.lstrip('/')) dest = os.path.join(toplevel, gitdir.lstrip('/')) args = ['clone', '--mirror'] if reference is not None: reference = os.path.join(toplevel, reference.lstrip('/')) args.append('--reference') args.append(reference) args.append(source) args.append(dest) logger.info('Cloning %s into %s', source, dest) if reference is not None: logger.info('With reference to %s', reference) retcode, output, error = grokmirror.run_git_command(None, args) success = False if retcode == 0: success = True if error: # Put things we recognize into debug debug = [] warn = [] for line in error.split('\n'): if line.find('cloned an empty repository') > 0: debug.append(line) if line.find('into bare repository') > 0: debug.append(line) else: warn.append(line) if debug: logger.debug('Stderr: %s', '\n'.join(debug)) if warn: logger.warning('Stderr: %s', '\n'.join(warn)) return success
def run_git_repack(fullpath, config, level=1): # Returns false if we hit any errors on the way repack_ok = True is_precious = check_precious_objects(fullpath) # Figure out what our repack flags should be. repack_flags = list() if 'extra_repack_flags' in config and len(config['extra_repack_flags']): repack_flags += config['extra_repack_flags'].split() full_repack_flags = ['-f', '-b', '--pack-kept-objects'] if 'extra_repack_flags_full' in config and len(config['extra_repack_flags_full']): full_repack_flags += config['extra_repack_flags_full'].split() # Are any other repos using us in their objects/info/alternates? gitdir = '/' + os.path.relpath(fullpath, config['toplevel']).lstrip('/') if grokmirror.is_alt_repo(config['toplevel'], gitdir): # we are a "mother repo" if not is_precious and ('precious' in config and config['precious'] == 'yes'): is_precious = True set_precious_objects(fullpath) # are we using alternates ourselves? Multiple levels of alternates are # a bad idea in general due high possibility of corruption. if os.path.exists(os.path.join(fullpath, 'objects', 'info', 'alternates')): logger.warning('warning : has alternates and is used by others for alternates') logger.warning(' : this can cause grandchild corruption') repack_flags.append('-A') repack_flags.append('-l') else: repack_flags.append('-a') if not is_precious: repack_flags.append('-k') if level > 1: logger.info(' repack : performing a full repack for optimal deltas') repack_flags += full_repack_flags elif os.path.exists(os.path.join(fullpath, 'objects', 'info', 'alternates')): # we are a "child repo" repack_flags.append('-l') if level > 1: repack_flags.append('-A') else: # we have no relationships with other repos if level > 1: logger.info(' repack : performing a full repack for optimal deltas') repack_flags.append('-a') if not is_precious: repack_flags.append('-k') repack_flags += full_repack_flags if not is_precious: repack_flags.append('-d') args = ['repack'] + repack_flags logger.info(' repack : repacking with "%s"', ' '.join(repack_flags)) # We always tack on -q repack_flags.append('-q') retcode, output, error = grokmirror.run_git_command(fullpath, args) # With newer versions of git, repack may return warnings that are safe to ignore # so use the same strategy to weed out things we aren't interested in seeing if error: # Put things we recognize as fairly benign into debug debug = [] warn = [] for line in error.split('\n'): ignored = False for estring in config['ignore_errors']: if line.find(estring) != -1: ignored = True debug.append(line) break if not ignored: warn.append(line) if debug: logger.debug('Stderr: %s', '\n'.join(debug)) if warn: logger.critical('Repacking %s returned critical errors:', fullpath) repack_ok = False for entry in warn: logger.critical("\t%s", entry) check_reclone_error(fullpath, config, warn) if not repack_ok: # No need to repack refs if repo is broken return False # only repack refs on full repacks if level < 2: # do we still have loose objects after repacking? obj_info = get_repo_obj_info(fullpath) if obj_info['count'] != '0': return run_git_prune(fullpath, config) return repack_ok # repacking refs requires a separate command, so run it now args = ['pack-refs', '--all'] logger.info(' repack : repacking refs') retcode, output, error = grokmirror.run_git_command(fullpath, args) # pack-refs shouldn't return anything, but use the same ignore_errors block # to weed out any future potential benign warnings if error: # Put things we recognize as fairly benign into debug debug = [] warn = [] for line in error.split('\n'): ignored = False for estring in config['ignore_errors']: if line.find(estring) != -1: ignored = True debug.append(line) break if not ignored: warn.append(line) if debug: logger.debug('Stderr: %s', '\n'.join(debug)) if warn: logger.critical('Repacking refs %s returned critical errors:', fullpath) repack_ok = False for entry in warn: logger.critical("\t%s", entry) check_reclone_error(fullpath, config, warn) if repack_ok and 'prune' in config and config['prune'] == 'yes': # run prune now return run_git_prune(fullpath, config) return repack_ok
def check_precious_objects(fullpath): args = ['config', '--get', 'extensions.preciousObjects'] retcode, output, error = grokmirror.run_git_command(fullpath, args) if output.strip().lower() == 'true': return True return False
def set_precious_objects(fullpath): # It's better to just set it blindly without checking first, # as this results in one fewer shell-out. logger.debug('Setting preciousObjects for %s', fullpath) args = ['config', 'extensions.preciousObjects', 'true'] grokmirror.run_git_command(fullpath, args)
def run_git_repack(fullpath, config, level=1): # Returns false if we hit any errors on the way repack_ok = True is_precious = False set_precious = False gen_commitgraph = True # Figure out what our repack flags should be. repack_flags = list() if 'extra_repack_flags' in config and len(config['extra_repack_flags']): repack_flags += config['extra_repack_flags'].split() full_repack_flags = ['-f', '-b', '--pack-kept-objects'] if 'extra_repack_flags_full' in config and len( config['extra_repack_flags_full']): full_repack_flags += config['extra_repack_flags_full'].split() # Are any other repos using us in their objects/info/alternates? gitdir = '/' + os.path.relpath(fullpath, config['toplevel']).lstrip('/') if grokmirror.is_alt_repo(config['toplevel'], gitdir): # we are a "mother repo" # Force preciousObjects if precious is "always" if config['precious'] == 'always': is_precious = True set_precious_objects(fullpath, enabled=True) else: # Turn precious off during repacks set_precious_objects(fullpath, enabled=False) # Turn it back on after we're done set_precious = True # are we using alternates ourselves? Multiple levels of alternates are # a bad idea in general due high possibility of corruption. if os.path.exists( os.path.join(fullpath, 'objects', 'info', 'alternates')): logger.warning( 'warning : has alternates and is used by others for alternates' ) logger.warning(' : this can cause grandchild corruption') repack_flags.append('-A') repack_flags.append('-l') else: repack_flags.append('-a') if not is_precious: repack_flags.append('-k') if level > 1: logger.info( ' repack : performing a full repack for optimal deltas') repack_flags += full_repack_flags elif os.path.exists(os.path.join(fullpath, 'objects', 'info', 'alternates')): # we are a "child repo" gen_commitgraph = False repack_flags.append('-l') if level > 1: repack_flags.append('-A') else: # we have no relationships with other repos if level > 1: logger.info( ' repack : performing a full repack for optimal deltas') repack_flags.append('-a') if not is_precious: repack_flags.append('-k') repack_flags += full_repack_flags if not is_precious: repack_flags.append('-d') args = ['repack'] + repack_flags logger.info(' repack : repacking with "%s"', ' '.join(repack_flags)) # We always tack on -q repack_flags.append('-q') retcode, output, error = grokmirror.run_git_command(fullpath, args) if set_precious: set_precious_objects(fullpath, enabled=True) # With newer versions of git, repack may return warnings that are safe to ignore # so use the same strategy to weed out things we aren't interested in seeing if error: # Put things we recognize as fairly benign into debug debug = [] warn = [] for line in error.split('\n'): ignored = False for estring in config['ignore_errors']: if line.find(estring) != -1: ignored = True debug.append(line) break if not ignored: warn.append(line) if debug: logger.debug('Stderr: %s', '\n'.join(debug)) if warn: logger.critical('Repacking %s returned critical errors:', fullpath) repack_ok = False for entry in warn: logger.critical("\t%s", entry) check_reclone_error(fullpath, config, warn) if not repack_ok: # No need to repack refs if repo is broken return False if gen_commitgraph and config['commitgraph'] == 'yes': run_git_commit_graph(fullpath) # only repack refs on full repacks if level < 2: # do we still have loose objects after repacking? obj_info = get_repo_obj_info(fullpath) if obj_info['count'] != '0': return run_git_prune(fullpath, config) return repack_ok # repacking refs requires a separate command, so run it now args = ['pack-refs', '--all'] logger.info(' repack : repacking refs') retcode, output, error = grokmirror.run_git_command(fullpath, args) # pack-refs shouldn't return anything, but use the same ignore_errors block # to weed out any future potential benign warnings if error: # Put things we recognize as fairly benign into debug debug = [] warn = [] for line in error.split('\n'): ignored = False for estring in config['ignore_errors']: if line.find(estring) != -1: ignored = True debug.append(line) break if not ignored: warn.append(line) if debug: logger.debug('Stderr: %s', '\n'.join(debug)) if warn: logger.critical('Repacking refs %s returned critical errors:', fullpath) repack_ok = False for entry in warn: logger.critical("\t%s", entry) check_reclone_error(fullpath, config, warn) if repack_ok and 'prune' in config and config['prune'] == 'yes': # run prune now return run_git_prune(fullpath, config) return repack_ok