def full_init(self, repo_name_p4client=None): """Initialize the repo.""" # Ensure we have a sane environment. p4gf_init.init(self.p4) self._get_config() # Initialize the repository if necessary. print( _("Initializing '{repo_name}'...").format( repo_name=self.repo_name)) self.init_repo(repo_name_p4client) print(_("Initialization complete.")) if not self.noclone: try: start_at = int(self.start.lstrip('@')) if self.start else 1 self._copy_p2g_with_start(start_at) except ValueError: raise RuntimeError( _('Invalid --start value: {start}').format( start=self.start)) except IndexError: raise RuntimeError( _("Could not find changes >= '{start_at}'").format( start_at=start_at))
def main(): """set up repo for a view""" parser = p4gf_util.create_arg_parser( "Initializes Git Fusion Perforce client and Git repository.") parser.add_argument('--start', metavar="", help='Changelist number to start repo history, default=1') parser.add_argument('view', metavar='view', help='name of view to be initialized') args = parser.parse_args() p4gf_version.log_version() view_name = p4gf_util.argv_to_view_name(args.view) p4gf_util.reset_git_enviro() p4 = connect_p4() if not p4: return 2 LOG.debug("connected to P4 at %s", p4.port) try: with p4gf_lock.view_lock(p4, view_name) as view_lock: # ensure we have a sane environment p4gf_init.init(p4) # now initialize the repository print("Initializing {}...".format(view_name)) r = init_repo(p4, view_name) if r > INIT_REPO_OK: return r print("Initialization complete.") if args.start: start = args.start.lstrip('@') print("Copying changes from {}...".format(start)) copy_p2g_with_start(view_name, start, view_lock) print("Copying completed.") except P4.P4Exception as e: sys.stderr.write("Error occurred: {}\n".format(e)) return 0
def main(): """set up repo for a view""" with ExceptionAuditLogger(): args = parse_args(sys.argv[1:]) if not args: return 1 # Record the p4 user in environment. We use environment to pass to # git-invoked hook. We don't have to set ctx.authenticated_p4user because # Context.__init__() reads it from environment, which we set here. os.environ[p4gf_const.P4GF_AUTH_P4USER_ENVAR] = args.user # print "args={}".format(args) view_name = args.options[-1] p4gf_util.reset_git_enviro() p4 = connect_p4() if not p4: return 2 LOG.debug("connected to P4: %s", p4) _check_lock_perm(p4) if not check_protects(p4): _raise_p4gf_perm() if run_special_command(view_name, p4, args.user): return 0 # Go no further, create NOTHING, if user not authorized. view_perm = p4gf_group.ViewPerm.for_user_and_view( p4, args.user, view_name) _check_authorization(view_perm, args.user, args.command[0], view_name) # Create Git Fusion server depot, user, config. NOPs if already created. p4gf_init.init(p4) with p4gf_lock.view_lock(p4, view_name) as view_lock: # Create Git Fusion per-repo client view mapping and config. # # NOPs if already created. # Create the empty directory that will hold the git repo. init_repo_status = p4gf_init_repo.init_repo(p4, view_name) if init_repo_status == p4gf_init_repo.INIT_REPO_OK: repo_created = True elif init_repo_status == p4gf_init_repo.INIT_REPO_EXISTS: repo_created = False else: return 1 # If authorization came from default, not explicit group # membership, copy that authorization to a group now. Could # not do this until after p4gf_init_repo() has a chance to # create not-yet-existing groups. view_perm.write_if(p4) # Now that we have valid git-fusion-user and # git-fusion-<view> client, replace our temporary P4 # connection with a more permanent Context, shared for the # remainder of this process. ctx = p4gf_context.create_context(view_name, view_lock) del p4 LOG.debug("reconnected to P4, p4gf=%s", ctx.p4gf) # Find directory paths to feed to git. ctx.view_dirs = p4gf_view_dirs.from_p4gf_dir( ctx.gitrootdir, view_name) ctx.log_context() # cd into the work directory. Not all git functions react well # to --work-tree=xxxx. cwd = os.getcwd() os.chdir(ctx.view_dirs.GIT_WORK_TREE) # Copy any recent changes from Perforce to Git. try: p4gf_copy_p2g.copy_p2g_ctx(ctx) except: # Dump failure to log, BEFORE cleanup, just in case # cleanup ALSO fails and throws its own error (which # happens if we're out of memory). LOG.error(traceback.format_exc()) if repo_created: # Return to the original working directory to allow the # config code to call os.getcwd() without dying, since # we are about to delete the current working directory. os.chdir(cwd) cleanup_client(ctx, view_name) raise # Detach git repo's workspace from master before calling # original git, otherwise we won't be able to push master. p4gf_util.checkout_detached_master() # Flush stderr before returning control to Git. # Otherwise Git's own output might interrupt ours. sys.stderr.flush() return _call_original_git(ctx, args)
def main(poll_only=False): """set up repo for a view view_name_git is the untranslated repo name view_name is the translated repo name """ p4gf_proc.install_stack_dumper() _log_environ(os.environ) with p4gf_server_common.ExceptionAuditLogger()\ , p4gf_create_p4.Closer(): LOG.debug(p4gf_log.memory_usage()) start_time = time.time() args = parse_args(sys.argv[1:]) if not args: return 1 is_push = 'upload' not in args.command[0] # Record the p4 user in environment. We use environment to pass to # git-invoked hook. We don't have to set ctx.authenticated_p4user because # Context.__init__() reads it from environment, which we set here. os.environ[p4gf_const.P4GF_AUTH_P4USER] = args.user # view_name_git is the untranslated repo name # view_name is the translated repo name # print "args={}".format(args) view_name_git = args.options[-1] # translate '/' ':' ' ' .. etc .. for internal view_name view_name = p4gf_translate.TranslateReponame.git_to_repo(view_name_git) LOG.debug("public view_name: {0} internal view_name: {1}". format(view_name_git, view_name)) p4gf_util.reset_git_enviro() p4 = p4gf_create_p4.create_p4() if not p4: return 2 LOG.debug("connected to P4: %s", p4) p4gf_server_common.check_readiness(p4) p4gf_server_common.check_lock_perm(p4) if not p4gf_server_common.check_protects(p4): p4gf_server_common.raise_p4gf_perm() if p4gf_server_common.run_special_command(view_name, p4, args.user): return 0 # Initialize the external process launcher early, before allocating lots # of memory, and just after all other conditions have been checked. p4gf_proc.init() # Prepare for possible spawn of GitMirror worker process by forking # now before allocating lots of memory. p4gf_gitmirror.setup_spawn(view_name) # Kick off garbage collection debugging, if enabled. p4gf_gc.init_gc() if poll_only: view_perm = None else: # Go no further, create NOTHING, if user not authorized. # We use the translated internal view name here for perm authorization required_perm = p4gf_server_common.COMMAND_TO_PERM[args.command[0]] view_perm = p4gf_group.ViewPerm.for_user_and_view(p4, args.user, view_name, required_perm) p4gf_server_common.check_authorization(p4, view_perm, args.user, args.command[0], view_name) # Create Git Fusion server depot, user, config. NOPs if already created. p4gf_init.init(p4) write_motd() # view_name is the internal view_name (identical when notExist special chars) before_lock_time = time.time() with p4gf_lock.view_lock(p4, view_name) as view_lock: after_lock_time = time.time() # Create Git Fusion per-repo client view mapping and config. # # NOPs if already created. # Create the empty directory that will hold the git repo. init_repo_status = p4gf_init_repo.init_repo(p4, view_name, view_lock) if init_repo_status == p4gf_init_repo.INIT_REPO_OK: repo_created = True elif init_repo_status == p4gf_init_repo.INIT_REPO_EXISTS: repo_created = False else: return 1 # If authorization came from default, not explicit group # membership, copy that authorization to a group now. Could # not do this until after p4gf_init_repo() has a chance to # create not-yet-existing groups. if view_perm: view_perm.write_if(p4) # Now that we have valid git-fusion-user and # git-fusion-<view> client, replace our temporary P4 # connection with a more permanent Context, shared for the # remainder of this process. with p4gf_context.create_context(view_name, view_lock) as ctx: LOG.debug("reconnected to P4, p4gf=%s", ctx.p4gf) # Find directory paths to feed to git. ctx.log_context() # cd into the work directory. Not all git functions react well # to --work-tree=xxxx. cwd = os.getcwd() os.chdir(ctx.view_dirs.GIT_WORK_TREE) # Only copy from Perforce to Git if no other process is cloning # from this Git repo right now. shared_in_progress = p4gf_lock.shared_host_view_lock_exists(ctx.p4, view_name) if not shared_in_progress: # Copy any recent changes from Perforce to Git. try: LOG.debug("bare: No git-upload-pack in progress, force non-bare" " before update Git from Perforce.") p4gf_git.set_bare(False) p4gf_copy_p2g.copy_p2g_ctx(ctx) p4gf_init_repo.process_imports(ctx) # Now is also an appropriate time to clear out any stale Git # Swarm reviews. We're pre-pull, pre-push, time when we've # got exclusive write access to the Git repo, GSReviewCollection.delete_refs_for_closed_reviews(ctx) except p4gf_lock.LockCanceled as lc: LOG.warning(str(lc)) except: # Dump failure to log, BEFORE cleanup, just in case # cleanup ALSO fails and throws its own error (which # happens if we're out of memory). LOG.error(traceback.format_exc()) if repo_created: # Return to the original working directory to allow the # config code to call os.getcwd() without dying, since # we are about to delete the current working directory. os.chdir(cwd) p4gf_server_common.cleanup_client(ctx, view_name) raise if poll_only: code = os.EX_OK else: git_caller = functools.partial(_call_git, args, ctx) try: # Deep in call_git(), we grab an 'p4 reviews' lock on # ctx.clientmap's LHS. Switch that clientmap to our # full union view to prevent simultaneous 'git push'es # from clobbering each other in some shared depot # branch. Must include all lightweight branches, too. ctx.switch_client_view_to_union() exclusive = 'upload' not in args.command[0] code = p4gf_call_git.call_git( git_caller, ctx, view_name, view_lock, exclusive) if is_push: GSReviewCollection.post_push(ctx) except p4gf_atomic_lock.LockConflict as lc: sys.stderr.write("{}\n".format(lc)) code = os.EX_SOFTWARE p4gf_gc.process_garbage(NTR('at end of auth_server')) if LOG.isEnabledFor(logging.DEBUG): end_time = time.time() frm = NTR("Runtime: preparation {} ms, lock acquisition {} ms," " processing {} ms") LOG.debug(frm.format(before_lock_time - start_time, after_lock_time - before_lock_time, end_time - after_lock_time)) return code
def _wsgi_app(environ, start_response): """ WSGI application to process the incoming Git client request. This is nearly equivalent to p4gf_auth_server.main() with the exception of input validation and error handling. """ p4gf_log.record_http(environ) p4gf_version.log_version() _log_environ(environ) p4gf_version.version_check() LOG.debug("processing HTTP request, pid={}".format(os.getpid())) # Keep the content type to exactly 'text/plain' so there is at least # the remote chance that Git might show our error messages (does not # appear to work in practice, however). headers = [('Content-Type', 'text/plain')] encoding = sys.getfilesystemencoding() if encoding == 'ascii': # This encoding is wrong and will eventually lead to problems. LOG.error("Using 'ascii' file encoding will ultimately result in errors, " "please set LANG/LC_ALL to 'utf-8' in web server configuration.") start_response(_('500 Internal Server Error'), headers) return [b"Filesystem encoding not set to acceptable value.\n"] # Sanity check the request. for (name, status, msg) in _REQUIRED_HTTP_PARAMS: if name not in environ: start_response(status, headers) return [msg.encode('UTF-8')] input_name = environ['wsgi.input'] # Extract the view_name_git by removing the expected git request suffixes path_info = environ['PATH_INFO'] git_suffixes = ['/info/refs', '/HEAD', '/git-upload-pack', '/git-receive-pack'] path_end = len(path_info) for suffix in git_suffixes: try: path_end = path_info.index(suffix) break except ValueError: pass # slice away the leading slash and the trailing git request suffixes view_name_git = path_info[1:path_end] # and remove the view_name_git from the front of PATH_INFO environ['PATH_INFO'] = path_info[path_end:] LOG.debug("new PATH_INFO {0} view_name_git {1}".format(environ['PATH_INFO'], view_name_git)) if not view_name_git: start_response(_('400 Bad Request'), headers) msg = _('Missing required repository name in URL\n') return [msg.encode('UTF-8')] # translate '/' ':' ' ' .. etc .. for internal view_name view_name = p4gf_translate.TranslateReponame.git_to_repo(view_name_git) LOG.debug("public view_name: {0} internal view_name: {1}".format(view_name_git, view_name)) audit_logger = p4gf_server_common.ExceptionAuditLogger() p4_closer = p4gf_create_p4.Closer() sink = OutputSink() temp_deleter = deleting(input_name) mirror_closer = unmirror(view_name) with audit_logger \ , p4_closer \ , sink \ , temp_deleter \ , mirror_closer: LOG.debug(p4gf_log.memory_usage()) start_time = time.time() p4gf_util.reset_git_enviro() p4 = p4gf_create_p4.create_p4() if not p4: start_response(_('500 Internal Server Error'), headers) return [b"Perforce connection failed\n"] LOG.debug("connected to P4: %s", p4) p4gf_server_common.check_readiness(p4) p4gf_server_common.check_lock_perm(p4) if not p4gf_server_common.check_protects(p4): p4gf_server_common.raise_p4gf_perm() user = environ['REMOTE_USER'] if p4gf_server_common.run_special_command(view_name, p4, user): start_response(_('200 OK'), headers) return [sink.readall()] command = _get_command(environ) if not command: start_response(_('400 Bad Request'), headers) return [b"Unrecognized service\n"] # Other places in the Perforce-to-Git phase will need to know the # name of client user, so set that here. As for Git-to-Perforce, # that is handled later by setting the REMOTE_USER envar. Notice # also that we're setting os.environ and not 'environ'. os.environ[p4gf_const.P4GF_AUTH_P4USER] = user # Likewise, some code needs a hint that the request is coming over # one protocol (HTTP) or the other (SSH). os.environ['REMOTE_ADDR'] = environ['REMOTE_ADDR'] # Initialize the external process launcher early, before allocating lots # of memory, and just after all other conditions have been checked. p4gf_proc.init() # Prepare for possible spawn of GitMirror worker process by forking # now before allocating lots of memory. p4gf_gitmirror.setup_spawn(view_name) # Kick off garbage collection debugging, if enabled. p4gf_gc.init_gc() # Go no further, create NOTHING, if user not authorized. # We use the translated internal view name here for perm authorization required_perm = p4gf_server_common.COMMAND_TO_PERM[command] view_perm = p4gf_group.ViewPerm.for_user_and_view(p4, user, view_name, required_perm) try: p4gf_server_common.check_authorization(p4, view_perm, user, command, view_name) except p4gf_server_common.CommandError as ce: start_response(_('403 Forbidden'), headers) return [str(ce).encode('UTF-8')] # Create Git Fusion server depot, user, config. NOPs if already created. p4gf_init.init(p4) before_lock_time = time.time() with p4gf_lock.view_lock(p4, view_name) as view_lock: after_lock_time = time.time() # Create Git Fusion per-repo client view mapping and config. init_repo_status = p4gf_init_repo.init_repo(p4, view_name, view_lock) if init_repo_status == p4gf_init_repo.INIT_REPO_OK: repo_created = True elif init_repo_status == p4gf_init_repo.INIT_REPO_EXISTS: repo_created = False elif init_repo_status == p4gf_init_repo.INIT_REPO_NOVIEW: start_response(_('404 Not Found'), headers) return [sink.readall()] else: start_response(_('500 Internal Server Error'), headers) return [b"Repository initialization failed\n"] # If authorization came from default, not explicit group # membership, copy that authorization to a group now. Could # not do this until after p4gf_init_repo() has a chance to # create not-yet-existing groups. if view_perm: view_perm.write_if(p4) # Now that we have valid git-fusion-user and # git-fusion-<view> client, replace our temporary P4 # connection with a more permanent Context, shared for the # remainder of this process. with p4gf_context.create_context(view_name, view_lock) as ctx: LOG.debug("reconnected to P4, p4gf=%s", ctx.p4gf) ctx.log_context() # cd into the work directory. Not all git functions react well # to --work-tree=xxxx. cwd = os.getcwd() os.chdir(ctx.view_dirs.GIT_WORK_TREE) # Only copy from Perforce to Git if no other process is cloning # from this Git repo right now. shared_in_progress = p4gf_lock.shared_host_view_lock_exists(ctx.p4, view_name) if not shared_in_progress: # Copy any recent changes from Perforce to Git. try: LOG.debug("bare: No git-upload-pack in progress, force non-bare" " before update Git from Perforce.") p4gf_git.set_bare(False) p4gf_copy_p2g.copy_p2g_ctx(ctx) p4gf_init_repo.process_imports(ctx) # Now is also an appropriate time to clear out any stale Git # Swarm reviews. We're pre-pull, pre-push, time when we've # got exclusive write access to the Git repo, GSReviewCollection.delete_refs_for_closed_reviews(ctx) except p4gf_lock.LockCanceled as lc: LOG.warning(str(lc)) except: # Dump failure to log, BEFORE cleanup, just in case # cleanup ALSO fails and throws its own error (which # happens if we're out of memory). LOG.error(traceback.format_exc()) if repo_created: # Return to the original working directory to allow the # config code to call os.getcwd() without dying, since # we are about to delete the current working directory. os.chdir(cwd) p4gf_server_common.cleanup_client(ctx, view_name) raise try: exclusive = 'upload' not in command is_push = 'upload' not in command git_caller = functools.partial(_call_git, input_name, environ, ctx) p4gf_call_git.call_git(git_caller, ctx, view_name, view_lock, exclusive) if is_push: GSReviewCollection.post_push(ctx) except p4gf_atomic_lock.LockConflict as lc: start_response(_('500 Internal Server Error'), headers) return ["{}".format(lc).encode('UTF-8')] p4gf_gc.process_garbage('at end of auth_server') if LOG.isEnabledFor(logging.DEBUG): end_time = time.time() frm = NTR('Runtime: preparation {} ms, lock acquisition {} ms, processing {} ms') LOG.debug(frm.format(before_lock_time - start_time, after_lock_time - before_lock_time, end_time - after_lock_time)) return []
def main(): """set up repo for a view""" p4gf_util.has_server_id_or_exit() args = _parse_argv() p4gf_version.log_version() log_l10n() # !!! view_name_git the untranslated repo name # !!! view_name the translated repo name view_name_p4client = None if args.p4client: view_name_p4client = p4gf_util.argv_to_view_name(args.p4client) view_name_git = p4gf_util.argv_to_view_name(args.view) #strip leading '/' to conform with p4gf_auth_server behavior if view_name_git[0] == '/': view_name_git = view_name_git[1:] view_name = p4gf_translate.TranslateReponame.git_to_repo(view_name_git) p4gf_gitmirror.setup_spawn(view_name) p4gf_util.reset_git_enviro() p4 = p4gf_create_p4.create_p4() if not p4: return INIT_REPO_NOVIEW LOG.debug("connected to P4 at %s", p4.port) p4gf_proc.init() try: with p4gf_create_p4.Closer(): p4gf_version.version_check() with p4gf_lock.view_lock(p4, view_name) as view_lock: # Ensure we have a sane environment. p4gf_init.init(p4) # Now that we can trust that the git-fusion--p4 client exists, # switch to that. Change takes effect immediately, don't need to # re-run p4.connect(). p4.client = p4gf_util.get_object_client_name() # If local config file specified, validate it and store in # Perforce now. Even if client exists (aka repo was already # inited), this is one way for an admin to modify an existing # repo's config. if args.config: if not os.path.exists(args.config): _print_stderr(_("error: missing config file '{}'").format(args.config)) return INIT_REPO_CONFIG_FILE_MISSING with Validator.from_local_file(view_name, p4, args.config) as validator: if not validator.is_valid(args.enablemismatchedrhs): return INIT_REPO_CONFIG_FILE_BAD p4gf_config.create_file_repo_with_contents(p4, view_name, args.config) elif args.charset and not Validator.valid_charset(args.charset): _print_stderr(_("error: invalid charset: {}").format(args.charset)) return INIT_REPO_BAD_CHARSET # Initialize the repository if necessary. print(_("Initializing '{}'...").format(view_name)) r = init_repo(p4, view_name, view_lock, args.charset, args.enablemismatchedrhs, view_name_p4client) if r > INIT_REPO_OK: return r print(_("Initialization complete.")) # Write --enablemismatchedrhs to config file if args.enablemismatchedrhs: config = p4gf_config.read_repo(p4, view_name) config[p4gf_config.SECTION_REPO]\ [p4gf_config.KEY_ENABLE_MISMATCHED_RHS] = str(True) p4gf_config.write_repo_if(p4, p4.fetch_client(), view_name, config) # Populate the repo from Perforce unless --noclone. if not args.noclone: return populate_repo(view_name, view_lock, args.start) except P4.P4Exception as e: _print_stderr(_('Error occurred: {}').format(e)) return INIT_REPO_EXISTS
def main(poll_only=False): """set up repo for a view view_name_git is the untranslated repo name view_name is the translated repo name """ p4gf_proc.install_stack_dumper() _log_environ(os.environ) with p4gf_server_common.ExceptionAuditLogger()\ , p4gf_create_p4.Closer(): LOG.debug(p4gf_log.memory_usage()) start_time = time.time() args = parse_args(sys.argv[1:]) if not args: return 1 is_push = 'upload' not in args.command[0] # Record the p4 user in environment. We use environment to pass to # git-invoked hook. We don't have to set ctx.authenticated_p4user because # Context.__init__() reads it from environment, which we set here. os.environ[p4gf_const.P4GF_AUTH_P4USER] = args.user # view_name_git is the untranslated repo name # view_name is the translated repo name # print "args={}".format(args) view_name_git = args.options[-1] # translate '/' ':' ' ' .. etc .. for internal view_name view_name = p4gf_translate.TranslateReponame.git_to_repo(view_name_git) LOG.debug("public view_name: {0} internal view_name: {1}".format( view_name_git, view_name)) p4gf_util.reset_git_enviro() p4 = p4gf_create_p4.create_p4() if not p4: return 2 LOG.debug("connected to P4: %s", p4) p4gf_server_common.check_readiness(p4) p4gf_server_common.check_lock_perm(p4) if not p4gf_server_common.check_protects(p4): p4gf_server_common.raise_p4gf_perm() if p4gf_server_common.run_special_command(view_name, p4, args.user): return 0 # Initialize the external process launcher early, before allocating lots # of memory, and just after all other conditions have been checked. p4gf_proc.init() # Prepare for possible spawn of GitMirror worker process by forking # now before allocating lots of memory. p4gf_gitmirror.setup_spawn(view_name) # Kick off garbage collection debugging, if enabled. p4gf_gc.init_gc() if poll_only: view_perm = None else: # Go no further, create NOTHING, if user not authorized. # We use the translated internal view name here for perm authorization required_perm = p4gf_server_common.COMMAND_TO_PERM[args.command[0]] view_perm = p4gf_group.ViewPerm.for_user_and_view( p4, args.user, view_name, required_perm) p4gf_server_common.check_authorization(p4, view_perm, args.user, args.command[0], view_name) # Create Git Fusion server depot, user, config. NOPs if already created. p4gf_init.init(p4) write_motd() # view_name is the internal view_name (identical when notExist special chars) before_lock_time = time.time() with p4gf_lock.view_lock(p4, view_name) as view_lock: after_lock_time = time.time() # Create Git Fusion per-repo client view mapping and config. # # NOPs if already created. # Create the empty directory that will hold the git repo. init_repo_status = p4gf_init_repo.init_repo( p4, view_name, view_lock) if init_repo_status == p4gf_init_repo.INIT_REPO_OK: repo_created = True elif init_repo_status == p4gf_init_repo.INIT_REPO_EXISTS: repo_created = False else: return 1 # If authorization came from default, not explicit group # membership, copy that authorization to a group now. Could # not do this until after p4gf_init_repo() has a chance to # create not-yet-existing groups. if view_perm: view_perm.write_if(p4) # Now that we have valid git-fusion-user and # git-fusion-<view> client, replace our temporary P4 # connection with a more permanent Context, shared for the # remainder of this process. with p4gf_context.create_context(view_name, view_lock) as ctx: LOG.debug("reconnected to P4, p4gf=%s", ctx.p4gf) # Find directory paths to feed to git. ctx.log_context() # cd into the work directory. Not all git functions react well # to --work-tree=xxxx. cwd = os.getcwd() os.chdir(ctx.view_dirs.GIT_WORK_TREE) # Only copy from Perforce to Git if no other process is cloning # from this Git repo right now. shared_in_progress = p4gf_lock.shared_host_view_lock_exists( ctx.p4, view_name) if not shared_in_progress: # Copy any recent changes from Perforce to Git. try: LOG.debug( "bare: No git-upload-pack in progress, force non-bare" " before update Git from Perforce.") p4gf_git.set_bare(False) p4gf_copy_p2g.copy_p2g_ctx(ctx) p4gf_init_repo.process_imports(ctx) # Now is also an appropriate time to clear out any stale Git # Swarm reviews. We're pre-pull, pre-push, time when we've # got exclusive write access to the Git repo, GSReviewCollection.delete_refs_for_closed_reviews(ctx) except p4gf_lock.LockCanceled as lc: LOG.warning(str(lc)) except: # Dump failure to log, BEFORE cleanup, just in case # cleanup ALSO fails and throws its own error (which # happens if we're out of memory). LOG.error(traceback.format_exc()) if repo_created: # Return to the original working directory to allow the # config code to call os.getcwd() without dying, since # we are about to delete the current working directory. os.chdir(cwd) p4gf_server_common.cleanup_client(ctx, view_name) raise if poll_only: code = os.EX_OK else: git_caller = functools.partial(_call_git, args, ctx) try: # Deep in call_git(), we grab an 'p4 reviews' lock on # ctx.clientmap's LHS. Switch that clientmap to our # full union view to prevent simultaneous 'git push'es # from clobbering each other in some shared depot # branch. Must include all lightweight branches, too. ctx.switch_client_view_to_union() exclusive = 'upload' not in args.command[0] code = p4gf_call_git.call_git(git_caller, ctx, view_name, view_lock, exclusive) if is_push: GSReviewCollection.post_push(ctx) except p4gf_atomic_lock.LockConflict as lc: sys.stderr.write("{}\n".format(lc)) code = os.EX_SOFTWARE p4gf_gc.process_garbage(NTR('at end of auth_server')) if LOG.isEnabledFor(logging.DEBUG): end_time = time.time() frm = NTR("Runtime: preparation {} ms, lock acquisition {} ms," " processing {} ms") LOG.debug( frm.format(before_lock_time - start_time, after_lock_time - before_lock_time, end_time - after_lock_time)) return code
def main(): """set up repo for a view""" p4gf_util.has_server_id_or_exit() args = _parse_argv() p4gf_version.log_version() log_l10n() # !!! view_name_git the untranslated repo name # !!! view_name the translated repo name view_name_p4client = None if args.p4client: view_name_p4client = p4gf_util.argv_to_view_name(args.p4client) view_name_git = p4gf_util.argv_to_view_name(args.view) #strip leading '/' to conform with p4gf_auth_server behavior if view_name_git[0] == '/': view_name_git = view_name_git[1:] view_name = p4gf_translate.TranslateReponame.git_to_repo(view_name_git) p4gf_gitmirror.setup_spawn(view_name) p4gf_util.reset_git_enviro() p4 = p4gf_create_p4.create_p4() if not p4: return INIT_REPO_NOVIEW LOG.debug("connected to P4 at %s", p4.port) p4gf_proc.init() try: with p4gf_create_p4.Closer(): p4gf_version.version_check() with p4gf_lock.view_lock(p4, view_name) as view_lock: # Ensure we have a sane environment. p4gf_init.init(p4) # Now that we can trust that the git-fusion--p4 client exists, # switch to that. Change takes effect immediately, don't need to # re-run p4.connect(). p4.client = p4gf_util.get_object_client_name() # If local config file specified, validate it and store in # Perforce now. Even if client exists (aka repo was already # inited), this is one way for an admin to modify an existing # repo's config. if args.config: if not os.path.exists(args.config): _print_stderr( _("error: missing config file '{}'").format( args.config)) return INIT_REPO_CONFIG_FILE_MISSING with Validator.from_local_file(view_name, p4, args.config) as validator: if not validator.is_valid(args.enablemismatchedrhs): return INIT_REPO_CONFIG_FILE_BAD p4gf_config.create_file_repo_with_contents( p4, view_name, args.config) elif args.charset and not Validator.valid_charset( args.charset): _print_stderr( _("error: invalid charset: {}").format(args.charset)) return INIT_REPO_BAD_CHARSET # Initialize the repository if necessary. print(_("Initializing '{}'...").format(view_name)) r = init_repo(p4, view_name, view_lock, args.charset, args.enablemismatchedrhs, view_name_p4client) if r > INIT_REPO_OK: return r print(_("Initialization complete.")) # Write --enablemismatchedrhs to config file if args.enablemismatchedrhs: config = p4gf_config.read_repo(p4, view_name) config[p4gf_config.SECTION_REPO]\ [p4gf_config.KEY_ENABLE_MISMATCHED_RHS] = str(True) p4gf_config.write_repo_if(p4, p4.fetch_client(), view_name, config) # Populate the repo from Perforce unless --noclone. if not args.noclone: return populate_repo(view_name, view_lock, args.start) except P4.P4Exception as e: _print_stderr(_('Error occurred: {}').format(e)) return INIT_REPO_EXISTS
def main(): """set up repo for a view""" with ExceptionAuditLogger(): args = parse_args(sys.argv[1:]) if not args: return 1 # Record the p4 user in environment. We use environment to pass to # git-invoked hook. We don't have to set ctx.authenticated_p4user because # Context.__init__() reads it from environment, which we set here. os.environ[p4gf_const.P4GF_AUTH_P4USER_ENVAR] = args.user # print "args={}".format(args) view_name = args.options[-1] p4gf_util.reset_git_enviro() p4 = connect_p4() if not p4: return 2 LOG.debug("connected to P4: %s", p4) _check_lock_perm(p4) if not check_protects(p4): _raise_p4gf_perm() if run_special_command(view_name, p4, args.user): return 0 # Go no further, create NOTHING, if user not authorized. view_perm = p4gf_group.ViewPerm.for_user_and_view(p4, args.user, view_name) _check_authorization(view_perm, args.user, args.command[0], view_name) # Create Git Fusion server depot, user, config. NOPs if already created. p4gf_init.init(p4) with p4gf_lock.view_lock(p4, view_name) as view_lock: # Create Git Fusion per-repo client view mapping and config. # # NOPs if already created. # Create the empty directory that will hold the git repo. init_repo_status = p4gf_init_repo.init_repo(p4, view_name) if init_repo_status == p4gf_init_repo.INIT_REPO_OK: repo_created = True elif init_repo_status == p4gf_init_repo.INIT_REPO_EXISTS: repo_created = False else: return 1 # If authorization came from default, not explicit group # membership, copy that authorization to a group now. Could # not do this until after p4gf_init_repo() has a chance to # create not-yet-existing groups. view_perm.write_if(p4) # Now that we have valid git-fusion-user and # git-fusion-<view> client, replace our temporary P4 # connection with a more permanent Context, shared for the # remainder of this process. ctx = p4gf_context.create_context(view_name, view_lock) del p4 LOG.debug("reconnected to P4, p4gf=%s", ctx.p4gf) # Find directory paths to feed to git. ctx.view_dirs = p4gf_view_dirs.from_p4gf_dir(ctx.gitrootdir, view_name) ctx.log_context() # cd into the work directory. Not all git functions react well # to --work-tree=xxxx. cwd = os.getcwd() os.chdir(ctx.view_dirs.GIT_WORK_TREE) # Copy any recent changes from Perforce to Git. try: p4gf_copy_p2g.copy_p2g_ctx(ctx) except: # Dump failure to log, BEFORE cleanup, just in case # cleanup ALSO fails and throws its own error (which # happens if we're out of memory). LOG.error(traceback.format_exc()) if repo_created: # Return to the original working directory to allow the # config code to call os.getcwd() without dying, since # we are about to delete the current working directory. os.chdir(cwd) cleanup_client(ctx, view_name) raise # Detach git repo's workspace from master before calling # original git, otherwise we won't be able to push master. p4gf_util.checkout_detached_master() # Flush stderr before returning control to Git. # Otherwise Git's own output might interrupt ours. sys.stderr.flush() return _call_original_git(ctx, args)
def _wsgi_app(environ, start_response): """ WSGI application to process the incoming Git client request. This is nearly equivalent to p4gf_auth_server.main() with the exception of input validation and error handling. """ p4gf_log.record_http(environ) p4gf_version.log_version() _log_environ(environ) p4gf_version.version_check() LOG.debug("processing HTTP request, pid={}".format(os.getpid())) # Keep the content type to exactly 'text/plain' so there is at least # the remote chance that Git might show our error messages (does not # appear to work in practice, however). headers = [('Content-Type', 'text/plain')] encoding = sys.getfilesystemencoding() if encoding == 'ascii': # This encoding is wrong and will eventually lead to problems. LOG.error( "Using 'ascii' file encoding will ultimately result in errors, " "please set LANG/LC_ALL to 'utf-8' in web server configuration.") start_response(_('500 Internal Server Error'), headers) return [b"Filesystem encoding not set to acceptable value.\n"] # Sanity check the request. for (name, status, msg) in _REQUIRED_HTTP_PARAMS: if name not in environ: start_response(status, headers) return [msg.encode('UTF-8')] input_name = environ['wsgi.input'] # Extract the view_name_git by removing the expected git request suffixes path_info = environ['PATH_INFO'] git_suffixes = [ '/info/refs', '/HEAD', '/git-upload-pack', '/git-receive-pack' ] path_end = len(path_info) for suffix in git_suffixes: try: path_end = path_info.index(suffix) break except ValueError: pass # slice away the leading slash and the trailing git request suffixes view_name_git = path_info[1:path_end] # and remove the view_name_git from the front of PATH_INFO environ['PATH_INFO'] = path_info[path_end:] LOG.debug("new PATH_INFO {0} view_name_git {1}".format( environ['PATH_INFO'], view_name_git)) if not view_name_git: start_response(_('400 Bad Request'), headers) msg = _('Missing required repository name in URL\n') return [msg.encode('UTF-8')] # translate '/' ':' ' ' .. etc .. for internal view_name view_name = p4gf_translate.TranslateReponame.git_to_repo(view_name_git) LOG.debug("public view_name: {0} internal view_name: {1}".format( view_name_git, view_name)) audit_logger = p4gf_server_common.ExceptionAuditLogger() p4_closer = p4gf_create_p4.Closer() sink = OutputSink() temp_deleter = deleting(input_name) mirror_closer = unmirror(view_name) with audit_logger \ , p4_closer \ , sink \ , temp_deleter \ , mirror_closer: LOG.debug(p4gf_log.memory_usage()) start_time = time.time() p4gf_util.reset_git_enviro() p4 = p4gf_create_p4.create_p4() if not p4: start_response(_('500 Internal Server Error'), headers) return [b"Perforce connection failed\n"] LOG.debug("connected to P4: %s", p4) p4gf_server_common.check_readiness(p4) p4gf_server_common.check_lock_perm(p4) if not p4gf_server_common.check_protects(p4): p4gf_server_common.raise_p4gf_perm() user = environ['REMOTE_USER'] if p4gf_server_common.run_special_command(view_name, p4, user): start_response(_('200 OK'), headers) return [sink.readall()] command = _get_command(environ) if not command: start_response(_('400 Bad Request'), headers) return [b"Unrecognized service\n"] # Other places in the Perforce-to-Git phase will need to know the # name of client user, so set that here. As for Git-to-Perforce, # that is handled later by setting the REMOTE_USER envar. Notice # also that we're setting os.environ and not 'environ'. os.environ[p4gf_const.P4GF_AUTH_P4USER] = user # Likewise, some code needs a hint that the request is coming over # one protocol (HTTP) or the other (SSH). os.environ['REMOTE_ADDR'] = environ['REMOTE_ADDR'] # Initialize the external process launcher early, before allocating lots # of memory, and just after all other conditions have been checked. p4gf_proc.init() # Prepare for possible spawn of GitMirror worker process by forking # now before allocating lots of memory. p4gf_gitmirror.setup_spawn(view_name) # Kick off garbage collection debugging, if enabled. p4gf_gc.init_gc() # Go no further, create NOTHING, if user not authorized. # We use the translated internal view name here for perm authorization required_perm = p4gf_server_common.COMMAND_TO_PERM[command] view_perm = p4gf_group.ViewPerm.for_user_and_view( p4, user, view_name, required_perm) try: p4gf_server_common.check_authorization(p4, view_perm, user, command, view_name) except p4gf_server_common.CommandError as ce: start_response(_('403 Forbidden'), headers) return [str(ce).encode('UTF-8')] # Create Git Fusion server depot, user, config. NOPs if already created. p4gf_init.init(p4) before_lock_time = time.time() with p4gf_lock.view_lock(p4, view_name) as view_lock: after_lock_time = time.time() # Create Git Fusion per-repo client view mapping and config. init_repo_status = p4gf_init_repo.init_repo( p4, view_name, view_lock) if init_repo_status == p4gf_init_repo.INIT_REPO_OK: repo_created = True elif init_repo_status == p4gf_init_repo.INIT_REPO_EXISTS: repo_created = False elif init_repo_status == p4gf_init_repo.INIT_REPO_NOVIEW: start_response(_('404 Not Found'), headers) return [sink.readall()] else: start_response(_('500 Internal Server Error'), headers) return [b"Repository initialization failed\n"] # If authorization came from default, not explicit group # membership, copy that authorization to a group now. Could # not do this until after p4gf_init_repo() has a chance to # create not-yet-existing groups. if view_perm: view_perm.write_if(p4) # Now that we have valid git-fusion-user and # git-fusion-<view> client, replace our temporary P4 # connection with a more permanent Context, shared for the # remainder of this process. with p4gf_context.create_context(view_name, view_lock) as ctx: LOG.debug("reconnected to P4, p4gf=%s", ctx.p4gf) ctx.log_context() # cd into the work directory. Not all git functions react well # to --work-tree=xxxx. cwd = os.getcwd() os.chdir(ctx.view_dirs.GIT_WORK_TREE) # Only copy from Perforce to Git if no other process is cloning # from this Git repo right now. shared_in_progress = p4gf_lock.shared_host_view_lock_exists( ctx.p4, view_name) if not shared_in_progress: # Copy any recent changes from Perforce to Git. try: LOG.debug( "bare: No git-upload-pack in progress, force non-bare" " before update Git from Perforce.") p4gf_git.set_bare(False) p4gf_copy_p2g.copy_p2g_ctx(ctx) p4gf_init_repo.process_imports(ctx) # Now is also an appropriate time to clear out any stale Git # Swarm reviews. We're pre-pull, pre-push, time when we've # got exclusive write access to the Git repo, GSReviewCollection.delete_refs_for_closed_reviews(ctx) except p4gf_lock.LockCanceled as lc: LOG.warning(str(lc)) except: # Dump failure to log, BEFORE cleanup, just in case # cleanup ALSO fails and throws its own error (which # happens if we're out of memory). LOG.error(traceback.format_exc()) if repo_created: # Return to the original working directory to allow the # config code to call os.getcwd() without dying, since # we are about to delete the current working directory. os.chdir(cwd) p4gf_server_common.cleanup_client(ctx, view_name) raise try: exclusive = 'upload' not in command is_push = 'upload' not in command git_caller = functools.partial(_call_git, input_name, environ, ctx) p4gf_call_git.call_git(git_caller, ctx, view_name, view_lock, exclusive) if is_push: GSReviewCollection.post_push(ctx) except p4gf_atomic_lock.LockConflict as lc: start_response(_('500 Internal Server Error'), headers) return ["{}".format(lc).encode('UTF-8')] p4gf_gc.process_garbage('at end of auth_server') if LOG.isEnabledFor(logging.DEBUG): end_time = time.time() frm = NTR( 'Runtime: preparation {} ms, lock acquisition {} ms, processing {} ms' ) LOG.debug( frm.format(before_lock_time - start_time, after_lock_time - before_lock_time, end_time - after_lock_time)) return []