def full_init(self, repo_name_p4client=None):
"""Initialize the repo."""
# Ensure we have a sane environment.
p4gf_init.init(self.p4)
self._get_config()
# Initialize the repository if necessary.
print(
_("Initializing '{repo_name}'...").format(
repo_name=self.repo_name))
self.init_repo(repo_name_p4client)
print(_("Initialization complete."))
if not self.noclone:
try:
start_at = int(self.start.lstrip('@')) if self.start else 1
self._copy_p2g_with_start(start_at)
except ValueError:
raise RuntimeError(
_('Invalid --start value: {start}').format(
start=self.start))
except IndexError:
raise RuntimeError(
_("Could not find changes >= '{start_at}'").format(
start_at=start_at))
def main():
"""set up repo for a view"""
parser = p4gf_util.create_arg_parser(
"Initializes Git Fusion Perforce client and Git repository.")
parser.add_argument('--start', metavar="",
help='Changelist number to start repo history, default=1')
parser.add_argument('view', metavar='view',
help='name of view to be initialized')
args = parser.parse_args()
p4gf_version.log_version()
view_name = p4gf_util.argv_to_view_name(args.view)
p4gf_util.reset_git_enviro()
p4 = connect_p4()
if not p4:
return 2
LOG.debug("connected to P4 at %s", p4.port)
try:
with p4gf_lock.view_lock(p4, view_name) as view_lock:
# ensure we have a sane environment
p4gf_init.init(p4)
# now initialize the repository
print("Initializing {}...".format(view_name))
r = init_repo(p4, view_name)
if r > INIT_REPO_OK:
return r
print("Initialization complete.")
if args.start:
start = args.start.lstrip('@')
print("Copying changes from {}...".format(start))
copy_p2g_with_start(view_name, start, view_lock)
print("Copying completed.")
except P4.P4Exception as e:
sys.stderr.write("Error occurred: {}\n".format(e))
return 0
def main():
"""set up repo for a view"""
with ExceptionAuditLogger():
args = parse_args(sys.argv[1:])
if not args:
return 1
# Record the p4 user in environment. We use environment to pass to
# git-invoked hook. We don't have to set ctx.authenticated_p4user because
# Context.__init__() reads it from environment, which we set here.
os.environ[p4gf_const.P4GF_AUTH_P4USER_ENVAR] = args.user
# print "args={}".format(args)
view_name = args.options[-1]
p4gf_util.reset_git_enviro()
p4 = connect_p4()
if not p4:
return 2
LOG.debug("connected to P4: %s", p4)
_check_lock_perm(p4)
if not check_protects(p4):
_raise_p4gf_perm()
if run_special_command(view_name, p4, args.user):
return 0
# Go no further, create NOTHING, if user not authorized.
view_perm = p4gf_group.ViewPerm.for_user_and_view(
p4, args.user, view_name)
_check_authorization(view_perm, args.user, args.command[0], view_name)
# Create Git Fusion server depot, user, config. NOPs if already created.
p4gf_init.init(p4)
with p4gf_lock.view_lock(p4, view_name) as view_lock:
# Create Git Fusion per-repo client view mapping and config.
#
# NOPs if already created.
# Create the empty directory that will hold the git repo.
init_repo_status = p4gf_init_repo.init_repo(p4, view_name)
if init_repo_status == p4gf_init_repo.INIT_REPO_OK:
repo_created = True
elif init_repo_status == p4gf_init_repo.INIT_REPO_EXISTS:
repo_created = False
else:
return 1
# If authorization came from default, not explicit group
# membership, copy that authorization to a group now. Could
# not do this until after p4gf_init_repo() has a chance to
# create not-yet-existing groups.
view_perm.write_if(p4)
# Now that we have valid git-fusion-user and
# git-fusion-<view> client, replace our temporary P4
# connection with a more permanent Context, shared for the
# remainder of this process.
ctx = p4gf_context.create_context(view_name, view_lock)
del p4
LOG.debug("reconnected to P4, p4gf=%s", ctx.p4gf)
# Find directory paths to feed to git.
ctx.view_dirs = p4gf_view_dirs.from_p4gf_dir(
ctx.gitrootdir, view_name)
ctx.log_context()
# cd into the work directory. Not all git functions react well
# to --work-tree=xxxx.
cwd = os.getcwd()
os.chdir(ctx.view_dirs.GIT_WORK_TREE)
# Copy any recent changes from Perforce to Git.
try:
p4gf_copy_p2g.copy_p2g_ctx(ctx)
except:
# Dump failure to log, BEFORE cleanup, just in case
# cleanup ALSO fails and throws its own error (which
# happens if we're out of memory).
LOG.error(traceback.format_exc())
if repo_created:
# Return to the original working directory to allow the
# config code to call os.getcwd() without dying, since
# we are about to delete the current working directory.
os.chdir(cwd)
cleanup_client(ctx, view_name)
raise
# Detach git repo's workspace from master before calling
# original git, otherwise we won't be able to push master.
p4gf_util.checkout_detached_master()
# Flush stderr before returning control to Git.
# Otherwise Git's own output might interrupt ours.
sys.stderr.flush()
return _call_original_git(ctx, args)
def main(poll_only=False):
"""set up repo for a view
view_name_git is the untranslated repo name
view_name is the translated repo name
"""
p4gf_proc.install_stack_dumper()
_log_environ(os.environ)
with p4gf_server_common.ExceptionAuditLogger()\
, p4gf_create_p4.Closer():
LOG.debug(p4gf_log.memory_usage())
start_time = time.time()
args = parse_args(sys.argv[1:])
if not args:
return 1
is_push = 'upload' not in args.command[0]
# Record the p4 user in environment. We use environment to pass to
# git-invoked hook. We don't have to set ctx.authenticated_p4user because
# Context.__init__() reads it from environment, which we set here.
os.environ[p4gf_const.P4GF_AUTH_P4USER] = args.user
# view_name_git is the untranslated repo name
# view_name is the translated repo name
# print "args={}".format(args)
view_name_git = args.options[-1]
# translate '/' ':' ' ' .. etc .. for internal view_name
view_name = p4gf_translate.TranslateReponame.git_to_repo(view_name_git)
LOG.debug("public view_name: {0} internal view_name: {1}".
format(view_name_git, view_name))
p4gf_util.reset_git_enviro()
p4 = p4gf_create_p4.create_p4()
if not p4:
return 2
LOG.debug("connected to P4: %s", p4)
p4gf_server_common.check_readiness(p4)
p4gf_server_common.check_lock_perm(p4)
if not p4gf_server_common.check_protects(p4):
p4gf_server_common.raise_p4gf_perm()
if p4gf_server_common.run_special_command(view_name, p4, args.user):
return 0
# Initialize the external process launcher early, before allocating lots
# of memory, and just after all other conditions have been checked.
p4gf_proc.init()
# Prepare for possible spawn of GitMirror worker process by forking
# now before allocating lots of memory.
p4gf_gitmirror.setup_spawn(view_name)
# Kick off garbage collection debugging, if enabled.
p4gf_gc.init_gc()
if poll_only:
view_perm = None
else:
# Go no further, create NOTHING, if user not authorized.
# We use the translated internal view name here for perm authorization
required_perm = p4gf_server_common.COMMAND_TO_PERM[args.command[0]]
view_perm = p4gf_group.ViewPerm.for_user_and_view(p4, args.user,
view_name, required_perm)
p4gf_server_common.check_authorization(p4, view_perm, args.user, args.command[0],
view_name)
# Create Git Fusion server depot, user, config. NOPs if already created.
p4gf_init.init(p4)
write_motd()
# view_name is the internal view_name (identical when notExist special chars)
before_lock_time = time.time()
with p4gf_lock.view_lock(p4, view_name) as view_lock:
after_lock_time = time.time()
# Create Git Fusion per-repo client view mapping and config.
#
# NOPs if already created.
# Create the empty directory that will hold the git repo.
init_repo_status = p4gf_init_repo.init_repo(p4, view_name, view_lock)
if init_repo_status == p4gf_init_repo.INIT_REPO_OK:
repo_created = True
elif init_repo_status == p4gf_init_repo.INIT_REPO_EXISTS:
repo_created = False
else:
return 1
# If authorization came from default, not explicit group
# membership, copy that authorization to a group now. Could
# not do this until after p4gf_init_repo() has a chance to
# create not-yet-existing groups.
if view_perm:
view_perm.write_if(p4)
# Now that we have valid git-fusion-user and
# git-fusion-<view> client, replace our temporary P4
# connection with a more permanent Context, shared for the
# remainder of this process.
with p4gf_context.create_context(view_name, view_lock) as ctx:
LOG.debug("reconnected to P4, p4gf=%s", ctx.p4gf)
# Find directory paths to feed to git.
ctx.log_context()
# cd into the work directory. Not all git functions react well
# to --work-tree=xxxx.
cwd = os.getcwd()
os.chdir(ctx.view_dirs.GIT_WORK_TREE)
# Only copy from Perforce to Git if no other process is cloning
# from this Git repo right now.
shared_in_progress = p4gf_lock.shared_host_view_lock_exists(ctx.p4, view_name)
if not shared_in_progress:
# Copy any recent changes from Perforce to Git.
try:
LOG.debug("bare: No git-upload-pack in progress, force non-bare"
" before update Git from Perforce.")
p4gf_git.set_bare(False)
p4gf_copy_p2g.copy_p2g_ctx(ctx)
p4gf_init_repo.process_imports(ctx)
# Now is also an appropriate time to clear out any stale Git
# Swarm reviews. We're pre-pull, pre-push, time when we've
# got exclusive write access to the Git repo,
GSReviewCollection.delete_refs_for_closed_reviews(ctx)
except p4gf_lock.LockCanceled as lc:
LOG.warning(str(lc))
except:
# Dump failure to log, BEFORE cleanup, just in case
# cleanup ALSO fails and throws its own error (which
# happens if we're out of memory).
LOG.error(traceback.format_exc())
if repo_created:
# Return to the original working directory to allow the
# config code to call os.getcwd() without dying, since
# we are about to delete the current working directory.
os.chdir(cwd)
p4gf_server_common.cleanup_client(ctx, view_name)
raise
if poll_only:
code = os.EX_OK
else:
git_caller = functools.partial(_call_git, args, ctx)
try:
# Deep in call_git(), we grab an 'p4 reviews' lock on
# ctx.clientmap's LHS. Switch that clientmap to our
# full union view to prevent simultaneous 'git push'es
# from clobbering each other in some shared depot
# branch. Must include all lightweight branches, too.
ctx.switch_client_view_to_union()
exclusive = 'upload' not in args.command[0]
code = p4gf_call_git.call_git(
git_caller, ctx, view_name, view_lock, exclusive)
if is_push:
GSReviewCollection.post_push(ctx)
except p4gf_atomic_lock.LockConflict as lc:
sys.stderr.write("{}\n".format(lc))
code = os.EX_SOFTWARE
p4gf_gc.process_garbage(NTR('at end of auth_server'))
if LOG.isEnabledFor(logging.DEBUG):
end_time = time.time()
frm = NTR("Runtime: preparation {} ms, lock acquisition {} ms,"
" processing {} ms")
LOG.debug(frm.format(before_lock_time - start_time,
after_lock_time - before_lock_time,
end_time - after_lock_time))
return code
def _wsgi_app(environ, start_response):
"""
WSGI application to process the incoming Git client request. This is
nearly equivalent to p4gf_auth_server.main() with the exception of
input validation and error handling.
"""
p4gf_log.record_http(environ)
p4gf_version.log_version()
_log_environ(environ)
p4gf_version.version_check()
LOG.debug("processing HTTP request, pid={}".format(os.getpid()))
# Keep the content type to exactly 'text/plain' so there is at least
# the remote chance that Git might show our error messages (does not
# appear to work in practice, however).
headers = [('Content-Type', 'text/plain')]
encoding = sys.getfilesystemencoding()
if encoding == 'ascii':
# This encoding is wrong and will eventually lead to problems.
LOG.error("Using 'ascii' file encoding will ultimately result in errors, "
"please set LANG/LC_ALL to 'utf-8' in web server configuration.")
start_response(_('500 Internal Server Error'), headers)
return [b"Filesystem encoding not set to acceptable value.\n"]
# Sanity check the request.
for (name, status, msg) in _REQUIRED_HTTP_PARAMS:
if name not in environ:
start_response(status, headers)
return [msg.encode('UTF-8')]
input_name = environ['wsgi.input']
# Extract the view_name_git by removing the expected git request suffixes
path_info = environ['PATH_INFO']
git_suffixes = ['/info/refs', '/HEAD', '/git-upload-pack', '/git-receive-pack']
path_end = len(path_info)
for suffix in git_suffixes:
try:
path_end = path_info.index(suffix)
break
except ValueError:
pass
# slice away the leading slash and the trailing git request suffixes
view_name_git = path_info[1:path_end]
# and remove the view_name_git from the front of PATH_INFO
environ['PATH_INFO'] = path_info[path_end:]
LOG.debug("new PATH_INFO {0} view_name_git {1}".format(environ['PATH_INFO'], view_name_git))
if not view_name_git:
start_response(_('400 Bad Request'), headers)
msg = _('Missing required repository name in URL\n')
return [msg.encode('UTF-8')]
# translate '/' ':' ' ' .. etc .. for internal view_name
view_name = p4gf_translate.TranslateReponame.git_to_repo(view_name_git)
LOG.debug("public view_name: {0} internal view_name: {1}".format(view_name_git, view_name))
audit_logger = p4gf_server_common.ExceptionAuditLogger()
p4_closer = p4gf_create_p4.Closer()
sink = OutputSink()
temp_deleter = deleting(input_name)
mirror_closer = unmirror(view_name)
with audit_logger \
, p4_closer \
, sink \
, temp_deleter \
, mirror_closer:
LOG.debug(p4gf_log.memory_usage())
start_time = time.time()
p4gf_util.reset_git_enviro()
p4 = p4gf_create_p4.create_p4()
if not p4:
start_response(_('500 Internal Server Error'), headers)
return [b"Perforce connection failed\n"]
LOG.debug("connected to P4: %s", p4)
p4gf_server_common.check_readiness(p4)
p4gf_server_common.check_lock_perm(p4)
if not p4gf_server_common.check_protects(p4):
p4gf_server_common.raise_p4gf_perm()
user = environ['REMOTE_USER']
if p4gf_server_common.run_special_command(view_name, p4, user):
start_response(_('200 OK'), headers)
return [sink.readall()]
command = _get_command(environ)
if not command:
start_response(_('400 Bad Request'), headers)
return [b"Unrecognized service\n"]
# Other places in the Perforce-to-Git phase will need to know the
# name of client user, so set that here. As for Git-to-Perforce,
# that is handled later by setting the REMOTE_USER envar. Notice
# also that we're setting os.environ and not 'environ'.
os.environ[p4gf_const.P4GF_AUTH_P4USER] = user
# Likewise, some code needs a hint that the request is coming over
# one protocol (HTTP) or the other (SSH).
os.environ['REMOTE_ADDR'] = environ['REMOTE_ADDR']
# Initialize the external process launcher early, before allocating lots
# of memory, and just after all other conditions have been checked.
p4gf_proc.init()
# Prepare for possible spawn of GitMirror worker process by forking
# now before allocating lots of memory.
p4gf_gitmirror.setup_spawn(view_name)
# Kick off garbage collection debugging, if enabled.
p4gf_gc.init_gc()
# Go no further, create NOTHING, if user not authorized.
# We use the translated internal view name here for perm authorization
required_perm = p4gf_server_common.COMMAND_TO_PERM[command]
view_perm = p4gf_group.ViewPerm.for_user_and_view(p4, user, view_name, required_perm)
try:
p4gf_server_common.check_authorization(p4, view_perm, user, command, view_name)
except p4gf_server_common.CommandError as ce:
start_response(_('403 Forbidden'), headers)
return [str(ce).encode('UTF-8')]
# Create Git Fusion server depot, user, config. NOPs if already created.
p4gf_init.init(p4)
before_lock_time = time.time()
with p4gf_lock.view_lock(p4, view_name) as view_lock:
after_lock_time = time.time()
# Create Git Fusion per-repo client view mapping and config.
init_repo_status = p4gf_init_repo.init_repo(p4, view_name, view_lock)
if init_repo_status == p4gf_init_repo.INIT_REPO_OK:
repo_created = True
elif init_repo_status == p4gf_init_repo.INIT_REPO_EXISTS:
repo_created = False
elif init_repo_status == p4gf_init_repo.INIT_REPO_NOVIEW:
start_response(_('404 Not Found'), headers)
return [sink.readall()]
else:
start_response(_('500 Internal Server Error'), headers)
return [b"Repository initialization failed\n"]
# If authorization came from default, not explicit group
# membership, copy that authorization to a group now. Could
# not do this until after p4gf_init_repo() has a chance to
# create not-yet-existing groups.
if view_perm:
view_perm.write_if(p4)
# Now that we have valid git-fusion-user and
# git-fusion-<view> client, replace our temporary P4
# connection with a more permanent Context, shared for the
# remainder of this process.
with p4gf_context.create_context(view_name, view_lock) as ctx:
LOG.debug("reconnected to P4, p4gf=%s", ctx.p4gf)
ctx.log_context()
# cd into the work directory. Not all git functions react well
# to --work-tree=xxxx.
cwd = os.getcwd()
os.chdir(ctx.view_dirs.GIT_WORK_TREE)
# Only copy from Perforce to Git if no other process is cloning
# from this Git repo right now.
shared_in_progress = p4gf_lock.shared_host_view_lock_exists(ctx.p4, view_name)
if not shared_in_progress:
# Copy any recent changes from Perforce to Git.
try:
LOG.debug("bare: No git-upload-pack in progress, force non-bare"
" before update Git from Perforce.")
p4gf_git.set_bare(False)
p4gf_copy_p2g.copy_p2g_ctx(ctx)
p4gf_init_repo.process_imports(ctx)
# Now is also an appropriate time to clear out any stale Git
# Swarm reviews. We're pre-pull, pre-push, time when we've
# got exclusive write access to the Git repo,
GSReviewCollection.delete_refs_for_closed_reviews(ctx)
except p4gf_lock.LockCanceled as lc:
LOG.warning(str(lc))
except:
# Dump failure to log, BEFORE cleanup, just in case
# cleanup ALSO fails and throws its own error (which
# happens if we're out of memory).
LOG.error(traceback.format_exc())
if repo_created:
# Return to the original working directory to allow the
# config code to call os.getcwd() without dying, since
# we are about to delete the current working directory.
os.chdir(cwd)
p4gf_server_common.cleanup_client(ctx, view_name)
raise
try:
exclusive = 'upload' not in command
is_push = 'upload' not in command
git_caller = functools.partial(_call_git, input_name, environ, ctx)
p4gf_call_git.call_git(git_caller, ctx, view_name, view_lock, exclusive)
if is_push:
GSReviewCollection.post_push(ctx)
except p4gf_atomic_lock.LockConflict as lc:
start_response(_('500 Internal Server Error'), headers)
return ["{}".format(lc).encode('UTF-8')]
p4gf_gc.process_garbage('at end of auth_server')
if LOG.isEnabledFor(logging.DEBUG):
end_time = time.time()
frm = NTR('Runtime: preparation {} ms, lock acquisition {} ms, processing {} ms')
LOG.debug(frm.format(before_lock_time - start_time,
after_lock_time - before_lock_time,
end_time - after_lock_time))
return []
def main():
"""set up repo for a view"""
p4gf_util.has_server_id_or_exit()
args = _parse_argv()
p4gf_version.log_version()
log_l10n()
# !!! view_name_git the untranslated repo name
# !!! view_name the translated repo name
view_name_p4client = None
if args.p4client:
view_name_p4client = p4gf_util.argv_to_view_name(args.p4client)
view_name_git = p4gf_util.argv_to_view_name(args.view)
#strip leading '/' to conform with p4gf_auth_server behavior
if view_name_git[0] == '/':
view_name_git = view_name_git[1:]
view_name = p4gf_translate.TranslateReponame.git_to_repo(view_name_git)
p4gf_gitmirror.setup_spawn(view_name)
p4gf_util.reset_git_enviro()
p4 = p4gf_create_p4.create_p4()
if not p4:
return INIT_REPO_NOVIEW
LOG.debug("connected to P4 at %s", p4.port)
p4gf_proc.init()
try:
with p4gf_create_p4.Closer():
p4gf_version.version_check()
with p4gf_lock.view_lock(p4, view_name) as view_lock:
# Ensure we have a sane environment.
p4gf_init.init(p4)
# Now that we can trust that the git-fusion--p4 client exists,
# switch to that. Change takes effect immediately, don't need to
# re-run p4.connect().
p4.client = p4gf_util.get_object_client_name()
# If local config file specified, validate it and store in
# Perforce now. Even if client exists (aka repo was already
# inited), this is one way for an admin to modify an existing
# repo's config.
if args.config:
if not os.path.exists(args.config):
_print_stderr(_("error: missing config file '{}'").format(args.config))
return INIT_REPO_CONFIG_FILE_MISSING
with Validator.from_local_file(view_name, p4, args.config) as validator:
if not validator.is_valid(args.enablemismatchedrhs):
return INIT_REPO_CONFIG_FILE_BAD
p4gf_config.create_file_repo_with_contents(p4, view_name, args.config)
elif args.charset and not Validator.valid_charset(args.charset):
_print_stderr(_("error: invalid charset: {}").format(args.charset))
return INIT_REPO_BAD_CHARSET
# Initialize the repository if necessary.
print(_("Initializing '{}'...").format(view_name))
r = init_repo(p4, view_name, view_lock, args.charset, args.enablemismatchedrhs,
view_name_p4client)
if r > INIT_REPO_OK:
return r
print(_("Initialization complete."))
# Write --enablemismatchedrhs to config file
if args.enablemismatchedrhs:
config = p4gf_config.read_repo(p4, view_name)
config[p4gf_config.SECTION_REPO]\
[p4gf_config.KEY_ENABLE_MISMATCHED_RHS] = str(True)
p4gf_config.write_repo_if(p4, p4.fetch_client(), view_name, config)
# Populate the repo from Perforce unless --noclone.
if not args.noclone:
return populate_repo(view_name, view_lock, args.start)
except P4.P4Exception as e:
_print_stderr(_('Error occurred: {}').format(e))
return INIT_REPO_EXISTS
def main(poll_only=False):
"""set up repo for a view
view_name_git is the untranslated repo name
view_name is the translated repo name
"""
p4gf_proc.install_stack_dumper()
_log_environ(os.environ)
with p4gf_server_common.ExceptionAuditLogger()\
, p4gf_create_p4.Closer():
LOG.debug(p4gf_log.memory_usage())
start_time = time.time()
args = parse_args(sys.argv[1:])
if not args:
return 1
is_push = 'upload' not in args.command[0]
# Record the p4 user in environment. We use environment to pass to
# git-invoked hook. We don't have to set ctx.authenticated_p4user because
# Context.__init__() reads it from environment, which we set here.
os.environ[p4gf_const.P4GF_AUTH_P4USER] = args.user
# view_name_git is the untranslated repo name
# view_name is the translated repo name
# print "args={}".format(args)
view_name_git = args.options[-1]
# translate '/' ':' ' ' .. etc .. for internal view_name
view_name = p4gf_translate.TranslateReponame.git_to_repo(view_name_git)
LOG.debug("public view_name: {0} internal view_name: {1}".format(
view_name_git, view_name))
p4gf_util.reset_git_enviro()
p4 = p4gf_create_p4.create_p4()
if not p4:
return 2
LOG.debug("connected to P4: %s", p4)
p4gf_server_common.check_readiness(p4)
p4gf_server_common.check_lock_perm(p4)
if not p4gf_server_common.check_protects(p4):
p4gf_server_common.raise_p4gf_perm()
if p4gf_server_common.run_special_command(view_name, p4, args.user):
return 0
# Initialize the external process launcher early, before allocating lots
# of memory, and just after all other conditions have been checked.
p4gf_proc.init()
# Prepare for possible spawn of GitMirror worker process by forking
# now before allocating lots of memory.
p4gf_gitmirror.setup_spawn(view_name)
# Kick off garbage collection debugging, if enabled.
p4gf_gc.init_gc()
if poll_only:
view_perm = None
else:
# Go no further, create NOTHING, if user not authorized.
# We use the translated internal view name here for perm authorization
required_perm = p4gf_server_common.COMMAND_TO_PERM[args.command[0]]
view_perm = p4gf_group.ViewPerm.for_user_and_view(
p4, args.user, view_name, required_perm)
p4gf_server_common.check_authorization(p4, view_perm, args.user,
args.command[0], view_name)
# Create Git Fusion server depot, user, config. NOPs if already created.
p4gf_init.init(p4)
write_motd()
# view_name is the internal view_name (identical when notExist special chars)
before_lock_time = time.time()
with p4gf_lock.view_lock(p4, view_name) as view_lock:
after_lock_time = time.time()
# Create Git Fusion per-repo client view mapping and config.
#
# NOPs if already created.
# Create the empty directory that will hold the git repo.
init_repo_status = p4gf_init_repo.init_repo(
p4, view_name, view_lock)
if init_repo_status == p4gf_init_repo.INIT_REPO_OK:
repo_created = True
elif init_repo_status == p4gf_init_repo.INIT_REPO_EXISTS:
repo_created = False
else:
return 1
# If authorization came from default, not explicit group
# membership, copy that authorization to a group now. Could
# not do this until after p4gf_init_repo() has a chance to
# create not-yet-existing groups.
if view_perm:
view_perm.write_if(p4)
# Now that we have valid git-fusion-user and
# git-fusion-<view> client, replace our temporary P4
# connection with a more permanent Context, shared for the
# remainder of this process.
with p4gf_context.create_context(view_name, view_lock) as ctx:
LOG.debug("reconnected to P4, p4gf=%s", ctx.p4gf)
# Find directory paths to feed to git.
ctx.log_context()
# cd into the work directory. Not all git functions react well
# to --work-tree=xxxx.
cwd = os.getcwd()
os.chdir(ctx.view_dirs.GIT_WORK_TREE)
# Only copy from Perforce to Git if no other process is cloning
# from this Git repo right now.
shared_in_progress = p4gf_lock.shared_host_view_lock_exists(
ctx.p4, view_name)
if not shared_in_progress:
# Copy any recent changes from Perforce to Git.
try:
LOG.debug(
"bare: No git-upload-pack in progress, force non-bare"
" before update Git from Perforce.")
p4gf_git.set_bare(False)
p4gf_copy_p2g.copy_p2g_ctx(ctx)
p4gf_init_repo.process_imports(ctx)
# Now is also an appropriate time to clear out any stale Git
# Swarm reviews. We're pre-pull, pre-push, time when we've
# got exclusive write access to the Git repo,
GSReviewCollection.delete_refs_for_closed_reviews(ctx)
except p4gf_lock.LockCanceled as lc:
LOG.warning(str(lc))
except:
# Dump failure to log, BEFORE cleanup, just in case
# cleanup ALSO fails and throws its own error (which
# happens if we're out of memory).
LOG.error(traceback.format_exc())
if repo_created:
# Return to the original working directory to allow the
# config code to call os.getcwd() without dying, since
# we are about to delete the current working directory.
os.chdir(cwd)
p4gf_server_common.cleanup_client(ctx, view_name)
raise
if poll_only:
code = os.EX_OK
else:
git_caller = functools.partial(_call_git, args, ctx)
try:
# Deep in call_git(), we grab an 'p4 reviews' lock on
# ctx.clientmap's LHS. Switch that clientmap to our
# full union view to prevent simultaneous 'git push'es
# from clobbering each other in some shared depot
# branch. Must include all lightweight branches, too.
ctx.switch_client_view_to_union()
exclusive = 'upload' not in args.command[0]
code = p4gf_call_git.call_git(git_caller, ctx,
view_name, view_lock,
exclusive)
if is_push:
GSReviewCollection.post_push(ctx)
except p4gf_atomic_lock.LockConflict as lc:
sys.stderr.write("{}\n".format(lc))
code = os.EX_SOFTWARE
p4gf_gc.process_garbage(NTR('at end of auth_server'))
if LOG.isEnabledFor(logging.DEBUG):
end_time = time.time()
frm = NTR("Runtime: preparation {} ms, lock acquisition {} ms,"
" processing {} ms")
LOG.debug(
frm.format(before_lock_time - start_time,
after_lock_time - before_lock_time,
end_time - after_lock_time))
return code
def main():
"""set up repo for a view"""
p4gf_util.has_server_id_or_exit()
args = _parse_argv()
p4gf_version.log_version()
log_l10n()
# !!! view_name_git the untranslated repo name
# !!! view_name the translated repo name
view_name_p4client = None
if args.p4client:
view_name_p4client = p4gf_util.argv_to_view_name(args.p4client)
view_name_git = p4gf_util.argv_to_view_name(args.view)
#strip leading '/' to conform with p4gf_auth_server behavior
if view_name_git[0] == '/':
view_name_git = view_name_git[1:]
view_name = p4gf_translate.TranslateReponame.git_to_repo(view_name_git)
p4gf_gitmirror.setup_spawn(view_name)
p4gf_util.reset_git_enviro()
p4 = p4gf_create_p4.create_p4()
if not p4:
return INIT_REPO_NOVIEW
LOG.debug("connected to P4 at %s", p4.port)
p4gf_proc.init()
try:
with p4gf_create_p4.Closer():
p4gf_version.version_check()
with p4gf_lock.view_lock(p4, view_name) as view_lock:
# Ensure we have a sane environment.
p4gf_init.init(p4)
# Now that we can trust that the git-fusion--p4 client exists,
# switch to that. Change takes effect immediately, don't need to
# re-run p4.connect().
p4.client = p4gf_util.get_object_client_name()
# If local config file specified, validate it and store in
# Perforce now. Even if client exists (aka repo was already
# inited), this is one way for an admin to modify an existing
# repo's config.
if args.config:
if not os.path.exists(args.config):
_print_stderr(
_("error: missing config file '{}'").format(
args.config))
return INIT_REPO_CONFIG_FILE_MISSING
with Validator.from_local_file(view_name, p4,
args.config) as validator:
if not validator.is_valid(args.enablemismatchedrhs):
return INIT_REPO_CONFIG_FILE_BAD
p4gf_config.create_file_repo_with_contents(
p4, view_name, args.config)
elif args.charset and not Validator.valid_charset(
args.charset):
_print_stderr(
_("error: invalid charset: {}").format(args.charset))
return INIT_REPO_BAD_CHARSET
# Initialize the repository if necessary.
print(_("Initializing '{}'...").format(view_name))
r = init_repo(p4, view_name, view_lock, args.charset,
args.enablemismatchedrhs, view_name_p4client)
if r > INIT_REPO_OK:
return r
print(_("Initialization complete."))
# Write --enablemismatchedrhs to config file
if args.enablemismatchedrhs:
config = p4gf_config.read_repo(p4, view_name)
config[p4gf_config.SECTION_REPO]\
[p4gf_config.KEY_ENABLE_MISMATCHED_RHS] = str(True)
p4gf_config.write_repo_if(p4, p4.fetch_client(), view_name,
config)
# Populate the repo from Perforce unless --noclone.
if not args.noclone:
return populate_repo(view_name, view_lock, args.start)
except P4.P4Exception as e:
_print_stderr(_('Error occurred: {}').format(e))
return INIT_REPO_EXISTS
def main():
"""set up repo for a view"""
with ExceptionAuditLogger():
args = parse_args(sys.argv[1:])
if not args:
return 1
# Record the p4 user in environment. We use environment to pass to
# git-invoked hook. We don't have to set ctx.authenticated_p4user because
# Context.__init__() reads it from environment, which we set here.
os.environ[p4gf_const.P4GF_AUTH_P4USER_ENVAR] = args.user
# print "args={}".format(args)
view_name = args.options[-1]
p4gf_util.reset_git_enviro()
p4 = connect_p4()
if not p4:
return 2
LOG.debug("connected to P4: %s", p4)
_check_lock_perm(p4)
if not check_protects(p4):
_raise_p4gf_perm()
if run_special_command(view_name, p4, args.user):
return 0
# Go no further, create NOTHING, if user not authorized.
view_perm = p4gf_group.ViewPerm.for_user_and_view(p4, args.user, view_name)
_check_authorization(view_perm, args.user, args.command[0], view_name)
# Create Git Fusion server depot, user, config. NOPs if already created.
p4gf_init.init(p4)
with p4gf_lock.view_lock(p4, view_name) as view_lock:
# Create Git Fusion per-repo client view mapping and config.
#
# NOPs if already created.
# Create the empty directory that will hold the git repo.
init_repo_status = p4gf_init_repo.init_repo(p4, view_name)
if init_repo_status == p4gf_init_repo.INIT_REPO_OK:
repo_created = True
elif init_repo_status == p4gf_init_repo.INIT_REPO_EXISTS:
repo_created = False
else:
return 1
# If authorization came from default, not explicit group
# membership, copy that authorization to a group now. Could
# not do this until after p4gf_init_repo() has a chance to
# create not-yet-existing groups.
view_perm.write_if(p4)
# Now that we have valid git-fusion-user and
# git-fusion-<view> client, replace our temporary P4
# connection with a more permanent Context, shared for the
# remainder of this process.
ctx = p4gf_context.create_context(view_name, view_lock)
del p4
LOG.debug("reconnected to P4, p4gf=%s", ctx.p4gf)
# Find directory paths to feed to git.
ctx.view_dirs = p4gf_view_dirs.from_p4gf_dir(ctx.gitrootdir, view_name)
ctx.log_context()
# cd into the work directory. Not all git functions react well
# to --work-tree=xxxx.
cwd = os.getcwd()
os.chdir(ctx.view_dirs.GIT_WORK_TREE)
# Copy any recent changes from Perforce to Git.
try:
p4gf_copy_p2g.copy_p2g_ctx(ctx)
except:
# Dump failure to log, BEFORE cleanup, just in case
# cleanup ALSO fails and throws its own error (which
# happens if we're out of memory).
LOG.error(traceback.format_exc())
if repo_created:
# Return to the original working directory to allow the
# config code to call os.getcwd() without dying, since
# we are about to delete the current working directory.
os.chdir(cwd)
cleanup_client(ctx, view_name)
raise
# Detach git repo's workspace from master before calling
# original git, otherwise we won't be able to push master.
p4gf_util.checkout_detached_master()
# Flush stderr before returning control to Git.
# Otherwise Git's own output might interrupt ours.
sys.stderr.flush()
return _call_original_git(ctx, args)
def _wsgi_app(environ, start_response):
"""
WSGI application to process the incoming Git client request. This is
nearly equivalent to p4gf_auth_server.main() with the exception of
input validation and error handling.
"""
p4gf_log.record_http(environ)
p4gf_version.log_version()
_log_environ(environ)
p4gf_version.version_check()
LOG.debug("processing HTTP request, pid={}".format(os.getpid()))
# Keep the content type to exactly 'text/plain' so there is at least
# the remote chance that Git might show our error messages (does not
# appear to work in practice, however).
headers = [('Content-Type', 'text/plain')]
encoding = sys.getfilesystemencoding()
if encoding == 'ascii':
# This encoding is wrong and will eventually lead to problems.
LOG.error(
"Using 'ascii' file encoding will ultimately result in errors, "
"please set LANG/LC_ALL to 'utf-8' in web server configuration.")
start_response(_('500 Internal Server Error'), headers)
return [b"Filesystem encoding not set to acceptable value.\n"]
# Sanity check the request.
for (name, status, msg) in _REQUIRED_HTTP_PARAMS:
if name not in environ:
start_response(status, headers)
return [msg.encode('UTF-8')]
input_name = environ['wsgi.input']
# Extract the view_name_git by removing the expected git request suffixes
path_info = environ['PATH_INFO']
git_suffixes = [
'/info/refs', '/HEAD', '/git-upload-pack', '/git-receive-pack'
]
path_end = len(path_info)
for suffix in git_suffixes:
try:
path_end = path_info.index(suffix)
break
except ValueError:
pass
# slice away the leading slash and the trailing git request suffixes
view_name_git = path_info[1:path_end]
# and remove the view_name_git from the front of PATH_INFO
environ['PATH_INFO'] = path_info[path_end:]
LOG.debug("new PATH_INFO {0} view_name_git {1}".format(
environ['PATH_INFO'], view_name_git))
if not view_name_git:
start_response(_('400 Bad Request'), headers)
msg = _('Missing required repository name in URL\n')
return [msg.encode('UTF-8')]
# translate '/' ':' ' ' .. etc .. for internal view_name
view_name = p4gf_translate.TranslateReponame.git_to_repo(view_name_git)
LOG.debug("public view_name: {0} internal view_name: {1}".format(
view_name_git, view_name))
audit_logger = p4gf_server_common.ExceptionAuditLogger()
p4_closer = p4gf_create_p4.Closer()
sink = OutputSink()
temp_deleter = deleting(input_name)
mirror_closer = unmirror(view_name)
with audit_logger \
, p4_closer \
, sink \
, temp_deleter \
, mirror_closer:
LOG.debug(p4gf_log.memory_usage())
start_time = time.time()
p4gf_util.reset_git_enviro()
p4 = p4gf_create_p4.create_p4()
if not p4:
start_response(_('500 Internal Server Error'), headers)
return [b"Perforce connection failed\n"]
LOG.debug("connected to P4: %s", p4)
p4gf_server_common.check_readiness(p4)
p4gf_server_common.check_lock_perm(p4)
if not p4gf_server_common.check_protects(p4):
p4gf_server_common.raise_p4gf_perm()
user = environ['REMOTE_USER']
if p4gf_server_common.run_special_command(view_name, p4, user):
start_response(_('200 OK'), headers)
return [sink.readall()]
command = _get_command(environ)
if not command:
start_response(_('400 Bad Request'), headers)
return [b"Unrecognized service\n"]
# Other places in the Perforce-to-Git phase will need to know the
# name of client user, so set that here. As for Git-to-Perforce,
# that is handled later by setting the REMOTE_USER envar. Notice
# also that we're setting os.environ and not 'environ'.
os.environ[p4gf_const.P4GF_AUTH_P4USER] = user
# Likewise, some code needs a hint that the request is coming over
# one protocol (HTTP) or the other (SSH).
os.environ['REMOTE_ADDR'] = environ['REMOTE_ADDR']
# Initialize the external process launcher early, before allocating lots
# of memory, and just after all other conditions have been checked.
p4gf_proc.init()
# Prepare for possible spawn of GitMirror worker process by forking
# now before allocating lots of memory.
p4gf_gitmirror.setup_spawn(view_name)
# Kick off garbage collection debugging, if enabled.
p4gf_gc.init_gc()
# Go no further, create NOTHING, if user not authorized.
# We use the translated internal view name here for perm authorization
required_perm = p4gf_server_common.COMMAND_TO_PERM[command]
view_perm = p4gf_group.ViewPerm.for_user_and_view(
p4, user, view_name, required_perm)
try:
p4gf_server_common.check_authorization(p4, view_perm, user,
command, view_name)
except p4gf_server_common.CommandError as ce:
start_response(_('403 Forbidden'), headers)
return [str(ce).encode('UTF-8')]
# Create Git Fusion server depot, user, config. NOPs if already created.
p4gf_init.init(p4)
before_lock_time = time.time()
with p4gf_lock.view_lock(p4, view_name) as view_lock:
after_lock_time = time.time()
# Create Git Fusion per-repo client view mapping and config.
init_repo_status = p4gf_init_repo.init_repo(
p4, view_name, view_lock)
if init_repo_status == p4gf_init_repo.INIT_REPO_OK:
repo_created = True
elif init_repo_status == p4gf_init_repo.INIT_REPO_EXISTS:
repo_created = False
elif init_repo_status == p4gf_init_repo.INIT_REPO_NOVIEW:
start_response(_('404 Not Found'), headers)
return [sink.readall()]
else:
start_response(_('500 Internal Server Error'), headers)
return [b"Repository initialization failed\n"]
# If authorization came from default, not explicit group
# membership, copy that authorization to a group now. Could
# not do this until after p4gf_init_repo() has a chance to
# create not-yet-existing groups.
if view_perm:
view_perm.write_if(p4)
# Now that we have valid git-fusion-user and
# git-fusion-<view> client, replace our temporary P4
# connection with a more permanent Context, shared for the
# remainder of this process.
with p4gf_context.create_context(view_name, view_lock) as ctx:
LOG.debug("reconnected to P4, p4gf=%s", ctx.p4gf)
ctx.log_context()
# cd into the work directory. Not all git functions react well
# to --work-tree=xxxx.
cwd = os.getcwd()
os.chdir(ctx.view_dirs.GIT_WORK_TREE)
# Only copy from Perforce to Git if no other process is cloning
# from this Git repo right now.
shared_in_progress = p4gf_lock.shared_host_view_lock_exists(
ctx.p4, view_name)
if not shared_in_progress:
# Copy any recent changes from Perforce to Git.
try:
LOG.debug(
"bare: No git-upload-pack in progress, force non-bare"
" before update Git from Perforce.")
p4gf_git.set_bare(False)
p4gf_copy_p2g.copy_p2g_ctx(ctx)
p4gf_init_repo.process_imports(ctx)
# Now is also an appropriate time to clear out any stale Git
# Swarm reviews. We're pre-pull, pre-push, time when we've
# got exclusive write access to the Git repo,
GSReviewCollection.delete_refs_for_closed_reviews(ctx)
except p4gf_lock.LockCanceled as lc:
LOG.warning(str(lc))
except:
# Dump failure to log, BEFORE cleanup, just in case
# cleanup ALSO fails and throws its own error (which
# happens if we're out of memory).
LOG.error(traceback.format_exc())
if repo_created:
# Return to the original working directory to allow the
# config code to call os.getcwd() without dying, since
# we are about to delete the current working directory.
os.chdir(cwd)
p4gf_server_common.cleanup_client(ctx, view_name)
raise
try:
exclusive = 'upload' not in command
is_push = 'upload' not in command
git_caller = functools.partial(_call_git, input_name,
environ, ctx)
p4gf_call_git.call_git(git_caller, ctx, view_name,
view_lock, exclusive)
if is_push:
GSReviewCollection.post_push(ctx)
except p4gf_atomic_lock.LockConflict as lc:
start_response(_('500 Internal Server Error'), headers)
return ["{}".format(lc).encode('UTF-8')]
p4gf_gc.process_garbage('at end of auth_server')
if LOG.isEnabledFor(logging.DEBUG):
end_time = time.time()
frm = NTR(
'Runtime: preparation {} ms, lock acquisition {} ms, processing {} ms'
)
LOG.debug(
frm.format(before_lock_time - start_time,
after_lock_time - before_lock_time,
end_time - after_lock_time))
return []
