def copy_p2g_with_start(view_name, start, view_lock):
"""Invoked 'p4gf_init_repo.py --start=NNN': copy changes from @NNN to @now."""
ctx = p4gf_context.create_context(view_name, view_lock)
LOG.debug("connected to P4, p4gf=%s", ctx.p4gf)
# Copy any recent changes from Perforce to Git.
p4gf_copy_p2g.copy_p2g_ctx(ctx, start)
def _copy_submodule(self, subtxt, local_path, change_num, user_3tuple):
"""Copy from Perforce to Git the submodule changes.
Arguments:
subtxt -- context for submodule repo.
local_path -- path within parent repo where submodule will go.
user_3tuple -- (p4user, email, fullname) for Git Fusion user
Returns the new SHA1 of the parent repo and an error string, or None
if successful.
"""
cwd = os.getcwd()
os.chdir(subtxt.repo_dirs.GIT_WORK_TREE)
repo_name = subtxt.config.repo_name
LOG.debug('_copy_submodule() marking submodule %s as read-only', repo_name)
subtxt.repo_config.set(p4gf_config.SECTION_REPO, p4gf_config.KEY_READ_ONLY, 'yes')
if self.ctx.is_lfs_enabled:
LOG.debug('_copy_submodule() marking submodule %s as git-lfs-enable', repo_name)
subtxt.is_lfs_enabled = self.ctx.is_lfs_enabled
subtxt.repo_config.set(p4gf_config.SECTION_REPO,
p4gf_config.KEY_GIT_LFS_ENABLE, 'yes')
initial_tracking = self.ctx.repo_config.get(
p4gf_config.SECTION_PERFORCE_TO_GIT,
p4gf_config.KEY_GIT_LFS_INITIAL_TRACK)
if initial_tracking:
subtxt.repo_config.set(p4gf_config.SECTION_REPO,
p4gf_config.KEY_GIT_LFS_INITIAL_TRACK, initial_tracking)
subtxt.repo_config.write_repo_if(subtxt.p4gf)
LOG.debug('_copy_submodule() copying changes for %s', repo_name)
p4gf_copy_p2g.copy_p2g_ctx(subtxt)
# if available, use the requested change to get the corresponding SHA1 of the submodule
commit_ot = None
latest_change = subtxt.union_view_highest_change_num(at_or_before_change_num=change_num)
if latest_change:
LOG.debug('_copy_submodule() latest change: %s', latest_change)
commit_ot = p4gf_object_type.ObjectType.change_num_to_commit(
subtxt, latest_change, None)
if commit_ot:
sub_sha1 = commit_ot.sha1
LOG.debug('_copy_submodule() using commit %s', sub_sha1)
else:
# otherwise use the latest commit
sub_sha1 = p4gf_pygit2.head_commit_sha1(subtxt.repo)
LOG.debug('_copy_submodule() using HEAD: %s', sub_sha1)
os.chdir(cwd)
url = _submodule_url(subtxt.repo_config)
if local_path.endswith('...'):
local_path = local_path[:-3]
local_path = local_path.rstrip('/')
if p4gf_git.add_submodule(self.ctx.repo, repo_name, local_path, sub_sha1, url, user_3tuple):
LOG.debug('_copy_submodule() added submodule %s to %s as %s',
local_path, repo_name, user_3tuple[0])
def copy_submodule(ctx, repo_name, subtxt, local_path, change_num,
user_3tuple):
"""Copy from Perforce to Git the submodule changes.
Arguments:
ctx -- parent repo context.
repo_name -- name of submodule repo.
subtxt -- context for submodule repo.
local_path -- path within parent repo where submodule will go.
user_3tuple -- (p4user, email, fullname) for Git Fusion user
Returns the new SHA1 of the parent repo and an error string, or None
if successful.
"""
cwd = os.getcwd()
if subtxt.view_repo is None:
subtxt.get_view_repo()
os.chdir(subtxt.view_dirs.GIT_WORK_TREE)
LOG.debug('copy_submodule() copying changes for {}'.format(repo_name))
p4gf_copy_p2g.copy_p2g_ctx(subtxt)
# if available, use the requested change to get the corresponding SHA1 of the submodule
commit_ot = None
changes = subtxt.p4run(
['changes', '-m1',
subtxt.client_view_path(change_num)])
if changes:
real_dict = p4gf_util.first_dict_with_key(changes, 'change')
if real_dict:
real_change = real_dict['change']
commit_ot = p4gf_object_type.ObjectType.commit_for_change(
subtxt, real_change, None)
if commit_ot:
sub_sha1 = commit_ot.sha1
LOG.debug2('copy_submodule() using commit {}'.format(sub_sha1))
else:
# otherwise use the latest commit
sub_sha1 = subtxt.view_repo.head.hex
LOG.debug2('copy_submodule() using HEAD: {}'.format(sub_sha1))
os.chdir(cwd)
url = _submodule_url(subtxt)
if local_path.endswith('...'):
local_path = local_path[:-3]
local_path = local_path.rstrip('/')
LOG.debug('adding submodule {} to {} as {}'.format(local_path, repo_name,
user_3tuple[0]))
p4gf_git.add_submodule(ctx.view_repo, repo_name, local_path, sub_sha1, url,
user_3tuple)
def _copy_p2g_with_start(self, start):
"""Invoked 'p4gf_init_repo.py --start=NNN': copy changes from @NNN to #head."""
with self.connector() as ctx:
LOG.debug("connected to P4, p4gf=%s", ctx.p4gf)
# Check that there are changes to be copied from any branch.
r = ctx.union_view_highest_change_num(after_change_num=int(start))
if r:
# Copy any recent changes from Perforce to Git.
print(
_("Copying changes from '{start}'...").format(start=start))
p4gf_copy_p2g.copy_p2g_ctx(ctx, start)
print(_('Copying completed.'))
else:
msg = _("No changes above '{start}'.").format(start=start)
if int(start) == 1:
LOG.debug(msg)
else:
LOG.info(msg)
raise IndexError(msg)
def copy_submodule(ctx, repo_name, subtxt, local_path, change_num, user_3tuple):
"""Copy from Perforce to Git the submodule changes.
Arguments:
ctx -- parent repo context.
repo_name -- name of submodule repo.
subtxt -- context for submodule repo.
local_path -- path within parent repo where submodule will go.
user_3tuple -- (p4user, email, fullname) for Git Fusion user
Returns the new SHA1 of the parent repo and an error string, or None
if successful.
"""
cwd = os.getcwd()
if subtxt.view_repo is None:
subtxt.get_view_repo()
os.chdir(subtxt.view_dirs.GIT_WORK_TREE)
LOG.debug('copy_submodule() copying changes for {}'.format(repo_name))
p4gf_copy_p2g.copy_p2g_ctx(subtxt)
# if available, use the requested change to get the corresponding SHA1 of the submodule
commit_ot = None
changes = subtxt.p4run(['changes', '-m1', subtxt.client_view_path(change_num)])
if changes:
real_dict = p4gf_util.first_dict_with_key(changes, 'change')
if real_dict:
real_change = real_dict['change']
commit_ot = p4gf_object_type.ObjectType.commit_for_change(subtxt, real_change, None)
if commit_ot:
sub_sha1 = commit_ot.sha1
LOG.debug2('copy_submodule() using commit {}'.format(sub_sha1))
else:
# otherwise use the latest commit
sub_sha1 = subtxt.view_repo.head.hex
LOG.debug2('copy_submodule() using HEAD: {}'.format(sub_sha1))
os.chdir(cwd)
url = _submodule_url(subtxt)
if local_path.endswith('...'):
local_path = local_path[:-3]
local_path = local_path.rstrip('/')
LOG.debug('adding submodule {} to {} as {}'.format(local_path, repo_name, user_3tuple[0]))
p4gf_git.add_submodule(ctx.view_repo, repo_name, local_path, sub_sha1, url, user_3tuple)
def copy_p2g_with_start(view_name, start, view_lock, ctx=None):
"""Invoked 'p4gf_init_repo.py --start=NNN': copy changes from @NNN to #head."""
if ctx is None:
ctx = p4gf_context.create_context(view_name, view_lock)
with ctx:
LOG.debug("connected to P4, p4gf=%s", ctx.p4gf)
# Check that there are changes to be copied from any branch.
ctx.switch_client_view_to_union()
path = ctx.client_view_path()
changes_result = ctx.p4.run("changes", "-m1", "{}@{},#head".format(path, start))
if len(changes_result):
# Copy any recent changes from Perforce to Git.
print(_("Copying changes from '{}'...").format(start))
p4gf_copy_p2g.copy_p2g_ctx(ctx, start)
print(_('Copying completed.'))
else:
msg = _("No changes above '{}'.").format(start)
if int(start) == 1:
LOG.debug(msg)
else:
LOG.info(msg)
raise IndexError(msg)
def copy_p2g_with_start(view_name, start, view_lock, ctx=None):
"""Invoked 'p4gf_init_repo.py --start=NNN': copy changes from @NNN to #head."""
if ctx is None:
ctx = p4gf_context.create_context(view_name, view_lock)
with ctx:
LOG.debug("connected to P4, p4gf=%s", ctx.p4gf)
# Check that there are changes to be copied from any branch.
ctx.switch_client_view_to_union()
path = ctx.client_view_path()
changes_result = ctx.p4.run("changes", "-m1",
"{}@{},#head".format(path, start))
if len(changes_result):
# Copy any recent changes from Perforce to Git.
print(_("Copying changes from '{}'...").format(start))
p4gf_copy_p2g.copy_p2g_ctx(ctx, start)
print(_('Copying completed.'))
else:
msg = _("No changes above '{}'.").format(start)
if int(start) == 1:
LOG.debug(msg)
else:
LOG.info(msg)
raise IndexError(msg)
def main():
"""set up repo for a view"""
with ExceptionAuditLogger():
args = parse_args(sys.argv[1:])
if not args:
return 1
# Record the p4 user in environment. We use environment to pass to
# git-invoked hook. We don't have to set ctx.authenticated_p4user because
# Context.__init__() reads it from environment, which we set here.
os.environ[p4gf_const.P4GF_AUTH_P4USER_ENVAR] = args.user
# print "args={}".format(args)
view_name = args.options[-1]
p4gf_util.reset_git_enviro()
p4 = connect_p4()
if not p4:
return 2
LOG.debug("connected to P4: %s", p4)
_check_lock_perm(p4)
if not check_protects(p4):
_raise_p4gf_perm()
if run_special_command(view_name, p4, args.user):
return 0
# Go no further, create NOTHING, if user not authorized.
view_perm = p4gf_group.ViewPerm.for_user_and_view(
p4, args.user, view_name)
_check_authorization(view_perm, args.user, args.command[0], view_name)
# Create Git Fusion server depot, user, config. NOPs if already created.
p4gf_init.init(p4)
with p4gf_lock.view_lock(p4, view_name) as view_lock:
# Create Git Fusion per-repo client view mapping and config.
#
# NOPs if already created.
# Create the empty directory that will hold the git repo.
init_repo_status = p4gf_init_repo.init_repo(p4, view_name)
if init_repo_status == p4gf_init_repo.INIT_REPO_OK:
repo_created = True
elif init_repo_status == p4gf_init_repo.INIT_REPO_EXISTS:
repo_created = False
else:
return 1
# If authorization came from default, not explicit group
# membership, copy that authorization to a group now. Could
# not do this until after p4gf_init_repo() has a chance to
# create not-yet-existing groups.
view_perm.write_if(p4)
# Now that we have valid git-fusion-user and
# git-fusion-<view> client, replace our temporary P4
# connection with a more permanent Context, shared for the
# remainder of this process.
ctx = p4gf_context.create_context(view_name, view_lock)
del p4
LOG.debug("reconnected to P4, p4gf=%s", ctx.p4gf)
# Find directory paths to feed to git.
ctx.view_dirs = p4gf_view_dirs.from_p4gf_dir(
ctx.gitrootdir, view_name)
ctx.log_context()
# cd into the work directory. Not all git functions react well
# to --work-tree=xxxx.
cwd = os.getcwd()
os.chdir(ctx.view_dirs.GIT_WORK_TREE)
# Copy any recent changes from Perforce to Git.
try:
p4gf_copy_p2g.copy_p2g_ctx(ctx)
except:
# Dump failure to log, BEFORE cleanup, just in case
# cleanup ALSO fails and throws its own error (which
# happens if we're out of memory).
LOG.error(traceback.format_exc())
if repo_created:
# Return to the original working directory to allow the
# config code to call os.getcwd() without dying, since
# we are about to delete the current working directory.
os.chdir(cwd)
cleanup_client(ctx, view_name)
raise
# Detach git repo's workspace from master before calling
# original git, otherwise we won't be able to push master.
p4gf_util.checkout_detached_master()
# Flush stderr before returning control to Git.
# Otherwise Git's own output might interrupt ours.
sys.stderr.flush()
return _call_original_git(ctx, args)
def main(poll_only=False):
"""set up repo for a view
view_name_git is the untranslated repo name
view_name is the translated repo name
"""
p4gf_proc.install_stack_dumper()
_log_environ(os.environ)
with p4gf_server_common.ExceptionAuditLogger()\
, p4gf_create_p4.Closer():
LOG.debug(p4gf_log.memory_usage())
start_time = time.time()
args = parse_args(sys.argv[1:])
if not args:
return 1
is_push = 'upload' not in args.command[0]
# Record the p4 user in environment. We use environment to pass to
# git-invoked hook. We don't have to set ctx.authenticated_p4user because
# Context.__init__() reads it from environment, which we set here.
os.environ[p4gf_const.P4GF_AUTH_P4USER] = args.user
# view_name_git is the untranslated repo name
# view_name is the translated repo name
# print "args={}".format(args)
view_name_git = args.options[-1]
# translate '/' ':' ' ' .. etc .. for internal view_name
view_name = p4gf_translate.TranslateReponame.git_to_repo(view_name_git)
LOG.debug("public view_name: {0} internal view_name: {1}".
format(view_name_git, view_name))
p4gf_util.reset_git_enviro()
p4 = p4gf_create_p4.create_p4()
if not p4:
return 2
LOG.debug("connected to P4: %s", p4)
p4gf_server_common.check_readiness(p4)
p4gf_server_common.check_lock_perm(p4)
if not p4gf_server_common.check_protects(p4):
p4gf_server_common.raise_p4gf_perm()
if p4gf_server_common.run_special_command(view_name, p4, args.user):
return 0
# Initialize the external process launcher early, before allocating lots
# of memory, and just after all other conditions have been checked.
p4gf_proc.init()
# Prepare for possible spawn of GitMirror worker process by forking
# now before allocating lots of memory.
p4gf_gitmirror.setup_spawn(view_name)
# Kick off garbage collection debugging, if enabled.
p4gf_gc.init_gc()
if poll_only:
view_perm = None
else:
# Go no further, create NOTHING, if user not authorized.
# We use the translated internal view name here for perm authorization
required_perm = p4gf_server_common.COMMAND_TO_PERM[args.command[0]]
view_perm = p4gf_group.ViewPerm.for_user_and_view(p4, args.user,
view_name, required_perm)
p4gf_server_common.check_authorization(p4, view_perm, args.user, args.command[0],
view_name)
# Create Git Fusion server depot, user, config. NOPs if already created.
p4gf_init.init(p4)
write_motd()
# view_name is the internal view_name (identical when notExist special chars)
before_lock_time = time.time()
with p4gf_lock.view_lock(p4, view_name) as view_lock:
after_lock_time = time.time()
# Create Git Fusion per-repo client view mapping and config.
#
# NOPs if already created.
# Create the empty directory that will hold the git repo.
init_repo_status = p4gf_init_repo.init_repo(p4, view_name, view_lock)
if init_repo_status == p4gf_init_repo.INIT_REPO_OK:
repo_created = True
elif init_repo_status == p4gf_init_repo.INIT_REPO_EXISTS:
repo_created = False
else:
return 1
# If authorization came from default, not explicit group
# membership, copy that authorization to a group now. Could
# not do this until after p4gf_init_repo() has a chance to
# create not-yet-existing groups.
if view_perm:
view_perm.write_if(p4)
# Now that we have valid git-fusion-user and
# git-fusion-<view> client, replace our temporary P4
# connection with a more permanent Context, shared for the
# remainder of this process.
with p4gf_context.create_context(view_name, view_lock) as ctx:
LOG.debug("reconnected to P4, p4gf=%s", ctx.p4gf)
# Find directory paths to feed to git.
ctx.log_context()
# cd into the work directory. Not all git functions react well
# to --work-tree=xxxx.
cwd = os.getcwd()
os.chdir(ctx.view_dirs.GIT_WORK_TREE)
# Only copy from Perforce to Git if no other process is cloning
# from this Git repo right now.
shared_in_progress = p4gf_lock.shared_host_view_lock_exists(ctx.p4, view_name)
if not shared_in_progress:
# Copy any recent changes from Perforce to Git.
try:
LOG.debug("bare: No git-upload-pack in progress, force non-bare"
" before update Git from Perforce.")
p4gf_git.set_bare(False)
p4gf_copy_p2g.copy_p2g_ctx(ctx)
p4gf_init_repo.process_imports(ctx)
# Now is also an appropriate time to clear out any stale Git
# Swarm reviews. We're pre-pull, pre-push, time when we've
# got exclusive write access to the Git repo,
GSReviewCollection.delete_refs_for_closed_reviews(ctx)
except p4gf_lock.LockCanceled as lc:
LOG.warning(str(lc))
except:
# Dump failure to log, BEFORE cleanup, just in case
# cleanup ALSO fails and throws its own error (which
# happens if we're out of memory).
LOG.error(traceback.format_exc())
if repo_created:
# Return to the original working directory to allow the
# config code to call os.getcwd() without dying, since
# we are about to delete the current working directory.
os.chdir(cwd)
p4gf_server_common.cleanup_client(ctx, view_name)
raise
if poll_only:
code = os.EX_OK
else:
git_caller = functools.partial(_call_git, args, ctx)
try:
# Deep in call_git(), we grab an 'p4 reviews' lock on
# ctx.clientmap's LHS. Switch that clientmap to our
# full union view to prevent simultaneous 'git push'es
# from clobbering each other in some shared depot
# branch. Must include all lightweight branches, too.
ctx.switch_client_view_to_union()
exclusive = 'upload' not in args.command[0]
code = p4gf_call_git.call_git(
git_caller, ctx, view_name, view_lock, exclusive)
if is_push:
GSReviewCollection.post_push(ctx)
except p4gf_atomic_lock.LockConflict as lc:
sys.stderr.write("{}\n".format(lc))
code = os.EX_SOFTWARE
p4gf_gc.process_garbage(NTR('at end of auth_server'))
if LOG.isEnabledFor(logging.DEBUG):
end_time = time.time()
frm = NTR("Runtime: preparation {} ms, lock acquisition {} ms,"
" processing {} ms")
LOG.debug(frm.format(before_lock_time - start_time,
after_lock_time - before_lock_time,
end_time - after_lock_time))
return code
def _wsgi_app(environ, start_response):
"""
WSGI application to process the incoming Git client request. This is
nearly equivalent to p4gf_auth_server.main() with the exception of
input validation and error handling.
"""
p4gf_log.record_http(environ)
p4gf_version.log_version()
_log_environ(environ)
p4gf_version.version_check()
LOG.debug("processing HTTP request, pid={}".format(os.getpid()))
# Keep the content type to exactly 'text/plain' so there is at least
# the remote chance that Git might show our error messages (does not
# appear to work in practice, however).
headers = [('Content-Type', 'text/plain')]
encoding = sys.getfilesystemencoding()
if encoding == 'ascii':
# This encoding is wrong and will eventually lead to problems.
LOG.error("Using 'ascii' file encoding will ultimately result in errors, "
"please set LANG/LC_ALL to 'utf-8' in web server configuration.")
start_response(_('500 Internal Server Error'), headers)
return [b"Filesystem encoding not set to acceptable value.\n"]
# Sanity check the request.
for (name, status, msg) in _REQUIRED_HTTP_PARAMS:
if name not in environ:
start_response(status, headers)
return [msg.encode('UTF-8')]
input_name = environ['wsgi.input']
# Extract the view_name_git by removing the expected git request suffixes
path_info = environ['PATH_INFO']
git_suffixes = ['/info/refs', '/HEAD', '/git-upload-pack', '/git-receive-pack']
path_end = len(path_info)
for suffix in git_suffixes:
try:
path_end = path_info.index(suffix)
break
except ValueError:
pass
# slice away the leading slash and the trailing git request suffixes
view_name_git = path_info[1:path_end]
# and remove the view_name_git from the front of PATH_INFO
environ['PATH_INFO'] = path_info[path_end:]
LOG.debug("new PATH_INFO {0} view_name_git {1}".format(environ['PATH_INFO'], view_name_git))
if not view_name_git:
start_response(_('400 Bad Request'), headers)
msg = _('Missing required repository name in URL\n')
return [msg.encode('UTF-8')]
# translate '/' ':' ' ' .. etc .. for internal view_name
view_name = p4gf_translate.TranslateReponame.git_to_repo(view_name_git)
LOG.debug("public view_name: {0} internal view_name: {1}".format(view_name_git, view_name))
audit_logger = p4gf_server_common.ExceptionAuditLogger()
p4_closer = p4gf_create_p4.Closer()
sink = OutputSink()
temp_deleter = deleting(input_name)
mirror_closer = unmirror(view_name)
with audit_logger \
, p4_closer \
, sink \
, temp_deleter \
, mirror_closer:
LOG.debug(p4gf_log.memory_usage())
start_time = time.time()
p4gf_util.reset_git_enviro()
p4 = p4gf_create_p4.create_p4()
if not p4:
start_response(_('500 Internal Server Error'), headers)
return [b"Perforce connection failed\n"]
LOG.debug("connected to P4: %s", p4)
p4gf_server_common.check_readiness(p4)
p4gf_server_common.check_lock_perm(p4)
if not p4gf_server_common.check_protects(p4):
p4gf_server_common.raise_p4gf_perm()
user = environ['REMOTE_USER']
if p4gf_server_common.run_special_command(view_name, p4, user):
start_response(_('200 OK'), headers)
return [sink.readall()]
command = _get_command(environ)
if not command:
start_response(_('400 Bad Request'), headers)
return [b"Unrecognized service\n"]
# Other places in the Perforce-to-Git phase will need to know the
# name of client user, so set that here. As for Git-to-Perforce,
# that is handled later by setting the REMOTE_USER envar. Notice
# also that we're setting os.environ and not 'environ'.
os.environ[p4gf_const.P4GF_AUTH_P4USER] = user
# Likewise, some code needs a hint that the request is coming over
# one protocol (HTTP) or the other (SSH).
os.environ['REMOTE_ADDR'] = environ['REMOTE_ADDR']
# Initialize the external process launcher early, before allocating lots
# of memory, and just after all other conditions have been checked.
p4gf_proc.init()
# Prepare for possible spawn of GitMirror worker process by forking
# now before allocating lots of memory.
p4gf_gitmirror.setup_spawn(view_name)
# Kick off garbage collection debugging, if enabled.
p4gf_gc.init_gc()
# Go no further, create NOTHING, if user not authorized.
# We use the translated internal view name here for perm authorization
required_perm = p4gf_server_common.COMMAND_TO_PERM[command]
view_perm = p4gf_group.ViewPerm.for_user_and_view(p4, user, view_name, required_perm)
try:
p4gf_server_common.check_authorization(p4, view_perm, user, command, view_name)
except p4gf_server_common.CommandError as ce:
start_response(_('403 Forbidden'), headers)
return [str(ce).encode('UTF-8')]
# Create Git Fusion server depot, user, config. NOPs if already created.
p4gf_init.init(p4)
before_lock_time = time.time()
with p4gf_lock.view_lock(p4, view_name) as view_lock:
after_lock_time = time.time()
# Create Git Fusion per-repo client view mapping and config.
init_repo_status = p4gf_init_repo.init_repo(p4, view_name, view_lock)
if init_repo_status == p4gf_init_repo.INIT_REPO_OK:
repo_created = True
elif init_repo_status == p4gf_init_repo.INIT_REPO_EXISTS:
repo_created = False
elif init_repo_status == p4gf_init_repo.INIT_REPO_NOVIEW:
start_response(_('404 Not Found'), headers)
return [sink.readall()]
else:
start_response(_('500 Internal Server Error'), headers)
return [b"Repository initialization failed\n"]
# If authorization came from default, not explicit group
# membership, copy that authorization to a group now. Could
# not do this until after p4gf_init_repo() has a chance to
# create not-yet-existing groups.
if view_perm:
view_perm.write_if(p4)
# Now that we have valid git-fusion-user and
# git-fusion-<view> client, replace our temporary P4
# connection with a more permanent Context, shared for the
# remainder of this process.
with p4gf_context.create_context(view_name, view_lock) as ctx:
LOG.debug("reconnected to P4, p4gf=%s", ctx.p4gf)
ctx.log_context()
# cd into the work directory. Not all git functions react well
# to --work-tree=xxxx.
cwd = os.getcwd()
os.chdir(ctx.view_dirs.GIT_WORK_TREE)
# Only copy from Perforce to Git if no other process is cloning
# from this Git repo right now.
shared_in_progress = p4gf_lock.shared_host_view_lock_exists(ctx.p4, view_name)
if not shared_in_progress:
# Copy any recent changes from Perforce to Git.
try:
LOG.debug("bare: No git-upload-pack in progress, force non-bare"
" before update Git from Perforce.")
p4gf_git.set_bare(False)
p4gf_copy_p2g.copy_p2g_ctx(ctx)
p4gf_init_repo.process_imports(ctx)
# Now is also an appropriate time to clear out any stale Git
# Swarm reviews. We're pre-pull, pre-push, time when we've
# got exclusive write access to the Git repo,
GSReviewCollection.delete_refs_for_closed_reviews(ctx)
except p4gf_lock.LockCanceled as lc:
LOG.warning(str(lc))
except:
# Dump failure to log, BEFORE cleanup, just in case
# cleanup ALSO fails and throws its own error (which
# happens if we're out of memory).
LOG.error(traceback.format_exc())
if repo_created:
# Return to the original working directory to allow the
# config code to call os.getcwd() without dying, since
# we are about to delete the current working directory.
os.chdir(cwd)
p4gf_server_common.cleanup_client(ctx, view_name)
raise
try:
exclusive = 'upload' not in command
is_push = 'upload' not in command
git_caller = functools.partial(_call_git, input_name, environ, ctx)
p4gf_call_git.call_git(git_caller, ctx, view_name, view_lock, exclusive)
if is_push:
GSReviewCollection.post_push(ctx)
except p4gf_atomic_lock.LockConflict as lc:
start_response(_('500 Internal Server Error'), headers)
return ["{}".format(lc).encode('UTF-8')]
p4gf_gc.process_garbage('at end of auth_server')
if LOG.isEnabledFor(logging.DEBUG):
end_time = time.time()
frm = NTR('Runtime: preparation {} ms, lock acquisition {} ms, processing {} ms')
LOG.debug(frm.format(before_lock_time - start_time,
after_lock_time - before_lock_time,
end_time - after_lock_time))
return []
def main():
"""Update the disk usage p4 keys for one or more repositories."""
desc = _("Set/reset the total and pending p4 keys.")
epilog = _("Without the -y/--reset option, only displays current values.")
parser = p4gf_util.create_arg_parser(desc, epilog=epilog)
parser.add_argument('-a', '--all', action='store_true',
help=_('process all known Git Fusion repositories'))
parser.add_argument('-y', '--reset', action='store_true',
help=_('perform the reset of the p4 keys'))
parser.add_argument(NTR('repos'), metavar=NTR('repo'), nargs='*',
help=_('name of repository to be updated'))
args = parser.parse_args()
# Check that either --all, or 'repos' was specified.
if not args.all and len(args.repos) == 0:
sys.stderr.write(_('Missing repo names; try adding --all option.\n'))
sys.exit(2)
if args.all and len(args.repos) > 0:
sys.stderr.write(_('Ambiguous arguments. Choose --all or a repo name.\n'))
sys.exit(2)
with p4gf_create_p4.Closer():
p4 = p4gf_create_p4.create_p4_temp_client()
if not p4:
sys.exit(2)
# Sanity check the connection (e.g. user logged in?) before proceeding.
try:
p4.fetch_client()
except P4.P4Exception as e:
sys.stderr.write(_('P4 exception occurred: {exception}').format(exception=e))
sys.exit(1)
if args.all:
repos = p4gf_util.repo_config_list(p4)
if len(repos) == 0:
print(_('No Git Fusion repositories found, nothing to do.'))
sys.exit(0)
else:
repos = args.repos
p4gf_create_p4.p4_disconnect(p4)
for repo in repos:
repo_name = p4gf_translate.TranslateReponame.git_to_repo(repo)
print(_("Processing repository {repo_name}... ").format(repo_name=repo_name), end='')
ctx = p4gf_context.create_context(repo_name)
with ExitStack() as stack:
stack.enter_context(ctx)
ctx.repo_lock = p4gf_lock.RepoLock(ctx.p4gf, repo_name, blocking=False)
stack.enter_context(ctx.repo_lock)
limits = PushLimits(ctx)
if args.reset:
# Copy any Perforce changes down to this Git repository.
p4gf_copy_p2g.copy_p2g_ctx(ctx)
# Attempt to trim any unreferenced objects.
p4gf_proc.popen(['git', '--git-dir=' + ctx.repo.path, 'prune'])
limits.post_copy()
# Display current key values and disk usage.
pending_mb = limits.get_pending_mb()
total_mb = limits.get_total_mb()
current_mb = limits.space_total
print(
_('{total_mb:.2f}M total, {pending_mb:.2f}M pending, '
'{current_mb:.2f}M current')
.format(total_mb=total_mb,
pending_mb=pending_mb,
current_mb=current_mb), end='')
print("")
def main(poll_only=False):
"""set up repo for a view
view_name_git is the untranslated repo name
view_name is the translated repo name
"""
p4gf_proc.install_stack_dumper()
_log_environ(os.environ)
with p4gf_server_common.ExceptionAuditLogger()\
, p4gf_create_p4.Closer():
LOG.debug(p4gf_log.memory_usage())
start_time = time.time()
args = parse_args(sys.argv[1:])
if not args:
return 1
is_push = 'upload' not in args.command[0]
# Record the p4 user in environment. We use environment to pass to
# git-invoked hook. We don't have to set ctx.authenticated_p4user because
# Context.__init__() reads it from environment, which we set here.
os.environ[p4gf_const.P4GF_AUTH_P4USER] = args.user
# view_name_git is the untranslated repo name
# view_name is the translated repo name
# print "args={}".format(args)
view_name_git = args.options[-1]
# translate '/' ':' ' ' .. etc .. for internal view_name
view_name = p4gf_translate.TranslateReponame.git_to_repo(view_name_git)
LOG.debug("public view_name: {0} internal view_name: {1}".format(
view_name_git, view_name))
p4gf_util.reset_git_enviro()
p4 = p4gf_create_p4.create_p4()
if not p4:
return 2
LOG.debug("connected to P4: %s", p4)
p4gf_server_common.check_readiness(p4)
p4gf_server_common.check_lock_perm(p4)
if not p4gf_server_common.check_protects(p4):
p4gf_server_common.raise_p4gf_perm()
if p4gf_server_common.run_special_command(view_name, p4, args.user):
return 0
# Initialize the external process launcher early, before allocating lots
# of memory, and just after all other conditions have been checked.
p4gf_proc.init()
# Prepare for possible spawn of GitMirror worker process by forking
# now before allocating lots of memory.
p4gf_gitmirror.setup_spawn(view_name)
# Kick off garbage collection debugging, if enabled.
p4gf_gc.init_gc()
if poll_only:
view_perm = None
else:
# Go no further, create NOTHING, if user not authorized.
# We use the translated internal view name here for perm authorization
required_perm = p4gf_server_common.COMMAND_TO_PERM[args.command[0]]
view_perm = p4gf_group.ViewPerm.for_user_and_view(
p4, args.user, view_name, required_perm)
p4gf_server_common.check_authorization(p4, view_perm, args.user,
args.command[0], view_name)
# Create Git Fusion server depot, user, config. NOPs if already created.
p4gf_init.init(p4)
write_motd()
# view_name is the internal view_name (identical when notExist special chars)
before_lock_time = time.time()
with p4gf_lock.view_lock(p4, view_name) as view_lock:
after_lock_time = time.time()
# Create Git Fusion per-repo client view mapping and config.
#
# NOPs if already created.
# Create the empty directory that will hold the git repo.
init_repo_status = p4gf_init_repo.init_repo(
p4, view_name, view_lock)
if init_repo_status == p4gf_init_repo.INIT_REPO_OK:
repo_created = True
elif init_repo_status == p4gf_init_repo.INIT_REPO_EXISTS:
repo_created = False
else:
return 1
# If authorization came from default, not explicit group
# membership, copy that authorization to a group now. Could
# not do this until after p4gf_init_repo() has a chance to
# create not-yet-existing groups.
if view_perm:
view_perm.write_if(p4)
# Now that we have valid git-fusion-user and
# git-fusion-<view> client, replace our temporary P4
# connection with a more permanent Context, shared for the
# remainder of this process.
with p4gf_context.create_context(view_name, view_lock) as ctx:
LOG.debug("reconnected to P4, p4gf=%s", ctx.p4gf)
# Find directory paths to feed to git.
ctx.log_context()
# cd into the work directory. Not all git functions react well
# to --work-tree=xxxx.
cwd = os.getcwd()
os.chdir(ctx.view_dirs.GIT_WORK_TREE)
# Only copy from Perforce to Git if no other process is cloning
# from this Git repo right now.
shared_in_progress = p4gf_lock.shared_host_view_lock_exists(
ctx.p4, view_name)
if not shared_in_progress:
# Copy any recent changes from Perforce to Git.
try:
LOG.debug(
"bare: No git-upload-pack in progress, force non-bare"
" before update Git from Perforce.")
p4gf_git.set_bare(False)
p4gf_copy_p2g.copy_p2g_ctx(ctx)
p4gf_init_repo.process_imports(ctx)
# Now is also an appropriate time to clear out any stale Git
# Swarm reviews. We're pre-pull, pre-push, time when we've
# got exclusive write access to the Git repo,
GSReviewCollection.delete_refs_for_closed_reviews(ctx)
except p4gf_lock.LockCanceled as lc:
LOG.warning(str(lc))
except:
# Dump failure to log, BEFORE cleanup, just in case
# cleanup ALSO fails and throws its own error (which
# happens if we're out of memory).
LOG.error(traceback.format_exc())
if repo_created:
# Return to the original working directory to allow the
# config code to call os.getcwd() without dying, since
# we are about to delete the current working directory.
os.chdir(cwd)
p4gf_server_common.cleanup_client(ctx, view_name)
raise
if poll_only:
code = os.EX_OK
else:
git_caller = functools.partial(_call_git, args, ctx)
try:
# Deep in call_git(), we grab an 'p4 reviews' lock on
# ctx.clientmap's LHS. Switch that clientmap to our
# full union view to prevent simultaneous 'git push'es
# from clobbering each other in some shared depot
# branch. Must include all lightweight branches, too.
ctx.switch_client_view_to_union()
exclusive = 'upload' not in args.command[0]
code = p4gf_call_git.call_git(git_caller, ctx,
view_name, view_lock,
exclusive)
if is_push:
GSReviewCollection.post_push(ctx)
except p4gf_atomic_lock.LockConflict as lc:
sys.stderr.write("{}\n".format(lc))
code = os.EX_SOFTWARE
p4gf_gc.process_garbage(NTR('at end of auth_server'))
if LOG.isEnabledFor(logging.DEBUG):
end_time = time.time()
frm = NTR("Runtime: preparation {} ms, lock acquisition {} ms,"
" processing {} ms")
LOG.debug(
frm.format(before_lock_time - start_time,
after_lock_time - before_lock_time,
end_time - after_lock_time))
return code
def main():
"""set up repo for a view"""
with ExceptionAuditLogger():
args = parse_args(sys.argv[1:])
if not args:
return 1
# Record the p4 user in environment. We use environment to pass to
# git-invoked hook. We don't have to set ctx.authenticated_p4user because
# Context.__init__() reads it from environment, which we set here.
os.environ[p4gf_const.P4GF_AUTH_P4USER_ENVAR] = args.user
# print "args={}".format(args)
view_name = args.options[-1]
p4gf_util.reset_git_enviro()
p4 = connect_p4()
if not p4:
return 2
LOG.debug("connected to P4: %s", p4)
_check_lock_perm(p4)
if not check_protects(p4):
_raise_p4gf_perm()
if run_special_command(view_name, p4, args.user):
return 0
# Go no further, create NOTHING, if user not authorized.
view_perm = p4gf_group.ViewPerm.for_user_and_view(p4, args.user, view_name)
_check_authorization(view_perm, args.user, args.command[0], view_name)
# Create Git Fusion server depot, user, config. NOPs if already created.
p4gf_init.init(p4)
with p4gf_lock.view_lock(p4, view_name) as view_lock:
# Create Git Fusion per-repo client view mapping and config.
#
# NOPs if already created.
# Create the empty directory that will hold the git repo.
init_repo_status = p4gf_init_repo.init_repo(p4, view_name)
if init_repo_status == p4gf_init_repo.INIT_REPO_OK:
repo_created = True
elif init_repo_status == p4gf_init_repo.INIT_REPO_EXISTS:
repo_created = False
else:
return 1
# If authorization came from default, not explicit group
# membership, copy that authorization to a group now. Could
# not do this until after p4gf_init_repo() has a chance to
# create not-yet-existing groups.
view_perm.write_if(p4)
# Now that we have valid git-fusion-user and
# git-fusion-<view> client, replace our temporary P4
# connection with a more permanent Context, shared for the
# remainder of this process.
ctx = p4gf_context.create_context(view_name, view_lock)
del p4
LOG.debug("reconnected to P4, p4gf=%s", ctx.p4gf)
# Find directory paths to feed to git.
ctx.view_dirs = p4gf_view_dirs.from_p4gf_dir(ctx.gitrootdir, view_name)
ctx.log_context()
# cd into the work directory. Not all git functions react well
# to --work-tree=xxxx.
cwd = os.getcwd()
os.chdir(ctx.view_dirs.GIT_WORK_TREE)
# Copy any recent changes from Perforce to Git.
try:
p4gf_copy_p2g.copy_p2g_ctx(ctx)
except:
# Dump failure to log, BEFORE cleanup, just in case
# cleanup ALSO fails and throws its own error (which
# happens if we're out of memory).
LOG.error(traceback.format_exc())
if repo_created:
# Return to the original working directory to allow the
# config code to call os.getcwd() without dying, since
# we are about to delete the current working directory.
os.chdir(cwd)
cleanup_client(ctx, view_name)
raise
# Detach git repo's workspace from master before calling
# original git, otherwise we won't be able to push master.
p4gf_util.checkout_detached_master()
# Flush stderr before returning control to Git.
# Otherwise Git's own output might interrupt ours.
sys.stderr.flush()
return _call_original_git(ctx, args)
def _wsgi_app(environ, start_response):
"""
WSGI application to process the incoming Git client request. This is
nearly equivalent to p4gf_auth_server.main() with the exception of
input validation and error handling.
"""
p4gf_log.record_http(environ)
p4gf_version.log_version()
_log_environ(environ)
p4gf_version.version_check()
LOG.debug("processing HTTP request, pid={}".format(os.getpid()))
# Keep the content type to exactly 'text/plain' so there is at least
# the remote chance that Git might show our error messages (does not
# appear to work in practice, however).
headers = [('Content-Type', 'text/plain')]
encoding = sys.getfilesystemencoding()
if encoding == 'ascii':
# This encoding is wrong and will eventually lead to problems.
LOG.error(
"Using 'ascii' file encoding will ultimately result in errors, "
"please set LANG/LC_ALL to 'utf-8' in web server configuration.")
start_response(_('500 Internal Server Error'), headers)
return [b"Filesystem encoding not set to acceptable value.\n"]
# Sanity check the request.
for (name, status, msg) in _REQUIRED_HTTP_PARAMS:
if name not in environ:
start_response(status, headers)
return [msg.encode('UTF-8')]
input_name = environ['wsgi.input']
# Extract the view_name_git by removing the expected git request suffixes
path_info = environ['PATH_INFO']
git_suffixes = [
'/info/refs', '/HEAD', '/git-upload-pack', '/git-receive-pack'
]
path_end = len(path_info)
for suffix in git_suffixes:
try:
path_end = path_info.index(suffix)
break
except ValueError:
pass
# slice away the leading slash and the trailing git request suffixes
view_name_git = path_info[1:path_end]
# and remove the view_name_git from the front of PATH_INFO
environ['PATH_INFO'] = path_info[path_end:]
LOG.debug("new PATH_INFO {0} view_name_git {1}".format(
environ['PATH_INFO'], view_name_git))
if not view_name_git:
start_response(_('400 Bad Request'), headers)
msg = _('Missing required repository name in URL\n')
return [msg.encode('UTF-8')]
# translate '/' ':' ' ' .. etc .. for internal view_name
view_name = p4gf_translate.TranslateReponame.git_to_repo(view_name_git)
LOG.debug("public view_name: {0} internal view_name: {1}".format(
view_name_git, view_name))
audit_logger = p4gf_server_common.ExceptionAuditLogger()
p4_closer = p4gf_create_p4.Closer()
sink = OutputSink()
temp_deleter = deleting(input_name)
mirror_closer = unmirror(view_name)
with audit_logger \
, p4_closer \
, sink \
, temp_deleter \
, mirror_closer:
LOG.debug(p4gf_log.memory_usage())
start_time = time.time()
p4gf_util.reset_git_enviro()
p4 = p4gf_create_p4.create_p4()
if not p4:
start_response(_('500 Internal Server Error'), headers)
return [b"Perforce connection failed\n"]
LOG.debug("connected to P4: %s", p4)
p4gf_server_common.check_readiness(p4)
p4gf_server_common.check_lock_perm(p4)
if not p4gf_server_common.check_protects(p4):
p4gf_server_common.raise_p4gf_perm()
user = environ['REMOTE_USER']
if p4gf_server_common.run_special_command(view_name, p4, user):
start_response(_('200 OK'), headers)
return [sink.readall()]
command = _get_command(environ)
if not command:
start_response(_('400 Bad Request'), headers)
return [b"Unrecognized service\n"]
# Other places in the Perforce-to-Git phase will need to know the
# name of client user, so set that here. As for Git-to-Perforce,
# that is handled later by setting the REMOTE_USER envar. Notice
# also that we're setting os.environ and not 'environ'.
os.environ[p4gf_const.P4GF_AUTH_P4USER] = user
# Likewise, some code needs a hint that the request is coming over
# one protocol (HTTP) or the other (SSH).
os.environ['REMOTE_ADDR'] = environ['REMOTE_ADDR']
# Initialize the external process launcher early, before allocating lots
# of memory, and just after all other conditions have been checked.
p4gf_proc.init()
# Prepare for possible spawn of GitMirror worker process by forking
# now before allocating lots of memory.
p4gf_gitmirror.setup_spawn(view_name)
# Kick off garbage collection debugging, if enabled.
p4gf_gc.init_gc()
# Go no further, create NOTHING, if user not authorized.
# We use the translated internal view name here for perm authorization
required_perm = p4gf_server_common.COMMAND_TO_PERM[command]
view_perm = p4gf_group.ViewPerm.for_user_and_view(
p4, user, view_name, required_perm)
try:
p4gf_server_common.check_authorization(p4, view_perm, user,
command, view_name)
except p4gf_server_common.CommandError as ce:
start_response(_('403 Forbidden'), headers)
return [str(ce).encode('UTF-8')]
# Create Git Fusion server depot, user, config. NOPs if already created.
p4gf_init.init(p4)
before_lock_time = time.time()
with p4gf_lock.view_lock(p4, view_name) as view_lock:
after_lock_time = time.time()
# Create Git Fusion per-repo client view mapping and config.
init_repo_status = p4gf_init_repo.init_repo(
p4, view_name, view_lock)
if init_repo_status == p4gf_init_repo.INIT_REPO_OK:
repo_created = True
elif init_repo_status == p4gf_init_repo.INIT_REPO_EXISTS:
repo_created = False
elif init_repo_status == p4gf_init_repo.INIT_REPO_NOVIEW:
start_response(_('404 Not Found'), headers)
return [sink.readall()]
else:
start_response(_('500 Internal Server Error'), headers)
return [b"Repository initialization failed\n"]
# If authorization came from default, not explicit group
# membership, copy that authorization to a group now. Could
# not do this until after p4gf_init_repo() has a chance to
# create not-yet-existing groups.
if view_perm:
view_perm.write_if(p4)
# Now that we have valid git-fusion-user and
# git-fusion-<view> client, replace our temporary P4
# connection with a more permanent Context, shared for the
# remainder of this process.
with p4gf_context.create_context(view_name, view_lock) as ctx:
LOG.debug("reconnected to P4, p4gf=%s", ctx.p4gf)
ctx.log_context()
# cd into the work directory. Not all git functions react well
# to --work-tree=xxxx.
cwd = os.getcwd()
os.chdir(ctx.view_dirs.GIT_WORK_TREE)
# Only copy from Perforce to Git if no other process is cloning
# from this Git repo right now.
shared_in_progress = p4gf_lock.shared_host_view_lock_exists(
ctx.p4, view_name)
if not shared_in_progress:
# Copy any recent changes from Perforce to Git.
try:
LOG.debug(
"bare: No git-upload-pack in progress, force non-bare"
" before update Git from Perforce.")
p4gf_git.set_bare(False)
p4gf_copy_p2g.copy_p2g_ctx(ctx)
p4gf_init_repo.process_imports(ctx)
# Now is also an appropriate time to clear out any stale Git
# Swarm reviews. We're pre-pull, pre-push, time when we've
# got exclusive write access to the Git repo,
GSReviewCollection.delete_refs_for_closed_reviews(ctx)
except p4gf_lock.LockCanceled as lc:
LOG.warning(str(lc))
except:
# Dump failure to log, BEFORE cleanup, just in case
# cleanup ALSO fails and throws its own error (which
# happens if we're out of memory).
LOG.error(traceback.format_exc())
if repo_created:
# Return to the original working directory to allow the
# config code to call os.getcwd() without dying, since
# we are about to delete the current working directory.
os.chdir(cwd)
p4gf_server_common.cleanup_client(ctx, view_name)
raise
try:
exclusive = 'upload' not in command
is_push = 'upload' not in command
git_caller = functools.partial(_call_git, input_name,
environ, ctx)
p4gf_call_git.call_git(git_caller, ctx, view_name,
view_lock, exclusive)
if is_push:
GSReviewCollection.post_push(ctx)
except p4gf_atomic_lock.LockConflict as lc:
start_response(_('500 Internal Server Error'), headers)
return ["{}".format(lc).encode('UTF-8')]
p4gf_gc.process_garbage('at end of auth_server')
if LOG.isEnabledFor(logging.DEBUG):
end_time = time.time()
frm = NTR(
'Runtime: preparation {} ms, lock acquisition {} ms, processing {} ms'
)
LOG.debug(
frm.format(before_lock_time - start_time,
after_lock_time - before_lock_time,
end_time - after_lock_time))
return []
