def _upgrade_p4gf(p4):
"""Perform upgrade from earlier versions of P4GF.
This should be invoked using _maybe_perform_init() to avoid race conditions
across hosts.
"""
# If updating from 12.2 to 13.1 we need to create global config file
# (this does nothing if file already exists)
p4gf_config.GlobalConfig.init(p4)
with p4.at_exception_level(p4.RAISE_ERROR):
if p4gf_config.GlobalConfig.write_if(p4):
_info(_("Global config file '{path}' created/updated.")
.format(path=p4gf_config.depot_path_global()))
else:
_info(_("Global config file '{path}' already exists.")
.format(path=p4gf_config.depot_path_global()))
# Ensure the time zone name has been set, else default to something sensible.
tzname = P4Key.get(p4, p4gf_const.P4GF_P4KEY_TIME_ZONE_NAME)
if tzname == "0" or tzname is None:
msg = _("p4 key '{key}' not set, using UTC as default."
" Change this to your Perforce server's time zone.") \
.format(key=p4gf_const.P4GF_P4KEY_TIME_ZONE_NAME)
LOG.warning(msg)
sys.stderr.write(_('Git Fusion: {message}\n').format(message=msg))
tzname = None
else:
# Sanity check the time zone name.
try:
pytz.timezone(tzname)
except pytz.exceptions.UnknownTimeZoneError:
LOG.warning("Time zone name '{}' unrecognized, using UTC as default".format(tzname))
tzname = None
if tzname is None:
P4Key.set(p4, p4gf_const.P4GF_P4KEY_TIME_ZONE_NAME, 'UTC')
def do_acquire(self):
"""Acquire the lock and set the owner key if successful."""
if not super(ReviewsLock, self).do_acquire():
return False
val = json.dumps(self.content)
P4Key.set(self.p4, p4gf_const.P4GF_REVIEWS_COMMON_LOCK_OWNER, val)
return True
def _prevent_access(p4):
"""Prevent further access to Git Fusion while deleting everything.
Return the previous value of the p4key so it can be restored later.
"""
old_value = P4Key.get(p4, p4gf_const.P4GF_P4KEY_PREVENT_NEW_SESSIONS)
P4Key.set(p4, p4gf_const.P4GF_P4KEY_PREVENT_NEW_SESSIONS, 'true')
return old_value
def set_proxy_protects_key():
"""Get server's dm.proxy_protects and store as a key for Git Fusion."""
v = p4gf_util.first_value_for_key(
p4.run('configure', 'show', CONFIGURABLE_PROXY_PROTECTS),
KEY_VALUE)
v = 'false' if v == '0' else 'true'
P4Key.set(p4, p4gf_const.P4GF_P4KEY_PROXY_PROTECTS, v)
Verbosity.report(
Verbosity.INFO, _("Configurable '{configurable}' is set to {value}.")
.format(configurable=CONFIGURABLE_PROXY_PROTECTS, value=v))
def _write_p4key(self, change_num, val_dict):
"""Convert a dict of depot_path#rev ==> sha1 to a single string,
write that to this changelist's p4key.
"""
val = _format_p4key_val(val_dict)
if len(val) > 1024:
# For sufficiently large values, compress and base64 encode to
# save space (about 40% at 1kb, and generally increasing with
# input size).
val = binascii.b2a_base64(zlib.compress(val.encode()))
P4Key.set(self.ctx, self.p4key_name(change_num), val)
def create_default_perm(p4, perm=DEFAULT_PERM):
"""Create the 'stick all users into this pull/push permission group' default p4key.
If p4key already exists with non-zero value, leave it unchanged.
"""
p4key = P4Key.get(p4, p4gf_const.P4GF_P4KEY_PERMISSION_GROUP_DEFAULT)
if p4key is not None and p4key != '0':
# Somebody already set it.
return False
P4Key.set(p4, p4gf_const.P4GF_P4KEY_PERMISSION_GROUP_DEFAULT, perm)
return True
def update_last_change_num(p4, change_num):
"""Update the last change number with which we synced the keys to the
SSH configuration file. The change_num must be a positive number.
"""
fmt = _("invalid change number '{change_num}'")
try:
v = int(change_num)
if v < 1:
raise RuntimeError(fmt.format(change_num=change_num))
P4Key.set(p4, _get_p4key_name(), v)
except ValueError:
raise RuntimeError(fmt.format(change_num=change_num))
def update_last_change_num(ctx, commit_ot):
"""Update p4 key that tracks the last change_num on a branch."""
# unconditionally add a p4key mapping change_num -> commit sha1
ObjectType.write_index_p4key(ctx, commit_ot)
branch_id = commit_ot.branch_id
# only update last change_num p4key if this commit has a higher change_num
if branch_id in ObjectType.last_commits_cache and\
(int(ObjectType.last_commits_cache[branch_id].split(',')[0]) >
int(commit_ot.change_num)):
return
(key_pattern, value) = commit_ot.to_index_last_key_value()
P4Key.set(ctx.p4gf, key_pattern, value)
ObjectType.last_commits_cache[branch_id] = value
def create_p4_temp_client(port=None, user=None, skip_count=False):
"""Create a connected P4 instance with a generic temporary client.
Dropping the connection will automatically delete the client. This is
useful for owning the locks and permitting reliable lock stealing by
other processes.
:return: P4API instance.
"""
p4 = create_p4(port, user, warn_no_client=False)
if p4 is None:
# Propagate the error (that has already been reported).
return None
name = p4gf_const.P4GF_OBJECT_CLIENT_UNIQUE.format(
server_id=p4gf_util.get_server_id(), uuid=str(uuid.uuid1()))
client = p4.fetch_client(name)
client['Owner'] = p4gf_const.P4GF_USER
client['LineEnd'] = NTR('unix')
client['View'] = [
'//{0}/... //{1}/...'.format(p4gf_const.P4GF_DEPOT, name)
]
# to prevent the mirrored git commit/tree objects from being retained in the
# git-fusion workspace, set client option 'rmdir' and sync #none in p4gf_gitmirror
client['Options'] = p4gf_const.CLIENT_OPTIONS.replace("normdir", "rmdir")
client['Root'] = p4gf_const.P4GF_HOME
# The -x option is a deep undoc feature that signals to p4d that this
# is a temporary client, which will be automatically deleted upon
# disconnect. Requires passing the client specification using -i flag.
# N.B. this client cannot shelve changes. See @465851 for details.
# N.B. only one temporary client per connection will be auto-deleted
p4.client = name
p4.save_client(client, '-x')
LOG.debug("create_p4_temp_client() created temp client %s", name)
if 'P4T4TEST_ORIG_LANG' in os.environ and not skip_count:
# In the testing environment, we check that each process created no
# more than one temporary client (concurrently). Keep the highest
# value, rather than whatever the last count happened to be.
prefix = p4gf_const.P4GF_OBJECT_CLIENT_UNIQUE.format(
server_id=p4gf_util.get_server_id(), uuid='')
count = _count_active(prefix)
# Some tests set up an unnatural environment, so don't let those
# blow up in unexpected ways (i.e. fail gracefully elsewhere).
with p4.at_exception_level(P4.P4.RAISE_NONE):
value = p4gf_p4key.get(
p4, "git-fusion-temp-clients-{}".format(os.getpid()))
if int(value) < count:
p4gf_p4key.set(
p4, "git-fusion-temp-clients-{}".format(os.getpid()),
str(count))
return p4
def overwrite_last_copied_tag(p4, repo_name, change_num):
"""Set the changelist number for the most recent tags change.
:type p4: :class:`P4API`
:param p4: P4 API instance
:type repo_name: str
:param repo_name: name fo the Git Fusion repository
:type change_num: int || str
:param change_num: changelist number to assign to tags key.
"""
keyname = _calc_last_copied_tag_p4key_name(repo_name,
p4gf_util.get_server_id())
LOG.debug('overwrite_last_copied_tag() assigning {} to {}'.format(
change_num, keyname))
return P4Key.set(p4, keyname, change_num)
def write_index_p4key(ctx, commit_ot):
"""Record the p4key index that goes with this commit."""
(key_name, value) = commit_ot.to_index_key_value()
P4Key.set(ctx.p4gf, key_name, value)
def delete_all(args, p4, metrics):
"""Remove all Git Fusion clients, as well as the object cache.
Keyword arguments:
args -- parsed command line arguments
p4 -- Git user's Perforce client
"""
# pylint:disable=too-many-branches
p4.user = p4gf_const.P4GF_USER
group_list = [p4gf_const.P4GF_GROUP_PULL, p4gf_const.P4GF_GROUP_PUSH]
print(_('Connected to {P4PORT}').format(P4PORT=p4.port))
print_verbose(args, _('Scanning for Git Fusion clients...'))
client_name = p4gf_util.get_object_client_name()
locks = _lock_all_repos(p4)
if args.delete:
was_prevented = _prevent_access(p4)
else:
was_prevented = None
delete_clients(args, p4, metrics)
# Retrieve the names of the initialization/upgrade "lock" p4keys.
p4keys = [
p4gf_const.P4GF_P4KEY_ALL_PENDING_MB,
p4gf_const.P4GF_P4KEY_ALL_REMAINING_MB
]
# Key patterns NOT published in p4gf_const because they have trailing *
# wildcards and it's not worth cluttering p4gf_const for this one use.
p4key_patterns = [
'git-fusion-init-started*', 'git-fusion-init-complete*',
'git-fusion-upgrade-started*', 'git-fusion-upgrade-complete*',
'git-fusion-index-*'
]
for p4key_pattern in p4key_patterns:
d = P4Key.get_all(p4, p4key_pattern)
p4keys.extend(sorted(d.keys()))
localroot = get_p4gf_localroot(p4)
if not args.delete:
if localroot:
if args.no_obliterate:
print(NTR('p4 sync -f #none'))
else:
print(NTR('p4 client -f -d {}').format(client_name))
print(NTR('rm -rf {}').format(localroot))
if not args.no_obliterate:
print(
NTR('p4 obliterate -hay //{}/objects/...').format(
p4gf_const.P4GF_DEPOT))
for p4key in p4keys:
print(NTR('p4 key -d {}').format(p4key))
for group in group_list:
print(NTR('p4 group -a -d {}').format(group))
else:
if localroot:
if not args.no_obliterate:
# Need this in order to use --gc later on
# client should not exist; this is likely a NOOP
p4gf_util.p4_client_df(p4, client_name)
metrics.clients += 1
print_verbose(
args,
_("Deleting client '{client_name}'s workspace...").format(
client_name=client_name))
_remove_local_root(localroot)
_delete_cache(args, p4, metrics)
print_verbose(args, _('Removing initialization p4keys...'))
for p4key in p4keys:
delete_p4key(p4, p4key, metrics)
for group in group_list:
delete_group(args, p4, group, metrics)
_release_locks(locks)
if was_prevented is not None:
if was_prevented != '0':
P4Key.set(p4, p4gf_const.P4GF_P4KEY_PREVENT_NEW_SESSIONS,
was_prevented)
else:
P4Key.delete(p4, p4gf_const.P4GF_P4KEY_PREVENT_NEW_SESSIONS)
def _write_last_copied_tag(ctx, change_num):
"""Update the changelist number for the most recent tags change for this
Git Fusion server.
"""
return P4Key.set(ctx.p4gf, _last_copied_tag_p4key_name(ctx), change_num)
def _set_key(self, key, value):
"""Set a key."""
if '{repo_name}' in key:
key = key.format(repo_name=self.repo_name)
p4gf_p4key.set(self.p4, key, str(value))
def main():
"""Parse the command-line arguments and report on locks."""
# pylint: disable=too-many-statements
desc = _("Report the currently held locks in Git Fusion.")
parser = p4gf_util.create_arg_parser(desc=desc)
parser.add_argument('--test',
action='store_true',
help=_('invoke test mode, acquire locks and report'))
parser.add_argument(
'--test2',
action='store_true',
help=_(
'invoke test mode, acquire locks and report, set dead processes.'))
args = parser.parse_args()
p4gf_util.has_server_id_or_exit()
server_id = p4gf_util.get_server_id()
p4 = p4gf_create_p4.create_p4_temp_client()
if not p4:
sys.exit(1)
print("Connecting to P4PORT={} as P4USER={}".format(p4.port, p4.user))
if args.test or args.test2:
repo_name = "p4gf_test_status_repo"
status_key_name = p4gf_p4key.calc_repo_status_p4key_name(
repo_name, None)
p4gf_p4key.set(p4, status_key_name, 'Push 1 completed successfully')
pushid_key_name = p4gf_p4key.calc_repo_push_id_p4key_name(repo_name)
p4gf_p4key.set(p4, pushid_key_name, '1')
# create a process and kill it and set its dead pid as a RepoLock owner below.
if args.test:
# A test with nothing stale
with ExitStack() as stack:
stack.enter_context(p4gf_lock.ReviewsLock(p4))
stack.enter_context(p4gf_lock.RepoLock(p4, repo_name))
stack.enter_context(p4gf_git_repo_lock.read_lock(repo_name))
stack.enter_context(
p4gf_git_repo_lock.write_lock(repo_name, upgrade=True))
print_lock_status(p4, server_id)
else: # if args.test2
# Now a test with some DEAD processes and a stale view Lock
dead_process = subprocess.Popen(['echo', 'x'],
stdout=subprocess.DEVNULL)
dead_process.kill()
while dead_process.returncode is None:
dead_process.communicate()
lock2 = None
with ExitStack() as stack:
stack.enter_context(p4gf_lock.ReviewsLock(p4))
# first lock owner
lock1 = p4gf_lock.RepoLock(p4, repo_name)
# second lock owner with same group_id and a dead pid
lock2 = p4gf_lock.RepoLock(p4,
repo_name,
group_id=lock1.group_id)
lock2.process_id = dead_process.pid
# acquire the first RepoLock
stack.enter_context(lock1)
# Use low level method to add this DEAD pid to the group's lock owners
lock2.do_acquire()
stack.enter_context(p4gf_git_repo_lock.read_lock(repo_name))
stack.enter_context(
p4gf_git_repo_lock.write_lock(repo_name, upgrade=True))
print("Test 1:")
print_lock_status(p4, server_id)
p4gf_p4key.set(p4, pushid_key_name, '2')
p4gf_p4key.set(p4, status_key_name,
'Push 2 failed: some error')
# Finally lets set the P4GF_P4KEY_LOCK_VIEW - the least likley to be stale
p4gf_p4key.set(p4, lock2.lock_key, '1')
print("Test 2:")
print_lock_status(p4, server_id)
# Cant exit the ExistStack unless we clean this
p4gf_p4key.delete(p4, lock2.lock_key)
# Clean up this lock so the test may be run again
p4gf_p4key.delete(p4, lock2.owners_key)
# remove test keys
p4gf_p4key.delete(p4, status_key_name)
p4gf_p4key.delete(p4, pushid_key_name)
else:
print_lock_status(p4, server_id)
def _write(self, content):
"""Write content to the key."""
val = json.dumps(content)
P4Key.set(self.p4, self.owners_key, val)
def ensure_depot():
"""Create depot P4GF_DEPOT if not already exists."""
# We use P4GF_P4KEY_P4GF_DEPOT to send the depot name
# to p4gf_submit_trigger.py. Ensure that P4GF_P4KEY_P4GF_DEPOT matches
# requested depot name from P4GF_ENV.
# pylint: disable=line-too-long
p4gf_depot_key = P4Key.get(p4, p4gf_const.P4GF_P4KEY_P4GF_DEPOT)
if p4gf_depot_key != '0' and p4gf_depot_key != p4gf_const.P4GF_DEPOT:
Verbosity.report(
Verbosity.ERROR,
_("\n**ERROR: There is a configuration conflict for this Git Fusion instance of the P4GF_DEPOT.\n"
" P4GF_DEPOT is configured to a value different than that recorded in the p4d server.\n"
" You must correct this as described here.\n"
" This key must either be:\n"
" 1) unset, in which case this initialization will set it to the correct value, or\n"
" 2) match the value set in the P4GF_ENV configuration file, or\n"
" 3) set to the default '.git-fusion' if not set to a different value in the P4GF_ENV configuration file.\n"
"\n"
" The p4d server p4key '{key}' is set to '{value}'.\n")
.format(key=p4gf_const.P4GF_P4KEY_P4GF_DEPOT, value=p4gf_depot_key))
depot_configured = (p4gf_const.P4GF_ENV and p4gf_env_config.P4GF_ENV_CONFIG_DICT and
p4gf_const.P4GF_DEPOT_NAME in p4gf_env_config.P4GF_ENV_CONFIG_DICT)
if depot_configured:
msg = _(" This Git Fusion instance is configured with Git Fusion depot = '{depot}'"
"\n from P4GF_ENV file '{env}' containing these settings:\n") \
.format(depot=p4gf_const.P4GF_DEPOT, env=p4gf_const.P4GF_ENV)
for k, v in p4gf_env_config.P4GF_ENV_CONFIG_DICT.items():
msg = msg + " {0} = {1}\n".format(k, v)
else:
msg = _(" This Git Fusion instance is configured for the default P4GF_DEPOT : '{depot}'.\n")\
.format(depot=p4gf_const.P4GF_DEPOT)
Verbosity.report(Verbosity.ERROR, msg)
Verbosity.report(
Verbosity.ERROR,
_(" To correct this mismatch , determine the required P4GF_DEPOT value.\n"
" 1) If the value of p4key '{key}'='{value}' is different than the required value, remove the p4key.\n"
" Make certain the current p4key value was not correctly set by another Git Fusion instance.\n"
" 2) If the required value is the default 'git-fusion', and there is a P4GF_DEPOT setting in the P4GF_ENV file, remove it.\n"
" 2) If the required value is not the default 'git-fusion', set P4GF_DEPOT to the required value in the P4GF_ENV file.\n"
" 3) Re-run this configure-git-fusion.sh script.")
.format(key=p4gf_const.P4GF_P4KEY_P4GF_DEPOT, value=p4gf_depot_key))
sys.exit(1)
try:
created = p4gf_p4spec.ensure_depot_gf(p4)
if created:
Verbosity.report(Verbosity.INFO, _("Depot '{depot}' created.")
.format(depot=p4gf_const.P4GF_DEPOT))
else:
Verbosity.report(
Verbosity.INFO, _("Depot '{depot}' already exists. Not creating.")
.format(depot=p4gf_const.P4GF_DEPOT))
except Exception as depote:
Verbosity.report(Verbosity.INFO, _("Error creating Depot '{depot}'.")
.format(depot=p4gf_const.P4GF_DEPOT))
raise depote
# Set the P4GF_DEPOT p4key to send the depot name to p4gf_submit_trigger.py.
# This will either be the default or the value set from P4GF_ENV config file
P4Key.set(p4, p4gf_const.P4GF_P4KEY_P4GF_DEPOT, p4gf_const.P4GF_DEPOT)
return created
def set_server_id_p4key(server_id):
"""Set the server-id p4key value to the hostname to identify GF hosts."""
if server_id:
P4Key.set(p4, p4gf_const.P4GF_P4KEY_SERVER_ID + server_id,
p4gf_util.get_hostname())