def print_reviews_lock_status(p4, server_id, pfunc=print):
"""Report on the status of the reviews locks.
:param p4: P4API to query Perforce.
:param server_id: identifier for this Git Fusion instance.
:param pfunc: print function - either 'print' or some logger : 'LOG.debug'
"""
raw_value = p4gf_p4key.get(p4, p4gf_const.P4GF_REVIEWS_COMMON_LOCK_OWNER)
if raw_value and raw_value != '0':
try:
content = json.loads(raw_value)
except ValueError:
# 15.1 had a bug in which the key value was not properly formed JSON
# (in fact, it was just a str()-formatted Python dict).
pfunc("Reviews common lock invalid format.\nRaw value: {}".format(
raw_value))
return
if 'server_id' not in content:
pfunc("Reviews common lock malformed")
return
lock_server_id = content['server_id']
pid = content['process_id']
start_time = content['start_time']
status = _pid_status(pid) if lock_server_id == server_id else "UNKNOWN"
pfunc("Reviews common: server {}, PID {}, started at {}, status {}\n".
format(lock_server_id, pid, start_time, status))
def for_user_and_repo(p4, p4user, repo_name, required_perm):
"""Factory to fetch user's permissions on a repo."""
LOG.debug("for_user_and_repo() {u} {r} {p}".format(u=p4user,
r=repo_name,
p=required_perm))
group_list = _groups_i(p4, p4user)
group_dict = {group['group']: group for group in group_list}
LOG.debug3("group_dict.keys()={}".format(group_dict.keys()))
vp = RepoPerm()
vp.p4user_name = p4user
vp.repo_name = repo_name
vp.repo_pull = p4gf_const.P4GF_GROUP_REPO_PULL.format(
repo=repo_name) in group_dict
vp.repo_push = p4gf_const.P4GF_GROUP_REPO_PUSH.format(
repo=repo_name) in group_dict
vp.global_pull = p4gf_const.P4GF_GROUP_PULL in group_dict
vp.global_push = p4gf_const.P4GF_GROUP_PUSH in group_dict
value = P4Key.get(p4, p4gf_const.P4GF_P4KEY_PERMISSION_GROUP_DEFAULT)
if value == '0':
value = DEFAULT_PERM
vp.default_pull = value == PERM_PULL
vp.default_push = value == PERM_PUSH
LOG.debug("p4key={}".format(value))
LOG.debug(vp)
return vp
def _upgrade_p4gf(p4):
"""Perform upgrade from earlier versions of P4GF.
This should be invoked using _maybe_perform_init() to avoid race conditions
across hosts.
"""
# If updating from 12.2 to 13.1 we need to create global config file
# (this does nothing if file already exists)
p4gf_config.GlobalConfig.init(p4)
with p4.at_exception_level(p4.RAISE_ERROR):
if p4gf_config.GlobalConfig.write_if(p4):
_info(_("Global config file '{path}' created/updated.")
.format(path=p4gf_config.depot_path_global()))
else:
_info(_("Global config file '{path}' already exists.")
.format(path=p4gf_config.depot_path_global()))
# Ensure the time zone name has been set, else default to something sensible.
tzname = P4Key.get(p4, p4gf_const.P4GF_P4KEY_TIME_ZONE_NAME)
if tzname == "0" or tzname is None:
msg = _("p4 key '{key}' not set, using UTC as default."
" Change this to your Perforce server's time zone.") \
.format(key=p4gf_const.P4GF_P4KEY_TIME_ZONE_NAME)
LOG.warning(msg)
sys.stderr.write(_('Git Fusion: {message}\n').format(message=msg))
tzname = None
else:
# Sanity check the time zone name.
try:
pytz.timezone(tzname)
except pytz.exceptions.UnknownTimeZoneError:
LOG.warning("Time zone name '{}' unrecognized, using UTC as default".format(tzname))
tzname = None
if tzname is None:
P4Key.set(p4, p4gf_const.P4GF_P4KEY_TIME_ZONE_NAME, 'UTC')
def _prevent_access(p4):
"""Prevent further access to Git Fusion while deleting everything.
Return the previous value of the p4key so it can be restored later.
"""
old_value = P4Key.get(p4, p4gf_const.P4GF_P4KEY_PREVENT_NEW_SESSIONS)
P4Key.set(p4, p4gf_const.P4GF_P4KEY_PREVENT_NEW_SESSIONS, 'true')
return old_value
def _perm_key_default(p4):
"""Run `p4 key git-fusion-permission-group-default` to get
default permissions. Once.
"""
global _PERM_KEY_DEFAULT
if _PERM_KEY_DEFAULT is None:
_PERM_KEY_DEFAULT = P4Key.get(
p4, p4gf_const.P4GF_P4KEY_PERMISSION_GROUP_DEFAULT)
return _PERM_KEY_DEFAULT
def _read(self):
"""Read the current content from key."""
try:
s = P4Key.get(self.p4, self.owners_key)
if s == "0":
return {}
return json.loads(s)
except ValueError:
raise LockCorrupt(self)
def get_last_change(p4):
"""Fetch the last change number with which we synced the keys to the
SSH configuration file. Returns a positive number, or zero if no saved
p4key value.
"""
v = P4Key.get(p4, _get_p4key_name())
if v:
return int(v)
else:
return 0
def print_space_lock_status(p4, pfunc=print):
"""Report on the status of the disk usage lock.
:param p4: P4API to query Perforce.
:param pfunc: print function - either 'print' or some logger : 'LOG.debug'
"""
raw_value = p4gf_p4key.get(p4, p4gf_const.P4GF_P4KEY_LOCK_SPACE)
if raw_value and raw_value != '0':
pfunc("Space lock ({}) is set".format(
p4gf_const.P4GF_P4KEY_LOCK_SPACE))
def create_default_perm(p4, perm=DEFAULT_PERM):
"""Create the 'stick all users into this pull/push permission group' default p4key.
If p4key already exists with non-zero value, leave it unchanged.
"""
p4key = P4Key.get(p4, p4gf_const.P4GF_P4KEY_PERMISSION_GROUP_DEFAULT)
if p4key is not None and p4key != '0':
# Somebody already set it.
return False
P4Key.set(p4, p4gf_const.P4GF_P4KEY_PERMISSION_GROUP_DEFAULT, perm)
return True
def create_p4_temp_client(port=None, user=None, skip_count=False):
"""Create a connected P4 instance with a generic temporary client.
Dropping the connection will automatically delete the client. This is
useful for owning the locks and permitting reliable lock stealing by
other processes.
:return: P4API instance.
"""
p4 = create_p4(port, user, warn_no_client=False)
if p4 is None:
# Propagate the error (that has already been reported).
return None
name = p4gf_const.P4GF_OBJECT_CLIENT_UNIQUE.format(
server_id=p4gf_util.get_server_id(), uuid=str(uuid.uuid1()))
client = p4.fetch_client(name)
client['Owner'] = p4gf_const.P4GF_USER
client['LineEnd'] = NTR('unix')
client['View'] = [
'//{0}/... //{1}/...'.format(p4gf_const.P4GF_DEPOT, name)
]
# to prevent the mirrored git commit/tree objects from being retained in the
# git-fusion workspace, set client option 'rmdir' and sync #none in p4gf_gitmirror
client['Options'] = p4gf_const.CLIENT_OPTIONS.replace("normdir", "rmdir")
client['Root'] = p4gf_const.P4GF_HOME
# The -x option is a deep undoc feature that signals to p4d that this
# is a temporary client, which will be automatically deleted upon
# disconnect. Requires passing the client specification using -i flag.
# N.B. this client cannot shelve changes. See @465851 for details.
# N.B. only one temporary client per connection will be auto-deleted
p4.client = name
p4.save_client(client, '-x')
LOG.debug("create_p4_temp_client() created temp client %s", name)
if 'P4T4TEST_ORIG_LANG' in os.environ and not skip_count:
# In the testing environment, we check that each process created no
# more than one temporary client (concurrently). Keep the highest
# value, rather than whatever the last count happened to be.
prefix = p4gf_const.P4GF_OBJECT_CLIENT_UNIQUE.format(
server_id=p4gf_util.get_server_id(), uuid='')
count = _count_active(prefix)
# Some tests set up an unnatural environment, so don't let those
# blow up in unexpected ways (i.e. fail gracefully elsewhere).
with p4.at_exception_level(P4.P4.RAISE_NONE):
value = p4gf_p4key.get(
p4, "git-fusion-temp-clients-{}".format(os.getpid()))
if int(value) < count:
p4gf_p4key.set(
p4, "git-fusion-temp-clients-{}".format(os.getpid()),
str(count))
return p4
def _get_key(self, key, value_type=None, default=None):
"""Get a key.
If value_type is specified, convert result to type.
If key is empty/missing, default will be returned.
Will raise TypeError if key contains a value that can't convert to type.
"""
if '{repo_name}' in key:
key = key.format(repo_name=self.repo_name)
value = p4gf_p4key.get(self.p4, key) or default
# will raise TypeError if can't convert to requested type
if value_type and value is not None:
value = value_type(value)
return value
def server_tzname(ctx):
"""Return the Perforce server's timezone name, such as "US/Pacific".
Does NOT return ambiguous and annoying local abbreviations such as
"EST" or "PDT".
Cache this result, or use get_server_time(), to avoid pointless
round-trips to the server.
"""
value = P4Key.get(ctx.p4gf, p4gf_const.P4GF_P4KEY_TIME_ZONE_NAME)
if value == '0' or value is None:
# Upgrade from an EA system, perhaps, in which the upgrade p4keys
# have been set but the later changes where not applied during init.
msg = _("p4key '{key}' not set, using UTC as default."
" Change this to your Perforce server's time zone.") \
.format(key=p4gf_const.P4GF_P4KEY_TIME_ZONE_NAME)
LOG.warning(msg)
value = NTR('UTC')
return value
def print_broken_p4key_lock(p4, pfunc=print):
"""Report on all of the repo locks that appear to be broken.
:param p4: P4API to query Perforce.
:param pfunc: either 'print' or some logger : 'LOG.debug'
"""
# Get all of the existing lock keys, sleep briefly, then check again.
pattern = p4gf_const.P4GF_P4KEY_LOCK_VIEW.format(repo_name='*')
repo_locks = p4gf_p4key.get_all(p4, pattern)
time.sleep(1)
lock_name_re = re.compile(r'git-fusion-view-(.*)-lock')
for name, old_value in repo_locks.items():
new_value = p4gf_p4key.get(p4, name)
# If the value is unchanged or increasing, that is a bad sign.
if int(new_value) >= int(old_value):
repo_name = lock_name_re.match(name).group(1)
pfunc("Possibly broken repo lock: {}".format(repo_name))
pfunc("May need to delete key {} if repo is inaccessible".format(
name))
def _read_p4key(self, change_num):
"""Read a p4key value previously record()ed.
Parse into a dict depot_path#rev ==> sha1
Returned dict can be empty: Perforce changelists can contain zero
file actions after 'p4 obliterate'.
Return P4KEY_NOT_SET if p4key not set.
Never returns None.
"""
val = P4Key.get(self.ctx, self.p4key_name(change_num))
if (not val) or (val == '0'):
return P4KEY_NOT_SET
try:
# Assume compressed format, catch error if not
val = zlib.decompress(binascii.a2b_base64(val)).decode()
except (binascii.Error, zlib.error, ValueError):
# Uncompressed string format
pass
return _parse_p4key_val(val)
def __get_timezone_offset(self, timestamp):
"""
Determine the time zone offset for the given timestamp, using the.
time zone name set in the P4GF time zone p4key. For example, with
time zone 'US/Pacific' and timestamp 1371663968, the offset returned
will be the string '-0700'.
"""
try:
ts = int(timestamp)
except ValueError:
LOG.error(
"__get_timezone_offset() given non-numeric input {}".format(
timestamp))
if self.__tzname is None:
value = P4Key.get(self.ctx.p4,
p4gf_const.P4GF_P4KEY_TIME_ZONE_NAME)
if value == '0' or value is None:
# Upgrade from an EA system, perhaps, in which the upgrade p4keys
# have been set but the later changes where not applied during init.
LOG.warning(
"p4key '{}' not set, using UTC as default."
" Change this to your Perforce server's time zone.".format(
p4gf_const.P4GF_P4KEY_TIME_ZONE_NAME))
value = NTR('UTC')
self.__tzname = value
try:
mytz = pytz.timezone(self.__tzname)
except pytz.exceptions.UnknownTimeZoneError:
LOG.warning(
"Time zone '{}' not found, using UTC as default".format(
self.__tzname))
mytz = pytz.utc
LOG.debug("__get_timezone_offset({}) with {}".format(ts, mytz))
dt = datetime.datetime.fromtimestamp(ts, tz=pytz.utc)
ct = dt.astimezone(mytz)
return ct.strftime('%z')
def check_triggers():
"""Check all of the GF triggers are installed and the trigger version is correct."""
# pylint: disable=too-many-branches
triggers = fetch_triggers()
if not triggers:
Verbosity.report(Verbosity.INFO, _('Git Fusion Triggers are not installed.'))
return
# Do we have a "p4gf_submit_trigger.py change-blah" entry for each
# of the P4_TRIGGER_NAMES that we expect? If so, then the trigger
# is fully installed.
seen = set()
for trig in triggers:
if "p4gf_submit_trigger.py" not in trig:
continue
for tn in P4_TRIGGER_NAMES:
if tn in trig:
seen.add(tn)
have_all_triggers = seen.issuperset(P4_TRIGGER_NAMES)
if not have_all_triggers:
Verbosity.report(Verbosity.INFO, _('Git Fusion Triggers are not installed.'))
return
# Is the installed version what we require?
version = P4Key.get(p4, p4gf_const.P4GF_P4KEY_TRIGGER_VERSION)
if version != '0':
version = version.split(":")[0].strip()
if version != p4gf_const.P4GF_TRIGGER_VERSION:
Verbosity.report(Verbosity.INFO, _('Git Fusion Triggers are not up to date.'))
Verbosity.report(Verbosity.INFO,
_('This version of Git Fusion expects p4 key {key}={value}')
.format(key=p4gf_const.P4GF_P4KEY_TRIGGER_VERSION,
value=p4gf_const.P4GF_TRIGGER_VERSION))
return
else:
Verbosity.report(Verbosity.INFO, _('Git Fusion triggers are up to date.'))
return
def print_p4key_lock_status(p4, server_id, pfunc=print):
"""Report on all of the repo locks and their status.
:param p4: P4API to query Perforce.
:param server_id: identifier for this Git Fusion instance.
:param pfunc: either 'print' or some logger : 'LOG.debug'
"""
# pylint: disable=too-many-branches, too-many-statements, maybe-no-member
# Instance of 'bool' has no 'group' member
pattern = p4gf_const.P4GF_P4KEY_LOCK_VIEW_OWNERS.format(repo_name='*')
repo_locks = p4gf_p4key.get_all(p4, pattern)
dead_processes_exist = False
for name, raw_value in repo_locks.items():
content = json.loads(raw_value)
if "owners" not in content:
pfunc(_("Malformed lock {lock_name}").format(lock_name=name))
else:
repo_name = LOCK_OWNERS_NAME_RE.match(name).group(1)
pfunc(
_("***************** {repo_name} Status *****************").
format(repo_name=repo_name))
lock_key_name = p4gf_const.P4GF_P4KEY_LOCK_VIEW.format(
repo_name=repo_name)
have_lock_key = p4gf_p4key.get(p4, lock_key_name) != '0'
if have_lock_key:
pfunc(
_("View lock {lock_key_name} is set").format(
lock_key_name=lock_key_name))
status_key_name = p4gf_p4key.calc_repo_status_p4key_name(
repo_name, None)
repo_status = p4gf_p4key.get(p4, status_key_name)
pushid_key_name = p4gf_p4key.calc_repo_push_id_p4key_name(
repo_name)
repo_pushid = p4gf_p4key.get(p4, pushid_key_name)
failed_push = False
if repo_pushid != '0':
pfunc(
_("Most recent push for repo '{repo_name}'... '{push_id}'"
).format(repo_name=repo_name, push_id=repo_pushid))
if repo_status != '0':
pfunc(
_("Most recent push status for repo '{repo_name}'... '{status}'"
).format(repo_name=repo_name, status=repo_status))
failed_push = FAILED_PUSH_RE.match(repo_status)
if failed_push:
repo_pushid = failed_push.group(1)
pfunc(FAILED_PUSH_MSG.format(push_id=repo_pushid))
else:
successful_push = SUCCESSFUL_PUSH_RE.match(repo_status)
if successful_push:
repo_pushid = successful_push.group(1)
pfunc(
_("P4 key based locks for '{repo_name}'...").format(
repo_name=repo_name))
lock_server_id = content["server_id"]
pfunc(
_(" Owning instance: {server_id}").format(
server_id=lock_server_id))
pfunc(
_(" Initial process: {pid}").format(pid=content["group_id"]))
process_number = 0
dead_process_count = 0
for owner in content["owners"]:
process_number += 1
pid = owner["process_id"]
start_time = owner["start_time"]
status = _pid_status(
pid) if lock_server_id == server_id else "UNKNOWN"
pfunc(
_(" Owner #{process_number}: PID {pid}, started at "
"{start_time}, status {status}").format(
process_number=process_number,
pid=pid,
start_time=start_time,
status=status))
if status == 'DEAD':
dead_processes_exist = True
dead_process_count += 1
pfunc(DEAD_PROC_MSG.format(pid=pid))
check_syslog_for_sigkill(pid, pfunc)
else:
pfunc(ACTIVE_PROC_MSG.format(pid=pid))
if dead_processes_exist:
pfunc('\n')
numkeys = 2 if have_lock_key else 1
if process_number == dead_process_count:
if failed_push:
pfunc(
ALL_DEAD_AND_FAILED_PROCESS.format(
repo_name=repo_name, push_id=repo_pushid))
else:
pfunc(
ALL_DEAD_WITH_LOCK.format(repo_name=repo_name,
push_id=repo_pushid))
pfunc(NEED_TO_RELEASE_MSG.format(numkeys=numkeys))
if have_lock_key:
pfunc(RELEASE_VIEW_LOCK_MSG.format(lock_key_name))
pfunc(RELEASE_OWNERS_LOCK_MSG.format(name))
else:
if failed_push:
pfunc(
SOME_DEAD_AND_FAILED_PUSH_MSG.format(
repo_name=repo_name, push_id=repo_pushid))
else:
pfunc(
SOME_DEAD_AND_SUCCESSFUL_MSG.format(
repo_name=repo_name, push_id=repo_pushid))
pfunc(NEED_TO_RELEASE_MSG.format(numkeys=numkeys))
if have_lock_key:
pfunc(RELEASE_VIEW_LOCK_MSG.format(lock_key_name))
pfunc(RELEASE_OWNERS_LOCK_MSG.format(name))
if len(repo_locks):
pfunc("")
def _read_last_copied_tag(ctx):
"""Return the changelist number for the most recent tags change for this
Git Fusion server.
"""
return P4Key.get(ctx.p4gf, _last_copied_tag_p4key_name(ctx))
def ensure_depot():
"""Create depot P4GF_DEPOT if not already exists."""
# We use P4GF_P4KEY_P4GF_DEPOT to send the depot name
# to p4gf_submit_trigger.py. Ensure that P4GF_P4KEY_P4GF_DEPOT matches
# requested depot name from P4GF_ENV.
# pylint: disable=line-too-long
p4gf_depot_key = P4Key.get(p4, p4gf_const.P4GF_P4KEY_P4GF_DEPOT)
if p4gf_depot_key != '0' and p4gf_depot_key != p4gf_const.P4GF_DEPOT:
Verbosity.report(
Verbosity.ERROR,
_("\n**ERROR: There is a configuration conflict for this Git Fusion instance of the P4GF_DEPOT.\n"
" P4GF_DEPOT is configured to a value different than that recorded in the p4d server.\n"
" You must correct this as described here.\n"
" This key must either be:\n"
" 1) unset, in which case this initialization will set it to the correct value, or\n"
" 2) match the value set in the P4GF_ENV configuration file, or\n"
" 3) set to the default '.git-fusion' if not set to a different value in the P4GF_ENV configuration file.\n"
"\n"
" The p4d server p4key '{key}' is set to '{value}'.\n")
.format(key=p4gf_const.P4GF_P4KEY_P4GF_DEPOT, value=p4gf_depot_key))
depot_configured = (p4gf_const.P4GF_ENV and p4gf_env_config.P4GF_ENV_CONFIG_DICT and
p4gf_const.P4GF_DEPOT_NAME in p4gf_env_config.P4GF_ENV_CONFIG_DICT)
if depot_configured:
msg = _(" This Git Fusion instance is configured with Git Fusion depot = '{depot}'"
"\n from P4GF_ENV file '{env}' containing these settings:\n") \
.format(depot=p4gf_const.P4GF_DEPOT, env=p4gf_const.P4GF_ENV)
for k, v in p4gf_env_config.P4GF_ENV_CONFIG_DICT.items():
msg = msg + " {0} = {1}\n".format(k, v)
else:
msg = _(" This Git Fusion instance is configured for the default P4GF_DEPOT : '{depot}'.\n")\
.format(depot=p4gf_const.P4GF_DEPOT)
Verbosity.report(Verbosity.ERROR, msg)
Verbosity.report(
Verbosity.ERROR,
_(" To correct this mismatch , determine the required P4GF_DEPOT value.\n"
" 1) If the value of p4key '{key}'='{value}' is different than the required value, remove the p4key.\n"
" Make certain the current p4key value was not correctly set by another Git Fusion instance.\n"
" 2) If the required value is the default 'git-fusion', and there is a P4GF_DEPOT setting in the P4GF_ENV file, remove it.\n"
" 2) If the required value is not the default 'git-fusion', set P4GF_DEPOT to the required value in the P4GF_ENV file.\n"
" 3) Re-run this configure-git-fusion.sh script.")
.format(key=p4gf_const.P4GF_P4KEY_P4GF_DEPOT, value=p4gf_depot_key))
sys.exit(1)
try:
created = p4gf_p4spec.ensure_depot_gf(p4)
if created:
Verbosity.report(Verbosity.INFO, _("Depot '{depot}' created.")
.format(depot=p4gf_const.P4GF_DEPOT))
else:
Verbosity.report(
Verbosity.INFO, _("Depot '{depot}' already exists. Not creating.")
.format(depot=p4gf_const.P4GF_DEPOT))
except Exception as depote:
Verbosity.report(Verbosity.INFO, _("Error creating Depot '{depot}'.")
.format(depot=p4gf_const.P4GF_DEPOT))
raise depote
# Set the P4GF_DEPOT p4key to send the depot name to p4gf_submit_trigger.py.
# This will either be the default or the value set from P4GF_ENV config file
P4Key.set(p4, p4gf_const.P4GF_P4KEY_P4GF_DEPOT, p4gf_const.P4GF_DEPOT)
return created
