def print_reviews_lock_status(p4, server_id, pfunc=print): """Report on the status of the reviews locks. :param p4: P4API to query Perforce. :param server_id: identifier for this Git Fusion instance. :param pfunc: print function - either 'print' or some logger : 'LOG.debug' """ raw_value = p4gf_p4key.get(p4, p4gf_const.P4GF_REVIEWS_COMMON_LOCK_OWNER) if raw_value and raw_value != '0': try: content = json.loads(raw_value) except ValueError: # 15.1 had a bug in which the key value was not properly formed JSON # (in fact, it was just a str()-formatted Python dict). pfunc("Reviews common lock invalid format.\nRaw value: {}".format( raw_value)) return if 'server_id' not in content: pfunc("Reviews common lock malformed") return lock_server_id = content['server_id'] pid = content['process_id'] start_time = content['start_time'] status = _pid_status(pid) if lock_server_id == server_id else "UNKNOWN" pfunc("Reviews common: server {}, PID {}, started at {}, status {}\n". format(lock_server_id, pid, start_time, status))
def for_user_and_repo(p4, p4user, repo_name, required_perm): """Factory to fetch user's permissions on a repo.""" LOG.debug("for_user_and_repo() {u} {r} {p}".format(u=p4user, r=repo_name, p=required_perm)) group_list = _groups_i(p4, p4user) group_dict = {group['group']: group for group in group_list} LOG.debug3("group_dict.keys()={}".format(group_dict.keys())) vp = RepoPerm() vp.p4user_name = p4user vp.repo_name = repo_name vp.repo_pull = p4gf_const.P4GF_GROUP_REPO_PULL.format( repo=repo_name) in group_dict vp.repo_push = p4gf_const.P4GF_GROUP_REPO_PUSH.format( repo=repo_name) in group_dict vp.global_pull = p4gf_const.P4GF_GROUP_PULL in group_dict vp.global_push = p4gf_const.P4GF_GROUP_PUSH in group_dict value = P4Key.get(p4, p4gf_const.P4GF_P4KEY_PERMISSION_GROUP_DEFAULT) if value == '0': value = DEFAULT_PERM vp.default_pull = value == PERM_PULL vp.default_push = value == PERM_PUSH LOG.debug("p4key={}".format(value)) LOG.debug(vp) return vp
def _upgrade_p4gf(p4): """Perform upgrade from earlier versions of P4GF. This should be invoked using _maybe_perform_init() to avoid race conditions across hosts. """ # If updating from 12.2 to 13.1 we need to create global config file # (this does nothing if file already exists) p4gf_config.GlobalConfig.init(p4) with p4.at_exception_level(p4.RAISE_ERROR): if p4gf_config.GlobalConfig.write_if(p4): _info(_("Global config file '{path}' created/updated.") .format(path=p4gf_config.depot_path_global())) else: _info(_("Global config file '{path}' already exists.") .format(path=p4gf_config.depot_path_global())) # Ensure the time zone name has been set, else default to something sensible. tzname = P4Key.get(p4, p4gf_const.P4GF_P4KEY_TIME_ZONE_NAME) if tzname == "0" or tzname is None: msg = _("p4 key '{key}' not set, using UTC as default." " Change this to your Perforce server's time zone.") \ .format(key=p4gf_const.P4GF_P4KEY_TIME_ZONE_NAME) LOG.warning(msg) sys.stderr.write(_('Git Fusion: {message}\n').format(message=msg)) tzname = None else: # Sanity check the time zone name. try: pytz.timezone(tzname) except pytz.exceptions.UnknownTimeZoneError: LOG.warning("Time zone name '{}' unrecognized, using UTC as default".format(tzname)) tzname = None if tzname is None: P4Key.set(p4, p4gf_const.P4GF_P4KEY_TIME_ZONE_NAME, 'UTC')
def _prevent_access(p4): """Prevent further access to Git Fusion while deleting everything. Return the previous value of the p4key so it can be restored later. """ old_value = P4Key.get(p4, p4gf_const.P4GF_P4KEY_PREVENT_NEW_SESSIONS) P4Key.set(p4, p4gf_const.P4GF_P4KEY_PREVENT_NEW_SESSIONS, 'true') return old_value
def _perm_key_default(p4): """Run `p4 key git-fusion-permission-group-default` to get default permissions. Once. """ global _PERM_KEY_DEFAULT if _PERM_KEY_DEFAULT is None: _PERM_KEY_DEFAULT = P4Key.get( p4, p4gf_const.P4GF_P4KEY_PERMISSION_GROUP_DEFAULT) return _PERM_KEY_DEFAULT
def _read(self): """Read the current content from key.""" try: s = P4Key.get(self.p4, self.owners_key) if s == "0": return {} return json.loads(s) except ValueError: raise LockCorrupt(self)
def get_last_change(p4): """Fetch the last change number with which we synced the keys to the SSH configuration file. Returns a positive number, or zero if no saved p4key value. """ v = P4Key.get(p4, _get_p4key_name()) if v: return int(v) else: return 0
def print_space_lock_status(p4, pfunc=print): """Report on the status of the disk usage lock. :param p4: P4API to query Perforce. :param pfunc: print function - either 'print' or some logger : 'LOG.debug' """ raw_value = p4gf_p4key.get(p4, p4gf_const.P4GF_P4KEY_LOCK_SPACE) if raw_value and raw_value != '0': pfunc("Space lock ({}) is set".format( p4gf_const.P4GF_P4KEY_LOCK_SPACE))
def create_default_perm(p4, perm=DEFAULT_PERM): """Create the 'stick all users into this pull/push permission group' default p4key. If p4key already exists with non-zero value, leave it unchanged. """ p4key = P4Key.get(p4, p4gf_const.P4GF_P4KEY_PERMISSION_GROUP_DEFAULT) if p4key is not None and p4key != '0': # Somebody already set it. return False P4Key.set(p4, p4gf_const.P4GF_P4KEY_PERMISSION_GROUP_DEFAULT, perm) return True
def create_p4_temp_client(port=None, user=None, skip_count=False): """Create a connected P4 instance with a generic temporary client. Dropping the connection will automatically delete the client. This is useful for owning the locks and permitting reliable lock stealing by other processes. :return: P4API instance. """ p4 = create_p4(port, user, warn_no_client=False) if p4 is None: # Propagate the error (that has already been reported). return None name = p4gf_const.P4GF_OBJECT_CLIENT_UNIQUE.format( server_id=p4gf_util.get_server_id(), uuid=str(uuid.uuid1())) client = p4.fetch_client(name) client['Owner'] = p4gf_const.P4GF_USER client['LineEnd'] = NTR('unix') client['View'] = [ '//{0}/... //{1}/...'.format(p4gf_const.P4GF_DEPOT, name) ] # to prevent the mirrored git commit/tree objects from being retained in the # git-fusion workspace, set client option 'rmdir' and sync #none in p4gf_gitmirror client['Options'] = p4gf_const.CLIENT_OPTIONS.replace("normdir", "rmdir") client['Root'] = p4gf_const.P4GF_HOME # The -x option is a deep undoc feature that signals to p4d that this # is a temporary client, which will be automatically deleted upon # disconnect. Requires passing the client specification using -i flag. # N.B. this client cannot shelve changes. See @465851 for details. # N.B. only one temporary client per connection will be auto-deleted p4.client = name p4.save_client(client, '-x') LOG.debug("create_p4_temp_client() created temp client %s", name) if 'P4T4TEST_ORIG_LANG' in os.environ and not skip_count: # In the testing environment, we check that each process created no # more than one temporary client (concurrently). Keep the highest # value, rather than whatever the last count happened to be. prefix = p4gf_const.P4GF_OBJECT_CLIENT_UNIQUE.format( server_id=p4gf_util.get_server_id(), uuid='') count = _count_active(prefix) # Some tests set up an unnatural environment, so don't let those # blow up in unexpected ways (i.e. fail gracefully elsewhere). with p4.at_exception_level(P4.P4.RAISE_NONE): value = p4gf_p4key.get( p4, "git-fusion-temp-clients-{}".format(os.getpid())) if int(value) < count: p4gf_p4key.set( p4, "git-fusion-temp-clients-{}".format(os.getpid()), str(count)) return p4
def _get_key(self, key, value_type=None, default=None): """Get a key. If value_type is specified, convert result to type. If key is empty/missing, default will be returned. Will raise TypeError if key contains a value that can't convert to type. """ if '{repo_name}' in key: key = key.format(repo_name=self.repo_name) value = p4gf_p4key.get(self.p4, key) or default # will raise TypeError if can't convert to requested type if value_type and value is not None: value = value_type(value) return value
def server_tzname(ctx): """Return the Perforce server's timezone name, such as "US/Pacific". Does NOT return ambiguous and annoying local abbreviations such as "EST" or "PDT". Cache this result, or use get_server_time(), to avoid pointless round-trips to the server. """ value = P4Key.get(ctx.p4gf, p4gf_const.P4GF_P4KEY_TIME_ZONE_NAME) if value == '0' or value is None: # Upgrade from an EA system, perhaps, in which the upgrade p4keys # have been set but the later changes where not applied during init. msg = _("p4key '{key}' not set, using UTC as default." " Change this to your Perforce server's time zone.") \ .format(key=p4gf_const.P4GF_P4KEY_TIME_ZONE_NAME) LOG.warning(msg) value = NTR('UTC') return value
def print_broken_p4key_lock(p4, pfunc=print): """Report on all of the repo locks that appear to be broken. :param p4: P4API to query Perforce. :param pfunc: either 'print' or some logger : 'LOG.debug' """ # Get all of the existing lock keys, sleep briefly, then check again. pattern = p4gf_const.P4GF_P4KEY_LOCK_VIEW.format(repo_name='*') repo_locks = p4gf_p4key.get_all(p4, pattern) time.sleep(1) lock_name_re = re.compile(r'git-fusion-view-(.*)-lock') for name, old_value in repo_locks.items(): new_value = p4gf_p4key.get(p4, name) # If the value is unchanged or increasing, that is a bad sign. if int(new_value) >= int(old_value): repo_name = lock_name_re.match(name).group(1) pfunc("Possibly broken repo lock: {}".format(repo_name)) pfunc("May need to delete key {} if repo is inaccessible".format( name))
def _read_p4key(self, change_num): """Read a p4key value previously record()ed. Parse into a dict depot_path#rev ==> sha1 Returned dict can be empty: Perforce changelists can contain zero file actions after 'p4 obliterate'. Return P4KEY_NOT_SET if p4key not set. Never returns None. """ val = P4Key.get(self.ctx, self.p4key_name(change_num)) if (not val) or (val == '0'): return P4KEY_NOT_SET try: # Assume compressed format, catch error if not val = zlib.decompress(binascii.a2b_base64(val)).decode() except (binascii.Error, zlib.error, ValueError): # Uncompressed string format pass return _parse_p4key_val(val)
def __get_timezone_offset(self, timestamp): """ Determine the time zone offset for the given timestamp, using the. time zone name set in the P4GF time zone p4key. For example, with time zone 'US/Pacific' and timestamp 1371663968, the offset returned will be the string '-0700'. """ try: ts = int(timestamp) except ValueError: LOG.error( "__get_timezone_offset() given non-numeric input {}".format( timestamp)) if self.__tzname is None: value = P4Key.get(self.ctx.p4, p4gf_const.P4GF_P4KEY_TIME_ZONE_NAME) if value == '0' or value is None: # Upgrade from an EA system, perhaps, in which the upgrade p4keys # have been set but the later changes where not applied during init. LOG.warning( "p4key '{}' not set, using UTC as default." " Change this to your Perforce server's time zone.".format( p4gf_const.P4GF_P4KEY_TIME_ZONE_NAME)) value = NTR('UTC') self.__tzname = value try: mytz = pytz.timezone(self.__tzname) except pytz.exceptions.UnknownTimeZoneError: LOG.warning( "Time zone '{}' not found, using UTC as default".format( self.__tzname)) mytz = pytz.utc LOG.debug("__get_timezone_offset({}) with {}".format(ts, mytz)) dt = datetime.datetime.fromtimestamp(ts, tz=pytz.utc) ct = dt.astimezone(mytz) return ct.strftime('%z')
def check_triggers(): """Check all of the GF triggers are installed and the trigger version is correct.""" # pylint: disable=too-many-branches triggers = fetch_triggers() if not triggers: Verbosity.report(Verbosity.INFO, _('Git Fusion Triggers are not installed.')) return # Do we have a "p4gf_submit_trigger.py change-blah" entry for each # of the P4_TRIGGER_NAMES that we expect? If so, then the trigger # is fully installed. seen = set() for trig in triggers: if "p4gf_submit_trigger.py" not in trig: continue for tn in P4_TRIGGER_NAMES: if tn in trig: seen.add(tn) have_all_triggers = seen.issuperset(P4_TRIGGER_NAMES) if not have_all_triggers: Verbosity.report(Verbosity.INFO, _('Git Fusion Triggers are not installed.')) return # Is the installed version what we require? version = P4Key.get(p4, p4gf_const.P4GF_P4KEY_TRIGGER_VERSION) if version != '0': version = version.split(":")[0].strip() if version != p4gf_const.P4GF_TRIGGER_VERSION: Verbosity.report(Verbosity.INFO, _('Git Fusion Triggers are not up to date.')) Verbosity.report(Verbosity.INFO, _('This version of Git Fusion expects p4 key {key}={value}') .format(key=p4gf_const.P4GF_P4KEY_TRIGGER_VERSION, value=p4gf_const.P4GF_TRIGGER_VERSION)) return else: Verbosity.report(Verbosity.INFO, _('Git Fusion triggers are up to date.')) return
def print_p4key_lock_status(p4, server_id, pfunc=print): """Report on all of the repo locks and their status. :param p4: P4API to query Perforce. :param server_id: identifier for this Git Fusion instance. :param pfunc: either 'print' or some logger : 'LOG.debug' """ # pylint: disable=too-many-branches, too-many-statements, maybe-no-member # Instance of 'bool' has no 'group' member pattern = p4gf_const.P4GF_P4KEY_LOCK_VIEW_OWNERS.format(repo_name='*') repo_locks = p4gf_p4key.get_all(p4, pattern) dead_processes_exist = False for name, raw_value in repo_locks.items(): content = json.loads(raw_value) if "owners" not in content: pfunc(_("Malformed lock {lock_name}").format(lock_name=name)) else: repo_name = LOCK_OWNERS_NAME_RE.match(name).group(1) pfunc( _("***************** {repo_name} Status *****************"). format(repo_name=repo_name)) lock_key_name = p4gf_const.P4GF_P4KEY_LOCK_VIEW.format( repo_name=repo_name) have_lock_key = p4gf_p4key.get(p4, lock_key_name) != '0' if have_lock_key: pfunc( _("View lock {lock_key_name} is set").format( lock_key_name=lock_key_name)) status_key_name = p4gf_p4key.calc_repo_status_p4key_name( repo_name, None) repo_status = p4gf_p4key.get(p4, status_key_name) pushid_key_name = p4gf_p4key.calc_repo_push_id_p4key_name( repo_name) repo_pushid = p4gf_p4key.get(p4, pushid_key_name) failed_push = False if repo_pushid != '0': pfunc( _("Most recent push for repo '{repo_name}'... '{push_id}'" ).format(repo_name=repo_name, push_id=repo_pushid)) if repo_status != '0': pfunc( _("Most recent push status for repo '{repo_name}'... '{status}'" ).format(repo_name=repo_name, status=repo_status)) failed_push = FAILED_PUSH_RE.match(repo_status) if failed_push: repo_pushid = failed_push.group(1) pfunc(FAILED_PUSH_MSG.format(push_id=repo_pushid)) else: successful_push = SUCCESSFUL_PUSH_RE.match(repo_status) if successful_push: repo_pushid = successful_push.group(1) pfunc( _("P4 key based locks for '{repo_name}'...").format( repo_name=repo_name)) lock_server_id = content["server_id"] pfunc( _(" Owning instance: {server_id}").format( server_id=lock_server_id)) pfunc( _(" Initial process: {pid}").format(pid=content["group_id"])) process_number = 0 dead_process_count = 0 for owner in content["owners"]: process_number += 1 pid = owner["process_id"] start_time = owner["start_time"] status = _pid_status( pid) if lock_server_id == server_id else "UNKNOWN" pfunc( _(" Owner #{process_number}: PID {pid}, started at " "{start_time}, status {status}").format( process_number=process_number, pid=pid, start_time=start_time, status=status)) if status == 'DEAD': dead_processes_exist = True dead_process_count += 1 pfunc(DEAD_PROC_MSG.format(pid=pid)) check_syslog_for_sigkill(pid, pfunc) else: pfunc(ACTIVE_PROC_MSG.format(pid=pid)) if dead_processes_exist: pfunc('\n') numkeys = 2 if have_lock_key else 1 if process_number == dead_process_count: if failed_push: pfunc( ALL_DEAD_AND_FAILED_PROCESS.format( repo_name=repo_name, push_id=repo_pushid)) else: pfunc( ALL_DEAD_WITH_LOCK.format(repo_name=repo_name, push_id=repo_pushid)) pfunc(NEED_TO_RELEASE_MSG.format(numkeys=numkeys)) if have_lock_key: pfunc(RELEASE_VIEW_LOCK_MSG.format(lock_key_name)) pfunc(RELEASE_OWNERS_LOCK_MSG.format(name)) else: if failed_push: pfunc( SOME_DEAD_AND_FAILED_PUSH_MSG.format( repo_name=repo_name, push_id=repo_pushid)) else: pfunc( SOME_DEAD_AND_SUCCESSFUL_MSG.format( repo_name=repo_name, push_id=repo_pushid)) pfunc(NEED_TO_RELEASE_MSG.format(numkeys=numkeys)) if have_lock_key: pfunc(RELEASE_VIEW_LOCK_MSG.format(lock_key_name)) pfunc(RELEASE_OWNERS_LOCK_MSG.format(name)) if len(repo_locks): pfunc("")
def _read_last_copied_tag(ctx): """Return the changelist number for the most recent tags change for this Git Fusion server. """ return P4Key.get(ctx.p4gf, _last_copied_tag_p4key_name(ctx))
def ensure_depot(): """Create depot P4GF_DEPOT if not already exists.""" # We use P4GF_P4KEY_P4GF_DEPOT to send the depot name # to p4gf_submit_trigger.py. Ensure that P4GF_P4KEY_P4GF_DEPOT matches # requested depot name from P4GF_ENV. # pylint: disable=line-too-long p4gf_depot_key = P4Key.get(p4, p4gf_const.P4GF_P4KEY_P4GF_DEPOT) if p4gf_depot_key != '0' and p4gf_depot_key != p4gf_const.P4GF_DEPOT: Verbosity.report( Verbosity.ERROR, _("\n**ERROR: There is a configuration conflict for this Git Fusion instance of the P4GF_DEPOT.\n" " P4GF_DEPOT is configured to a value different than that recorded in the p4d server.\n" " You must correct this as described here.\n" " This key must either be:\n" " 1) unset, in which case this initialization will set it to the correct value, or\n" " 2) match the value set in the P4GF_ENV configuration file, or\n" " 3) set to the default '.git-fusion' if not set to a different value in the P4GF_ENV configuration file.\n" "\n" " The p4d server p4key '{key}' is set to '{value}'.\n") .format(key=p4gf_const.P4GF_P4KEY_P4GF_DEPOT, value=p4gf_depot_key)) depot_configured = (p4gf_const.P4GF_ENV and p4gf_env_config.P4GF_ENV_CONFIG_DICT and p4gf_const.P4GF_DEPOT_NAME in p4gf_env_config.P4GF_ENV_CONFIG_DICT) if depot_configured: msg = _(" This Git Fusion instance is configured with Git Fusion depot = '{depot}'" "\n from P4GF_ENV file '{env}' containing these settings:\n") \ .format(depot=p4gf_const.P4GF_DEPOT, env=p4gf_const.P4GF_ENV) for k, v in p4gf_env_config.P4GF_ENV_CONFIG_DICT.items(): msg = msg + " {0} = {1}\n".format(k, v) else: msg = _(" This Git Fusion instance is configured for the default P4GF_DEPOT : '{depot}'.\n")\ .format(depot=p4gf_const.P4GF_DEPOT) Verbosity.report(Verbosity.ERROR, msg) Verbosity.report( Verbosity.ERROR, _(" To correct this mismatch , determine the required P4GF_DEPOT value.\n" " 1) If the value of p4key '{key}'='{value}' is different than the required value, remove the p4key.\n" " Make certain the current p4key value was not correctly set by another Git Fusion instance.\n" " 2) If the required value is the default 'git-fusion', and there is a P4GF_DEPOT setting in the P4GF_ENV file, remove it.\n" " 2) If the required value is not the default 'git-fusion', set P4GF_DEPOT to the required value in the P4GF_ENV file.\n" " 3) Re-run this configure-git-fusion.sh script.") .format(key=p4gf_const.P4GF_P4KEY_P4GF_DEPOT, value=p4gf_depot_key)) sys.exit(1) try: created = p4gf_p4spec.ensure_depot_gf(p4) if created: Verbosity.report(Verbosity.INFO, _("Depot '{depot}' created.") .format(depot=p4gf_const.P4GF_DEPOT)) else: Verbosity.report( Verbosity.INFO, _("Depot '{depot}' already exists. Not creating.") .format(depot=p4gf_const.P4GF_DEPOT)) except Exception as depote: Verbosity.report(Verbosity.INFO, _("Error creating Depot '{depot}'.") .format(depot=p4gf_const.P4GF_DEPOT)) raise depote # Set the P4GF_DEPOT p4key to send the depot name to p4gf_submit_trigger.py. # This will either be the default or the value set from P4GF_ENV config file P4Key.set(p4, p4gf_const.P4GF_P4KEY_P4GF_DEPOT, p4gf_const.P4GF_DEPOT) return created