def __set_up_p4gf_paths(self):
"""set up depot and local paths for P4GF
These paths are derived from the client root and client view.
"""
client = self.p4gf.fetch_client()
self.client_spec_gf = client
self.gitrootdir = client_spec_to_root(client)
self.clientmap_gf = Map(client["View"])
lhs = self.clientmap_gf.lhs()
check_client_view_gf(lhs)
assert len(lhs) == 1, _('view must contain only one line')
rpath = self.clientmap_gf.translate(lhs[0])
self.gitlocalroot = strip_wild(
client_path_to_local(rpath, self.p4gf.client, self.gitrootdir))
def __set_up_content_paths(self):
"""set up depot and local paths for both content and P4GF
These paths are derived from the client root and client view.
"""
client = self.p4.fetch_client()
self.clientmap = Map(client["View"])
# If the len of the client Views differs from the len of the Map
# then the P4 disabmbiguator added exclusionary mappings - note this here
# for reporting a message back to the user.
self.client_exclusions_added = len(client["View"]) != len(
self.clientmap.as_array())
# local syntax client root, force trailing /
self.contentlocalroot = client["Root"]
if not self.contentlocalroot.endswith("/"):
self.contentlocalroot += '/'
# client sytax client root with wildcard
self.contentclientroot = '//' + self.p4.client + '/...'
def connect(self):
self.p4 = P4Server(self.P4PORT, self.P4USER, self.P4PASSWD,
log_level=self.cli_arguments.verbose)
self.p4.exception_level = P4.RAISE_ERROR
self.p4.client = self.P4CLIENT
clientspec = self.p4.fetch_client(self.p4.client)
self.stream = clientspec.get('Stream')
self.root = clientspec._root
self.p4.cwd = self.root
self.clientmap = Map(clientspec._view)
ctr = Map('//%s/... %s/...' % (clientspec._client,
clientspec._root))
self.localmap = Map.join(self.clientmap, ctr)
self.depotmap = self.localmap.reverse()
self.maskdepotmap = None
if self.P4MASKCLIENT:
maskclientspec = self.p4.fetch_client(self.P4MASKCLIENT)
maskclientmap = Map(maskclientspec._view)
ctr = Map('//%s/... %s/...' % (maskclientspec._client,
maskclientspec._root))
masklocalmap = Map.join(maskclientmap, ctr)
self.maskdepotmap = masklocalmap.reverse()
def has_intersecting_views(current_reviews, clientmap):
"""Determine whether the clientmap intersects the
current set of reviews for this GF reviews user.
"""
reviews_map = Map()
for v in current_reviews:
reviews_map.insert(v)
repo_map = Map()
for l in clientmap.lhs():
repo_map.insert(l)
joined = Map.join(reviews_map, repo_map)
#for l in joined.lhs():
# if not l.startswith('-'):
# return INTERSECT
return NO_INTERSECT
def __set_up_p4gf_paths(self):
"""set up depot and local paths for P4GF
These paths are derived from the client root and client view.
"""
client = self.p4gf.fetch_client()
self.client_spec_gf = client
self.gitrootdir = client_spec_to_root(client)
self.clientmap_gf = Map(client["View"])
lhs = self.clientmap_gf.lhs()
check_client_view_gf(lhs)
assert len(lhs) == 1, _('view must contain only one line')
rpath = self.clientmap_gf.translate(lhs[0])
self.gitlocalroot = strip_wild(client_path_to_local(
rpath, self.p4gf.client, self.gitrootdir))
def __set_up_content_paths(self):
"""set up depot and local paths for both content and P4GF
These paths are derived from the client root and client view.
"""
client = self.p4.fetch_client()
self.clientmap = Map(client["View"])
# If the len of the client Views differs from the len of the Map
# then the P4 disabmbiguator added exclusionary mappings - note this here
# for reporting a message back to the user.
self.client_exclusions_added = len(client["View"]) != len(self.clientmap.as_array())
# local syntax client root, force trailing /
self.contentlocalroot = client["Root"]
if not self.contentlocalroot.endswith("/"):
self.contentlocalroot += '/'
# client sytax client root with wildcard
self.contentclientroot = '//' + self.p4.client + '/...'
def __set_up_p4gf_paths(self):
"""set up depot and local paths for P4GF
These paths are derived from the client root and client view.
"""
client = self.p4gf.fetch_client()
# client root, minus any trailing /
self.gitrootdir = client["Root"]
if self.gitrootdir.endswith("/"):
self.gitrootdir = self.gitrootdir[:-1]
self.clientmap_gf = Map(client["View"])
lhs = self.clientmap_gf.lhs()
check_client_view_gf(lhs)
for lpath in lhs:
rpath = self.clientmap_gf.translate(lpath)
self.gitlocalroot = strip_wild(
client_path_to_local(rpath, self.p4gf.client, self.gitrootdir))
class Context:
"""a single git-fusion view/repo context"""
# pylint:disable=R0915
# too many statements
def __init__(self, config):
self.config = config
# connected by default:
self.p4 = None
self.p4gf = None
# not connected by default:
self.p4gf_reviews = None
self.p4gf_reviews_non_gf = None
try:
from p4gf_gitmirror import GitMirror
self.mirror = GitMirror(config.view_name)
except:
LOG.error("failed to create GitMirror:\n{}".format(
traceback.format_exc()))
raise
self.timezone = None
self.server_version = None
self._user_to_protect = None
self.view_dirs = None
self.view_lock = None
self.view_repo = None
self.tempdir = tempfile.TemporaryDirectory(
prefix=p4gf_const.P4GF_TEMP_DIR_PREFIX)
# RAII object to operate on a numbered changelist with p4run and p4gfrun
# set in p4gf_util by NumberedChangelist
self.numbered_change = None
self.numbered_change_gf = None
# Environment variable set by p4gf_auth_server.py.
self.authenticated_p4user = os.environ.get(p4gf_const.P4GF_AUTH_P4USER)
# gf_branch_name ==> p4gf_branch.Branch
# Lazy-loaded by branch_dict()
self._branch_dict = None
self.branch_creation = None
self.merge_commits = None
self.submodules = None
self.owner_is_author = None
# DepotBranchInfoIndex of all known depot branches that house
# files from lightweight branches, even ones we don't own.
# Lazy-loaded by depot_branch_info_index()
self._depot_branch_info_index = None
# paths set up by set_up_paths()
self.gitdepotroot = "//{}/".format(p4gf_const.P4GF_DEPOT)
self.gitlocalroot = None
self.client_spec_gf = None
self.gitrootdir = None
self.contentlocalroot = None
self.contentclientroot = None
self.clientmap = None
self.clientmap_gf = None
self.client_exclusions_added = False
# Avoid unnecessary view switches.
self.last_view_lines = None
self.last_client_spec = None
# Seconds since the epoch when we last polled for remote kill switch.
self._last_heartbeat_cancel_check_time = None
self._heartbeat_time = None
# A set of temporary Perforce client specs, each mapped to one
# branch's view. Use these to query Perforce rather than switching
# ctx.p4 back and forth just to run 'p4 files //client/...' for some
# random branch other than our current branch.
self._client_pool = None
# By default, temp clients in pool will be deleted when Context exits.
# For 'nested' Contexts, set this to False to avoid deleting clients
# out from under the surrounding Context.
self.cleanup_client_pool = True
# A single, shared temporary Perforce branch, useful for integrations.
self._temp_branch = None
# Minimize the number of 'p4 files //branch-client/...@n' calls.
self.branch_files_cache = BranchFilesCache()
# Set by G2PMatrix during a 'git push' to remember the most recent
# changelist integrated from each branch to each other branch.
# Instance of IntegratedUpTo
self.integrated_up_to = None
# Last N p4run() commands. Reported in _dump_on_failure()
self.p4run_history = deque(maxlen=20)
# Last N p4run() commands. Reported in _dump_on_failure()
self.p4gfrun_history = deque(maxlen=20)
# Admin-configured option to reject unworthy commits.
self._preflight_hook = None
# pylint:enable=R0915
def __enter__(self):
self.connect()
return self
def __exit__(self, _exc_type, _exc_value, _traceback):
self.disconnect()
if self.tempdir:
self.tempdir.cleanup()
return False # False == do not squelch any current exception
def connect(self):
'''
Connect the p4 and p4gf connections.
Note: p4gf_reviews and p4gf_reviews_non_gf are not connected!
If this is the first time connecting, complete context initialization.
'''
# if previously connected, just reconnect
if self.p4:
if not self.p4.connected():
p4_connect(self.p4)
if not self.p4gf.connected():
p4_connect(self.p4gf)
return
# create connections and use them to complete initialization of context
self.p4 = self.__make_p4(client=self.config.p4client)
self.p4gf = self.__make_p4(client=self.config.p4client_gf)
self.p4gf_reviews = self.__make_p4(
user=p4gf_util.gf_reviews_user_name(), connect=False)
self.p4gf_reviews_non_gf = self.__make_p4(
user=p4gf_const.P4GF_REVIEWS__NON_GF, connect=False)
self._client_pool = ClientPool(self)
self.__wrangle_charset()
self.get_timezone_serverversion()
self.__set_branch_creation()
self.__set_merge_commits()
self.__set_submodules()
self.__set_change_owner()
self.__set_up_paths()
def disconnect(self):
'''
disconnect any p4 connections
'''
LOG.debug("context disconnecting {} {}".format(
self.p4gf_reviews.connected(),
self.p4gf_reviews_non_gf.connected()))
# clean up temp clients before disconnecting p4gf
if self.cleanup_client_pool:
self._client_pool.cleanup()
if self.p4.connected():
p4_disconnect(self.p4)
if self.p4gf.connected():
p4_disconnect(self.p4gf)
if self.p4gf_reviews.connected():
p4_disconnect(self.p4gf_reviews)
if self.p4gf_reviews_non_gf.connected():
p4_disconnect(self.p4gf_reviews_non_gf)
def last_copied_change_counter_name(self):
'''
Return a counter that holds the highest changelist number
copied to the our repo, on our Git Fusion server.
'''
return calc_last_copied_change_counter_name(self.config.view_name,
p4gf_util.get_server_id())
def read_last_copied_change(self):
'''
Return the highest changelist number copied to a repo on a server.
'''
r = self.p4run(
['counter', '-u',
self.last_copied_change_counter_name()])
return r[0]['value']
def write_last_copied_change(self, change_num):
'''
Return the highest changelist number copied to a repo on a server.
'''
self.p4run([
'counter', '-u',
self.last_copied_change_counter_name(), change_num
])
def user_to_protect(self, user):
"""Return a p4gf_protect.Protect instance that knows
the given user's permissions."""
# Lazy-create the user_to_protect instance since not all
# Context-using code requires it.
if not self._user_to_protect:
self._user_to_protect = p4gf_protect.UserToProtect(self.p4)
return self._user_to_protect.user_to_protect(user)
def get_view_repo(self):
'''
Lazy-create the pygit2 repo object.
Ideally we rename this to become a property getter "view_repo" and then
data member self.view_repo becomes self._view_repo.
Some future refactor day.
'''
if not self.view_repo:
self.view_repo = pygit2.Repository(self.view_dirs.GIT_DIR)
return self.view_repo
def __make_p4(self, client=None, user=None, connect=True):
"""create a connection to the perforce server"""
if not user:
user = self.config.p4user
if not client:
client = self.config.p4client
return create_p4(port=self.config.p4port,
user=user,
client=client,
connect=connect)
def __set_branch_creation(self):
"""Configure branch creation"""
config = p4gf_config.get_repo(self.p4gf, self.config.view_name)
self.branch_creation = config.getboolean(
p4gf_config.SECTION_REPO, p4gf_config.KEY_ENABLE_BRANCH_CREATION)
LOG.debug('Enable repo branch creation = {0}'.format(
self.branch_creation))
def __set_merge_commits(self):
"""Configure merge commits"""
config = p4gf_config.get_repo(self.p4gf, self.config.view_name)
self.merge_commits = config.getboolean(
p4gf_config.SECTION_REPO, p4gf_config.KEY_ENABLE_MERGE_COMMITS)
LOG.debug('Enable repo merge commits = {0}'.format(self.merge_commits))
def __set_submodules(self):
"""Configure submodule support"""
config = p4gf_config.get_repo(self.p4gf, self.config.view_name)
self.submodules = config.getboolean(p4gf_config.SECTION_REPO,
p4gf_config.KEY_ENABLE_SUBMODULES)
LOG.debug('Enable repo submodules = {0}'.format(self.submodules))
def __set_change_owner(self):
"""Configure change ownership setting"""
config = p4gf_config.get_repo(self.p4gf, self.config.view_name)
value = config.get(p4gf_config.SECTION_REPO,
p4gf_config.KEY_CHANGE_OWNER)
value = str(value).lower()
if value not in [p4gf_config.VALUE_AUTHOR, p4gf_config.VALUE_PUSHER]:
LOG.warn(
"change-owner config setting has invalid value, defaulting to author"
)
value = p4gf_config.VALUE_AUTHOR
self.owner_is_author = True if value == 'author' else False
LOG.debug('Set change owner to {0}'.format(value))
def __wrangle_charset(self):
"""figure out if server is unicode and if it is, set charset"""
if not self.p4.server_unicode:
return
# we have a unicode server
# first, always use utf8 for the gf connection
# use that connection to fetch the config for the repo
self.p4gf.charset = 'utf8'
config = p4gf_config.get_repo(self.p4gf, self.config.view_name)
# then set the repo-specific charset for their connection
self.p4.charset = config.get(p4gf_config.SECTION_REPO,
p4gf_config.KEY_CHARSET)
LOG.debug('repo charset will be: ' + self.p4.charset)
def client_view_path(self, change_num=None):
'''
Return "//{client}/..." : the client path for whole view,
including ... wildcard
Optional change_num, if supplied, appended as "@N"
Because I'm sick of constructing that over and over.
'''
if change_num:
return '{}@{}'.format(self.contentclientroot, change_num)
return self.contentclientroot
def client_view_union(self):
"""Return the client view to the union of all branches in this repo
"""
p4map = p4gf_branch.calc_branch_union_client_view(
self.config.p4client, self.branch_dict())
return p4map.as_array()
def switch_client_view_to_union(self):
"""Change the repo's Perforce client view to the union of all branches
in this repo.
Warning: other objects retain pointers into OUR self.clientmap object.
If we swap in a new object, those other objects point to stale data.
If we change the values in the same object, those other objects point
to new data. In neither case do these other objects get notification
of change.
"""
view_lines = self.client_view_union()
if LOG.isEnabledFor(logging.DEBUG2):
LOG.debug2('switch_client_view_to_union:\n{}'.format(
'\n'.join(view_lines)))
else:
LOG.debug('switch_client_view_to_union')
self.switch_client_view_lines(view_lines)
def switch_client_view_to_branch(self, branch):
"""Change the repo's Perforce client view to view of
the given Branch object.
branch is a p4gf_branch instance. See branch_dict() if you need to
lookup by name.
"""
LOG.debug('switch_client_view_to_branch() {}'.format(branch.to_log()))
if branch.stream_name:
self.switch_client_to_stream(branch)
else:
self.switch_client_view_lines(branch.view_lines)
def switch_client_view_lines(self, lines):
"""Change this repo's Perforce client view to the given line list.
Update our clientmap object, with the new lines. Update clientmap in
place so that other object with pointers to OUR clientmap object will
see the new data.
"""
_lines = to_lines(lines)
if _lines == self.last_view_lines:
LOG.debug2('switch_client_view_lines() already switched. NOP.')
return
LOG.debug2('switch_client_view_lines() client={} {}'.format(
self.config.p4client, _lines))
self.last_client_spec = p4gf_util.set_spec(
self.p4,
'client',
spec_id=self.config.p4client,
values={
'View': _lines,
'Stream': None
},
cached_vardict=self.last_client_spec)
self.clientmap.clear()
for line in _lines:
self.clientmap.insert(line)
self.last_view_lines = copy.copy(_lines)
def switch_client_to_stream(self, branch):
"""Change this repo's Perforce client view to the given line list.
Update our clientmap object, with the new lines. Update clientmap in
place so that other object with pointers to OUR clientmap object will
see the new data.
"""
LOG.debug2('switch_client_view_stream() client={} stream={}'.format(
self.config.p4client, branch.stream_name))
self.p4run([
'client', '-f', '-s', '-S', branch.stream_name,
self.config.p4client
])
lines = branch.view_lines
_lines = to_lines(lines)
if _lines == self.last_view_lines:
return
LOG.debug2('switch_client_view_streams() client={} {}'.format(
self.config.p4client, _lines))
self.clientmap.clear()
for line in _lines:
self.clientmap.insert(line)
self.last_view_lines = copy.copy(_lines)
def checkout_master_ish(self):
'''
Switch Git to the first branch view defined in p4gf_config.
This is often master, but not always.
Since P4GF defaults to configuring a 'master' branch, which may
not exist, try the other configured branches before giving up.
NOP if no such branch (perhaps an empty p4gf_config?).
'''
branches = self.branch_dict()
br = p4gf_branch.most_equal(branches)
if br and br.git_branch_name:
if not p4gf_util.git_checkout(br.git_branch_name):
# Perhaps the most-equal branch does not exist?
# Try the others until we achieve success.
success = False
for br in branches.values():
if br.git_branch_name and p4gf_util.git_checkout(
br.git_branch_name):
success = True
break
if not success:
LOG.warn(
'Unable to checkout sensible branch for {}'.format(
self.config.view_name))
def git_branch_name_to_branch(self, git_branch_name):
'''
If we have an undeleted branch with the requested name, return it.
If not, return None.
O(n) scan.
'''
assert git_branch_name
name = git_branch_name
if name.startswith('refs/heads/'):
name = name[len('refs/heads/'):]
for branch in self.branch_dict().values():
if branch.deleted:
continue
if branch.git_branch_name == name:
return branch
return None
def branch_dict(self):
"""Return all known Git<->Perforce branch associations.
Lazy-loaded from config file.
Loads DepotBranchInfo dict as a side effect so that we can
reconnect Branch.depot_branch pointers.
"""
if not self._branch_dict:
# Load p4gf_config: the fully populated branches.
config = p4gf_config.get_repo(self.p4gf, self.config.view_name)
self._branch_dict = p4gf_branch.dict_from_config(config, self.p4gf)
# First branch listed in p4gf_config becomes our default HEAD.
# This is usually 'master', but not required.
bsl = p4gf_config.branch_section_list(config)
if bsl:
self._branch_dict[bsl[0]].more_equal = True
# Load the lightweight and stream-based branch config data into the
# branch_dict. This is stored in p4gf_config2. For lightweight
# branches, the full branch def is there. For stream-based branches
# all we care about is the original-view, which gets merged with
# any config stored in p4gf_config.
config2 = p4gf_config.get_repo2(self.p4gf, self.config.view_name)
if config2:
branch_dict2 = p4gf_branch.dict_from_config(config2, self.p4gf)
lwb_dict = {}
for branch in branch_dict2.values():
if branch.stream_name:
if branch.branch_id in self._branch_dict:
self._branch_dict[branch.branch_id].original_view_lines = \
branch.original_view_lines
else:
branch.is_lightweight = True
if (branch.depot_branch
and isinstance(branch.depot_branch, str)):
branch.depot_branch = self.depot_branch_info_index() \
.find_depot_branch_id(branch.depot_branch)
lwb_dict[branch.branch_id] = branch
self._branch_dict.update(lwb_dict)
for b in self._branch_dict.values():
b.set_rhs_client(self.config.p4client)
LOG.debug('branch_dict() lazy-loaded ct={}'.format(
len(self._branch_dict)))
if LOG.isEnabledFor(logging.DEBUG2):
for b in self._branch_dict.values():
LOG.debug2('\n' + b.to_log(LOG))
return self._branch_dict
def undeleted_branches(self):
'''
An iterator/generator of all Branch values in branch_dict
that are not deleted.
'''
for branch in self.branch_dict().values():
if branch.deleted:
continue
yield branch
def depot_branch_info_index(self):
'''
Return all known depot branches that house files for lightweight
branches. This includes depot branches that other Git Fusion repos
created: we must stay lightweight even when sharing across repos.
Lazy-loaded from all depot branch-info files.
'''
if not self._depot_branch_info_index:
self._depot_branch_info_index = p4gf_depot_branch.DepotBranchInfoIndex(
)
root = p4gf_const.P4GF_DEPOT_BRANCH_INFO_ROOT.format(
P4GF_DEPOT=p4gf_const.P4GF_DEPOT)
root = root + '/...'
with p4gf_util.RawEncoding(self.p4):
file_data = self.p4.run('print', root)
delete = False
started = False
file_contents = ''
for item in file_data:
if isinstance(item, dict):
if started: # finish with the current branch info
dbi = p4gf_depot_branch.depot_branch_info_from_string(
file_contents)
self._depot_branch_info_index.add(dbi)
if item['action'] == 'delete':
started = False
delete = True
else:
file_contents = ''
delete = False
started = True
else:
if delete:
continue
new_item = item.decode().strip()
if len(new_item):
file_contents = file_contents + new_item
if started:
dbi = p4gf_depot_branch.depot_branch_info_from_string(
file_contents)
self._depot_branch_info_index.add(dbi)
if LOG.isEnabledFor(logging.DEBUG):
for dpid in self._depot_branch_info_index.by_id:
LOG.debug("DBI index LAZY loaded: {0} {1}".format(
dpid, self._depot_branch_info_index.by_id[dpid]))
### YAGNI until we have a test that proves it.
### p4 print utf8 //P4GF_DEPOT/branches/branch-info/...
return self._depot_branch_info_index
def get_timezone_serverversion(self):
"""get server's timezone and server version via p4 info"""
r = self.p4.run_info()
server_date = p4gf_util.first_value_for_key(r, 'serverDate')
self.timezone = server_date.split(" ")[2]
self.server_version = p4gf_util.first_value_for_key(r, 'serverVersion')
def __set_up_paths(self):
"""set up depot and local paths for both content and P4GF
These paths are derived from the client root and client view.
"""
self.__set_up_content_paths()
self.__set_up_p4gf_paths()
self.view_dirs = p4gf_view_dirs.from_p4gf_dir(self.gitrootdir,
self.config.view_name)
def __set_up_content_paths(self):
"""set up depot and local paths for both content and P4GF
These paths are derived from the client root and client view.
"""
client = self.p4.fetch_client()
self.clientmap = Map(client["View"])
# If the len of the client Views differs from the len of the Map
# then the P4 disabmbiguator added exclusionary mappings - note this here
# for reporting a message back to the user.
self.client_exclusions_added = len(client["View"]) != len(
self.clientmap.as_array())
# local syntax client root, force trailing /
self.contentlocalroot = client["Root"]
if not self.contentlocalroot.endswith("/"):
self.contentlocalroot += '/'
# client sytax client root with wildcard
self.contentclientroot = '//' + self.p4.client + '/...'
def __set_up_p4gf_paths(self):
"""set up depot and local paths for P4GF
These paths are derived from the client root and client view.
"""
client = self.p4gf.fetch_client()
self.client_spec_gf = client
self.gitrootdir = client_spec_to_root(client)
self.clientmap_gf = Map(client["View"])
lhs = self.clientmap_gf.lhs()
check_client_view_gf(lhs)
assert len(lhs) == 1, _('view must contain only one line')
rpath = self.clientmap_gf.translate(lhs[0])
self.gitlocalroot = strip_wild(
client_path_to_local(rpath, self.p4gf.client, self.gitrootdir))
def __str__(self):
return "\n".join([
"Git data in Perforce: " + self.gitdepotroot + "...",
" " + self.gitlocalroot + "...",
"Exported Perforce tree: " + self.contentlocalroot + "...",
" " + self.contentclientroot,
"timezone: " + self.timezone
])
def __repr__(self):
return str(self) + "\n" + repr(self.mirror)
def log_context(self):
"""Dump connection info, client info, directories, all to log category
'context' as INFO."""
log = logging.getLogger('context')
if not log.isEnabledFor(logging.INFO):
return
# Dump client spec as raw untagged text.
self.p4.tagged = 0
client_lines_raw = self.p4.run('client', '-o')[0].splitlines()
self.p4.tagged = 1
# Strip comment header
client_lines = [l for l in client_lines_raw if not l.startswith('#')]
# Dump p4 info, tagged, since that includes more pairs than untagged.
p4info = p4gf_util.first_dict(self.p4.run('info'))
key_len_max = max(len(k) for k in p4info.keys())
info_template = NTR('%-{}s : %s').format(key_len_max)
log.info(info_template, 'P4PORT', self.p4.port)
log.info(info_template, 'P4USER', self.p4.user)
log.info(info_template, 'P4CLIENT', self.p4.client)
log.info(info_template, 'p4gfclient', self.p4gf.client)
for k in sorted(p4info.keys(), key=str.lower):
log.info(info_template, k, p4info[k])
for line in client_lines:
log.info(line)
def _check_heartbeat_canceled(self):
'''
Ask the server if someone else wants us to stop.
Remote kill switch: delete our counter and we'll voluntarily stop.
'p4 counter -u git-fusion-view-{repo}-lock'
Throttled back to check only once every N seconds.
'''
# Check no more than once every N seconds.
now = time.time()
if self._last_heartbeat_cancel_check_time:
since_secs = now - self._last_heartbeat_cancel_check_time
if since_secs < _CANCEL_POLL_PERIOD_SECS:
return
self._last_heartbeat_cancel_check_time += _CANCEL_POLL_PERIOD_SECS
else:
self._last_heartbeat_cancel_check_time = now
if self.view_lock.canceled():
raise p4gf_lock.LockCanceled(
_("Canceling: lock '{}' lost.").format(
self.view_lock.counter_name()))
def heartbeat(self):
'''
If we have a view lock, update its heartbeat.
If our lock is cleared, then raise a RuntimeException
canceling our current task.
'''
if not self.view_lock:
return
self._check_heartbeat_canceled()
self.view_lock.update_heartbeat()
self.log_memory_usage()
def log_memory_usage(self):
'''Log our memory usage on a regular basis.'''
if LOG.isEnabledFor(logging.DEBUG):
now = time.time()
if self._heartbeat_time and now - self._heartbeat_time < MEMLOG_HEART_RATE:
return
self._heartbeat_time = now
LOG.debug(p4gf_log.memory_usage())
def _convert_path(self, clazz, path):
'''Return a path object that can convert to other formats.'''
return clazz(
self.clientmap,
self.config.p4client,
self.contentlocalroot[:-1] # -1 to strip trailing /
,
path)
def depot_path(self, path):
'''Return an object that can convert from depot to other syntax.'''
return self._convert_path(p4gf_path_convert.DepotPath, path)
def client_path(self, path):
'''Return an object that can convert from client to other syntax.'''
return self._convert_path(p4gf_path_convert.ClientPath, path)
def gwt_path(self, path):
'''
Return an object that can convert from Git work tree to other syntax.
'''
return self._convert_path(p4gf_path_convert.GWTPath, path)
def gwt_to_depot_path(self, gwt_path):
'''
Optimized version of ctx.gwt_path(gwt).to_depot().
Avoid creating the convert and goes straight to P4.Map.translate().
We call this once for every row in G2PMatrix. This function runs in
about 60% of ctx.gwt_path(x).to_depot() time. That works out to about 5%
of total wall clock time for many-file repos such as james.
'''
gwt_esc = p4gf_util.escape_path(gwt_path)
client_path = '//{}/'.format(self.config.p4client) + gwt_esc
return self.clientmap.translate(client_path, self.clientmap.RIGHT2LEFT)
def depot_to_gwt_path(self, depot_path):
'''
Optimized version of ctx.depot_path(dp).to_gwt().
Avoid creating the convert and goes straight to P4.Map.translate().
We call this once for every row in G2PMatrix. This function runs in
about 60% of ctx.depot_path(x).to_gwt() time. That works out to about 5%
of total wall clock time for many-file repos such as james.
'''
client_path = self.clientmap.translate(depot_path,
self.clientmap.LEFT2RIGHT)
gwt_esc = client_path[3 + len(self.config.p4client):]
return p4gf_util.unescape_path(gwt_esc)
def local_path(self, path):
'''
Return an object that can convert from absolute local filesystem
to other syntax.
'''
return self._convert_path(p4gf_path_convert.LocalPath, path)
# pylint:disable=R0913
# Too many arguments
# We really should just dial that up to 11
# if we're going to keep ignoring its advice.
@staticmethod
def _p4run(p4, numbered_change, run_history, cmd, log_warnings,
log_errors):
'''
Record the command in history, then perform it.
'''
if numbered_change:
cmd = numbered_change.add_change_option(cmd)
run_history.append(cmd)
return p4gf_util.p4run_logged(p4,
cmd,
log_warnings=log_warnings,
log_errors=log_errors)
# pylint:enable=R0913
def p4run(self,
cmd,
log_warnings=logging.WARNING,
log_errors=logging.ERROR):
'''
Run a command, with logging.
'''
return self._p4run(p4=self.p4,
numbered_change=self.numbered_change,
run_history=self.p4run_history,
cmd=cmd,
log_warnings=log_warnings,
log_errors=log_errors)
def p4gfrun(self,
cmd,
log_warnings=logging.WARNING,
log_errors=logging.ERROR):
'''
Run a command, with logging.
'''
return self._p4run(p4=self.p4gf,
numbered_change=self.numbered_change_gf,
run_history=self.p4gfrun_history,
cmd=cmd,
log_warnings=log_warnings,
log_errors=log_errors)
def switched_to_view_lines(self, view_lines):
'''
Return an RAII object to switch p4 connection to a different,
temporary, client spec, with the requested view lines, then
restore p4 connection to original client on exit.
Use this ONLY for read-only commands, no opening for edit or submit.
Ignore any client_path values: they will contain the name of the
temporary client, not our real client.
'''
return View(self, view_lines)
def switched_to_union(self):
'''
Return an RAII object to switch p4 connection to a different,
temporary, client spec, with the requested view lines, then
restore p4 connection to original client on exit.
Use this ONLY for read-only commands, no opening for edit or submit.
Ignore any client_path values: they will contain the name of the
temporary client, not our real client.
'''
return View(self, self.client_view_union())
def switched_to_branch(self, branch):
'''
Return an RAII object to switch p4 connection to a different,
temporary, client spec, with the requested view lines, then
restore p4 connection to original client on exit.
Use this ONLY for read-only commands, no opening for edit or submit.
Ignore any client_path values: they will contain the name of the
temporary client, not our real client.
'''
return View(self, branch.view_lines)
def temp_branch(self, create_if_none=True):
'''
Retrieve the shared temporary branch associated with this context.
If create_if_none is False and there is no temporary branch already,
None will be returned.
'''
if self._temp_branch is None and create_if_none:
self._temp_branch = TempP4BranchMapping()
return self._temp_branch
def is_feature_enabled(self, feature):
'''
Return whether feature is enabled for this repo.
Looks in @features section of config, repo first then global.
If a feature is not set in config it defaults to not enabled.
'''
config = p4gf_config.get_repo(self.p4gf, self.config.view_name)
return p4gf_config.is_feature_enabled(config, feature)
@property
def preflight_hook(self):
'''
Return our preflight hook, creating if necessary.
'''
if not self._preflight_hook:
self._preflight_hook = PreflightHook.from_context(self)
return self._preflight_hook
class Context:
"""a single git-fusion view/repo context"""
# pylint:disable=R0915
# too many statements
def __init__(self, config):
self.config = config
# connected by default:
self.p4 = None
self.p4gf = None
# not connected by default:
self.p4gf_reviews = None
self.p4gf_reviews_non_gf = None
try:
from p4gf_gitmirror import GitMirror
self.mirror = GitMirror(config.view_name)
except:
LOG.error("failed to create GitMirror:\n{}".format(traceback.format_exc()))
raise
self.timezone = None
self.server_version = None
self._user_to_protect = None
self.view_dirs = None
self.view_lock = None
self.view_repo = None
self.tempdir = tempfile.TemporaryDirectory(
prefix=p4gf_const.P4GF_TEMP_DIR_PREFIX)
# RAII object to operate on a numbered changelist with p4run and p4gfrun
# set in p4gf_util by NumberedChangelist
self.numbered_change = None
self.numbered_change_gf = None
# Environment variable set by p4gf_auth_server.py.
self.authenticated_p4user = os.environ.get(p4gf_const.P4GF_AUTH_P4USER)
# gf_branch_name ==> p4gf_branch.Branch
# Lazy-loaded by branch_dict()
self._branch_dict = None
self.branch_creation = None
self.merge_commits = None
self.submodules = None
self.owner_is_author = None
# DepotBranchInfoIndex of all known depot branches that house
# files from lightweight branches, even ones we don't own.
# Lazy-loaded by depot_branch_info_index()
self._depot_branch_info_index = None
# paths set up by set_up_paths()
self.gitdepotroot = "//{}/".format(p4gf_const.P4GF_DEPOT)
self.gitlocalroot = None
self.client_spec_gf = None
self.gitrootdir = None
self.contentlocalroot = None
self.contentclientroot = None
self.clientmap = None
self.clientmap_gf = None
self.client_exclusions_added = False
# Avoid unnecessary view switches.
self.last_view_lines = None
self.last_client_spec = None
# Seconds since the epoch when we last polled for remote kill switch.
self._last_heartbeat_cancel_check_time = None
self._heartbeat_time = None
# A set of temporary Perforce client specs, each mapped to one
# branch's view. Use these to query Perforce rather than switching
# ctx.p4 back and forth just to run 'p4 files //client/...' for some
# random branch other than our current branch.
self._client_pool = None
# By default, temp clients in pool will be deleted when Context exits.
# For 'nested' Contexts, set this to False to avoid deleting clients
# out from under the surrounding Context.
self.cleanup_client_pool = True
# A single, shared temporary Perforce branch, useful for integrations.
self._temp_branch = None
# Minimize the number of 'p4 files //branch-client/...@n' calls.
self.branch_files_cache = BranchFilesCache()
# Set by G2PMatrix during a 'git push' to remember the most recent
# changelist integrated from each branch to each other branch.
# Instance of IntegratedUpTo
self.integrated_up_to = None
# Last N p4run() commands. Reported in _dump_on_failure()
self.p4run_history = deque(maxlen=20)
# Last N p4run() commands. Reported in _dump_on_failure()
self.p4gfrun_history = deque(maxlen=20)
# Admin-configured option to reject unworthy commits.
self._preflight_hook = None
# pylint:enable=R0915
def __enter__(self):
self.connect()
return self
def __exit__(self, _exc_type, _exc_value, _traceback):
self.disconnect()
if self.tempdir:
self.tempdir.cleanup()
return False # False == do not squelch any current exception
def connect(self):
'''
Connect the p4 and p4gf connections.
Note: p4gf_reviews and p4gf_reviews_non_gf are not connected!
If this is the first time connecting, complete context initialization.
'''
# if previously connected, just reconnect
if self.p4:
if not self.p4.connected():
p4_connect(self.p4)
if not self.p4gf.connected():
p4_connect(self.p4gf)
return
# create connections and use them to complete initialization of context
self.p4 = self.__make_p4( client=self.config.p4client )
self.p4gf = self.__make_p4( client=self.config.p4client_gf )
self.p4gf_reviews = self.__make_p4( user = p4gf_util.gf_reviews_user_name()
, connect = False )
self.p4gf_reviews_non_gf = self.__make_p4( user = p4gf_const.P4GF_REVIEWS__NON_GF
, connect = False )
self._client_pool = ClientPool(self)
self.__wrangle_charset()
self.get_timezone_serverversion()
self.__set_branch_creation()
self.__set_merge_commits()
self.__set_submodules()
self.__set_change_owner()
self.__set_up_paths()
def disconnect(self):
'''
disconnect any p4 connections
'''
LOG.debug("context disconnecting {} {}"
.format( self.p4gf_reviews.connected()
, self.p4gf_reviews_non_gf.connected() ))
# clean up temp clients before disconnecting p4gf
if self.cleanup_client_pool:
self._client_pool.cleanup()
if self.p4.connected():
p4_disconnect(self.p4)
if self.p4gf.connected():
p4_disconnect(self.p4gf)
if self.p4gf_reviews.connected():
p4_disconnect(self.p4gf_reviews)
if self.p4gf_reviews_non_gf.connected():
p4_disconnect(self.p4gf_reviews_non_gf)
def last_copied_change_counter_name(self):
'''
Return a counter that holds the highest changelist number
copied to the our repo, on our Git Fusion server.
'''
return calc_last_copied_change_counter_name(
self.config.view_name
, p4gf_util.get_server_id())
def read_last_copied_change(self):
'''
Return the highest changelist number copied to a repo on a server.
'''
r = self.p4run(['counter', '-u', self.last_copied_change_counter_name()])
return r[0]['value']
def write_last_copied_change(self, change_num):
'''
Return the highest changelist number copied to a repo on a server.
'''
self.p4run([ 'counter', '-u', self.last_copied_change_counter_name()
, change_num])
def user_to_protect(self, user):
"""Return a p4gf_protect.Protect instance that knows
the given user's permissions."""
# Lazy-create the user_to_protect instance since not all
# Context-using code requires it.
if not self._user_to_protect:
self._user_to_protect = p4gf_protect.UserToProtect(self.p4)
return self._user_to_protect.user_to_protect(user)
def get_view_repo(self):
'''
Lazy-create the pygit2 repo object.
Ideally we rename this to become a property getter "view_repo" and then
data member self.view_repo becomes self._view_repo.
Some future refactor day.
'''
if not self.view_repo:
self.view_repo = pygit2.Repository(self.view_dirs.GIT_DIR)
return self.view_repo
def __make_p4(self, client=None, user=None, connect=True):
"""create a connection to the perforce server"""
if not user:
user = self.config.p4user
if not client:
client = self.config.p4client
return create_p4(port=self.config.p4port, user=user, client=client, connect=connect)
def __set_branch_creation(self):
"""Configure branch creation"""
config = p4gf_config.get_repo(self.p4gf, self.config.view_name)
self.branch_creation = config.getboolean(p4gf_config.SECTION_REPO,
p4gf_config.KEY_ENABLE_BRANCH_CREATION)
LOG.debug('Enable repo branch creation = {0}'.format(self.branch_creation))
def __set_merge_commits(self):
"""Configure merge commits"""
config = p4gf_config.get_repo(self.p4gf, self.config.view_name)
self.merge_commits = config.getboolean(p4gf_config.SECTION_REPO,
p4gf_config.KEY_ENABLE_MERGE_COMMITS)
LOG.debug('Enable repo merge commits = {0}'.format(self.merge_commits))
def __set_submodules(self):
"""Configure submodule support"""
config = p4gf_config.get_repo(self.p4gf, self.config.view_name)
self.submodules = config.getboolean(p4gf_config.SECTION_REPO,
p4gf_config.KEY_ENABLE_SUBMODULES)
LOG.debug('Enable repo submodules = {0}'.format(self.submodules))
def __set_change_owner(self):
"""Configure change ownership setting"""
config = p4gf_config.get_repo(self.p4gf, self.config.view_name)
value = config.get(p4gf_config.SECTION_REPO, p4gf_config.KEY_CHANGE_OWNER)
value = str(value).lower()
if value not in [p4gf_config.VALUE_AUTHOR, p4gf_config.VALUE_PUSHER]:
LOG.warn("change-owner config setting has invalid value, defaulting to author")
value = p4gf_config.VALUE_AUTHOR
self.owner_is_author = True if value == 'author' else False
LOG.debug('Set change owner to {0}'.format(value))
def __wrangle_charset(self):
"""figure out if server is unicode and if it is, set charset"""
if not self.p4.server_unicode:
return
# we have a unicode server
# first, always use utf8 for the gf connection
# use that connection to fetch the config for the repo
self.p4gf.charset = 'utf8'
config = p4gf_config.get_repo(self.p4gf, self.config.view_name)
# then set the repo-specific charset for their connection
self.p4.charset = config.get(p4gf_config.SECTION_REPO, p4gf_config.KEY_CHARSET)
LOG.debug('repo charset will be: '+self.p4.charset)
def client_view_path(self, change_num=None):
'''
Return "//{client}/..." : the client path for whole view,
including ... wildcard
Optional change_num, if supplied, appended as "@N"
Because I'm sick of constructing that over and over.
'''
if change_num:
return '{}@{}'.format(self.contentclientroot, change_num)
return self.contentclientroot
def client_view_union(self):
"""Return the client view to the union of all branches in this repo
"""
p4map = p4gf_branch.calc_branch_union_client_view( self.config.p4client
, self.branch_dict())
return p4map.as_array()
def switch_client_view_to_union(self):
"""Change the repo's Perforce client view to the union of all branches
in this repo.
Warning: other objects retain pointers into OUR self.clientmap object.
If we swap in a new object, those other objects point to stale data.
If we change the values in the same object, those other objects point
to new data. In neither case do these other objects get notification
of change.
"""
view_lines = self.client_view_union()
if LOG.isEnabledFor(logging.DEBUG2):
LOG.debug2('switch_client_view_to_union:\n{}'.format('\n'.join(view_lines)))
else:
LOG.debug('switch_client_view_to_union')
self.switch_client_view_lines(view_lines)
def switch_client_view_to_branch(self, branch):
"""Change the repo's Perforce client view to view of
the given Branch object.
branch is a p4gf_branch instance. See branch_dict() if you need to
lookup by name.
"""
LOG.debug('switch_client_view_to_branch() {}'.format(branch.to_log()))
if branch.stream_name:
self.switch_client_to_stream(branch)
else:
self.switch_client_view_lines(branch.view_lines)
def switch_client_view_lines(self, lines):
"""Change this repo's Perforce client view to the given line list.
Update our clientmap object, with the new lines. Update clientmap in
place so that other object with pointers to OUR clientmap object will
see the new data.
"""
_lines = to_lines(lines)
if _lines == self.last_view_lines:
LOG.debug2('switch_client_view_lines() already switched. NOP.')
return
LOG.debug2('switch_client_view_lines() client={} {}'
.format(self.config.p4client, _lines))
self.last_client_spec = p4gf_util.set_spec(
self.p4, 'client'
, spec_id = self.config.p4client
, values = {'View': _lines, 'Stream': None}
, cached_vardict = self.last_client_spec)
self.clientmap.clear()
for line in _lines:
self.clientmap.insert(line)
self.last_view_lines = copy.copy(_lines)
def switch_client_to_stream(self, branch):
"""Change this repo's Perforce client view to the given line list.
Update our clientmap object, with the new lines. Update clientmap in
place so that other object with pointers to OUR clientmap object will
see the new data.
"""
LOG.debug2('switch_client_view_stream() client={} stream={}'
.format(self.config.p4client, branch.stream_name))
self.p4run(['client', '-f', '-s', '-S', branch.stream_name, self.config.p4client])
lines = branch.view_lines
_lines = to_lines(lines)
if _lines == self.last_view_lines:
return
LOG.debug2('switch_client_view_streams() client={} {}'
.format(self.config.p4client, _lines))
self.clientmap.clear()
for line in _lines:
self.clientmap.insert(line)
self.last_view_lines = copy.copy(_lines)
def checkout_master_ish(self):
'''
Switch Git to the first branch view defined in p4gf_config.
This is often master, but not always.
Since P4GF defaults to configuring a 'master' branch, which may
not exist, try the other configured branches before giving up.
NOP if no such branch (perhaps an empty p4gf_config?).
'''
branches = self.branch_dict()
br = p4gf_branch.most_equal(branches)
if br and br.git_branch_name:
if not p4gf_util.git_checkout(br.git_branch_name):
# Perhaps the most-equal branch does not exist?
# Try the others until we achieve success.
success = False
for br in branches.values():
if br.git_branch_name and p4gf_util.git_checkout(br.git_branch_name):
success = True
break
if not success:
LOG.warn('Unable to checkout sensible branch for {}'.format(
self.config.view_name))
def git_branch_name_to_branch(self, git_branch_name):
'''
If we have an undeleted branch with the requested name, return it.
If not, return None.
O(n) scan.
'''
assert git_branch_name
name = git_branch_name
if name.startswith('refs/heads/'):
name = name[len('refs/heads/'):]
for branch in self.branch_dict().values():
if branch.deleted:
continue
if branch.git_branch_name == name:
return branch
return None
def branch_dict(self):
"""Return all known Git<->Perforce branch associations.
Lazy-loaded from config file.
Loads DepotBranchInfo dict as a side effect so that we can
reconnect Branch.depot_branch pointers.
"""
if not self._branch_dict:
# Load p4gf_config: the fully populated branches.
config = p4gf_config.get_repo(self.p4gf, self.config.view_name)
self._branch_dict = p4gf_branch.dict_from_config(config, self.p4gf)
# First branch listed in p4gf_config becomes our default HEAD.
# This is usually 'master', but not required.
bsl = p4gf_config.branch_section_list(config)
if bsl:
self._branch_dict[bsl[0]].more_equal = True
# Load the lightweight and stream-based branch config data into the
# branch_dict. This is stored in p4gf_config2. For lightweight
# branches, the full branch def is there. For stream-based branches
# all we care about is the original-view, which gets merged with
# any config stored in p4gf_config.
config2 = p4gf_config.get_repo2(self.p4gf, self.config.view_name)
if config2:
branch_dict2 = p4gf_branch.dict_from_config(config2, self.p4gf)
lwb_dict = {}
for branch in branch_dict2.values():
if branch.stream_name:
if branch.branch_id in self._branch_dict:
self._branch_dict[branch.branch_id].original_view_lines = \
branch.original_view_lines
else:
branch.is_lightweight = True
if ( branch.depot_branch
and isinstance(branch.depot_branch, str)):
branch.depot_branch = self.depot_branch_info_index() \
.find_depot_branch_id(branch.depot_branch)
lwb_dict[branch.branch_id] = branch
self._branch_dict.update(lwb_dict)
for b in self._branch_dict.values():
b.set_rhs_client(self.config.p4client)
LOG.debug('branch_dict() lazy-loaded ct={}'
.format(len(self._branch_dict)))
if LOG.isEnabledFor(logging.DEBUG2):
for b in self._branch_dict.values():
LOG.debug2('\n' + b.to_log(LOG))
return self._branch_dict
def undeleted_branches(self):
'''
An iterator/generator of all Branch values in branch_dict
that are not deleted.
'''
for branch in self.branch_dict().values():
if branch.deleted:
continue
yield branch
def depot_branch_info_index(self):
'''
Return all known depot branches that house files for lightweight
branches. This includes depot branches that other Git Fusion repos
created: we must stay lightweight even when sharing across repos.
Lazy-loaded from all depot branch-info files.
'''
if not self._depot_branch_info_index:
self._depot_branch_info_index = p4gf_depot_branch.DepotBranchInfoIndex()
root = p4gf_const.P4GF_DEPOT_BRANCH_INFO_ROOT.format(
P4GF_DEPOT=p4gf_const.P4GF_DEPOT)
root = root + '/...'
with p4gf_util.RawEncoding(self.p4):
file_data = self.p4.run('print', root)
delete = False
started = False
file_contents = ''
for item in file_data:
if isinstance(item, dict):
if started: # finish with the current branch info
dbi = p4gf_depot_branch.depot_branch_info_from_string(file_contents)
self._depot_branch_info_index.add(dbi)
if item['action'] == 'delete':
started = False
delete = True
else:
file_contents = ''
delete = False
started = True
else:
if delete:
continue
new_item = item.decode().strip()
if len(new_item):
file_contents = file_contents + new_item
if started:
dbi = p4gf_depot_branch.depot_branch_info_from_string(file_contents)
self._depot_branch_info_index.add(dbi)
if LOG.isEnabledFor(logging.DEBUG):
for dpid in self._depot_branch_info_index.by_id:
LOG.debug("DBI index LAZY loaded: {0} {1}".format(dpid
, self._depot_branch_info_index.by_id[dpid]))
### YAGNI until we have a test that proves it.
### p4 print utf8 //P4GF_DEPOT/branches/branch-info/...
return self._depot_branch_info_index
def get_timezone_serverversion(self):
"""get server's timezone and server version via p4 info"""
r = self.p4.run_info()
server_date = p4gf_util.first_value_for_key(r, 'serverDate')
self.timezone = server_date.split(" ")[2]
self.server_version = p4gf_util.first_value_for_key(r, 'serverVersion')
def __set_up_paths(self):
"""set up depot and local paths for both content and P4GF
These paths are derived from the client root and client view.
"""
self.__set_up_content_paths()
self.__set_up_p4gf_paths()
self.view_dirs = p4gf_view_dirs.from_p4gf_dir(self.gitrootdir, self.config.view_name)
def __set_up_content_paths(self):
"""set up depot and local paths for both content and P4GF
These paths are derived from the client root and client view.
"""
client = self.p4.fetch_client()
self.clientmap = Map(client["View"])
# If the len of the client Views differs from the len of the Map
# then the P4 disabmbiguator added exclusionary mappings - note this here
# for reporting a message back to the user.
self.client_exclusions_added = len(client["View"]) != len(self.clientmap.as_array())
# local syntax client root, force trailing /
self.contentlocalroot = client["Root"]
if not self.contentlocalroot.endswith("/"):
self.contentlocalroot += '/'
# client sytax client root with wildcard
self.contentclientroot = '//' + self.p4.client + '/...'
def __set_up_p4gf_paths(self):
"""set up depot and local paths for P4GF
These paths are derived from the client root and client view.
"""
client = self.p4gf.fetch_client()
self.client_spec_gf = client
self.gitrootdir = client_spec_to_root(client)
self.clientmap_gf = Map(client["View"])
lhs = self.clientmap_gf.lhs()
check_client_view_gf(lhs)
assert len(lhs) == 1, _('view must contain only one line')
rpath = self.clientmap_gf.translate(lhs[0])
self.gitlocalroot = strip_wild(client_path_to_local(
rpath, self.p4gf.client, self.gitrootdir))
def __str__(self):
return "\n".join(["Git data in Perforce: " + self.gitdepotroot + "...",
" " + self.gitlocalroot + "...",
"Exported Perforce tree: " + self.contentlocalroot + "...",
" " + self.contentclientroot,
"timezone: " + self.timezone])
def __repr__(self):
return str(self) + "\n" + repr(self.mirror)
def log_context(self):
"""Dump connection info, client info, directories, all to log category
'context' as INFO."""
log = logging.getLogger('context')
if not log.isEnabledFor(logging.INFO):
return
# Dump client spec as raw untagged text.
self.p4.tagged = 0
client_lines_raw = self.p4.run('client', '-o')[0].splitlines()
self.p4.tagged = 1
# Strip comment header
client_lines = [l for l in client_lines_raw if not l.startswith('#')]
# Dump p4 info, tagged, since that includes more pairs than untagged.
p4info = p4gf_util.first_dict(self.p4.run('info'))
key_len_max = max(len(k) for k in p4info.keys())
info_template = NTR('%-{}s : %s').format(key_len_max)
log.info(info_template, 'P4PORT', self.p4.port)
log.info(info_template, 'P4USER', self.p4.user)
log.info(info_template, 'P4CLIENT', self.p4.client)
log.info(info_template, 'p4gfclient', self.p4gf.client)
for k in sorted(p4info.keys(), key=str.lower):
log.info(info_template, k, p4info[k])
for line in client_lines:
log.info(line)
def _check_heartbeat_canceled(self):
'''
Ask the server if someone else wants us to stop.
Remote kill switch: delete our counter and we'll voluntarily stop.
'p4 counter -u git-fusion-view-{repo}-lock'
Throttled back to check only once every N seconds.
'''
# Check no more than once every N seconds.
now = time.time()
if self._last_heartbeat_cancel_check_time:
since_secs = now - self._last_heartbeat_cancel_check_time
if since_secs < _CANCEL_POLL_PERIOD_SECS:
return
self._last_heartbeat_cancel_check_time += _CANCEL_POLL_PERIOD_SECS
else:
self._last_heartbeat_cancel_check_time = now
if self.view_lock.canceled():
raise p4gf_lock.LockCanceled(_("Canceling: lock '{}' lost.")
.format(self.view_lock.counter_name()))
def heartbeat(self):
'''
If we have a view lock, update its heartbeat.
If our lock is cleared, then raise a RuntimeException
canceling our current task.
'''
if not self.view_lock:
return
self._check_heartbeat_canceled()
self.view_lock.update_heartbeat()
self.log_memory_usage()
def log_memory_usage(self):
'''Log our memory usage on a regular basis.'''
if LOG.isEnabledFor(logging.DEBUG):
now = time.time()
if self._heartbeat_time and now - self._heartbeat_time < MEMLOG_HEART_RATE:
return
self._heartbeat_time = now
LOG.debug(p4gf_log.memory_usage())
def _convert_path(self, clazz, path):
'''Return a path object that can convert to other formats.'''
return clazz( self.clientmap
, self.config.p4client
, self.contentlocalroot[:-1] # -1 to strip trailing /
, path)
def depot_path(self, path):
'''Return an object that can convert from depot to other syntax.'''
return self._convert_path(p4gf_path_convert.DepotPath, path)
def client_path(self, path):
'''Return an object that can convert from client to other syntax.'''
return self._convert_path(p4gf_path_convert.ClientPath, path)
def gwt_path(self, path):
'''
Return an object that can convert from Git work tree to other syntax.
'''
return self._convert_path(p4gf_path_convert.GWTPath, path)
def gwt_to_depot_path(self, gwt_path):
'''
Optimized version of ctx.gwt_path(gwt).to_depot().
Avoid creating the convert and goes straight to P4.Map.translate().
We call this once for every row in G2PMatrix. This function runs in
about 60% of ctx.gwt_path(x).to_depot() time. That works out to about 5%
of total wall clock time for many-file repos such as james.
'''
gwt_esc = p4gf_util.escape_path(gwt_path)
client_path = '//{}/'.format(self.config.p4client) + gwt_esc
return self.clientmap.translate(client_path, self.clientmap.RIGHT2LEFT)
def depot_to_gwt_path(self, depot_path):
'''
Optimized version of ctx.depot_path(dp).to_gwt().
Avoid creating the convert and goes straight to P4.Map.translate().
We call this once for every row in G2PMatrix. This function runs in
about 60% of ctx.depot_path(x).to_gwt() time. That works out to about 5%
of total wall clock time for many-file repos such as james.
'''
client_path = self.clientmap.translate(depot_path, self.clientmap.LEFT2RIGHT)
gwt_esc = client_path[3+len(self.config.p4client):]
return p4gf_util.unescape_path(gwt_esc)
def local_path(self, path):
'''
Return an object that can convert from absolute local filesystem
to other syntax.
'''
return self._convert_path(p4gf_path_convert.LocalPath, path)
# pylint:disable=R0913
# Too many arguments
# We really should just dial that up to 11
# if we're going to keep ignoring its advice.
@staticmethod
def _p4run( p4
, numbered_change
, run_history
, cmd
, log_warnings
, log_errors ):
'''
Record the command in history, then perform it.
'''
if numbered_change:
cmd = numbered_change.add_change_option(cmd)
run_history.append(cmd)
return p4gf_util.p4run_logged( p4, cmd
, log_warnings = log_warnings
, log_errors = log_errors )
# pylint:enable=R0913
def p4run( self, cmd
, log_warnings = logging.WARNING
, log_errors = logging.ERROR):
'''
Run a command, with logging.
'''
return self._p4run( p4 = self.p4
, numbered_change = self.numbered_change
, run_history = self.p4run_history
, cmd = cmd
, log_warnings = log_warnings
, log_errors = log_errors
)
def p4gfrun(self, cmd
, log_warnings = logging.WARNING
, log_errors = logging.ERROR):
'''
Run a command, with logging.
'''
return self._p4run( p4 = self.p4gf
, numbered_change = self.numbered_change_gf
, run_history = self.p4gfrun_history
, cmd = cmd
, log_warnings = log_warnings
, log_errors = log_errors
)
def switched_to_view_lines(self, view_lines):
'''
Return an RAII object to switch p4 connection to a different,
temporary, client spec, with the requested view lines, then
restore p4 connection to original client on exit.
Use this ONLY for read-only commands, no opening for edit or submit.
Ignore any client_path values: they will contain the name of the
temporary client, not our real client.
'''
return View(self, view_lines)
def switched_to_union(self):
'''
Return an RAII object to switch p4 connection to a different,
temporary, client spec, with the requested view lines, then
restore p4 connection to original client on exit.
Use this ONLY for read-only commands, no opening for edit or submit.
Ignore any client_path values: they will contain the name of the
temporary client, not our real client.
'''
return View(self, self.client_view_union())
def switched_to_branch(self, branch):
'''
Return an RAII object to switch p4 connection to a different,
temporary, client spec, with the requested view lines, then
restore p4 connection to original client on exit.
Use this ONLY for read-only commands, no opening for edit or submit.
Ignore any client_path values: they will contain the name of the
temporary client, not our real client.
'''
return View(self, branch.view_lines)
def temp_branch(self, create_if_none=True):
'''
Retrieve the shared temporary branch associated with this context.
If create_if_none is False and there is no temporary branch already,
None will be returned.
'''
if self._temp_branch is None and create_if_none:
self._temp_branch = TempP4BranchMapping()
return self._temp_branch
def is_feature_enabled(self, feature):
'''
Return whether feature is enabled for this repo.
Looks in @features section of config, repo first then global.
If a feature is not set in config it defaults to not enabled.
'''
config = p4gf_config.get_repo(self.p4gf, self.config.view_name)
return p4gf_config.is_feature_enabled(config, feature)
@property
def preflight_hook(self):
'''
Return our preflight hook, creating if necessary.
'''
if not self._preflight_hook:
self._preflight_hook = PreflightHook.from_context(self)
return self._preflight_hook
class Context:
"""a single git-fusion view/repo context"""
def __init__(self, config):
self.config = config
self.p4 = self.__make_p4(client=self.config.p4client)
self.p4gf = self.__make_p4(client=self.config.p4client_gf)
self.mirror = GitMirror(config.view_name)
self.timezone = None
self.get_timezone()
self._user_to_protect = None
self.view_dirs = None
self.view_lock = None
self.tempdir = tempfile.TemporaryDirectory(prefix=p4gf_const.P4GF_TEMP_DIR_PREFIX)
# Environment variable set by p4gf_auth_server.py.
self.authenticated_p4user = os.environ.get(p4gf_const.P4GF_AUTH_P4USER_ENVAR)
# paths set up by set_up_paths()
self.gitdepotroot = "//" + p4gf_const.P4GF_DEPOT + "/"
self.gitlocalroot = None
self.gitrootdir = None
self.contentlocalroot = None
self.contentclientroot = None
self.clientmap = None
self.clientmap_gf = None
self.__set_up_paths()
def user_to_protect(self, user):
"""Return a p4gf_protect.Protect instance that knows
the given user's permissions."""
# Lazy-create the user_to_protect instance since not all
# Context-using code requires it.
if not self._user_to_protect:
self._user_to_protect = p4gf_protect.UserToProtect(self.p4)
return self._user_to_protect.user_to_protect(user)
def __make_p4(self, client=None):
"""create a connection to the perforce server"""
p4 = create_p4(port=self.config.p4port,
user=self.config.p4user)
if client:
p4.client = client
else:
p4.client = self.config.p4client
try:
p4.connect()
except P4Exception as e:
raise RuntimeError("Failed P4 connect: {}".format(str(e)))
p4.exception_level = 1
return p4
def client_view_path(self):
"""return client path for whole view, including ... wildcard"""
return self.contentclientroot
def get_timezone(self):
"""get server's timezone via p4 info"""
server_date = p4gf_util.first_value_for_key(self.p4.run("info"), 'serverDate')
self.timezone = server_date.split(" ")[2]
def __set_up_paths(self):
"""set up depot and local paths for both content and P4GF
These paths are derived from the client root and client view.
"""
self.__set_up_content_paths()
self.__set_up_p4gf_paths()
def __set_up_content_paths(self):
"""set up depot and local paths for both content and P4GF
These paths are derived from the client root and client view.
"""
client = self.p4.fetch_client()
self.clientmap = Map(client["View"])
# local syntax client root, force trailing /
self.contentlocalroot = client["Root"]
if not self.contentlocalroot.endswith("/"):
self.contentlocalroot += '/'
# client sytax client root with wildcard
self.contentclientroot = '//' + self.p4.client + '/...'
def __set_up_p4gf_paths(self):
"""set up depot and local paths for P4GF
These paths are derived from the client root and client view.
"""
client = self.p4gf.fetch_client()
# client root, minus any trailing /
self.gitrootdir = client["Root"]
if self.gitrootdir.endswith("/"):
self.gitrootdir = self.gitrootdir[:-1]
self.clientmap_gf = Map(client["View"])
lhs = self.clientmap_gf.lhs()
check_client_view_gf(lhs)
for lpath in lhs:
rpath = self.clientmap_gf.translate(lpath)
self.gitlocalroot = strip_wild(
client_path_to_local(rpath, self.p4gf.client, self.gitrootdir))
def __str__(self):
return "\n".join(["Git data in Perforce: " + self.gitdepotroot + "...",
" " + self.gitlocalroot + "...",
"Exported Perforce tree: " + self.contentlocalroot + "...",
" " + self.contentclientroot,
"timezone: " + self.timezone])
def __repr__(self):
return str(self) + "\n" + repr(self.mirror)
def log_context(self):
"""Dump connection info, client info, directories, all to log category
'context' as INFO."""
log = logging.getLogger('context')
if not log.isEnabledFor(logging.INFO):
return
# Dump client spec as raw untagged text.
self.p4.tagged = 0
client_lines_raw = self.p4.run('client', '-o')[0].splitlines()
self.p4.tagged = 1
# Strip comment header
client_lines = [l for l in client_lines_raw if not l.startswith('#')]
# Dump p4 info, tagged, since that includes more pairs than untagged.
p4info = p4gf_util.first_dict(self.p4.run('info'))
key_len_max = max(len(k) for k in p4info.keys())
info_template = "%-{}s : %s".format(key_len_max)
log.info(info_template % ('P4PORT', self.p4.port))
log.info(info_template % ('P4USER', self.p4.user))
log.info(info_template % ('P4CLIENT', self.p4.client))
log.info(info_template % ('p4gfclient', self.p4gf.client))
for k in sorted(p4info.keys(), key=str.lower):
log.info(info_template % (k, p4info[k]))
for line in client_lines:
log.info(line)
def heartbeat(self):
'''
If we have a view lock, update its heartbeat.
If our lock is cleared, then raise a RuntimeException
canceling our current task.
'''
if not self.view_lock:
return
if self.view_lock.canceled():
raise RuntimeError("Canceling: lock {} lost."
.format(self.view_lock.counter_name()))
self.view_lock.update_heartbeat()
# For some of the tests, slow down the operation by sleeping briefly.
key = p4gf_const.P4GF_TEST_LOCK_VIEW_SLEEP_AFTER_HEARTBEAT_SECONDS
test_vars = p4gf_util.test_vars()
if key in test_vars:
sleep_seconds = test_vars[key]
LOG.debug("Test: sleeping {} seconds...".format(sleep_seconds))
time.sleep(float(sleep_seconds))
LOG.debug("Test: sleeping {} seconds done".format(sleep_seconds))
