def _add_depot_branch_infos_to_p4(self, ctx):
'''
If we created any new depot branches, 'p4 add' their branch-info
file to Perforce. Does not submit.
If we edited any existing depot branches, 'p4 edit' them.
Return number of branch-info files added or edited.
'''
if not self.depot_branch_info_list:
return
add_path_list = []
edit_path_list = []
for dbi in self.depot_branch_info_list:
config = dbi.to_config()
depot_path = dbi.to_config_depot_path()
local_path = p4gf_util.depot_to_local_path( depot_path
, ctx.p4gf
, ctx.client_spec_gf )
p4gf_util.ensure_dir(p4gf_util.parent_dir(local_path))
p4gf_util.make_writable(local_path)
with open(local_path, 'w') as f:
config.write(f)
if dbi.needs_p4add:
add_path_list.append(local_path)
else:
edit_path_list.append(local_path)
p4gf_config.clean_up_parser(config)
del config
success_list = self._p4_add_in_bites(ctx, add_path_list)
success_list.extend(self._p4_sync_k_edit(ctx, edit_path_list))
return len(success_list)
def write(self):
"""Create a local file P4GF_HOME/changelists/{repo}/{change_num}
and tells GitMirror about it.
Does not yet open file for 'p4 add' or 'p4 edit': let
GitMirror.add_objects_to_p4() do that much later.
"""
lpath = self.local_path()
p4gf_util.ensure_dir(p4gf_util.parent_dir(lpath))
p4gf_util.make_writable(lpath)
with open(lpath, 'w') as f:
self._write(f)
def write(self):
'''
Create a local file P4GF_HOME/changelists/{repo}/{change_num}
and tells GitMirror about it.
Does not yet open file for 'p4 add' or 'p4 edit': let
GitMirror.add_objects_to_p4() do that much later.
'''
lpath = self.local_path()
p4gf_util.ensure_dir(p4gf_util.parent_dir(lpath))
p4gf_util.make_writable(lpath)
with open(lpath, 'w') as f:
self._write(f)
def check_and_create_default_p4gf_env_config():
"""If p4gf_env_config threw the MissingConfigPath exception,
because P4GF_ENV names a non-existing filepath
then save the required (two) default items
into the user configured P4GF_ENV environment config file.
"""
if not Create_P4GF_CONFIG:
LOG.debug('not creating configuration file')
return
LOG.debug('creating missing configuration file')
Verbosity.report(
Verbosity.INFO, _("Git Fusion environment var P4GF_ENV = {path} names a non-existing file.")
.format(path=p4gf_const.P4GF_ENV))
Verbosity.report(
Verbosity.INFO, _("Creating {path} with the default required items.")
.format(path=p4gf_const.P4GF_ENV))
Verbosity.report(
Verbosity.INFO, _("Review the file's comments and edit as needed."))
Verbosity.report(
Verbosity.INFO, _("You may unset P4GF_ENV to use no config file.")
.format(p4gf_const.P4GF_ENV))
config = configparser.ConfigParser(interpolation = None,
allow_no_value = True)
config.optionxform = str
config.add_section(p4gf_const.SECTION_ENVIRONMENT)
config.set(p4gf_const.SECTION_ENVIRONMENT, p4gf_const.P4GF_HOME_NAME, p4gf_const.P4GF_HOME)
Verbosity.report(
Verbosity.INFO, _("Setting {home_name} = {home} in {env}.")
.format(home_name=p4gf_const.P4GF_HOME_NAME,
home=p4gf_const.P4GF_HOME,
env=p4gf_const.P4GF_ENV))
config.set(p4gf_const.SECTION_ENVIRONMENT, NTR('P4PORT'), P4PORT)
Verbosity.report(
Verbosity.INFO, _("Setting {p4port} = {p4port_value} in {env}.")
.format(p4port=NTR('P4PORT'),
p4port_value=P4PORT,
env=p4gf_const.P4GF_ENV))
header = p4gf_util.read_bin_file(NTR('p4gf_env_config.txt'))
if header is False:
sys.stderr.write(_('no p4gf_env_config.txt found\n'))
header = _('# Missing p4gf_env_config.txt file!')
out = io.StringIO()
out.write(header)
config.write(out)
file_content = out.getvalue()
out.close()
p4gf_util.ensure_dir(p4gf_util.parent_dir(p4gf_const.P4GF_ENV))
with open(p4gf_const.P4GF_ENV, 'w') as f:
f.write(file_content)
LOG.debug('created configuration file %s', p4gf_const.P4GF_ENV)
def open_all(self):
"""Open zipfile and journal files for write.
Generates temp files for all of 'em.
"""
p4gf_util.ensure_dir(self._dir_abspath)
for j in self.jnl:
assert j.fp is None
j.fp = open(j.abspath, "w", encoding="utf-8")
self.zip.fp = zipfile.ZipFile(self.zip.abspath, "w"
#, compression = zipfile.ZIP_STORED
# ZIP_DEFLATED saves 50% temporary/working disk space,
# but costs 5% additional time for the compression.
, compression = zipfile.ZIP_DEFLATED
, allowZip64 = True )
def check_and_create_default_p4gf_env_config():
'''If p4gf_env_config threw the MissingConfigPath exception,
because P4GF_ENV names a non-existing filepath
then save the required (two) default items
into the user configured P4GF_ENV environment config file.
'''
if not Create_P4GF_CONFIG:
return
Verbosity.report(Verbosity.INFO,
_("Git Fusion environment var P4GF_ENV = {0} names a non-existing file.")
.format(p4gf_const.P4GF_ENV))
Verbosity.report(Verbosity.INFO, _("Creating {0} with the default required items.")
.format(p4gf_const.P4GF_ENV))
Verbosity.report(Verbosity.INFO, _("Review the file's comments and edit as needed."))
Verbosity.report(Verbosity.INFO, _("You may unset P4GF_ENV to use no config file.")
.format(p4gf_const.P4GF_ENV))
config = configparser.ConfigParser(interpolation = None,
allow_no_value = True)
config.optionxform = str
config.add_section(p4gf_const.SECTION_ENVIRONMENT)
config.set(p4gf_const.SECTION_ENVIRONMENT, p4gf_const.P4GF_HOME_NAME, p4gf_const.P4GF_HOME)
Verbosity.report(Verbosity.INFO, _("Setting {0} = {1} in {2}.")
.format(p4gf_const.P4GF_HOME_NAME, p4gf_const.P4GF_HOME, p4gf_const.P4GF_ENV))
config.set(p4gf_const.SECTION_ENVIRONMENT, NTR('P4PORT'), P4PORT)
Verbosity.report(Verbosity.INFO, _("Setting {0} = {1} in {2}.")
.format(NTR('P4PORT'), P4PORT, p4gf_const.P4GF_ENV))
header = p4gf_util.read_bin_file(NTR('p4gf_env_config.txt'))
if header is False:
sys.stderr.write(_('no p4gf_env_config.txt found\n'))
header = _('# Missing p4gf_env_config.txt file!')
out = io.StringIO()
out.write(header)
config.write(out)
file_content = out.getvalue()
out.close()
p4gf_util.ensure_dir(p4gf_util.parent_dir(p4gf_const.P4GF_ENV))
with open(p4gf_const.P4GF_ENV, 'w') as f:
f.write(file_content)
def create_dir(self):
"""Create .p4gf_gc directory, or fail if already exists."""
if os.path.exists(self.dir_abspath):
raise RuntimeError("Directory already exists: {}"
"\nEither:"
"\n* p4gf_gc.py --force to delete it and start clean, or"
"\n* p4gf_gc.py --cont to use whatever data is in .p4gf_gc"
.format(self.dir_abspath))
LOG.info("Creating work directory: {}".format(self.dir_abspath))
p4gf_util.ensure_dir(self.dir_abspath)
p4gf_util.ensure_dir(self.git_dir_abspath)
p4gf_util.ensure_dir(self.sql_dir_abspath)
def copy(self, start_at, stop_at, new_git_branches):
"""copy a set of changelists from perforce into git"""
LOG.debug('copy() start={} stop={} new_git_branches={}'.format(
start_at, stop_at, new_git_branches))
with self.p2g.perf.timer[OVERALL]:
self.p2g._log_memory('start')
# Stop early if nothing to copy.
# 'p4 changes -m1 //client/...'
repo_empty = p4gf_util.git_empty()
if not self._requires_copy(new_git_branches, repo_empty):
self.p2g.fastimport.cleanup()
LOG.debug("No changes since last copy.")
return
### Fake start. Needs proper setup with graft support.
self.p2g.rev_range = self.p2g.mc_rev_range()
self.ctx.view_repo = pygit2.Repository(self.ctx.view_dirs.GIT_DIR)
self.p2g._log_memory('pygit2')
# O(Bfp + 1) p4 changes: one for each Branch fully populated,
# + 1 for //.git-fusion/branches/repo/...
#
with self.p2g.perf.timer[CHANGES]:
self.change_num_on_branch_list \
= deque(self._p4_changes_each_branch())
LOG.debug('Found ChangeNumOnBranch count: {}'.format(
len(self.change_num_on_branch_list)))
self.p2g._log_memory('O(Bfp + 1) p4 changes')
#p4gf_gc.report_growth ('aftr O(Bfp + 1) p4 changes')
#p4gf_gc.report_objects('aftr O(Bfp + 1) p4 changes')
if not self.change_num_on_branch_list:
LOG.debug("No new changes found to copy")
return
# Prepare for the loop over each changelist x branch.
p4gf_util.ensure_dir(self.symlink_dir)
self.p2g._fill_head_marks_from_current_heads()
self.mark_to_branch_id = {}
self.branch_id_to_temp_name \
= self.p2g._create_branch_id_to_temp_name_dict()
self.known_dbi_set = {
branch.depot_branch
for branch in self.ctx.branch_dict().values()
if branch.depot_branch
}
# All string placeholders should have been replaced with
# pointers to full DepotBranchInfo instances before
# calling copy().
for dbi in self.known_dbi_set:
assert not isinstance(dbi, str)
LOG.error('known: {}'.format(dbi)) ##ZZ
# O(C) 'p4 changes -m1 + filelog + print'
#
# Print each file revision to its blob in the .git/objects/
# Write each changelist to git-fast-import script.
#
with ProgressReporter.Indeterminate():
while self.change_num_on_branch_list:
ProgressReporter.increment("MC Copying changelists...")
cnob = self.change_num_on_branch_list.pop()
self._copy_one(cnob)
# Explicitly delete the PrintHandler now so that it
# won't show up in any leak reports between now and
# P2GMemCapped's end-of-life.
if self.print_handler:
self.printed_byte_count = self.print_handler.total_byte_count
self.printed_rev_count = self.print_handler.printed_rev_count
self.print_handler = None
self.p2g._log_memory('P2G_MC.copy() loop')
p4gf_gc.report_growth('after P2G_MC.copy() loop')
p4gf_gc.report_objects('after P2G_MC.copy() loop')
#p4gf_gc.backref_objects_by_type(dict().__class__)
# Run git-fast-import to add everything to Git.
with self.p2g.perf.timer[FAST_IMPORT]:
LOG.info('Running git-fast-import')
marks = self.p2g.fastimport.run_fast_import()
# Remove all temporary Git branch refs.
# After git-fast-import, we no longer need them.
self._delete_temp_git_branch_refs()
# Record how much we've copied in a p4 counter so that
# future calls to _any_changes_since_last_copy() can
# tell if there's anything new to copy.
self.ctx.write_last_copied_change(self.highest_copied_change_num)
if repo_empty:
# If we are just now rebuilding the Git repository, also
# grab all of the tags that have been pushed in the past.
p4gf_tag.generate_tags(self.ctx)
self.p2g._log_memory('_generate_tags')
with self.p2g.perf.timer[MIRROR]:
self.p2g._mirror(marks, self.mark_to_branch_id)
self.p2g._log_memory('_mirror')
with self.p2g.perf.timer[BRANCH_REF]:
self.p2g._set_branch_refs(marks)
self.p2g._log_memory('_set_branch_refs')
with self.p2g.perf.timer[PACK]:
self.p2g._pack()
self.p2g._log_memory('_pack')
LOG.getChild("time").debug("\n" + str(self))
LOG.info('MC Done. Commits: {cnob_ct:,d} File Revisions: {rev_ct:,d}'
' Bytes: {byte_ct:,d} Seconds: {sec:,d}'.format(
cnob_ct=self.cnob_count,
rev_ct=self.printed_rev_count,
byte_ct=self.printed_byte_count,
sec=int(self.p2g.perf.timer[OVERALL].time)))
p4gf_gc.report_objects('after P2G MC copy()')
self.p2g._log_memory('copy() done')
def _add_branch_defs_to_p4(self, ctx):
'''
If we defined any new named+lightweight branches, update (or write the
first revision of) this repo's p4gf_config2 file with all the
currently defined named+lightweight branches.
'''
# Nothing to write? well maybe we have just deleted the remaining refs
have_branches = bool(self.branch_list)
# What does the file look like now?
p4 = ctx.p4gf # For less typing later.
old_content = None
new_content = None
depot_path = p4gf_config.depot_path_repo2(ctx.config.view_name)
local_path = p4gf_util.depot_to_local_path(depot_path, p4)
depot_exists = False
# 'p4 print' will fail if file doesn't exist yet. Okay.
with ctx.p4gf.at_exception_level(p4.RAISE_NONE):
b = p4gf_util.print_depot_path_raw(p4, depot_path)
if b:
old_content = b.decode() # as UTF-8
depot_exists = True
# What do we want the file to look like? ConfigParser writes only to
# file, not to string, so we have to give it a file path. Ooh! I know!
# How about writing to the very file that we have to 'p4 add' or 'p4
# edit' if its content differs?
if have_branches:
config = configparser.ConfigParser(interpolation=None)
for b in self.branch_list:
LOG.debug("add branch {0}".format(b))
b.add_to_config(config)
p4gf_util.ensure_dir(p4gf_util.parent_dir(local_path))
p4gf_util.make_writable(local_path)
with open(local_path, 'w') as f:
config.write(f)
with open(local_path, 'r') as f:
new_content = f.read()
p4gf_config.clean_up_parser(config)
del config
# Did nothing change? Then nothing to write.
if p4gf_config.compare_configs_string(old_content, new_content):
LOG.debug("No change to p4gf_config2 file")
return False
# Have to add or edit or delete the file.
if not have_branches:
ctx.p4gfrun(['sync', '-fkq', depot_path])
ctx.p4gfrun(['delete', depot_path])
LOG.debug("Deleted p4gf_config2 file")
else:
if depot_exists:
ctx.p4gfrun(['sync', '-fkq', depot_path])
ctx.p4gfrun(['edit', depot_path])
LOG.debug("Edited p4gf_config2 file")
else:
ctx.p4gfrun(['add', '-t', 'text', local_path])
LOG.debug("Added p4gf_config2 file")
return True
def _add_branch_defs_to_p4(self, ctx):
'''
If we defined any new named+lightweight branches, update (or write the
first revision of) this repo's p4gf_config2 file with all the
currently defined named+lightweight branches.
'''
# Nothing to write? well maybe we have just deleted the remaining refs
have_branches = bool(self.branch_list)
# What does the file look like now?
p4 = ctx.p4gf # For less typing later.
old_content = None
new_content = None
depot_path = p4gf_config.depot_path_repo2(ctx.config.view_name)
local_path = p4gf_util.depot_to_local_path(depot_path, p4)
depot_exists = False
# 'p4 print' will fail if file doesn't exist yet. Okay.
with ctx.p4gf.at_exception_level(p4.RAISE_NONE):
b = p4gf_util.print_depot_path_raw(p4, depot_path)
if b:
old_content = b.decode() # as UTF-8
depot_exists = True
# What do we want the file to look like? ConfigParser writes only to
# file, not to string, so we have to give it a file path. Ooh! I know!
# How about writing to the very file that we have to 'p4 add' or 'p4
# edit' if its content differs?
if have_branches:
config = configparser.ConfigParser(interpolation=None)
for b in self.branch_list:
LOG.debug("add branch {0}".format(b))
b.add_to_config(config)
p4gf_util.ensure_dir(p4gf_util.parent_dir(local_path))
p4gf_util.make_writable(local_path)
with open(local_path, 'w') as f:
config.write(f)
with open(local_path, 'r') as f:
new_content = f.read()
p4gf_config.clean_up_parser(config)
del config
# Did nothing change? Then nothing to write.
if p4gf_config.compare_configs_string(old_content, new_content):
LOG.debug("No change to p4gf_config2 file")
return False
# Have to add or edit or delete the file.
if not have_branches:
ctx.p4gfrun(['sync', '-fkq', depot_path])
ctx.p4gfrun(['delete', depot_path])
LOG.debug("Deleted p4gf_config2 file")
else:
if depot_exists:
ctx.p4gfrun(['sync', '-fkq', depot_path])
ctx.p4gfrun(['edit', depot_path])
LOG.debug("Edited p4gf_config2 file")
else:
ctx.p4gfrun(['add', '-t', 'text', local_path])
LOG.debug("Added p4gf_config2 file")
return True
def copy(self, start_at, stop_at, new_git_branches):
"""copy a set of changelists from perforce into git"""
LOG.debug('copy() start={} stop={} new_git_branches={}'
.format(start_at, stop_at, new_git_branches))
with self.p2g.perf.timer[OVERALL]:
self.p2g._log_memory('start')
# Stop early if nothing to copy.
# 'p4 changes -m1 //client/...'
repo_empty = p4gf_util.git_empty()
if not self._requires_copy(new_git_branches, repo_empty):
self.p2g.fastimport.cleanup()
LOG.debug("No changes since last copy.")
return
### Fake start. Needs proper setup with graft support.
self.p2g.rev_range = self.p2g.mc_rev_range()
self.ctx.view_repo = pygit2.Repository(self.ctx.view_dirs.GIT_DIR)
self.p2g._log_memory('pygit2')
# O(Bfp + 1) p4 changes: one for each Branch fully populated,
# + 1 for //.git-fusion/branches/repo/...
#
with self.p2g.perf.timer[CHANGES]:
self.change_num_on_branch_list \
= deque(self._p4_changes_each_branch())
LOG.debug('Found ChangeNumOnBranch count: {}'
.format(len(self.change_num_on_branch_list)))
self.p2g._log_memory( 'O(Bfp + 1) p4 changes')
#p4gf_gc.report_growth ('aftr O(Bfp + 1) p4 changes')
#p4gf_gc.report_objects('aftr O(Bfp + 1) p4 changes')
if not self.change_num_on_branch_list:
LOG.debug("No new changes found to copy")
return
# Prepare for the loop over each changelist x branch.
p4gf_util.ensure_dir(self.symlink_dir)
self.p2g._fill_head_marks_from_current_heads()
self.mark_to_branch_id = {}
self.branch_id_to_temp_name \
= self.p2g._create_branch_id_to_temp_name_dict()
self.known_dbi_set = { branch.depot_branch
for branch in self.ctx.branch_dict().values()
if branch.depot_branch }
# All string placeholders should have been replaced with
# pointers to full DepotBranchInfo instances before
# calling copy().
for dbi in self.known_dbi_set:
assert not isinstance(dbi, str)
LOG.error('known: {}'.format(dbi)) ##ZZ
# O(C) 'p4 changes -m1 + filelog + print'
#
# Print each file revision to its blob in the .git/objects/
# Write each changelist to git-fast-import script.
#
with ProgressReporter.Indeterminate():
while self.change_num_on_branch_list:
ProgressReporter.increment("MC Copying changelists...")
cnob = self.change_num_on_branch_list.pop()
self._copy_one(cnob)
# Explicitly delete the PrintHandler now so that it
# won't show up in any leak reports between now and
# P2GMemCapped's end-of-life.
if self.print_handler:
self.printed_byte_count = self.print_handler.total_byte_count
self.printed_rev_count = self.print_handler.printed_rev_count
self.print_handler = None
self.p2g._log_memory( 'P2G_MC.copy() loop')
p4gf_gc.report_growth ('after P2G_MC.copy() loop')
p4gf_gc.report_objects('after P2G_MC.copy() loop')
#p4gf_gc.backref_objects_by_type(dict().__class__)
# Run git-fast-import to add everything to Git.
with self.p2g.perf.timer[FAST_IMPORT]:
LOG.info('Running git-fast-import')
marks = self.p2g.fastimport.run_fast_import()
# Remove all temporary Git branch refs.
# After git-fast-import, we no longer need them.
self._delete_temp_git_branch_refs()
# Record how much we've copied in a p4 counter so that
# future calls to _any_changes_since_last_copy() can
# tell if there's anything new to copy.
self.ctx.write_last_copied_change(self.highest_copied_change_num)
if repo_empty:
# If we are just now rebuilding the Git repository, also
# grab all of the tags that have been pushed in the past.
p4gf_tag.generate_tags(self.ctx)
self.p2g._log_memory('_generate_tags')
with self.p2g.perf.timer[MIRROR]:
self.p2g._mirror(marks, self.mark_to_branch_id)
self.p2g._log_memory('_mirror')
with self.p2g.perf.timer[BRANCH_REF]:
self.p2g._set_branch_refs(marks)
self.p2g._log_memory('_set_branch_refs')
with self.p2g.perf.timer[PACK]:
self.p2g._pack()
self.p2g._log_memory('_pack')
LOG.getChild("time").debug("\n" + str(self))
LOG.info('MC Done. Commits: {cnob_ct:,d} File Revisions: {rev_ct:,d}'
' Bytes: {byte_ct:,d} Seconds: {sec:,d}'
.format( cnob_ct = self.cnob_count
, rev_ct = self.printed_rev_count
, byte_ct = self.printed_byte_count
, sec = int(self.p2g.perf.timer[OVERALL].time)
))
p4gf_gc.report_objects('after P2G MC copy()')
self.p2g._log_memory('copy() done')
def init_repo(self, repo_name_p4client=None, handle_imports=True):
"""Create repo if necessary, without copying from Perforce.
:param repo_name_p4client: name of actual p4 client on which to base this new repo;
if None - will be determined from repo_name if needed
:param handle_imports: if True, process stream imports as submodules
:return: True if repo created, False otherwise.
"""
# repo_name is the internal repo_name with special chars already translated
LOG.debug("init_repo : repo_name {0}".format(self.repo_name))
assert self.repo_config # Must be set by caller: we do not create one.
self._validate_repo_name()
repo_dirs = p4gf_repo_dirs.from_p4gf_dir(p4gf_const.P4GF_HOME,
self.repo_name)
client_root = repo_dirs.p4root
p4gf_util.ensure_dir(client_root)
# Check for cases where repo was created earlier and has changed in some way.
# pylint: disable=unused-variable
(status, head_rev, head_change) = self._discover_config_file()
if status == CONFIG_MISSING:
# just set counter to remember that this repo is gone
self._set_server_repo_config_rev(0)
raise InitRepoMissingConfigFile(
p4gf_const.MISSING_P4GF_CONFIG_MSG_TEMPLATE.format(
repo_name=self.repo_name))
if status == CONFIG_DELETED:
self._clean_deleted_config(self.repo_name)
# set counter so we won't try deleting it again
self._set_server_repo_config_rev(0)
raise InitRepoMissingConfigFile(
p4gf_const.DELETED_P4GF_CONFIG_MSG_TEMPLATE.format(
repo_name=self.repo_name))
if status == CONFIG_READDED:
self._clean_readded_config(self.repo_name, repo_dirs)
# Either nothing has been initialized, or it has been partially initialized.
created_repo = False
if status == CONFIG_NONE:
# No such config, so we must initialize everything now.
if p4gf_const.READ_ONLY and self.fail_when_read_only:
raise InitRepoReadOnly(
_("Cannot initialize repo in read-only instance."))
self._init_config(repo_name_p4client, handle_imports)
created_repo = True
else:
# Repo config file already checked into Perforce? Use that.
self._repo_from_config(handle_imports)
# Set the p4gf_config rev to the current p4gf_config rev
self._set_server_repo_config_rev(head_rev)
# Ensure everything else has been set up properly.
self._create_perm_groups()
with self.connector() as ctx:
self.repo_config.create_default_for_context(ctx, self.charset)
if created_repo:
map_tuple_list = p4gf_branch.calc_writable_branch_union_tuple_list(
ctx.p4.client, ctx.branch_dict(), self.repo_config)
p4gf_atomic_lock.update_all_gf_reviews(ctx, map_tuple_list)
if ctx.client_exclusions_added:
_print_stderr(
_("The referenced client view contains implicit exclusions.\n"
"The Git Fusion config will contain these as explicit exclusions."
))
_warn_if_ndpr_collision(ctx)
if p4gf_init_host.is_init_needed(repo_dirs):
p4gf_init_host.init_host(repo_dirs, ctx)
if created_repo:
LOG.debug("repository creation for %s complete", self.repo_name)
return created_repo