def _add_depot_branch_infos_to_p4(self, ctx): ''' If we created any new depot branches, 'p4 add' their branch-info file to Perforce. Does not submit. If we edited any existing depot branches, 'p4 edit' them. Return number of branch-info files added or edited. ''' if not self.depot_branch_info_list: return add_path_list = [] edit_path_list = [] for dbi in self.depot_branch_info_list: config = dbi.to_config() depot_path = dbi.to_config_depot_path() local_path = p4gf_util.depot_to_local_path( depot_path , ctx.p4gf , ctx.client_spec_gf ) p4gf_util.ensure_dir(p4gf_util.parent_dir(local_path)) p4gf_util.make_writable(local_path) with open(local_path, 'w') as f: config.write(f) if dbi.needs_p4add: add_path_list.append(local_path) else: edit_path_list.append(local_path) p4gf_config.clean_up_parser(config) del config success_list = self._p4_add_in_bites(ctx, add_path_list) success_list.extend(self._p4_sync_k_edit(ctx, edit_path_list)) return len(success_list)
def write(self): """Create a local file P4GF_HOME/changelists/{repo}/{change_num} and tells GitMirror about it. Does not yet open file for 'p4 add' or 'p4 edit': let GitMirror.add_objects_to_p4() do that much later. """ lpath = self.local_path() p4gf_util.ensure_dir(p4gf_util.parent_dir(lpath)) p4gf_util.make_writable(lpath) with open(lpath, 'w') as f: self._write(f)
def write(self): ''' Create a local file P4GF_HOME/changelists/{repo}/{change_num} and tells GitMirror about it. Does not yet open file for 'p4 add' or 'p4 edit': let GitMirror.add_objects_to_p4() do that much later. ''' lpath = self.local_path() p4gf_util.ensure_dir(p4gf_util.parent_dir(lpath)) p4gf_util.make_writable(lpath) with open(lpath, 'w') as f: self._write(f)
def check_and_create_default_p4gf_env_config(): """If p4gf_env_config threw the MissingConfigPath exception, because P4GF_ENV names a non-existing filepath then save the required (two) default items into the user configured P4GF_ENV environment config file. """ if not Create_P4GF_CONFIG: LOG.debug('not creating configuration file') return LOG.debug('creating missing configuration file') Verbosity.report( Verbosity.INFO, _("Git Fusion environment var P4GF_ENV = {path} names a non-existing file.") .format(path=p4gf_const.P4GF_ENV)) Verbosity.report( Verbosity.INFO, _("Creating {path} with the default required items.") .format(path=p4gf_const.P4GF_ENV)) Verbosity.report( Verbosity.INFO, _("Review the file's comments and edit as needed.")) Verbosity.report( Verbosity.INFO, _("You may unset P4GF_ENV to use no config file.") .format(p4gf_const.P4GF_ENV)) config = configparser.ConfigParser(interpolation = None, allow_no_value = True) config.optionxform = str config.add_section(p4gf_const.SECTION_ENVIRONMENT) config.set(p4gf_const.SECTION_ENVIRONMENT, p4gf_const.P4GF_HOME_NAME, p4gf_const.P4GF_HOME) Verbosity.report( Verbosity.INFO, _("Setting {home_name} = {home} in {env}.") .format(home_name=p4gf_const.P4GF_HOME_NAME, home=p4gf_const.P4GF_HOME, env=p4gf_const.P4GF_ENV)) config.set(p4gf_const.SECTION_ENVIRONMENT, NTR('P4PORT'), P4PORT) Verbosity.report( Verbosity.INFO, _("Setting {p4port} = {p4port_value} in {env}.") .format(p4port=NTR('P4PORT'), p4port_value=P4PORT, env=p4gf_const.P4GF_ENV)) header = p4gf_util.read_bin_file(NTR('p4gf_env_config.txt')) if header is False: sys.stderr.write(_('no p4gf_env_config.txt found\n')) header = _('# Missing p4gf_env_config.txt file!') out = io.StringIO() out.write(header) config.write(out) file_content = out.getvalue() out.close() p4gf_util.ensure_dir(p4gf_util.parent_dir(p4gf_const.P4GF_ENV)) with open(p4gf_const.P4GF_ENV, 'w') as f: f.write(file_content) LOG.debug('created configuration file %s', p4gf_const.P4GF_ENV)
def open_all(self): """Open zipfile and journal files for write. Generates temp files for all of 'em. """ p4gf_util.ensure_dir(self._dir_abspath) for j in self.jnl: assert j.fp is None j.fp = open(j.abspath, "w", encoding="utf-8") self.zip.fp = zipfile.ZipFile(self.zip.abspath, "w" #, compression = zipfile.ZIP_STORED # ZIP_DEFLATED saves 50% temporary/working disk space, # but costs 5% additional time for the compression. , compression = zipfile.ZIP_DEFLATED , allowZip64 = True )
def check_and_create_default_p4gf_env_config(): '''If p4gf_env_config threw the MissingConfigPath exception, because P4GF_ENV names a non-existing filepath then save the required (two) default items into the user configured P4GF_ENV environment config file. ''' if not Create_P4GF_CONFIG: return Verbosity.report(Verbosity.INFO, _("Git Fusion environment var P4GF_ENV = {0} names a non-existing file.") .format(p4gf_const.P4GF_ENV)) Verbosity.report(Verbosity.INFO, _("Creating {0} with the default required items.") .format(p4gf_const.P4GF_ENV)) Verbosity.report(Verbosity.INFO, _("Review the file's comments and edit as needed.")) Verbosity.report(Verbosity.INFO, _("You may unset P4GF_ENV to use no config file.") .format(p4gf_const.P4GF_ENV)) config = configparser.ConfigParser(interpolation = None, allow_no_value = True) config.optionxform = str config.add_section(p4gf_const.SECTION_ENVIRONMENT) config.set(p4gf_const.SECTION_ENVIRONMENT, p4gf_const.P4GF_HOME_NAME, p4gf_const.P4GF_HOME) Verbosity.report(Verbosity.INFO, _("Setting {0} = {1} in {2}.") .format(p4gf_const.P4GF_HOME_NAME, p4gf_const.P4GF_HOME, p4gf_const.P4GF_ENV)) config.set(p4gf_const.SECTION_ENVIRONMENT, NTR('P4PORT'), P4PORT) Verbosity.report(Verbosity.INFO, _("Setting {0} = {1} in {2}.") .format(NTR('P4PORT'), P4PORT, p4gf_const.P4GF_ENV)) header = p4gf_util.read_bin_file(NTR('p4gf_env_config.txt')) if header is False: sys.stderr.write(_('no p4gf_env_config.txt found\n')) header = _('# Missing p4gf_env_config.txt file!') out = io.StringIO() out.write(header) config.write(out) file_content = out.getvalue() out.close() p4gf_util.ensure_dir(p4gf_util.parent_dir(p4gf_const.P4GF_ENV)) with open(p4gf_const.P4GF_ENV, 'w') as f: f.write(file_content)
def create_dir(self): """Create .p4gf_gc directory, or fail if already exists.""" if os.path.exists(self.dir_abspath): raise RuntimeError("Directory already exists: {}" "\nEither:" "\n* p4gf_gc.py --force to delete it and start clean, or" "\n* p4gf_gc.py --cont to use whatever data is in .p4gf_gc" .format(self.dir_abspath)) LOG.info("Creating work directory: {}".format(self.dir_abspath)) p4gf_util.ensure_dir(self.dir_abspath) p4gf_util.ensure_dir(self.git_dir_abspath) p4gf_util.ensure_dir(self.sql_dir_abspath)
def copy(self, start_at, stop_at, new_git_branches): """copy a set of changelists from perforce into git""" LOG.debug('copy() start={} stop={} new_git_branches={}'.format( start_at, stop_at, new_git_branches)) with self.p2g.perf.timer[OVERALL]: self.p2g._log_memory('start') # Stop early if nothing to copy. # 'p4 changes -m1 //client/...' repo_empty = p4gf_util.git_empty() if not self._requires_copy(new_git_branches, repo_empty): self.p2g.fastimport.cleanup() LOG.debug("No changes since last copy.") return ### Fake start. Needs proper setup with graft support. self.p2g.rev_range = self.p2g.mc_rev_range() self.ctx.view_repo = pygit2.Repository(self.ctx.view_dirs.GIT_DIR) self.p2g._log_memory('pygit2') # O(Bfp + 1) p4 changes: one for each Branch fully populated, # + 1 for //.git-fusion/branches/repo/... # with self.p2g.perf.timer[CHANGES]: self.change_num_on_branch_list \ = deque(self._p4_changes_each_branch()) LOG.debug('Found ChangeNumOnBranch count: {}'.format( len(self.change_num_on_branch_list))) self.p2g._log_memory('O(Bfp + 1) p4 changes') #p4gf_gc.report_growth ('aftr O(Bfp + 1) p4 changes') #p4gf_gc.report_objects('aftr O(Bfp + 1) p4 changes') if not self.change_num_on_branch_list: LOG.debug("No new changes found to copy") return # Prepare for the loop over each changelist x branch. p4gf_util.ensure_dir(self.symlink_dir) self.p2g._fill_head_marks_from_current_heads() self.mark_to_branch_id = {} self.branch_id_to_temp_name \ = self.p2g._create_branch_id_to_temp_name_dict() self.known_dbi_set = { branch.depot_branch for branch in self.ctx.branch_dict().values() if branch.depot_branch } # All string placeholders should have been replaced with # pointers to full DepotBranchInfo instances before # calling copy(). for dbi in self.known_dbi_set: assert not isinstance(dbi, str) LOG.error('known: {}'.format(dbi)) ##ZZ # O(C) 'p4 changes -m1 + filelog + print' # # Print each file revision to its blob in the .git/objects/ # Write each changelist to git-fast-import script. # with ProgressReporter.Indeterminate(): while self.change_num_on_branch_list: ProgressReporter.increment("MC Copying changelists...") cnob = self.change_num_on_branch_list.pop() self._copy_one(cnob) # Explicitly delete the PrintHandler now so that it # won't show up in any leak reports between now and # P2GMemCapped's end-of-life. if self.print_handler: self.printed_byte_count = self.print_handler.total_byte_count self.printed_rev_count = self.print_handler.printed_rev_count self.print_handler = None self.p2g._log_memory('P2G_MC.copy() loop') p4gf_gc.report_growth('after P2G_MC.copy() loop') p4gf_gc.report_objects('after P2G_MC.copy() loop') #p4gf_gc.backref_objects_by_type(dict().__class__) # Run git-fast-import to add everything to Git. with self.p2g.perf.timer[FAST_IMPORT]: LOG.info('Running git-fast-import') marks = self.p2g.fastimport.run_fast_import() # Remove all temporary Git branch refs. # After git-fast-import, we no longer need them. self._delete_temp_git_branch_refs() # Record how much we've copied in a p4 counter so that # future calls to _any_changes_since_last_copy() can # tell if there's anything new to copy. self.ctx.write_last_copied_change(self.highest_copied_change_num) if repo_empty: # If we are just now rebuilding the Git repository, also # grab all of the tags that have been pushed in the past. p4gf_tag.generate_tags(self.ctx) self.p2g._log_memory('_generate_tags') with self.p2g.perf.timer[MIRROR]: self.p2g._mirror(marks, self.mark_to_branch_id) self.p2g._log_memory('_mirror') with self.p2g.perf.timer[BRANCH_REF]: self.p2g._set_branch_refs(marks) self.p2g._log_memory('_set_branch_refs') with self.p2g.perf.timer[PACK]: self.p2g._pack() self.p2g._log_memory('_pack') LOG.getChild("time").debug("\n" + str(self)) LOG.info('MC Done. Commits: {cnob_ct:,d} File Revisions: {rev_ct:,d}' ' Bytes: {byte_ct:,d} Seconds: {sec:,d}'.format( cnob_ct=self.cnob_count, rev_ct=self.printed_rev_count, byte_ct=self.printed_byte_count, sec=int(self.p2g.perf.timer[OVERALL].time))) p4gf_gc.report_objects('after P2G MC copy()') self.p2g._log_memory('copy() done')
def _add_branch_defs_to_p4(self, ctx): ''' If we defined any new named+lightweight branches, update (or write the first revision of) this repo's p4gf_config2 file with all the currently defined named+lightweight branches. ''' # Nothing to write? well maybe we have just deleted the remaining refs have_branches = bool(self.branch_list) # What does the file look like now? p4 = ctx.p4gf # For less typing later. old_content = None new_content = None depot_path = p4gf_config.depot_path_repo2(ctx.config.view_name) local_path = p4gf_util.depot_to_local_path(depot_path, p4) depot_exists = False # 'p4 print' will fail if file doesn't exist yet. Okay. with ctx.p4gf.at_exception_level(p4.RAISE_NONE): b = p4gf_util.print_depot_path_raw(p4, depot_path) if b: old_content = b.decode() # as UTF-8 depot_exists = True # What do we want the file to look like? ConfigParser writes only to # file, not to string, so we have to give it a file path. Ooh! I know! # How about writing to the very file that we have to 'p4 add' or 'p4 # edit' if its content differs? if have_branches: config = configparser.ConfigParser(interpolation=None) for b in self.branch_list: LOG.debug("add branch {0}".format(b)) b.add_to_config(config) p4gf_util.ensure_dir(p4gf_util.parent_dir(local_path)) p4gf_util.make_writable(local_path) with open(local_path, 'w') as f: config.write(f) with open(local_path, 'r') as f: new_content = f.read() p4gf_config.clean_up_parser(config) del config # Did nothing change? Then nothing to write. if p4gf_config.compare_configs_string(old_content, new_content): LOG.debug("No change to p4gf_config2 file") return False # Have to add or edit or delete the file. if not have_branches: ctx.p4gfrun(['sync', '-fkq', depot_path]) ctx.p4gfrun(['delete', depot_path]) LOG.debug("Deleted p4gf_config2 file") else: if depot_exists: ctx.p4gfrun(['sync', '-fkq', depot_path]) ctx.p4gfrun(['edit', depot_path]) LOG.debug("Edited p4gf_config2 file") else: ctx.p4gfrun(['add', '-t', 'text', local_path]) LOG.debug("Added p4gf_config2 file") return True
def _add_branch_defs_to_p4(self, ctx): ''' If we defined any new named+lightweight branches, update (or write the first revision of) this repo's p4gf_config2 file with all the currently defined named+lightweight branches. ''' # Nothing to write? well maybe we have just deleted the remaining refs have_branches = bool(self.branch_list) # What does the file look like now? p4 = ctx.p4gf # For less typing later. old_content = None new_content = None depot_path = p4gf_config.depot_path_repo2(ctx.config.view_name) local_path = p4gf_util.depot_to_local_path(depot_path, p4) depot_exists = False # 'p4 print' will fail if file doesn't exist yet. Okay. with ctx.p4gf.at_exception_level(p4.RAISE_NONE): b = p4gf_util.print_depot_path_raw(p4, depot_path) if b: old_content = b.decode() # as UTF-8 depot_exists = True # What do we want the file to look like? ConfigParser writes only to # file, not to string, so we have to give it a file path. Ooh! I know! # How about writing to the very file that we have to 'p4 add' or 'p4 # edit' if its content differs? if have_branches: config = configparser.ConfigParser(interpolation=None) for b in self.branch_list: LOG.debug("add branch {0}".format(b)) b.add_to_config(config) p4gf_util.ensure_dir(p4gf_util.parent_dir(local_path)) p4gf_util.make_writable(local_path) with open(local_path, 'w') as f: config.write(f) with open(local_path, 'r') as f: new_content = f.read() p4gf_config.clean_up_parser(config) del config # Did nothing change? Then nothing to write. if p4gf_config.compare_configs_string(old_content, new_content): LOG.debug("No change to p4gf_config2 file") return False # Have to add or edit or delete the file. if not have_branches: ctx.p4gfrun(['sync', '-fkq', depot_path]) ctx.p4gfrun(['delete', depot_path]) LOG.debug("Deleted p4gf_config2 file") else: if depot_exists: ctx.p4gfrun(['sync', '-fkq', depot_path]) ctx.p4gfrun(['edit', depot_path]) LOG.debug("Edited p4gf_config2 file") else: ctx.p4gfrun(['add', '-t', 'text', local_path]) LOG.debug("Added p4gf_config2 file") return True
def copy(self, start_at, stop_at, new_git_branches): """copy a set of changelists from perforce into git""" LOG.debug('copy() start={} stop={} new_git_branches={}' .format(start_at, stop_at, new_git_branches)) with self.p2g.perf.timer[OVERALL]: self.p2g._log_memory('start') # Stop early if nothing to copy. # 'p4 changes -m1 //client/...' repo_empty = p4gf_util.git_empty() if not self._requires_copy(new_git_branches, repo_empty): self.p2g.fastimport.cleanup() LOG.debug("No changes since last copy.") return ### Fake start. Needs proper setup with graft support. self.p2g.rev_range = self.p2g.mc_rev_range() self.ctx.view_repo = pygit2.Repository(self.ctx.view_dirs.GIT_DIR) self.p2g._log_memory('pygit2') # O(Bfp + 1) p4 changes: one for each Branch fully populated, # + 1 for //.git-fusion/branches/repo/... # with self.p2g.perf.timer[CHANGES]: self.change_num_on_branch_list \ = deque(self._p4_changes_each_branch()) LOG.debug('Found ChangeNumOnBranch count: {}' .format(len(self.change_num_on_branch_list))) self.p2g._log_memory( 'O(Bfp + 1) p4 changes') #p4gf_gc.report_growth ('aftr O(Bfp + 1) p4 changes') #p4gf_gc.report_objects('aftr O(Bfp + 1) p4 changes') if not self.change_num_on_branch_list: LOG.debug("No new changes found to copy") return # Prepare for the loop over each changelist x branch. p4gf_util.ensure_dir(self.symlink_dir) self.p2g._fill_head_marks_from_current_heads() self.mark_to_branch_id = {} self.branch_id_to_temp_name \ = self.p2g._create_branch_id_to_temp_name_dict() self.known_dbi_set = { branch.depot_branch for branch in self.ctx.branch_dict().values() if branch.depot_branch } # All string placeholders should have been replaced with # pointers to full DepotBranchInfo instances before # calling copy(). for dbi in self.known_dbi_set: assert not isinstance(dbi, str) LOG.error('known: {}'.format(dbi)) ##ZZ # O(C) 'p4 changes -m1 + filelog + print' # # Print each file revision to its blob in the .git/objects/ # Write each changelist to git-fast-import script. # with ProgressReporter.Indeterminate(): while self.change_num_on_branch_list: ProgressReporter.increment("MC Copying changelists...") cnob = self.change_num_on_branch_list.pop() self._copy_one(cnob) # Explicitly delete the PrintHandler now so that it # won't show up in any leak reports between now and # P2GMemCapped's end-of-life. if self.print_handler: self.printed_byte_count = self.print_handler.total_byte_count self.printed_rev_count = self.print_handler.printed_rev_count self.print_handler = None self.p2g._log_memory( 'P2G_MC.copy() loop') p4gf_gc.report_growth ('after P2G_MC.copy() loop') p4gf_gc.report_objects('after P2G_MC.copy() loop') #p4gf_gc.backref_objects_by_type(dict().__class__) # Run git-fast-import to add everything to Git. with self.p2g.perf.timer[FAST_IMPORT]: LOG.info('Running git-fast-import') marks = self.p2g.fastimport.run_fast_import() # Remove all temporary Git branch refs. # After git-fast-import, we no longer need them. self._delete_temp_git_branch_refs() # Record how much we've copied in a p4 counter so that # future calls to _any_changes_since_last_copy() can # tell if there's anything new to copy. self.ctx.write_last_copied_change(self.highest_copied_change_num) if repo_empty: # If we are just now rebuilding the Git repository, also # grab all of the tags that have been pushed in the past. p4gf_tag.generate_tags(self.ctx) self.p2g._log_memory('_generate_tags') with self.p2g.perf.timer[MIRROR]: self.p2g._mirror(marks, self.mark_to_branch_id) self.p2g._log_memory('_mirror') with self.p2g.perf.timer[BRANCH_REF]: self.p2g._set_branch_refs(marks) self.p2g._log_memory('_set_branch_refs') with self.p2g.perf.timer[PACK]: self.p2g._pack() self.p2g._log_memory('_pack') LOG.getChild("time").debug("\n" + str(self)) LOG.info('MC Done. Commits: {cnob_ct:,d} File Revisions: {rev_ct:,d}' ' Bytes: {byte_ct:,d} Seconds: {sec:,d}' .format( cnob_ct = self.cnob_count , rev_ct = self.printed_rev_count , byte_ct = self.printed_byte_count , sec = int(self.p2g.perf.timer[OVERALL].time) )) p4gf_gc.report_objects('after P2G MC copy()') self.p2g._log_memory('copy() done')
def init_repo(self, repo_name_p4client=None, handle_imports=True): """Create repo if necessary, without copying from Perforce. :param repo_name_p4client: name of actual p4 client on which to base this new repo; if None - will be determined from repo_name if needed :param handle_imports: if True, process stream imports as submodules :return: True if repo created, False otherwise. """ # repo_name is the internal repo_name with special chars already translated LOG.debug("init_repo : repo_name {0}".format(self.repo_name)) assert self.repo_config # Must be set by caller: we do not create one. self._validate_repo_name() repo_dirs = p4gf_repo_dirs.from_p4gf_dir(p4gf_const.P4GF_HOME, self.repo_name) client_root = repo_dirs.p4root p4gf_util.ensure_dir(client_root) # Check for cases where repo was created earlier and has changed in some way. # pylint: disable=unused-variable (status, head_rev, head_change) = self._discover_config_file() if status == CONFIG_MISSING: # just set counter to remember that this repo is gone self._set_server_repo_config_rev(0) raise InitRepoMissingConfigFile( p4gf_const.MISSING_P4GF_CONFIG_MSG_TEMPLATE.format( repo_name=self.repo_name)) if status == CONFIG_DELETED: self._clean_deleted_config(self.repo_name) # set counter so we won't try deleting it again self._set_server_repo_config_rev(0) raise InitRepoMissingConfigFile( p4gf_const.DELETED_P4GF_CONFIG_MSG_TEMPLATE.format( repo_name=self.repo_name)) if status == CONFIG_READDED: self._clean_readded_config(self.repo_name, repo_dirs) # Either nothing has been initialized, or it has been partially initialized. created_repo = False if status == CONFIG_NONE: # No such config, so we must initialize everything now. if p4gf_const.READ_ONLY and self.fail_when_read_only: raise InitRepoReadOnly( _("Cannot initialize repo in read-only instance.")) self._init_config(repo_name_p4client, handle_imports) created_repo = True else: # Repo config file already checked into Perforce? Use that. self._repo_from_config(handle_imports) # Set the p4gf_config rev to the current p4gf_config rev self._set_server_repo_config_rev(head_rev) # Ensure everything else has been set up properly. self._create_perm_groups() with self.connector() as ctx: self.repo_config.create_default_for_context(ctx, self.charset) if created_repo: map_tuple_list = p4gf_branch.calc_writable_branch_union_tuple_list( ctx.p4.client, ctx.branch_dict(), self.repo_config) p4gf_atomic_lock.update_all_gf_reviews(ctx, map_tuple_list) if ctx.client_exclusions_added: _print_stderr( _("The referenced client view contains implicit exclusions.\n" "The Git Fusion config will contain these as explicit exclusions." )) _warn_if_ndpr_collision(ctx) if p4gf_init_host.is_init_needed(repo_dirs): p4gf_init_host.init_host(repo_dirs, ctx) if created_repo: LOG.debug("repository creation for %s complete", self.repo_name) return created_repo