def collect_commits(self): self.exported_stores = [] for repo_num, input_repo in enumerate(self.input_repos, start=1): repo_name = basename(input_repo) remember_commits_filename = os.path.join(abspath(input_repo), ".git", "splice-commits.json") exported_store = self.open_export_file(repo_name, ".git-fast-export", "export", overwrite=True) self.exported_stores.append(exported_store) logging.info("Doing export on repository %s [%d]", repo_name, repo_num) stored_commits = [] export_mark_file = join(abspath(input_repo), ".git", "splice-export-marks") export_mark_args = ["--all", "--export-marks=%s" % export_mark_file] if exists(export_mark_file): export_mark_args.append("--import-marks=%s" % export_mark_file) export = fast_export_output(input_repo, export_mark_args) logging.info("_next_id is %d", _IDS._next_id) try: collect = FastExportFilter(reset_callback=lambda r: self.skip_reset(r), commit_callback=lambda c: stored_commits.append(self.jsonize_commit(c))) logging.info("offset for repo %d is %d", repo_num, collect._id_offset) kwargs = {} if repo_num == 2: kwargs["id_offset"] = 8 collect.run(export.stdout, exported_store.file, **kwargs) exported_store.close() with open(remember_commits_filename, 'wb') as remember_commits_file: json.dump(stored_commits, remember_commits_file, indent=4) except Exception, e: logging.error("Error in memorizing export for %s [%d]: %s", input_repo, repo_num, e) raise # TODO: handle the fact that marks are getting messed up by the two repos on an incremental import self.remember_commits(repo_num, stored_commits) logging.info("id_offset is %d", collect._id_offset) self.export_id_offsets[repo_num] = collect._id_offset
def run(self): self.target = fast_import_input(self.output_dir) input1 = fast_export_output(self.repo1) filter1 = FastExportFilter(reset_callback = lambda r: self.skip_reset(r), commit_callback = lambda c: self.hold_commit(c)) filter1.run(input1.stdout, self.target.stdin) input2 = fast_export_output(self.repo2) filter2 = FastExportFilter(commit_callback = lambda c: self.weave_commit(c)) filter2.run(input2.stdout, self.target.stdin) # Wait for git-fast-import to complete (only necessary since we passed # file objects to FastExportFilter.run; and even then the worst that # happens is git-fast-import completes after this python script does) self.target.stdin.close() self.target.wait()
def run(self): self.target = fast_import_input(self.output_dir) input1 = fast_export_output(self.repo1) filter1 = FastExportFilter( reset_callback=lambda r: self.skip_reset(r), commit_callback=lambda c: self.hold_commit(c)) filter1.run(input1.stdout, self.target.stdin) input2 = fast_export_output(self.repo2) filter2 = FastExportFilter( commit_callback=lambda c: self.weave_commit(c)) filter2.run(input2.stdout, self.target.stdin) # Wait for git-fast-import to complete (only necessary since we passed # file objects to FastExportFilter.run; and even then the worst that # happens is git-fast-import completes after this python script does) self.target.stdin.close() self.target.wait()
def run(self): ############################################################################# # Set members based on data from previous runs self._setup_files_and_excludes() # Setup the source and target processes. The source process will produce # fast-export output for the source repo, this output will be passed # through FastExportFilter which will manipulate the output using our # callbacks, finally, the manipulated output will be given to the # fast-import process and used to create the target repo. # (This should update sourcemarks and targetmarks) source = \ fast_export_output(self._source_repo, ["--export-marks=%s" % self._sourcemarks, "--import-marks=%s" % self._sourcemarks] + self._fast_export_args) target = \ fast_import_input( self._target_repo, ["--export-marks=%s" % self._targetmarks, "--import-marks=%s" % self._targetmarks]) filt = FastExportFilter(blob_callback = lambda b: self.blob_callback(b), commit_callback = lambda c: self.commit_callback(c)) filt.run(source.stdout, target.stdin) # Show progress if self._show_progress: sys.stdout.write("\nWaiting for git fast-import to complete...") sys.stdout.flush() target.stdin.close() target.wait() # need to wait for fast-import process to finish if self._show_progress: sys.stdout.write("done.\n") # Record the sourcemarks and targetmarks -- 2 steps # Step 1: Make sure the source and target marks have the same mark numbers. # Not doing this would allow one end of the grafting to reuse a number # that would then be misconnected on the other side. sourcemaps = self._get_maps(self._sourcemarks) targetmaps = self._get_maps(self._targetmarks) for key in sourcemaps.keys(): if key not in targetmaps: del sourcemaps[key] for key in targetmaps.keys(): if key not in sourcemaps: del targetmaps[key] # Step 2: Record the data for set_obj in [(sourcemaps, self._sourcemarks), (targetmaps, self._targetmarks)]: # get raw filename for source/target mapname = self._get_map_name(set_obj[1]) # create refs/collab if it's not there if not os.path.isdir(os.path.dirname(mapname)): os.mkdir(os.path.dirname(mapname)) # compute string content of commit-map content = ''.join([":%d %s\n" % (k,v) for k,v in set_obj[0].iteritems()]) # record content in the object database record_content(self._collab_git_dir, mapname, content) # Check if we are running from the target if self._target_repo == '.': # Record the excludes and includes so they can be reused next time for set_obj in [(self._excludes, 'excludes'), (self._includes, 'includes')]: filename = os.path.join(self._collab_git_dir, 'refs', 'collab', set_obj[1]) record_content(self._collab_git_dir, filename, '\n'.join(set_obj[0]) + '\n') # Record source_repo as the original repository filename = os.path.join(self._collab_git_dir, 'refs', 'collab', 'orig_repo') record_content(self._collab_git_dir, filename, self._source_repo+'\n')
def run(self): ############################################################################# # Set members based on data from previous runs self._setup_files_and_excludes() # Setup the source and target processes. The source process will produce # fast-export output for the source repo, this output will be passed # through FastExportFilter which will manipulate the output using our # callbacks, finally, the manipulated output will be given to the # fast-import process and used to create the target repo. # (This should update sourcemarks and targetmarks) source = \ fast_export_output(self._source_repo, ["--export-marks=%s" % self._sourcemarks, "--import-marks=%s" % self._sourcemarks] + self._fast_export_args) target = \ fast_import_input( self._target_repo, ["--export-marks=%s" % self._targetmarks, "--import-marks=%s" % self._targetmarks]) filt = FastExportFilter( blob_callback=lambda b: self.blob_callback(b), commit_callback=lambda c: self.commit_callback(c)) filt.run(source.stdout, target.stdin) # Show progress if self._show_progress: sys.stdout.write("\nWaiting for git fast-import to complete...") sys.stdout.flush() target.stdin.close() target.wait() # need to wait for fast-import process to finish if self._show_progress: sys.stdout.write("done.\n") # Record the sourcemarks and targetmarks -- 2 steps # Step 1: Make sure the source and target marks have the same mark numbers. # Not doing this would allow one end of the grafting to reuse a number # that would then be misconnected on the other side. sourcemaps = self._get_maps(self._sourcemarks) targetmaps = self._get_maps(self._targetmarks) for key in sourcemaps.keys(): if key not in targetmaps: del sourcemaps[key] for key in targetmaps.keys(): if key not in sourcemaps: del targetmaps[key] # Step 2: Record the data for set_obj in [(sourcemaps, self._sourcemarks), (targetmaps, self._targetmarks)]: # get raw filename for source/target mapname = self._get_map_name(set_obj[1]) # create refs/collab if it's not there if not os.path.isdir(os.path.dirname(mapname)): os.mkdir(os.path.dirname(mapname)) # compute string content of commit-map content = ''.join( [":%d %s\n" % (k, v) for k, v in set_obj[0].iteritems()]) # record content in the object database record_content(self._collab_git_dir, mapname, content) # Check if we are running from the target if self._target_repo == '.': # Record the excludes and includes so they can be reused next time for set_obj in [(self._excludes, 'excludes'), (self._includes, 'includes')]: filename = os.path.join(self._collab_git_dir, 'refs', 'collab', set_obj[1]) record_content(self._collab_git_dir, filename, '\n'.join(set_obj[0]) + '\n') # Record source_repo as the original repository filename = os.path.join(self._collab_git_dir, 'refs', 'collab', 'orig_repo') record_content(self._collab_git_dir, filename, self._source_repo + '\n')