def RunScenario(self, scenario_name): UNRUN_SCENARIOS.remove(scenario_name) scenario_base = os.path.join(SCENARIOS_DIR, scenario_name) args = dict( generated_codebase=self._MakeCodebase( os.path.join(scenario_base, 'generated')), public_codebase=self._MakeCodebase( os.path.join(scenario_base, 'public')), previous_codebase=self._MakeCodebase( os.path.join(scenario_base, 'previous')), ) config = merge_codebases.MergeCodebasesConfig( **args) context = merge_codebases.MergeCodebasesContext( config) context.Update() codebase1 = config.merged_codebase codebase2 = os.path.join(scenario_base, 'expected') different = base.AreCodebasesDifferent( codebase_utils.Codebase(codebase1), codebase_utils.Codebase(codebase2)) if different: # TODO(dbentley): this should describe how they differ. print 'DIFFERENT:', different self.fail("Codebases %s and %s differ" % (codebase1, codebase2))
def RunScenarioWithConfigFile(self, scenario_base, config_file): codebase = os.path.join(scenario_base, 'input') config_path = os.path.join(scenario_base, config_file) (_, input_files) = scrubber.CreateInputFileListFromDir(codebase) config = scrubber.ParseConfigFile(config_path, codebase, input_files) context = scrubber.ScrubberContext(config) context.Scan() context.WriteOutput() codebase1 = os.path.join(scenario_base, 'expected') if not os.path.exists(codebase1): self.assertTrue(context.Status(), 'Scrubber was expected to fail but did not') return if context.Status(): context.Report() self.fail('Scrubber returned non-zero status %d' % context.Status()) codebase2 = os.path.join(context._temp_dir, 'output') different = base.AreCodebasesDifferent( codebase_utils.Codebase(codebase1), codebase_utils.Codebase(codebase2)) if different: # TODO(dbentley): this should describe how they differ. self.fail('Codebases %s and %s differ' % (codebase1, codebase2))
def testCreateModifiableCopy(self): internal_creator = test_util.StaticCodebaseCreator( {'1001': 'simple_python'}) codebase = internal_creator.Create('1001') copy = codebase_utils.CreateModifiableCopy(codebase) self.assertFalse(base.AreCodebasesDifferent(codebase, copy)) self.assertFalse(codebase.Path() == copy.Path())
def testCodebaseDifference(self): codebase_internal = test_util.TestResourceFilename( os.path.join('diff_codebases', 'codebase_internal/')) codebase_public = test_util.TestResourceFilename( os.path.join('diff_codebases', 'codebase_public/')) codebase_diff_obj = base.AreCodebasesDifferent( codebase_utils.Codebase(codebase_internal), codebase_utils.Codebase(codebase_public), record_full_diffs=True) print 'codebase_diff_obj:', codebase_diff_obj expected_diff = open(test_util.TestResourceFilename( os.path.join('diff_codebases', 'codebase_diff'))).read() self.assertEquals(expected_diff.strip(), str(codebase_diff_obj).strip())
def main(unused_argv): project = db_client.MakeProjectContext() try: internal_codebase = project.internal_codebase_creator.Create( FLAGS.internal_revision > 0 and FLAGS.internal_revision or project.internal_repository.GetHeadRevision()) public_codebase = project.public_codebase_creator.Create( FLAGS.public_revision > 0 and FLAGS.public_revision or project.public_repository.GetHeadRevision()) diff_obj = base.AreCodebasesDifferent(internal_codebase, public_codebase, project.config.noisy_files_re, record_full_diffs=True) moe_app.RUN.ui.Info('\n===== Begin diff_codebases =====\n') moe_app.RUN.ui.Info(str(diff_obj)) finally: project.db.Disconnect()
def Perform(self, internal_codebase_creator, public_codebase_creator, report, db, temp_dir, actions, **unused_kwargs): """Perform the action, and determine next actions. Args: internal_codebase_creator: codebase_utils.CodebaseCreator public_codebase_creator: codebase_utils.CodebaseCreator report: config.MoeReport db: moe_db.MoeDbClient temp_dir: str, path to a tempoprary directory actions: sequence of Action Returns: StateUpdate, an update to the state of the Action sequence """ task = moe_app.RUN.ui.BeginIntermediateTask( 'equivalence_check', ('Checking for an Equivalence between internal revision %s and ' 'public revision %s') % (self.internal_revision, self.public_revision)) with task: internal = internal_codebase_creator.Create(self.internal_revision) generated = translators.TranslateToProjectSpace( internal, base.PUBLIC_STR, self.translators) public = public_codebase_creator.Create(self.public_revision) codebases_differ = None if not self.project.manual_equivalence_deltas: codebases_differ = base.AreCodebasesDifferent( generated, public, noisy_files_re=self.project.noisy_files_re) return self.result_dispatch(self, codebases_differ=codebases_differ, generated=generated, public=public, db=db, report=report, actions=actions)
def Perform(self, report=None, db=None, actions=None, **unused_kwargs): """Perform the action, and determine next actions. Args: report: base.MoeReport db: moe_db.MoeDbClient temp_dir: str, path to a tempoprary directory actions: seq of Action, the actions to run after this Returns: StateUpdate, an update to the state of the Action sequence """ actions = actions or [] if not self.revisions: return if self.num_revisions_to_migrate == -1: migration_revisions = self.revisions remaining_revisions = [] else: migration_revisions = self.revisions[:self.num_revisions_to_migrate] remaining_revisions = self.revisions[self.num_revisions_to_migrate:] up_to_revision = migration_revisions[-1] task = moe_app.RUN.ui.BeginIntermediateTask( 'migrate', 'Migrating up to revision %s by applying it against %s' % (up_to_revision.rev_id, self.applied_against.rev_id)) with task: if remaining_revisions: result_migration = Migration( up_to_revision, self.applied_against, remaining_revisions, self.project, self.translators, self.migration_config, self.mock_migration, 1) result_migration_list = [result_migration] result = StateUpdate(actions=[result_migration] + actions) else: result_migration = None result_migration_list = [] result = None previous_source = self.migration_config.source_codebase_creator.Create( self.previous_revision.rev_id) try: source = self.migration_config.source_codebase_creator.Create( up_to_revision.rev_id) translated_source = translators.TranslateToProjectSpace( source, self.migration_config.target_codebase_creator.ProjectSpace(), self.translators) except base.CodebaseCreationError: if not remaining_revisions: # This is an error at the last revision # It might be better if we just marked it as an error. raise # In this case, we can't migrate the up_to_revision. So instead, we # fold it into the next migration. action = Migration( self.previous_revision, self.applied_against, self.revisions, self.project, self.translators, self.migration_config, self.mock_migration, self.num_revisions_to_migrate + 1) return StateUpdate(actions=[action] + actions) should_migrate = bool(base.AreCodebasesDifferent( previous_source, source, noisy_files_re=self.project.noisy_files_re)) if not should_migrate: return result migration_strategy = copy.copy(self.migration_config.migration_strategy) if migration_strategy.merge_strategy == base.ERROR: raise base.Error('Attempted %s invalid with merge strategy.' % self.migration_config.direction) migration = db.FindMigration(up_to_revision, abbreviated=False) if (not migration or (migration.status == base.Migration.CANCELED)): changelog = base.ConcatenateChangelogs(migration_revisions) creator = self.migration_config.source_codebase_creator pre_approved = (migration_strategy.preapprove_public_changelogs and all([r.pre_approved for r in migration_revisions])) migration_id = db.StartMigration( self.migration_config.direction, up_to_revision, changelog=changelog, migrated_revisions=migration_revisions, pre_approved=pre_approved) elif migration.status == base.Migration.SUBMITTED: # TODO(dbentley): this should never happen find_step = moe_app.RUN.report.AddStep( 'migrate_changes', 'find_migration') find_step.SetResult('%s of revisions [%s, %s] has ' 'already been submitted in migration %s.' % (self.migration_config.direction, self.start_migrated_revision, end_migrated_revision, migration.migration_id)) return result else: changelog = migration.changelog migration_id = migration.migration_id if migration.status == base.Migration.APPROVED: # TODO(dbentley): should we do this only if it's somewhere in the # project config? migration_strategy.commit_strategy = base.COMMIT_REMOTELY if self.mock_migration: return result base_codebase = self.migration_config.target_codebase_creator.Create( self.applied_against.rev_id) merge_context = None if migration_strategy.merge_strategy == base.MERGE: merge_args = dict(previous_codebase=previous_source) if self.migration_config.IsExport(): merge_args['generated_codebase'] = translated_source merge_args['public_codebase'] = base_codebase elif self.migration_config.IsImport(): merge_args['generated_codebase'] = base_codebase merge_args['public_codebase'] = translated_source merge_config = merge_codebases.MergeCodebasesConfig(**merge_args) merge_context = merge_codebases.MergeCodebasesContext( merge_config) merge_context.Update() p_s = self.migration_config.target_codebase_creator.ProjectSpace() codebase_to_push = codebase_utils.Codebase( merge_context.config.merged_codebase, expanded_path=merge_context.config.merged_codebase, project_space=p_s) else: codebase_to_push = translated_source editor = base_codebase.MakeEditor(migration_strategy, migration_revisions) source_repository_config = self.migration_config.source_repository_config # TODO(dbentley): setting additional_files_re to this is almost # certainly superfluous now. But... I'm not completely sure I'm right # about that, so I'll do it in a separate change. additional_files_re = ( self.project.public_repository_config.additional_files_re) push_codebase_args = dict( source_codebase=codebase_to_push.Path(), destination_editor=editor.Root(), destination_revision=self.applied_against.rev_id, source_revision=up_to_revision.rev_id) pusher = push_codebase.CodebasePusher( source_codebase=codebase_to_push, destination_editor=editor, report=moe_app.RUN.report, files_to_ignore_re=additional_files_re, commit_message=changelog, migration_id=migration_id ) commit_id = pusher.Push() if pusher.pushed: db.UpdateMigrationDiff(migration_id, diff=editor.Diff(), link=editor.Link()) if commit_id: moe_app.RUN.ui.Info('%s completed' % self.migration_config.direction) r = self.migration_config.target_repository.MakeRevisionFromId( commit_id) db.FinishMigration(migration_id, r) if self.migration_config.direction == base.Migration.EXPORT: internal_rev = migration_revisions[-1].rev_id public_rev = commit_id elif self.migration_config.direction == base.Migration.IMPORT: internal_rev = commit_id public_rev = migration_revisions[-1].rev_id equivalence_check = EquivalenceCheck( internal_rev, public_rev, self.project, self.translators, EquivalenceCheck.NoteIfSame) update = StateUpdate(actions=[equivalence_check] + result_migration_list + actions) return update else: moe_app.RUN.ui.Info('%s ready for human intervention' % self.migration_config.direction) if merge_context and merge_context.failed_merges: moe_app.RUN.report.AddTodo( 'Resolve failed merges in %s' % base_codebase.Client().directory) moe_app.RUN.report.SetReturnCode(2) if self.migration_config.IsExport(): moe_app.RUN.report.AddTodo( "Alternately, visit your project's MOE dashboard at " '%s , approve each migration, then rerun ' 'manage_codebases.' % db.GetDashboardUrl()) if remaining_revisions: # We committed as requested, but this did not leave a permanent # commit. (probably because it wasn't approved on the db). # We should no longer continue committing this line of migrations, # but we should upload them to the db so they can be approved. # TODO(dbentley): no, we shouldn't. We should just quit. # But we should quit in a classy way. And explain to the user # why we're quitting. result_migration = Migration( up_to_revision, self.applied_against, remaining_revisions, self.project, self.translators, self.migration_config, True, 1 ) return StateUpdate(actions=[result_migration] + actions) else: # There are no revisions left to migrate in this direction, # so make sure we don't migrate the other way if actions: for action in actions: if isinstance(action, Migration): action.mock_migration = True return StateUpdate(actions=actions) else: return None else: db.CancelMigration(migration_id) # TODO(dbentley): should this Info() (and others in this file) be # in some way connected as the result of the current task? moe_app.RUN.ui.Info('%s resulted in a no-op' % self.migration_config.direction) return result