def RunScenario(self, scenario_name):
UNRUN_SCENARIOS.remove(scenario_name)
scenario_base = os.path.join(SCENARIOS_DIR, scenario_name)
args = dict(
generated_codebase=self._MakeCodebase(
os.path.join(scenario_base, 'generated')),
public_codebase=self._MakeCodebase(
os.path.join(scenario_base, 'public')),
previous_codebase=self._MakeCodebase(
os.path.join(scenario_base, 'previous')),
)
config = merge_codebases.MergeCodebasesConfig(
**args)
context = merge_codebases.MergeCodebasesContext(
config)
context.Update()
codebase1 = config.merged_codebase
codebase2 = os.path.join(scenario_base, 'expected')
different = base.AreCodebasesDifferent(
codebase_utils.Codebase(codebase1),
codebase_utils.Codebase(codebase2))
if different:
# TODO(dbentley): this should describe how they differ.
print 'DIFFERENT:', different
self.fail("Codebases %s and %s differ" % (codebase1, codebase2))
def RunScenarioWithConfigFile(self, scenario_base, config_file):
codebase = os.path.join(scenario_base, 'input')
config_path = os.path.join(scenario_base, config_file)
(_, input_files) = scrubber.CreateInputFileListFromDir(codebase)
config = scrubber.ParseConfigFile(config_path, codebase, input_files)
context = scrubber.ScrubberContext(config)
context.Scan()
context.WriteOutput()
codebase1 = os.path.join(scenario_base, 'expected')
if not os.path.exists(codebase1):
self.assertTrue(context.Status(),
'Scrubber was expected to fail but did not')
return
if context.Status():
context.Report()
self.fail('Scrubber returned non-zero status %d' % context.Status())
codebase2 = os.path.join(context._temp_dir, 'output')
different = base.AreCodebasesDifferent(
codebase_utils.Codebase(codebase1),
codebase_utils.Codebase(codebase2))
if different:
# TODO(dbentley): this should describe how they differ.
self.fail('Codebases %s and %s differ' % (codebase1, codebase2))
def testCreateModifiableCopy(self):
internal_creator = test_util.StaticCodebaseCreator(
{'1001': 'simple_python'})
codebase = internal_creator.Create('1001')
copy = codebase_utils.CreateModifiableCopy(codebase)
self.assertFalse(base.AreCodebasesDifferent(codebase, copy))
self.assertFalse(codebase.Path() == copy.Path())
def testCodebaseDifference(self):
codebase_internal = test_util.TestResourceFilename(
os.path.join('diff_codebases', 'codebase_internal/'))
codebase_public = test_util.TestResourceFilename(
os.path.join('diff_codebases', 'codebase_public/'))
codebase_diff_obj = base.AreCodebasesDifferent(
codebase_utils.Codebase(codebase_internal),
codebase_utils.Codebase(codebase_public),
record_full_diffs=True)
print 'codebase_diff_obj:', codebase_diff_obj
expected_diff = open(test_util.TestResourceFilename(
os.path.join('diff_codebases', 'codebase_diff'))).read()
self.assertEquals(expected_diff.strip(), str(codebase_diff_obj).strip())
def main(unused_argv):
project = db_client.MakeProjectContext()
try:
internal_codebase = project.internal_codebase_creator.Create(
FLAGS.internal_revision > 0 and FLAGS.internal_revision
or project.internal_repository.GetHeadRevision())
public_codebase = project.public_codebase_creator.Create(
FLAGS.public_revision > 0 and FLAGS.public_revision
or project.public_repository.GetHeadRevision())
diff_obj = base.AreCodebasesDifferent(internal_codebase,
public_codebase,
project.config.noisy_files_re,
record_full_diffs=True)
moe_app.RUN.ui.Info('\n===== Begin diff_codebases =====\n')
moe_app.RUN.ui.Info(str(diff_obj))
finally:
project.db.Disconnect()
def Perform(self, internal_codebase_creator, public_codebase_creator,
report, db, temp_dir, actions,
**unused_kwargs):
"""Perform the action, and determine next actions.
Args:
internal_codebase_creator: codebase_utils.CodebaseCreator
public_codebase_creator: codebase_utils.CodebaseCreator
report: config.MoeReport
db: moe_db.MoeDbClient
temp_dir: str, path to a tempoprary directory
actions: sequence of Action
Returns:
StateUpdate, an update to the state of the Action sequence
"""
task = moe_app.RUN.ui.BeginIntermediateTask(
'equivalence_check',
('Checking for an Equivalence between internal revision %s and '
'public revision %s') % (self.internal_revision, self.public_revision))
with task:
internal = internal_codebase_creator.Create(self.internal_revision)
generated = translators.TranslateToProjectSpace(
internal, base.PUBLIC_STR,
self.translators)
public = public_codebase_creator.Create(self.public_revision)
codebases_differ = None
if not self.project.manual_equivalence_deltas:
codebases_differ = base.AreCodebasesDifferent(
generated, public,
noisy_files_re=self.project.noisy_files_re)
return self.result_dispatch(self,
codebases_differ=codebases_differ,
generated=generated,
public=public,
db=db,
report=report,
actions=actions)
def Perform(self, report=None, db=None, actions=None, **unused_kwargs):
"""Perform the action, and determine next actions.
Args:
report: base.MoeReport
db: moe_db.MoeDbClient
temp_dir: str, path to a tempoprary directory
actions: seq of Action, the actions to run after this
Returns:
StateUpdate, an update to the state of the Action sequence
"""
actions = actions or []
if not self.revisions:
return
if self.num_revisions_to_migrate == -1:
migration_revisions = self.revisions
remaining_revisions = []
else:
migration_revisions = self.revisions[:self.num_revisions_to_migrate]
remaining_revisions = self.revisions[self.num_revisions_to_migrate:]
up_to_revision = migration_revisions[-1]
task = moe_app.RUN.ui.BeginIntermediateTask(
'migrate',
'Migrating up to revision %s by applying it against %s' %
(up_to_revision.rev_id, self.applied_against.rev_id))
with task:
if remaining_revisions:
result_migration = Migration(
up_to_revision,
self.applied_against,
remaining_revisions,
self.project,
self.translators,
self.migration_config, self.mock_migration, 1)
result_migration_list = [result_migration]
result = StateUpdate(actions=[result_migration] + actions)
else:
result_migration = None
result_migration_list = []
result = None
previous_source = self.migration_config.source_codebase_creator.Create(
self.previous_revision.rev_id)
try:
source = self.migration_config.source_codebase_creator.Create(
up_to_revision.rev_id)
translated_source = translators.TranslateToProjectSpace(
source,
self.migration_config.target_codebase_creator.ProjectSpace(),
self.translators)
except base.CodebaseCreationError:
if not remaining_revisions:
# This is an error at the last revision
# It might be better if we just marked it as an error.
raise
# In this case, we can't migrate the up_to_revision. So instead, we
# fold it into the next migration.
action = Migration(
self.previous_revision,
self.applied_against,
self.revisions,
self.project,
self.translators,
self.migration_config, self.mock_migration,
self.num_revisions_to_migrate + 1)
return StateUpdate(actions=[action] + actions)
should_migrate = bool(base.AreCodebasesDifferent(
previous_source, source, noisy_files_re=self.project.noisy_files_re))
if not should_migrate:
return result
migration_strategy = copy.copy(self.migration_config.migration_strategy)
if migration_strategy.merge_strategy == base.ERROR:
raise base.Error('Attempted %s invalid with merge strategy.' %
self.migration_config.direction)
migration = db.FindMigration(up_to_revision, abbreviated=False)
if (not migration or
(migration.status == base.Migration.CANCELED)):
changelog = base.ConcatenateChangelogs(migration_revisions)
creator = self.migration_config.source_codebase_creator
pre_approved = (migration_strategy.preapprove_public_changelogs and
all([r.pre_approved for r in migration_revisions]))
migration_id = db.StartMigration(
self.migration_config.direction,
up_to_revision,
changelog=changelog,
migrated_revisions=migration_revisions,
pre_approved=pre_approved)
elif migration.status == base.Migration.SUBMITTED:
# TODO(dbentley): this should never happen
find_step = moe_app.RUN.report.AddStep(
'migrate_changes', 'find_migration')
find_step.SetResult('%s of revisions [%s, %s] has '
'already been submitted in migration %s.' %
(self.migration_config.direction,
self.start_migrated_revision,
end_migrated_revision,
migration.migration_id))
return result
else:
changelog = migration.changelog
migration_id = migration.migration_id
if migration.status == base.Migration.APPROVED:
# TODO(dbentley): should we do this only if it's somewhere in the
# project config?
migration_strategy.commit_strategy = base.COMMIT_REMOTELY
if self.mock_migration:
return result
base_codebase = self.migration_config.target_codebase_creator.Create(
self.applied_against.rev_id)
merge_context = None
if migration_strategy.merge_strategy == base.MERGE:
merge_args = dict(previous_codebase=previous_source)
if self.migration_config.IsExport():
merge_args['generated_codebase'] = translated_source
merge_args['public_codebase'] = base_codebase
elif self.migration_config.IsImport():
merge_args['generated_codebase'] = base_codebase
merge_args['public_codebase'] = translated_source
merge_config = merge_codebases.MergeCodebasesConfig(**merge_args)
merge_context = merge_codebases.MergeCodebasesContext(
merge_config)
merge_context.Update()
p_s = self.migration_config.target_codebase_creator.ProjectSpace()
codebase_to_push = codebase_utils.Codebase(
merge_context.config.merged_codebase,
expanded_path=merge_context.config.merged_codebase,
project_space=p_s)
else:
codebase_to_push = translated_source
editor = base_codebase.MakeEditor(migration_strategy, migration_revisions)
source_repository_config = self.migration_config.source_repository_config
# TODO(dbentley): setting additional_files_re to this is almost
# certainly superfluous now. But... I'm not completely sure I'm right
# about that, so I'll do it in a separate change.
additional_files_re = (
self.project.public_repository_config.additional_files_re)
push_codebase_args = dict(
source_codebase=codebase_to_push.Path(),
destination_editor=editor.Root(),
destination_revision=self.applied_against.rev_id,
source_revision=up_to_revision.rev_id)
pusher = push_codebase.CodebasePusher(
source_codebase=codebase_to_push,
destination_editor=editor,
report=moe_app.RUN.report,
files_to_ignore_re=additional_files_re,
commit_message=changelog,
migration_id=migration_id
)
commit_id = pusher.Push()
if pusher.pushed:
db.UpdateMigrationDiff(migration_id, diff=editor.Diff(),
link=editor.Link())
if commit_id:
moe_app.RUN.ui.Info('%s completed' %
self.migration_config.direction)
r = self.migration_config.target_repository.MakeRevisionFromId(
commit_id)
db.FinishMigration(migration_id, r)
if self.migration_config.direction == base.Migration.EXPORT:
internal_rev = migration_revisions[-1].rev_id
public_rev = commit_id
elif self.migration_config.direction == base.Migration.IMPORT:
internal_rev = commit_id
public_rev = migration_revisions[-1].rev_id
equivalence_check = EquivalenceCheck(
internal_rev, public_rev, self.project, self.translators,
EquivalenceCheck.NoteIfSame)
update = StateUpdate(actions=[equivalence_check] +
result_migration_list + actions)
return update
else:
moe_app.RUN.ui.Info('%s ready for human intervention' %
self.migration_config.direction)
if merge_context and merge_context.failed_merges:
moe_app.RUN.report.AddTodo(
'Resolve failed merges in %s' %
base_codebase.Client().directory)
moe_app.RUN.report.SetReturnCode(2)
if self.migration_config.IsExport():
moe_app.RUN.report.AddTodo(
"Alternately, visit your project's MOE dashboard at "
'%s , approve each migration, then rerun '
'manage_codebases.' % db.GetDashboardUrl())
if remaining_revisions:
# We committed as requested, but this did not leave a permanent
# commit. (probably because it wasn't approved on the db).
# We should no longer continue committing this line of migrations,
# but we should upload them to the db so they can be approved.
# TODO(dbentley): no, we shouldn't. We should just quit.
# But we should quit in a classy way. And explain to the user
# why we're quitting.
result_migration = Migration(
up_to_revision,
self.applied_against,
remaining_revisions,
self.project,
self.translators,
self.migration_config,
True,
1
)
return StateUpdate(actions=[result_migration] + actions)
else:
# There are no revisions left to migrate in this direction,
# so make sure we don't migrate the other way
if actions:
for action in actions:
if isinstance(action, Migration):
action.mock_migration = True
return StateUpdate(actions=actions)
else:
return None
else:
db.CancelMigration(migration_id)
# TODO(dbentley): should this Info() (and others in this file) be
# in some way connected as the result of the current task?
moe_app.RUN.ui.Info('%s resulted in a no-op' %
self.migration_config.direction)
return result