def RunScenarioWithConfigFile(self, scenario_base, config_file):
codebase = os.path.join(scenario_base, 'input')
config_path = os.path.join(scenario_base, config_file)
(_, input_files) = scrubber.CreateInputFileListFromDir(codebase)
config = scrubber.ParseConfigFile(config_path, codebase, input_files)
context = scrubber.ScrubberContext(config)
context.Scan()
context.WriteOutput()
codebase1 = os.path.join(scenario_base, 'expected')
if not os.path.exists(codebase1):
self.assertTrue(context.Status(),
'Scrubber was expected to fail but did not')
return
if context.Status():
context.Report()
self.fail('Scrubber returned non-zero status %d' % context.Status())
codebase2 = os.path.join(context._temp_dir, 'output')
different = base.AreCodebasesDifferent(
codebase_utils.Codebase(codebase1),
codebase_utils.Codebase(codebase2))
if different:
# TODO(dbentley): this should describe how they differ.
self.fail('Codebases %s and %s differ' % (codebase1, codebase2))
def RunScenario(self, scenario_name):
UNRUN_SCENARIOS.remove(scenario_name)
scenario_base = os.path.join(SCENARIOS_DIR, scenario_name)
args = dict(
generated_codebase=self._MakeCodebase(
os.path.join(scenario_base, 'generated')),
public_codebase=self._MakeCodebase(
os.path.join(scenario_base, 'public')),
previous_codebase=self._MakeCodebase(
os.path.join(scenario_base, 'previous')),
)
config = merge_codebases.MergeCodebasesConfig(
**args)
context = merge_codebases.MergeCodebasesContext(
config)
context.Update()
codebase1 = config.merged_codebase
codebase2 = os.path.join(scenario_base, 'expected')
different = base.AreCodebasesDifferent(
codebase_utils.Codebase(codebase1),
codebase_utils.Codebase(codebase2))
if different:
# TODO(dbentley): this should describe how they differ.
print 'DIFFERENT:', different
self.fail("Codebases %s and %s differ" % (codebase1, codebase2))
def testCodebaseDifference(self):
codebase_internal = test_util.TestResourceFilename(
os.path.join('diff_codebases', 'codebase_internal/'))
codebase_public = test_util.TestResourceFilename(
os.path.join('diff_codebases', 'codebase_public/'))
codebase_diff_obj = base.AreCodebasesDifferent(
codebase_utils.Codebase(codebase_internal),
codebase_utils.Codebase(codebase_public),
record_full_diffs=True)
print 'codebase_diff_obj:', codebase_diff_obj
expected_diff = open(test_util.TestResourceFilename(
os.path.join('diff_codebases', 'codebase_diff'))).read()
self.assertEquals(expected_diff.strip(), str(codebase_diff_obj).strip())
def Create(self, revision, report=None):
codebase = self._revision_to_path_map[revision]
if not codebase:
raise base.CodebaseCreationError()
path = TestResourceFilename('codebases/%s/' % codebase)
return codebase_utils.Codebase(path, client_creator=self._client_creator,
project_space=self._project_space)
def Translate(self, codebase):
# TODO(dbentley): locate the scrubber
raise NotImplementedError
task = moe_app.RUN.ui.BeginImmediateTask(
'translate',
'Translating from %s project space to %s (using scrubber at %s)' %
(self._from_project_space, self._to_project_space, scrubber_path))
with task:
if codebase.ProjectSpace() != self._from_project_space:
raise base.Error(
('Attempting to translate Codebase %s from project space %s to %s, '
'but it is in project space %s') %
(codebase, self._from_project_space, self._to_project_space,
codebase.ProjectSpace()))
(output_tar_fd, output_tar_filename) = tempfile.mkstemp(
dir=moe_app.RUN.temp_dir,
prefix='translated_codebase_',
suffix='.tar')
os.close(output_tar_fd) # We use the name only, to pass to a subprocess.
# TODO(dbentley): should this be a CodebaseCreationError?
base.RunCmd(scrubber_path,
['--output_tar', output_tar_filename,
'--config_data', simplejson.dumps(self._scrubber_config),
codebase.ExpandedPath()])
return codebase_utils.Codebase(output_tar_filename,
project_space=self._to_project_space)
def testWalk(self):
internal_creator = test_util.StaticCodebaseCreator(
{'1001': 'simple_python'})
codebase = internal_creator.Create('1001')
self.assertEqual(codebase.Walk(), ['foo.py'])
codebase2 = codebase_utils.Codebase(codebase.ExpandedPath(),
additional_files_re='foo.py')
self.assertEqual(codebase2.Walk(), [])
def Translate(self, codebase):
task = moe_app.RUN.ui.BeginImmediateTask(
'translate',
'Translating from %s project space to %s (using identity translation)' %
(self._from_project_space, self._to_project_space))
with task:
if codebase.ProjectSpace() != self._from_project_space:
raise base.Error(
('Attempting to translate Codebase %s from project space %s to %s, '
'but it is in project space %s') %
(codebase, self._from_project_space, self._to_project_space,
codebase.ProjectSpace()))
return codebase_utils.Codebase(
codebase.ExpandedPath(),
additional_files_re=codebase.AdditionalFilesRe(),
project_space=self._to_project_space)
def _MakeCodebase(self, path): return codebase_utils.Codebase(path)
def main(unused_args):
project = db_client.MakeProjectContext()
try:
# TODO(dbentley): this code should use functions in the logic module
# to know what revision to apply against.
# TODO(dbentley): we shouldn't have to make a client just to get the config
# NB(dbentley): this also saves the project, so we call it all the time
# to save the config.
source_revision = FLAGS.source_revision
destination_revision = FLAGS.destination_revision
destination = FLAGS.destination
if destination not in base.REPOSITORIES:
raise app.UsageError('destination should be one of %s' %
str(base.REPOSITORIES))
print 'Pushing codebase for MOE project %s into %s' % (
project.config.name, destination)
if destination == base.INTERNAL_STR:
source_codebase_creator = project.public_codebase_creator
destination_codebase_creator = project.internal_codebase_creator
elif destination == base.PUBLIC_STR:
source_codebase_creator = project.internal_codebase_creator
destination_codebase_creator = project.public_codebase_creator
else:
raise base.Error('Unexpected destination: %s' % destination)
if FLAGS.codebase:
source_codebase = codebase_utils.Codebase(FLAGS.codebase)
else:
source_codebase = source_codebase_creator.Create(source_revision)
migration_strategy = config.MigrationStrategy(
merge_strategy=base.ERROR,
commit_strategy=base.LEAVE_PENDING,
separate_revisions=False,
copy_metadata=False)
# TODO(dbentley): this creates the codebase just to make an editor.
# We should make it possible to skip this step.
destination_editor = destination_codebase_creator.Create(
destination_revision).MakeEditor(migration_strategy)
# TODO(dbentley):
# We should be able to remove this additional_files_re, as it is
# accounted for in the call to Codebase.Walk(). Unfortunately, we can't.
# Why? Because a generated codebase is currently, incorrectly, getting
# generated with the additional_files_re from the internal codebase,
# even when it should be in the public project space.
additional_files_re = (
project.config.public_repository_config.additional_files_re)
pusher = CodebasePusher(source_codebase,
destination_editor,
report=None,
files_to_ignore_re=additional_files_re)
pusher.Push()
moe_app.RUN.report.PrintSummary()
finally:
project.db.Disconnect()
def Perform(self, report=None, db=None, actions=None, **unused_kwargs):
"""Perform the action, and determine next actions.
Args:
report: base.MoeReport
db: moe_db.MoeDbClient
temp_dir: str, path to a tempoprary directory
actions: seq of Action, the actions to run after this
Returns:
StateUpdate, an update to the state of the Action sequence
"""
actions = actions or []
if not self.revisions:
return
if self.num_revisions_to_migrate == -1:
migration_revisions = self.revisions
remaining_revisions = []
else:
migration_revisions = self.revisions[:self.num_revisions_to_migrate]
remaining_revisions = self.revisions[self.num_revisions_to_migrate:]
up_to_revision = migration_revisions[-1]
task = moe_app.RUN.ui.BeginIntermediateTask(
'migrate',
'Migrating up to revision %s by applying it against %s' %
(up_to_revision.rev_id, self.applied_against.rev_id))
with task:
if remaining_revisions:
result_migration = Migration(
up_to_revision,
self.applied_against,
remaining_revisions,
self.project,
self.translators,
self.migration_config, self.mock_migration, 1)
result_migration_list = [result_migration]
result = StateUpdate(actions=[result_migration] + actions)
else:
result_migration = None
result_migration_list = []
result = None
previous_source = self.migration_config.source_codebase_creator.Create(
self.previous_revision.rev_id)
try:
source = self.migration_config.source_codebase_creator.Create(
up_to_revision.rev_id)
translated_source = translators.TranslateToProjectSpace(
source,
self.migration_config.target_codebase_creator.ProjectSpace(),
self.translators)
except base.CodebaseCreationError:
if not remaining_revisions:
# This is an error at the last revision
# It might be better if we just marked it as an error.
raise
# In this case, we can't migrate the up_to_revision. So instead, we
# fold it into the next migration.
action = Migration(
self.previous_revision,
self.applied_against,
self.revisions,
self.project,
self.translators,
self.migration_config, self.mock_migration,
self.num_revisions_to_migrate + 1)
return StateUpdate(actions=[action] + actions)
should_migrate = bool(base.AreCodebasesDifferent(
previous_source, source, noisy_files_re=self.project.noisy_files_re))
if not should_migrate:
return result
migration_strategy = copy.copy(self.migration_config.migration_strategy)
if migration_strategy.merge_strategy == base.ERROR:
raise base.Error('Attempted %s invalid with merge strategy.' %
self.migration_config.direction)
migration = db.FindMigration(up_to_revision, abbreviated=False)
if (not migration or
(migration.status == base.Migration.CANCELED)):
changelog = base.ConcatenateChangelogs(migration_revisions)
creator = self.migration_config.source_codebase_creator
pre_approved = (migration_strategy.preapprove_public_changelogs and
all([r.pre_approved for r in migration_revisions]))
migration_id = db.StartMigration(
self.migration_config.direction,
up_to_revision,
changelog=changelog,
migrated_revisions=migration_revisions,
pre_approved=pre_approved)
elif migration.status == base.Migration.SUBMITTED:
# TODO(dbentley): this should never happen
find_step = moe_app.RUN.report.AddStep(
'migrate_changes', 'find_migration')
find_step.SetResult('%s of revisions [%s, %s] has '
'already been submitted in migration %s.' %
(self.migration_config.direction,
self.start_migrated_revision,
end_migrated_revision,
migration.migration_id))
return result
else:
changelog = migration.changelog
migration_id = migration.migration_id
if migration.status == base.Migration.APPROVED:
# TODO(dbentley): should we do this only if it's somewhere in the
# project config?
migration_strategy.commit_strategy = base.COMMIT_REMOTELY
if self.mock_migration:
return result
base_codebase = self.migration_config.target_codebase_creator.Create(
self.applied_against.rev_id)
merge_context = None
if migration_strategy.merge_strategy == base.MERGE:
merge_args = dict(previous_codebase=previous_source)
if self.migration_config.IsExport():
merge_args['generated_codebase'] = translated_source
merge_args['public_codebase'] = base_codebase
elif self.migration_config.IsImport():
merge_args['generated_codebase'] = base_codebase
merge_args['public_codebase'] = translated_source
merge_config = merge_codebases.MergeCodebasesConfig(**merge_args)
merge_context = merge_codebases.MergeCodebasesContext(
merge_config)
merge_context.Update()
p_s = self.migration_config.target_codebase_creator.ProjectSpace()
codebase_to_push = codebase_utils.Codebase(
merge_context.config.merged_codebase,
expanded_path=merge_context.config.merged_codebase,
project_space=p_s)
else:
codebase_to_push = translated_source
editor = base_codebase.MakeEditor(migration_strategy, migration_revisions)
source_repository_config = self.migration_config.source_repository_config
# TODO(dbentley): setting additional_files_re to this is almost
# certainly superfluous now. But... I'm not completely sure I'm right
# about that, so I'll do it in a separate change.
additional_files_re = (
self.project.public_repository_config.additional_files_re)
push_codebase_args = dict(
source_codebase=codebase_to_push.Path(),
destination_editor=editor.Root(),
destination_revision=self.applied_against.rev_id,
source_revision=up_to_revision.rev_id)
pusher = push_codebase.CodebasePusher(
source_codebase=codebase_to_push,
destination_editor=editor,
report=moe_app.RUN.report,
files_to_ignore_re=additional_files_re,
commit_message=changelog,
migration_id=migration_id
)
commit_id = pusher.Push()
if pusher.pushed:
db.UpdateMigrationDiff(migration_id, diff=editor.Diff(),
link=editor.Link())
if commit_id:
moe_app.RUN.ui.Info('%s completed' %
self.migration_config.direction)
r = self.migration_config.target_repository.MakeRevisionFromId(
commit_id)
db.FinishMigration(migration_id, r)
if self.migration_config.direction == base.Migration.EXPORT:
internal_rev = migration_revisions[-1].rev_id
public_rev = commit_id
elif self.migration_config.direction == base.Migration.IMPORT:
internal_rev = commit_id
public_rev = migration_revisions[-1].rev_id
equivalence_check = EquivalenceCheck(
internal_rev, public_rev, self.project, self.translators,
EquivalenceCheck.NoteIfSame)
update = StateUpdate(actions=[equivalence_check] +
result_migration_list + actions)
return update
else:
moe_app.RUN.ui.Info('%s ready for human intervention' %
self.migration_config.direction)
if merge_context and merge_context.failed_merges:
moe_app.RUN.report.AddTodo(
'Resolve failed merges in %s' %
base_codebase.Client().directory)
moe_app.RUN.report.SetReturnCode(2)
if self.migration_config.IsExport():
moe_app.RUN.report.AddTodo(
"Alternately, visit your project's MOE dashboard at "
'%s , approve each migration, then rerun '
'manage_codebases.' % db.GetDashboardUrl())
if remaining_revisions:
# We committed as requested, but this did not leave a permanent
# commit. (probably because it wasn't approved on the db).
# We should no longer continue committing this line of migrations,
# but we should upload them to the db so they can be approved.
# TODO(dbentley): no, we shouldn't. We should just quit.
# But we should quit in a classy way. And explain to the user
# why we're quitting.
result_migration = Migration(
up_to_revision,
self.applied_against,
remaining_revisions,
self.project,
self.translators,
self.migration_config,
True,
1
)
return StateUpdate(actions=[result_migration] + actions)
else:
# There are no revisions left to migrate in this direction,
# so make sure we don't migrate the other way
if actions:
for action in actions:
if isinstance(action, Migration):
action.mock_migration = True
return StateUpdate(actions=actions)
else:
return None
else:
db.CancelMigration(migration_id)
# TODO(dbentley): should this Info() (and others in this file) be
# in some way connected as the result of the current task?
moe_app.RUN.ui.Info('%s resulted in a no-op' %
self.migration_config.direction)
return result