def testRecurUntilMatchingRevision(self):
self.mox.StubOutWithMock(mercurial, 'RunHg')
mercurial.RunHg(['version'],
need_stdout=True).AndReturn('(version 1.8)')
self.mox.ReplayAll()
repos = mercurial.MercurialRepository('http://not_a_url',
'Dummy mercurial repository')
self.mox.UnsetStubs()
log_file = os.path.join(SCENARIOS_DIR, 'long_log', 'input')
log_text = file_util.Read(log_file)
def Revisionb05847911039(r):
return r.rev_id == 'b05847911039'
self.mox.StubOutWithMock(repos._client, 'RunHg')
repos._client.RunHg(['pull']).AndReturn(None)
repos._client.RunHg(
['log', '--style', 'default', '-v', '-l', '400', '-r', 'dummy:0'],
need_stdout=True).AndReturn(log_text)
self.mox.ReplayAll()
result = repos.RecurUntilMatchingRevision('dummy',
Revisionb05847911039)
self.assertEqual(3, len(result))
self.assertEqual('b05847911039', result[-1].rev_id)
self.mox.UnsetStubs()
def TestCommentExtractor(test_case, extractor, source_file, expected_file):
comments = extractor.ExtractComments(
test_util.FakeFile(filename=source_file))
expected_text = file_util.Read(expected_file)
expected_json = simplejson.loads(expected_text)
expected_comments = comment_scrubber.CommentsFromJson(expected_json)
test_case.assertListEqual(expected_comments, comments)
def Run(self, argv):
project = db_client.MakeProjectContext()
try:
revision = FLAGS.source_revision
repository = FLAGS.repository
if repository not in base.REPOSITORIES:
raise app.UsageError('repository should be one of %s' %
str(base.REPOSITORIES))
print 'Generating from and pushing into %s for MOE project %s' % (
repository, project.config.name)
if repository == base.INTERNAL_STR:
codebase_creator = project.internal_codebase_creator
elif destination == base.PUBLIC_STR:
codebase_creator = project.public_codebase_creator
else:
raise base.Error('Unexpected repository: %s' % destination)
original_codebase = codebase_creator.Create(revision)
codebase = codebase_utils.CreateModifiableCopy(original_codebase)
for relative_filename in codebase.Walk():
filename = codebase.FilePath(relative_filename)
if os.path.splitext(filename)[1] in EDITABLE_FILETYPES:
file_util.Write(filename,
file_util.Read(filename) + '\nMOE was here.\n')
migration_strategy = config.MigrationStrategy(
merge_strategy=base.ERROR,
commit_strategy=base.LEAVE_PENDING,
separate_revisions=False,
copy_metadata=False)
# TODO(dbentley): this creates the codebase just to make an editor.
# We should make it possible to skip this step.
destination_editor = codebase_creator.Create(
revision).MakeEditor(migration_strategy)
# TODO(dbentley):
# We should be able to remove this additional_files_re, as it is
# accounted for in the call to Codebase.Walk(). Unfortunately, we can't.
# Why? Because a generated codebase is currently, incorrectly, getting
# generated with the additional_files_re from the internal codebase,
# even when it should be in the public project space.
additional_files_re = (
project.config.public_repository_config.additional_files_re)
pusher = push_codebase.CodebasePusher(
codebase, destination_editor, files_to_ignore_re=additional_files_re)
pusher.Push()
moe_app.RUN.report.PrintSummary()
finally:
project.db.Disconnect()
def RunScenario(self, scenario_name, filter_to_test): UNRUN_SCENARIOS.remove(scenario_name) scenario_base = os.path.join(SCENARIOS_DIR, scenario_name) in_file = os.path.join(scenario_base, 'input') input_txt = file_util.Read(in_file) output = filter_to_test(input_txt) expected = os.path.join(scenario_base, 'expected') out_file = os.path.join(FLAGS.test_tmpdir, scenario_name + '.out.txt') file_util.Write(out_file, output) basetest.DiffTestFiles(expected, out_file)
def __init__(self, contents=None, filename=None):
if contents is not None:
self._contents = contents
self.filename = filename or ('%d_%f.txt' %
(hash(contents), time.time()))
elif filename is not None:
self._contents = file_util.Read(filename)
self.filename = os.path.basename(filename)
else:
raise base.Error(
'at least one of file or contents must be specified')
self._contents_filename = os.path.join(FLAGS.test_tmpdir,
os.path.basename(self.filename))
self.new_contents = contents
self.deleted = False
self.written = False
def testRecurUntilMatchingRevision(self):
repos = svn.SvnRepository('http://not_a_url', 'ummy svn repository')
log_file = os.path.join(SCENARIOS_DIR, 'long_log', 'input')
log_text = file_util.Read(log_file)
def Revision12(r):
return r.rev_id == '12'
self.mox.StubOutWithMock(svn, 'RunSvn')
svn.RunSvn(
['log', '--xml', '-l', '50', '-r', '15:1', 'http://not_a_url'],
need_stdout=True).AndReturn(log_text)
self.mox.ReplayAll()
result = repos.RecurUntilMatchingRevision('15', Revision12)
self.assertEqual(4, len(result))
self.assertEqual('12', result[-1].rev_id)
self.mox.UnsetStubs()
def testSuccessfulRead(self):
file_handle = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(__builtin__, 'open', use_mock_anything=True)
open(self.file_path).AndReturn(file_handle)
file_handle.__enter__().AndReturn(file_handle)
file_handle.read().AndReturn(self.sample_contents)
file_handle.__exit__(None, None, None)
self.mox.ReplayAll()
try:
self.assertEquals(file_util.Read(self.file_path), self.sample_contents)
self.mox.VerifyAll()
finally:
# Because we mock out the built-in open() function, which the unittest
# library depends on, we need to make sure we revert it before leaving the
# test, otherwise any test failures will cause further internal failures
# and yield no meaningful error output.
self.mox.ResetAll()
self.mox.UnsetStubs()
def ParseConfigFile(filename):
"""Parse a JSON config file."""
text = filename and file_util.Read(filename) or ''
filename = os.path.abspath(filename)
relative_name = config_utils.MakeConfigFilenameRelative(filename)
return ParseConfigText(text, relative_name)
def MakeProjectContextWithoutDefaults(create_project, acquire_lock,
allow_concurrent_instances, name,
config_file_path, default_moe_db_url):
"""Figure out the MOE project and its context.
This looks at flags, the running database, and the config file.
Args:
create_project: boolean, whether to create the project if it doesn't exist
acquire_lock: boolean, whether to acquire a lock on the project (None means
use the value implied by --allow_concurrent_instances)
Returns:
moe_project.MoeProjectContext
Raises:
base.Error if a project and client cannot be created. This may also happen
based on the current environment. E.g., if acquire_lock=True and another
process on this project is running. This is to prevent two people running
manage_codebases at the same time and pushing confusingly similar changes
(and populating the MOE db with confusingly similar but potentially
conflicting data).
NB:
This code is complicated because the user wants it to "Do The Right Thing."
An example: which server should we connect to? This could read from the flag
setting the moe_db_url, or the flag specifying the config file (which
contains a moe_db_url). And then we need to make sure to read that config
file before contacting the db. But if the project_config_file isn't set,
we need to talk to the db before reading the config.
The most rigorous definition of this function is its code and test
(in db_client_test).
"""
config_obj = None
if acquire_lock is None:
acquire_lock = not allow_concurrent_instances
if not name and not config_file_path:
raise base.Error(
'Must specify at least one of --project or --project_config_file')
if config_file_path:
# At this point in MOE setup, there is no moe_app.RUN, so we need to
# handle output on our own.
print 'Reading config file from', config_file_path
config_obj = config.ParseConfigFile(config_file_path)
if name and name != config_obj.name:
raise base.Error(
'Name "%s" from --project and name "%s" from config '
'differ.' % (name, config_obj.name))
moe_app._Init(name or config_obj.name)
url = (default_moe_db_url or (config_obj and config_obj.moe_db_url)
or FLAGS.moe_db_url)
if config_obj:
# If we have a local config file, post that to the server first.
# If a project is corrupt, GetStoredProject will fail, so updating
# the config file first from local data allows us to bootstrap ourselves
# back to a good state.
UpdateConfigFile(url, config_obj)
stored_project = GetStoredProject(url, name or config_obj.name)
if not stored_project:
if not create_project:
raise base.Error(
'Project %s does not exist. Create it on the MOE db.' %
(name or config_obj.name))
if not config_obj:
# The command-line contained --project but not --project_config_file.
# We'll read the current config file in from the filesystem, based on
# the stored project's filename.
if not stored_project.filename:
raise base.Error(
"Cannot determine where your project's config file lives. "
"Please run MOE once with --project_config_file <path_to_config_file>"
)
absolute_config_path = config_utils.MakeConfigFilenameAbsolute(
stored_project.filename)
try:
print 'Reading config file from', absolute_config_path
current_config_text = file_util.Read(absolute_config_path)
except IOError:
raise base.Error(
'Could not find config file "%s" (expected it at "%s")' %
(stored_project.filename, absolute_config_path))
config_obj = config.ParseConfigText(current_config_text,
filename=stored_project.filename)
UpdateConfigFile(url, config_obj)
db = ServerBackedMoeDbClient(config_obj,
record_process=acquire_lock,
url=url)
project_context = moe_project.MoeProjectContext(config_obj, db)
return project_context
def testGetResourceFilename(self):
self._CheckTestData(
lambda n: file_util.Read(resources.GetResourceFilename(n)))