def init():
'''
Return the hg repo object for this session
'''
bp_ = os.path.join(__opts__['cachedir'], 'hgfs')
repos = []
for _, opt in enumerate(__opts__['hgfs_remotes']):
repo_hash = hashlib.md5(opt).hexdigest()
rp_ = os.path.join(bp_, repo_hash)
if not os.path.isdir(rp_):
os.makedirs(rp_)
if not os.listdir(rp_):
# Only init if the directory is empty.
hglib.init(rp_)
try:
repo = hglib.open(rp_)
except hglib.error.ServerError:
log.error(
'Cache path {0} (corresponding remote: {1}) exists but is not '
'a valid mercurial repository. You will need to manually '
'delete this directory on the master to continue to use this '
'hgfs remote.'.format(rp_, opt))
continue
refs = repo.config(names='paths')
if not refs:
# Write an hgrc defining the remote URI
hgconfpath = os.path.join(rp_, '.hg', 'hgrc')
with salt.utils.fopen(hgconfpath, 'w+') as hgconfig:
hgconfig.write('[paths]\n')
hgconfig.write('default = {0}\n'.format(opt))
repos.append(repo)
repo.close()
return repos
def hg_setup_and_teardown():
"""
build up and tear down hg repos to test with.
"""
sourcedirPath = Path(__file__).resolve().parent.joinpath("files")
tempdir = tempfile.TemporaryDirectory()
tempsubdir = tempdir.name / Path("test2/")
tempsubdir2 = tempdir.name / Path("subdir/")
tempsubdir3 = tempdir.name / Path("subdir/test2/")
tempsubdir.mkdir()
tempsubdir2.mkdir()
tempsubdir3.mkdir()
tempdirPath = Path(tempdir.name)
filessrc = [
Path("top.sls"),
Path("test.sls"),
Path("test2/init.sls"),
]
for fnd in filessrc:
to = tempdirPath / fnd
to2 = tempsubdir2 / fnd
frm = sourcedirPath / fnd
shutil.copy(frm.as_posix(), to.as_posix())
shutil.copy(frm.as_posix(), to2.as_posix())
hglib.init(bytes(tempdirPath.as_posix(), encoding="utf8"))
repo = hglib.open(bytes(tempdirPath.as_posix(), encoding="utf8"))
repo.add(bytes(tempdirPath.as_posix(), encoding="utf8"))
repo.commit(b"init commit", user="******")
repo.tag(b"test", user="******")
repo.branch(b"test")
repo.commit(b"create test branch", user="******")
repo.bookmark(b"bookmark_test")
yield tempdirPath.as_uri()
tempdir.cleanup()
def hgfs_setup_and_teardown():
"""
build up and tear down hg repos to test with.
"""
source_dir = Path(__file__).resolve().parent.joinpath("files")
tempdir = tempfile.TemporaryDirectory()
tempsubdir = tempdir.name / Path("subdir/")
tempsubdir.mkdir()
tempdirPath = Path(tempdir.name)
for file in source_dir.iterdir():
to_file = tempdirPath / file.name
to_file2 = tempsubdir / file.name
shutil.copy(file.as_posix(), to_file.as_posix())
shutil.copy(file.as_posix(), to_file2.as_posix())
hglib.init(bytes(tempdirPath.as_posix(), encoding="utf8"))
repo = hglib.open(bytes(tempdirPath.as_posix(), encoding="utf8"))
repo.add(bytes(tempdirPath.as_posix(), encoding="utf8"))
repo.commit(b"init commit", user="******")
repo.tag(b"test", user="******")
repo.branch(b"test")
repo.commit(b"create test branch", user="******")
repo.bookmark(b"bookmark_test")
yield tempdirPath.as_uri()
tempdir.cleanup()
def RepoMock(tmpdir):
'''
Mock a local mercurial repo
'''
# Init empty repo
repo_dir = str(tmpdir.realpath())
hglib.init(repo_dir)
# Add default pull in Mercurial config
hgrc = tmpdir.join('.hg').join('hgrc')
hgrc.write('[paths]\ndefault = {}'.format(repo_dir))
# Open repo with config
repo = hglib.open(repo_dir)
# Commit a file on central
readme = tmpdir.join('README.md')
readme.write('Hello World')
repo.add(str(readme.realpath()).encode('utf-8'))
repo.branch(name=b'central', force=True)
repo.commit(message=b'Readme', user='******')
# Mock push to avoid reaching try server
repo.push = MagicMock(return_value=True)
return repo
def mock_repository(mock_config):
'''
Create a dummy mercurial repository
'''
# Init repo
hglib.init(mock_config.repo_dir)
# Init clean client
client = hglib.open(mock_config.repo_dir)
# Add test.txt file
path = os.path.join(mock_config.repo_dir, 'test.txt')
with open(path, 'w') as f:
f.write('Hello World\n')
# Initiall commit
client.add(path.encode('utf-8'))
client.commit(b'Hello World', user=b'Tester')
# Write dummy 3rd party file
third_party = os.path.join(mock_config.repo_dir, mock_config.third_party)
with open(third_party, 'w') as f:
f.write('test/dummy')
# Remove pull capabilities
client.pull = Mock(return_value=True)
return client
def build_repository(tmpdir, name):
"""
Mock a local mercurial repo
"""
# Init empty repo
repo_dir = str(tmpdir.mkdir(name).realpath())
hglib.init(repo_dir)
# Add default pull in Mercurial config
hgrc = tmpdir.join(name, ".hg", "hgrc")
hgrc.write("[paths]\ndefault = {}".format(repo_dir))
# Open repo with config
repo = hglib.open(repo_dir)
# Commit a file on central
readme = tmpdir.join(name, "README.md")
readme.write("Hello World")
repo.add(str(readme.realpath()).encode("utf-8"))
repo.branch(name=b"central", force=True)
repo.commit(message=b"Readme", user="******")
# Mock push to avoid reaching try server
repo.push = MagicMock(return_value=True)
return repo
def RepoMock(tmpdir):
'''
Mock a local mercurial repo
'''
# Init empty repo
repo_dir = str(tmpdir.realpath())
hglib.init(repo_dir)
# Add default pull in Mercurial config
hgrc = tmpdir.join('.hg').join('hgrc')
hgrc.write('[paths]\ndefault = {}'.format(repo_dir))
# Open repo with config
repo = hglib.open(repo_dir)
# Commit a file on central
readme = tmpdir.join('README.md')
readme.write('Hello World')
repo.add(str(readme.realpath()).encode('utf-8'))
repo.branch(name=b'central', force=True)
repo.commit(message=b'Readme', user='******')
# Mock push to avoid reaching try server
repo.push = MagicMock(return_value=True)
return repo
def init(self):
hglib.init(self.path)
with io.open(join(self.path, '.hg', 'hgrc'), 'w',
encoding="utf-8") as fd:
fd.write(_hg_config)
self._repo = hglib.open(self.path.encode(sys.getfilesystemencoding()),
encoding=self.encoding)
def fake_hg_repo(tmpdir):
def tobytes(x):
return bytes(x, 'ascii')
tmp_path = tmpdir.strpath
dest = os.path.join(tmp_path, 'repos')
local = os.path.join(dest, 'local')
remote = os.path.join(dest, 'remote')
for d in [local, remote]:
os.makedirs(d)
hglib.init(d)
os.environ['USER'] = '******'
oldcwd = os.getcwd()
os.chdir(local)
hg = hglib.open(local)
files = [
{'name': 'mozglue/build/dummy.cpp',
'size': 1},
{'name': 'toolkit/components/osfile/osfile.jsm',
'size': 2},
{'name': 'js/src/jit/JIT.cpp',
'size': 3},
{'name': 'toolkit/components/osfile/osfile-win.jsm',
'size': 4},
{'name': 'js/src/jit/BitSet.cpp',
'size': 5},
{'name': 'code_coverage_bot/cli.py',
'size': 6},
]
for c in '?!':
for f in files:
fname = f['name']
parent = os.path.dirname(fname)
if not os.path.exists(parent):
os.makedirs(parent)
with open(fname, 'w') as Out:
Out.write(c * f['size'])
hg.add(files=[tobytes(fname)])
hg.commit(message='Commit file {} with {} inside'.format(fname, c),
user='******')
hg.push(dest=tobytes(remote))
hg.close()
os.chdir(oldcwd)
shutil.copyfile(os.path.join(remote, '.hg/pushlog2.db'),
os.path.join(local, '.hg/pushlog2.db'))
return local
def mock_repo(
tmpdir: py.path.local, monkeypatch: MonkeyPatch
) -> tuple[py.path.local, py.path.local]:
"""Create an empty mercurial repo"""
local_dir = tmpdir / "local"
remote_dir = tmpdir / "remote"
# Setup the worker env to use that repo dir
monkeypatch.setattr(bugbug_http, "REPO_DIR", str(local_dir))
# Create the repo
hglib.init(str(local_dir))
with hglib.open(str(local_dir)) as repo:
(local_dir / ".hg-annotate-ignore-revs").write_text("", encoding="ascii")
repo.add(str(local_dir / ".hg-annotate-ignore-revs").encode("utf-8"))
# Add several commits on a test file to create some history
test_file = local_dir / "test.txt"
for i in range(4):
test_file.write_text(f"Version {i}", encoding="utf-8")
repo.add([str(test_file).encode("utf-8")])
repo.commit(f"Base history {i}", user="******")
# Copy initialized repo as remote
local_dir.copy(remote_dir)
# Configure remote on local
hgrc = local_dir / ".hg" / "hgrc"
hgrc.write_text("\n".join(["[paths]", f"default = {remote_dir}"]), "utf-8")
# Add extra commit on remote
with hglib.open(str(remote_dir)) as repo:
remote = remote_dir / "remote.txt"
remote.write_text("New remote file !", encoding="utf-8")
repo.add([str(remote).encode("utf-8")])
repo.commit("Pulled from remote", user="******")
# Allow using the local code analysis server.
responses.add_passthru("http://127.0.0.1")
orig_hgutil_cmdbuilder = hglib.util.cmdbuilder
def hglib_cmdbuilder(name, *args, **kwargs):
if name == "pull":
args = list(args)
args[0] = str(remote_dir).encode("ascii")
return orig_hgutil_cmdbuilder(name, *args, **kwargs)
monkeypatch.setattr(hglib.util, "cmdbuilder", hglib_cmdbuilder)
return local_dir, remote_dir
def mock_repo(tmpdir: py.path.local,
monkeypatch: MonkeyPatch) -> Tuple[str, str]:
"""Create an empty mercurial repo"""
local_dir = tmpdir / "local"
remote_dir = tmpdir / "remote"
# Setup the worker env to use that repo dir
monkeypatch.setattr(bugbug_http, "REPO_DIR", str(local_dir))
# Create the repo
hglib.init(str(local_dir))
with hglib.open(str(local_dir)) as repo:
(local_dir / ".hg-annotate-ignore-revs").write_text("",
encoding="ascii")
repo.add(str(local_dir / ".hg-annotate-ignore-revs").encode("utf-8"))
# Add several commits on a test file to create some history
test_file = local_dir / "test.txt"
for i in range(4):
test_file.write_text(f"Version {i}", encoding="utf-8")
repo.add([str(test_file).encode("utf-8")])
repo.commit(f"Base history {i}", user="******")
# Copy initialized repo as remote
local_dir.copy(remote_dir)
# Configure remote on local
hgrc = local_dir / ".hg" / "hgrc"
hgrc.write_text("\n".join(["[paths]", f"default = {remote_dir}"]), "utf-8")
# Add extra commit on remote
with hglib.open(str(remote_dir)) as repo:
remote = remote_dir / "remote.txt"
remote.write_text("New remote file !", encoding="utf-8")
repo.add([str(remote).encode("utf-8")])
repo.commit("Pulled from remote", user="******")
# Allow using the local code analysis server.
responses.add_passthru("http://127.0.0.1")
orig_hgclient_pull = hglib.client.hgclient.pull
def hglib_pull(hgclient, source=None, rev=None, update=False):
orig_hgclient_pull(hgclient,
source=str(remote_dir).encode("ascii"),
rev=rev,
update=update)
monkeypatch.setattr(hglib.client.hgclient, "pull", hglib_pull)
return local_dir, remote_dir
def fake_hg_repo(tmpdir):
tmp_path = tmpdir.strpath
dest = os.path.join(tmp_path, "repos")
local = os.path.join(dest, "local")
os.makedirs(local)
hglib.init(local)
os.environ["USER"] = "******"
hg = hglib.open(local)
yield hg, local
hg.close()
def setup_repository(self, path):
repo = self.new_repository(path)
LOG.debug("Setting up the repository at %s", repo.name)
repos_path = self.get_repos_path()
os.chdir(repos_path)
try:
hglib.init(repo.name)
except hglib.error.CommandError as err:
raise exc.VersionControlError(str(err))
repo.setup()
LOG.debug("Trying to add the repo to accessmanager")
self.app.accessmanager.add_repo(repo)
LOG.debug("Repo is now ready.")
def test_handlePushes_space_files(self):
repo = Repository.objects.create(name='mozilla-central',
url='file:///' + self.repo)
with hglib.init(self.repo).open() as hgrepo:
# deliberate trailing space in file name
with open(hgrepo.pathto('file.dtd '), 'w') as fh:
fh.write('''
<!ENTITY key1 "Hello">
<!ENTITY key2 "Cruel">
''')
hgrepo.commit(user="******",
message="initial commit",
addremove=True)
rev0 = hgrepo[0].node().decode('ascii')
timestamp = int(time.time())
pushjs0 = PushJS(100, {
'date': timestamp,
'changesets': [rev0],
'user': '******',
})
handlePushes(repo.pk, [pushjs0])
file_, = File.objects.all()
self.assertEqual(file_.path, 'file.dtd ')
def test_handlePushes_space_files(self):
repo = Repository.objects.create(
name='mozilla-central',
url='file:///' + self.repo
)
hgrepo = hglib.init(self.repo).open()
(open(hgrepo.pathto('file.dtd '), 'w') # deliberate trailing space
.write('''
<!ENTITY key1 "Hello">
<!ENTITY key2 "Cruel">
'''))
hgrepo.commit(user="******",
message="initial commit",
addremove=True)
rev0 = hgrepo[0].node()
timestamp = int(time.time())
pushjs0 = PushJS(100, {
'date': timestamp,
'changesets': [rev0],
'user': '******',
})
handlePushes(repo.pk, [pushjs0])
file_, = File.objects.all()
self.assertEqual(file_.path, 'file.dtd ')
def test_basic(self):
self.client.close()
self.client = None
shutil.rmtree('.hg')
self.client = hglib.init().open()
self.assertTrue(self.client.root().endswith(b('test_init')))
def hgfs_setup_and_teardown():
"""
build up and tear down hg repos to test with.
"""
initial_child_processes = psutil.Process().children()
source_dir = Path(__file__).resolve().parent.joinpath("files")
tempdir = tempfile.TemporaryDirectory()
tempsubdir = tempdir.name / Path("subdir/")
tempsubdir.mkdir()
tempdirPath = Path(tempdir.name)
for file in source_dir.iterdir():
to_file = tempdirPath / file.name
to_file2 = tempsubdir / file.name
shutil.copy(file.as_posix(), to_file.as_posix())
shutil.copy(file.as_posix(), to_file2.as_posix())
client = hglib.init(bytes(tempdirPath.as_posix(), encoding="utf8"))
client.close()
with hglib.open(bytes(tempdirPath.as_posix(), encoding="utf8")) as repo:
repo.add(bytes(tempdirPath.as_posix(), encoding="utf8"))
repo.commit(b"init commit", user="******")
repo.tag(b"test", user="******")
repo.branch(b"test")
repo.commit(b"create test branch", user="******")
repo.bookmark(b"bookmark_test")
try:
yield tempdirPath.as_uri()
finally:
tempdir.cleanup()
for child in psutil.Process().children():
if child not in initial_child_processes:
terminate_process(process=child, kill_children=True)
def test_error_handling(self):
"""Test various bad request parameters to the diff_app
and assure that it responds with the right error codes."""
url = reverse('pushes:diff')
response = self.client.get(url, {})
self.assertEqual(response.status_code, 400)
response = self.client.get(url, {'repo': 'junk'})
self.assertEqual(response.status_code, 404)
with hglib.init(self.repo).open() as hgrepo:
with open(hgrepo.pathto('file.dtd'), 'w') as fh:
fh.write('''
<!ENTITY key1 "Hello">
<!ENTITY key2 "Cruel">
''')
hgrepo.addremove()
hgrepo.commit(user="******",
message="initial commit")
rev0 = hgrepo[0].node().decode('ascii')
Repository.objects.create(name=self.repo_name,
url='http://localhost:8001/%s/' %
self.repo_name)
# missing 'from' and 'to'
response = self.client.get(url, {'repo': self.repo_name})
self.assertEqual(response.status_code, 400)
# missing 'to'
response = self.client.get(url, {'repo': self.repo_name, 'from': rev0})
self.assertEqual(response.status_code, 400)
def test_handlePushes_messedup_revisions(self):
repo = Repository.objects.create(
name='mozilla-central',
url='file:///' + self.repo
)
hgrepo = hglib.init(self.repo).open()
(open(hgrepo.pathto('file.dtd'), 'w')
.write('''
<!ENTITY key1 "Hello">
<!ENTITY key2 "Cruel">
'''))
hgrepo.commit(user="******",
message="initial commit",
addremove=True)
rev0 = hgrepo[0].node()
timestamp = int(time.time())
pushjs0 = PushJS(100, {
'date': timestamp,
'changesets': [rev0[::-1]],
'user': '******',
})
self.assertRaises(KeyError, handlePushes,
repo.pk, [pushjs0])
def test_handlePushes_cause_repoerror(self):
repo = Repository.objects.create(
name='mozilla-central',
url='file:///does/not/exist'
)
with hglib.init(self.repo).open() as hgrepo:
with open(hgrepo.pathto('file.dtd'), 'w') as fh:
fh.write('''
<!ENTITY key1 "Hello">
<!ENTITY key2 "Cruel">
''')
hgrepo.commit(user="******",
message="initial commit",
addremove=True)
rev0 = hgrepo[0].node().decode('ascii')
timestamp = int(time.time())
pushjs0 = PushJS(100, {
'date': timestamp,
'changesets': [rev0],
'user': '******',
})
self.assertRaises(hglib.error.CommandError, handlePushes,
repo.pk, [pushjs0])
def __init__(self, remote_versions_root_path, versions_root_path):
VersionControlBackend.__init__(
self, remote_versions_root_path, versions_root_path
)
client = hglib.init(
dest=self.remote_versions_root_path
)
client.close()
self._mercurial_backend = hglib.clone(
'{0}'.format(self.remote_versions_root_path),
'{0}'.format(self.versions_root_path)
)
self._mercurial_backend.close()
with open(os.path.join(versions_root_path, 'init.txt'), 'w+') as f:
f.write('# init\n')
self._mercurial_backend = hglib.open(
'{0}'.format(self.versions_root_path)
)
path = os.path.join(versions_root_path, 'init.txt').encode()
self._mercurial_backend.add(path)
self._mercurial_backend.commit(
message='first commit', user='******', include=path
)
self._mercurial_backend.push()
def setUp(self):
super(TestPaths4Revs, self).setUp()
hgrepo = hglib.init(self.repo).open()
# Initial commit, r=0
with open(hgrepo.pathto('README'), 'w') as f:
f.write('Initial commit')
hgrepo.commit(user='******',
message='Initial commit', addremove=True)
# Add l10n file, r=1
with open(hgrepo.pathto('f.ftl'), 'w') as f:
f.write('message = text\n')
hgrepo.commit(user='******',
message='Adding file', addremove=True)
# modify l10n file, r=2
with open(hgrepo.pathto('f.ftl'), 'w') as f:
f.write('message = othertext\n')
hgrepo.commit(user='******',
message='Modifying file', addremove=True)
# copy and edit, r=3
hgrepo.copy(hgrepo.pathto('f.ftl'), hgrepo.pathto('copied.ftl'))
with open(hgrepo.pathto('copied.ftl'), 'w') as f:
f.write('message = text\nnew_message = words\n')
hgrepo.commit(user='******',
message='Copying file', addremove=True)
# move and edit, r=4
hgrepo.move(hgrepo.pathto('f.ftl'), hgrepo.pathto('moved.ftl'))
with open(hgrepo.pathto('moved.ftl'), 'w') as f:
f.write('different = text\n')
hgrepo.commit(user='******',
message='Moving file', addremove=True)
# remove, r=5
hgrepo.remove([hgrepo.pathto('copied.ftl')])
hgrepo.commit(user='******',
message='Removing file', addremove=True)
def test_handlePushes_repeated(self):
repo = Repository.objects.create(name='mozilla-central',
url='file:///' + self.repo)
with hglib.init(self.repo).open() as hgrepo:
with open(hgrepo.pathto('file.dtd'), 'w') as fh:
fh.write('''
<!ENTITY key1 "Hello">
<!ENTITY key2 "Cruel">
''')
hgrepo.commit(user="******",
message="initial commit",
addremove=True)
rev0 = hgrepo[0].node().decode('ascii')
timestamp = int(time.time())
pushjs0 = PushJS(100, {
'date': timestamp,
'changesets': [rev0],
'user': '******',
})
# first time
pushes_initial = Push.objects.all().count()
result = handlePushes(repo.pk, [pushjs0])
self.assertEqual(result, 1)
pushes_after = Push.objects.all().count()
self.assertEqual(pushes_initial, pushes_after - 1)
# a second time should be harmless
result = handlePushes(repo.pk, [pushjs0])
self.assertEqual(result, 1)
pushes_after_after = Push.objects.all().count()
self.assertEqual(pushes_after, pushes_after_after)
def fake_hg_repo(tmpdir):
tmp_path = tmpdir.strpath
dest = os.path.join(tmp_path, "repos")
local = os.path.join(dest, "local")
remote = os.path.join(dest, "remote")
for d in [local, remote]:
os.makedirs(d)
hglib.init(d)
os.environ["USER"] = "******"
hg = hglib.open(local)
responses.add_passthru("http://localhost:8000")
yield hg, local, remote
hg.close()
def fake_hg_repo(tmpdir):
tmp_path = tmpdir.strpath
dest = os.path.join(tmp_path, 'repos')
local = os.path.join(dest, 'local')
remote = os.path.join(dest, 'remote')
for d in [local, remote]:
os.makedirs(d)
hglib.init(d)
os.environ['USER'] = '******'
hg = hglib.open(local)
responses.add_passthru('http://localhost:8000')
yield hg, local, remote
hg.close()
def fake_hg_repo(tmpdir):
tmp_path = tmpdir.strpath
dest = os.path.join(tmp_path, "repos")
local = os.path.join(dest, "local")
remote = os.path.join(dest, "remote")
for d in [local, remote]:
os.makedirs(d)
hglib.init(d)
os.environ["USER"] = "******"
hg = hglib.open(local)
hg.branch(b"central")
responses.add_passthru("http://localhost:8000")
yield hg, local, remote
hg.close()
def test_handlePushes_twice(self):
repo = Repository.objects.create(
name='mozilla-central',
url='file://' + self.repo
)
with hglib.init(self.repo).open() as hgrepo:
with open(hgrepo.pathto('file.dtd'), 'w') as fh:
fh.write('''
<!ENTITY key1 "Hello">
<!ENTITY key2 "Cruel">
''')
hgrepo.commit(user="******",
message="initial commit",
addremove=True)
rev0 = hgrepo[0].node().decode('ascii')
timestamp = int(time.time())
pushjs0 = PushJS(100, {
'date': timestamp,
'changesets': [rev0],
'user': '******',
})
result = handlePushes(repo.pk, [pushjs0])
with open(hgrepo.pathto('file.dtd'), 'w') as fh:
fh.write('''
<!ENTITY key1 "Hello">
<!ENTITY key2 "Cruel">
<!ENTITY key3 "World">
''')
hgrepo.commit(user="******",
message="Second commit")
rev1 = hgrepo[1].node().decode('ascii')
# a second time
timestamp = int(time.time())
pushjs0 = PushJS(101, {
'date': timestamp,
'changesets': [rev1],
'user': '******',
})
# re-fetch
repo = Repository.objects.get(pk=repo.pk)
self.assertEqual(repo.changesets.all().count(), 2)
result = handlePushes(repo.pk, [pushjs0])
self.assertEqual(result, 1)
# re-fetch
repo = Repository.objects.get(pk=repo.pk)
self.assertEqual(repo.changesets.all().count(), 3)
def test_handlePushes_twice(self):
repo = Repository.objects.create(
name='mozilla-central',
url='file://' + self.repo
)
hgrepo = hglib.init(self.repo).open()
(open(hgrepo.pathto('file.dtd'), 'w')
.write('''
<!ENTITY key1 "Hello">
<!ENTITY key2 "Cruel">
'''))
hgrepo.commit(user="******",
message="initial commit",
addremove=True)
rev0 = hgrepo[0].node()
timestamp = int(time.time())
pushjs0 = PushJS(100, {
'date': timestamp,
'changesets': [rev0],
'user': '******',
})
result = handlePushes(repo.pk, [pushjs0])
(open(hgrepo.pathto('file.dtd'), 'w')
.write('''
<!ENTITY key1 "Hello">
<!ENTITY key2 "Cruel">
<!ENTITY key3 "World">
'''))
hgrepo.commit(user="******",
message="Second commit")
rev1 = hgrepo[1].node()
# a second time
timestamp = int(time.time())
pushjs0 = PushJS(101, {
'date': timestamp,
'changesets': [rev1],
'user': '******',
})
# re-fetch
repo = Repository.objects.get(pk=repo.pk)
self.assertEqual(repo.changesets.all().count(), 2)
result = handlePushes(repo.pk, [pushjs0])
self.assertEqual(result, 1)
# re-fetch
repo = Repository.objects.get(pk=repo.pk)
self.assertEqual(repo.changesets.all().count(), 3)
def test_bogus_repo_hashes(self):
"""test to satisfy
https://bugzilla.mozilla.org/show_bug.cgi?id=750533
which says that passing unrecognized repo hashes
should yield a 400 Bad Request error.
"""
hgrepo = hglib.init(self.repo).open()
(open(hgrepo.pathto('file.dtd'), 'w')
.write('<!ENTITY key1 "Hello">\n'))
hgrepo.commit(user="******",
message="initial commit",
addremove=True)
rev0 = hgrepo[0].node()
# do this to trigger an exception on Mozilla.Parser.readContents
_content = '<!ENTITY key1 "Hell\xe3">\n'
(codecs.open(hgrepo.pathto('file.dtd'), 'w', 'latin1')
.write(_content))
hgrepo.commit(user="******",
message="Second commit")
rev1 = hgrepo[1].node()
repo_url = 'http://localhost:8001/' + self.repo_name + '/'
Repository.objects.create(
name=self.repo_name,
url=repo_url
)
url = reverse('pushes:diff')
response = self.client.get(url, {
'repo': self.repo_name,
'from': rev0,
'to': rev1
})
self.assertEqual(response.status_code, 200)
response = self.client.get(url, {
'repo': self.repo_name,
'from': 'xxx',
'to': rev1
})
self.assertEqual(response.status_code, 400)
response = self.client.get(url, {
'repo': self.repo_name,
'from': rev0,
'to': 'xxx'
})
self.assertEqual(response.status_code, 400)
def __init__(self, versions, remote_versions):
self.versions_root_path = versions.strpath
self.remote_versions_root_path = remote_versions.strpath
self.base_versions_dir_path = versions.dirname
client = hglib.init(dest=self.remote_versions_root_path)
client.close()
self._mercurial_backend = hglib.clone(
'{0}'.format(self.remote_versions_root_path),
'{0}'.format(self.versions_root_path))
self._repos = {}
def mock_repo(tmpdir, monkeypatch):
"""Create an empty mercurial repo"""
repo_dir = tmpdir / "repo"
# Setup the worker env to use that repo dir
monkeypatch.setattr(bugbug_http, "REPO_DIR", str(repo_dir))
# Silence the clean method
monkeypatch.setattr(bugbug.repository, "clean", lambda repo_dir: True)
# Create the repo
hglib.init(str(repo_dir))
# Add several commits on a test file to create some history
test_file = repo_dir / "test.txt"
repo = hglib.open(str(repo_dir))
for i in range(4):
test_file.write_text(f"Version {i}", encoding="utf-8")
repo.add([str(test_file).encode("utf-8")])
repo.commit(f"Base history {i}", user="******")
return repo_dir, repo
def init():
'''
Return the hg repo object for this session
'''
bp_ = os.path.join(__opts__['cachedir'], 'hgfs')
repos = []
for ind, opt in enumerate(__opts__['hgfs_remotes']):
rp_ = os.path.join(bp_, str(ind))
if not os.path.isdir(rp_):
os.makedirs(rp_)
hglib.init(rp_)
repo = hglib.open(rp_)
refs = repo.config(names='paths')
if not refs:
hgconfpath = os.path.join(rp_, '.hg', 'hgrc')
with salt.utils.fopen(hgconfpath, 'w+') as hgconfig:
hgconfig.write('[paths]\n')
hgconfig.write('default = {0}\n'.format(opt))
repos.append(repo)
repo.close()
return repos
def init():
'''
Return the hg repo object for this session
'''
bp_ = os.path.join(__opts__['cachedir'], 'hgfs')
repos = []
for ind, opt in enumerate(__opts__['hgfs_remotes']):
rp_ = os.path.join(bp_, str(ind))
if not os.path.isdir(rp_):
os.makedirs(rp_)
hglib.init(rp_)
repo = hglib.open(rp_)
refs = repo.config(names='paths')
if not refs:
hgconfpath = os.path.join(rp_, '.hg', 'hgrc')
with salt.utils.fopen(hgconfpath, 'w+') as hgconfig:
hgconfig.write('[paths]\n')
hgconfig.write('default = {0}\n'.format(opt))
repos.append(repo)
repo.close()
return repos
def mock_repository(tmpdir):
'''
Create a dummy mercurial repository
'''
# Init repo
repo_dir = str(tmpdir.mkdir('repo').realpath())
hglib.init(repo_dir)
# Init clean client
client = hglib.open(repo_dir)
client.directory = repo_dir
# Add test.txt file
path = os.path.join(repo_dir, 'test.txt')
with open(path, 'w') as f:
f.write('Hello World\n')
# Initiall commit
client.add(path.encode('utf-8'))
client.commit(b'Hello World', user=b'Tester')
return client
def mock_repository(mock_config):
'''
Create a dummy mercurial repository
'''
# Init repo
hglib.init(mock_config.repo_dir)
# Init clean client
client = hglib.open(mock_config.repo_dir)
# Add test.txt file
path = os.path.join(mock_config.repo_dir, 'test.txt')
with open(path, 'w') as f:
f.write('Hello World\n')
client.add(path.encode('utf-8'))
path = os.path.join(mock_config.repo_dir, 'test.cpp')
with open(path, 'w') as f:
f.write('Hello World\n')
client.add(path.encode('utf-8'))
# Initiall commit
client.commit(b'Hello World', user=b'Tester')
# Write dummy 3rd party file
third_party = os.path.join(mock_config.repo_dir, mock_config.third_party)
with open(third_party, 'w') as f:
f.write('test/dummy')
# Remove pull capabilities
client.pull = Mock(return_value=True)
# Mark clone is available
mock_config.has_local_clone = True
return client
def init():
'''
Return the hg repo object for this session
'''
bp_ = os.path.join(__opts__['cachedir'], 'hgfs')
repos = []
for _, opt in enumerate(__opts__['hgfs_remotes']):
repo_hash = hashlib.md5(opt).hexdigest()
rp_ = os.path.join(bp_, repo_hash)
if not os.path.isdir(rp_):
os.makedirs(rp_)
if not os.listdir(rp_):
# Only init if the directory is empty.
hglib.init(rp_)
try:
repo = hglib.open(rp_)
except hglib.error.ServerError:
log.error(
'Cache path {0} (corresponding remote: {1}) exists but is not '
'a valid mercurial repository. You will need to manually '
'delete this directory on the master to continue to use this '
'hgfs remote.'.format(rp_, opt)
)
continue
refs = repo.config(names='paths')
if not refs:
# Write an hgrc defining the remote URI
hgconfpath = os.path.join(rp_, '.hg', 'hgrc')
with salt.utils.fopen(hgconfpath, 'w+') as hgconfig:
hgconfig.write('[paths]\n')
hgconfig.write('default = {0}\n'.format(opt))
repos.append(repo)
repo.close()
return repos
def test_handlePushes(self):
repo = Repository.objects.create(
name='mozilla-central',
url='file:///' + self.repo
)
hgrepo = hglib.init(self.repo).open()
(open(hgrepo.pathto('file.dtd'), 'w')
.write('''
<!ENTITY key1 "Hello">
<!ENTITY key2 "Cruel">
'''))
hgrepo.commit(user="******",
message="initial commit",
addremove=True)
rev0 = hgrepo[0].node()
timestamp = int(time.time())
push_id = 100
username = '******'
pushjs0 = PushJS(push_id, {
'date': timestamp,
'changesets': [rev0],
'user': username,
})
result = handlePushes(repo.pk, [pushjs0])
self.assertEqual(result, 1)
# expect all of these to have been created
push, = Push.objects.all()
branch, = Branch.objects.all()
changeset, = push.changesets.all()
self.assertEqual(push.repository, repo)
self.assertEqual(push.push_id, push_id)
self.assertEqual(push.user, username)
self.assertEqual(
push.push_date,
datetime.datetime.utcfromtimestamp(timestamp)
)
self.assertEqual(changeset.description, 'initial commit')
self.assertEqual(changeset.user, 'Jane Doe <*****@*****.**>')
self.assertEqual(hglib.util.b(changeset.revision), rev0)
self.assertEqual(changeset.branch, branch)
self.assertEqual(branch.name, 'default')
def test_handlePushes(self):
repo = Repository.objects.create(
name='mozilla-central',
url='file:///' + self.repo
)
with hglib.init(self.repo).open() as hgrepo:
with open(hgrepo.pathto('file.dtd'), 'w') as fh:
fh.write('''
<!ENTITY key1 "Hello">
<!ENTITY key2 "Cruel">
''')
hgrepo.commit(user="******",
message="initial commit",
addremove=True)
rev0 = hgrepo[0].node().decode('ascii')
timestamp = int(time.time())
push_id = 100
username = '******'
pushjs0 = PushJS(push_id, {
'date': timestamp,
'changesets': [rev0],
'user': username,
})
result = handlePushes(repo.pk, [pushjs0])
self.assertEqual(result, 1)
# expect all of these to have been created
push, = Push.objects.all()
branch, = Branch.objects.all()
changeset, = push.changesets.all()
self.assertEqual(push.repository, repo)
self.assertEqual(push.push_id, push_id)
self.assertEqual(push.user, username)
self.assertEqual(
push.push_date,
datetime.datetime.utcfromtimestamp(timestamp)
)
self.assertEqual(changeset.description, 'initial commit')
self.assertEqual(changeset.user, 'Jane Doe <*****@*****.**>')
self.assertEqual(changeset.revision, rev0)
self.assertEqual(changeset.branch, branch)
self.assertEqual(branch.name, 'default')
def create_repo(self, repo_name, repo_type):
path = os.path.join(self.base_versions_dir_path, repo_name)
if repo_type == 'mercurial':
client = hglib.init(dest=path)
client.close()
elif repo_type == 'git':
repo = Repo.init(path=path)
repo.close()
else:
raise RuntimeError('Unknown repository type provided')
self._repos[repo_name] = {
'path': path,
'type': repo_type,
}
def test_get_modified_notebooks_deleted(self, d):
os.chdir(d.path)
client = hglib.init()
client.open()
d.makedir('first')
d.makedir('first/second')
path = d.write('first/second/1.ipynb', 'initial')
self.assertEqual(d.read('first/second/1.ipynb'), 'initial')
client.add('first/second/1.ipynb')
client.commit("message")
file = open(path, 'w')
file.write("modified")
file.close()
self.assertEqual(d.read('first/second/1.ipynb'), 'modified')
os.chdir(os.path.join(d.path, 'first'))
os.remove(path)
adapter = HgAdapter()
result = adapter.get_modified_notebooks()
self.assertTrue(len(result) == 0)
def __init__(self, path = '', src = None, uname = '', passwd = ''): """Initialize two repositories. Load local repository if exists. If not exists, clone from remote, or create new. :param path: :param src: :param uname: :param passwd: """ global hgUsefull self.User = uname self.Password = passwd self.AddRemove = True self._client = None self._local = True self._repoPath = os.path.join(path, '.hg') self._repoPathLocal = os.path.join(path, '.hg-local') self._repoPathRemote = os.path.join(path, '.hg-remote') if hgUsefull: conf = ['auth.def.prefix=*', '--config', 'auth.def.schemes=http https', '--config', 'auth.def.username=%s' % self.User, '--config', 'auth.def.password=%s' % self.Password] try: try: self._client = hglib.open(path, configs=conf) except hglib.error.ServerError: if src: self._client = hglib.clone(src, path, configs=conf) shutil.copytree(self._repoPath, self._repoPathRemote) self._client.open() else: self._client = hglib.init(path, configs=conf) shutil.copytree(self._repoPath, self._repoPathRemote) shutil.copytree(self._repoPath, self._repoPathLocal) self._client.open() except Exception: # not found HG? hgUsefull = False
def test_error_handling(self):
"""Test various bad request parameters to the diff_app
and assure that it responds with the right error codes."""
hgrepo = hglib.init(self.repo).open()
url = reverse('pushes:diff')
response = self.client.get(url, {})
self.assertEqual(response.status_code, 400)
response = self.client.get(url, {'repo': 'junk'})
self.assertEqual(response.status_code, 404)
(open(hgrepo.pathto('file.dtd'), 'w')
.write('''
<!ENTITY key1 "Hello">
<!ENTITY key2 "Cruel">
'''))
hgrepo.addremove()
hgrepo.commit(user="******",
message="initial commit")
rev0 = hgrepo[0].node()
hgrepo.close()
Repository.objects.create(
name=self.repo_name,
url='http://localhost:8001/%s/' % self.repo_name
)
# missing 'from' and 'to'
response = self.client.get(url, {'repo': self.repo_name})
self.assertEqual(response.status_code, 400)
# missing 'to'
response = self.client.get(url, {
'repo': self.repo_name,
'from': rev0
})
self.assertEqual(response.status_code, 400)
def test_handlePushes_repeated(self):
repo = Repository.objects.create(
name='mozilla-central',
url='file:///' + self.repo
)
hgrepo = hglib.init(self.repo).open()
(open(hgrepo.pathto('file.dtd'), 'w')
.write('''
<!ENTITY key1 "Hello">
<!ENTITY key2 "Cruel">
'''))
hgrepo.commit(user="******",
message="initial commit",
addremove=True)
rev0 = hgrepo[0].node()
timestamp = int(time.time())
pushjs0 = PushJS(100, {
'date': timestamp,
'changesets': [rev0],
'user': '******',
})
# first time
pushes_initial = Push.objects.all().count()
result = handlePushes(repo.pk, [pushjs0])
self.assertEqual(result, 1)
pushes_after = Push.objects.all().count()
self.assertEqual(pushes_initial, pushes_after - 1)
# a second time should be harmless
result = handlePushes(repo.pk, [pushjs0])
self.assertEqual(result, 1)
pushes_after_after = Push.objects.all().count()
self.assertEqual(pushes_after, pushes_after_after)
def init(self):
hglib.init(self.path)
with io.open(join(self.path, '.hg', 'hgrc'), 'w', encoding="utf-8") as fd:
fd.write(_hg_config)
self._repo = hglib.open(self.path)
def __init__(self,*args,**kwargs):
super(HgStoreMixin,self).__init__(*args,**kwargs)
if kwargs.get('create',None):
hglib.init(self.get_path(),encoding="utf-8")
self.repo = hglib.open(self.get_path(),encoding="utf-8")
def test_get_unmerged_notebooks_empty(self, d):
os.chdir(os.path.join(d.path))
hglib.init()
adapter = HgAdapter()
result = adapter.get_unmerged_notebooks()
self.assertTrue(result == [])
def init(self):
hglib.init(self.path)
with io.open(join(self.path, '.hg', 'hgrc'), 'w', encoding="utf-8") as fd:
fd.write(_hg_config)
self._repo = hglib.open(self.path.encode(sys.getfilesystemencoding()),
encoding=self.encoding)
def hg_env(post_db_setup, _django_cursor_wrapper):
from django.conf import settings
import pytest_pootle
from pytest_pootle.factories import (
ProjectDBFactory, TranslationProjectFactory)
from pootle_fs.utils import FSPlugin
from pootle_language.models import Language
import tempfile
with _django_cursor_wrapper:
project0 = ProjectDBFactory(
source_language=Language.objects.get(code="en"),
code="hg_project_0")
language0 = Language.objects.get(code="language0")
TranslationProjectFactory(project=project0, language=language0)
initial_src_path = os.path.abspath(
os.path.join(
os.path.dirname(pytest_pootle.__file__),
"data/fs/example_fs"))
fs_dir = tempfile.mkdtemp()
settings.POOTLE_FS_PATH = fs_dir
repo_path = os.path.join(fs_dir, "__hg_src_project_0__")
hglib.init(repo_path)
with tmp_hg(repo_path) as (tmp_repo_path, tmp_repo):
with get_dir_util() as dir_util:
dir_util.copy_tree(initial_src_path, tmp_repo_path)
tmp_repo.add()
tmp_repo.commit("Initial commit")
tmp_repo.push()
project0.config["pootle_fs.fs_type"] = "hg"
project0.config["pootle_fs.fs_url"] = repo_path
project0.config["pootle_fs.translation_paths"] = {
"default": "/<language_code>/<dir_path>/<filename>.<ext>"}
plugin = FSPlugin(project0)
plugin.add()
plugin.fetch()
plugin.sync()
# create_test_suite(plugin)
project1 = ProjectDBFactory(
source_language=Language.objects.get(code="en"),
code="hg_project_1")
TranslationProjectFactory(project=project1, language=language0)
repo_path = os.path.join(fs_dir, "__hg_src_project_1__")
hglib.init(repo_path)
with tmp_hg(repo_path) as (tmp_repo_path, tmp_repo):
with get_dir_util() as dir_util:
dir_util.copy_tree(initial_src_path, tmp_repo_path)
tmp_repo.add()
tmp_repo.commit("Initial commit")
tmp_repo.push()
project1.config["pootle_fs.fs_type"] = "hg"
project1.config["pootle_fs.fs_url"] = repo_path
project1.config["pootle_fs.translation_paths"] = {
"default": "/<language_code>/<dir_path>/<filename>.<ext>"}
def init():
'''
Return a list of hglib objects for the various hgfs remotes
'''
bp_ = os.path.join(__opts__['cachedir'], 'hgfs')
new_remote = False
repos = []
per_remote_defaults = {}
for param in PER_REMOTE_OVERRIDES:
per_remote_defaults[param] = \
six.text_type(__opts__['hgfs_{0}'.format(param)])
for remote in __opts__['hgfs_remotes']:
repo_conf = copy.deepcopy(per_remote_defaults)
if isinstance(remote, dict):
repo_url = next(iter(remote))
per_remote_conf = dict(
[(key, six.text_type(val)) for key, val in
six.iteritems(salt.utils.repack_dictlist(remote[repo_url]))]
)
if not per_remote_conf:
log.error(
'Invalid per-remote configuration for hgfs remote {0}. If '
'no per-remote parameters are being specified, there may '
'be a trailing colon after the URL, which should be '
'removed. Check the master configuration file.'
.format(repo_url)
)
_failhard()
branch_method = \
per_remote_conf.get('branch_method',
per_remote_defaults['branch_method'])
if branch_method not in VALID_BRANCH_METHODS:
log.error(
'Invalid branch_method \'{0}\' for remote {1}. Valid '
'branch methods are: {2}. This remote will be ignored.'
.format(branch_method, repo_url,
', '.join(VALID_BRANCH_METHODS))
)
_failhard()
per_remote_errors = False
for param in (x for x in per_remote_conf
if x not in PER_REMOTE_OVERRIDES):
log.error(
'Invalid configuration parameter \'{0}\' for remote {1}. '
'Valid parameters are: {2}. See the documentation for '
'further information.'.format(
param, repo_url, ', '.join(PER_REMOTE_OVERRIDES)
)
)
per_remote_errors = True
if per_remote_errors:
_failhard()
repo_conf.update(per_remote_conf)
else:
repo_url = remote
if not isinstance(repo_url, six.string_types):
log.error(
'Invalid hgfs remote {0}. Remotes must be strings, you may '
'need to enclose the URL in quotes'.format(repo_url)
)
_failhard()
try:
repo_conf['mountpoint'] = salt.utils.url.strip_proto(
repo_conf['mountpoint']
)
except TypeError:
# mountpoint not specified
pass
hash_type = getattr(hashlib, __opts__.get('hash_type', 'md5'))
repo_hash = hash_type(repo_url).hexdigest()
rp_ = os.path.join(bp_, repo_hash)
if not os.path.isdir(rp_):
os.makedirs(rp_)
if not os.listdir(rp_):
# Only init if the directory is empty.
hglib.init(rp_)
new_remote = True
try:
repo = hglib.open(rp_)
except hglib.error.ServerError:
log.error(
'Cache path {0} (corresponding remote: {1}) exists but is not '
'a valid mercurial repository. You will need to manually '
'delete this directory on the master to continue to use this '
'hgfs remote.'.format(rp_, repo_url)
)
_failhard()
except Exception as exc:
log.error(
'Exception \'{0}\' encountered while initializing hgfs remote '
'{1}'.format(exc, repo_url)
)
_failhard()
try:
refs = repo.config(names='paths')
except hglib.error.CommandError:
refs = None
# Do NOT put this if statement inside the except block above. Earlier
# versions of hglib did not raise an exception, so we need to do it
# this way to support both older and newer hglib.
if not refs:
# Write an hgrc defining the remote URL
hgconfpath = os.path.join(rp_, '.hg', 'hgrc')
with salt.utils.fopen(hgconfpath, 'w+') as hgconfig:
hgconfig.write('[paths]\n')
hgconfig.write('default = {0}\n'.format(repo_url))
repo_conf.update({
'repo': repo,
'url': repo_url,
'hash': repo_hash,
'cachedir': rp_,
'lockfile': os.path.join(__opts__['cachedir'],
'hgfs',
'{0}.update.lk'.format(repo_hash))
})
repos.append(repo_conf)
repo.close()
if new_remote:
remote_map = os.path.join(__opts__['cachedir'], 'hgfs/remote_map.txt')
try:
with salt.utils.fopen(remote_map, 'w+') as fp_:
timestamp = datetime.now().strftime('%d %b %Y %H:%M:%S.%f')
fp_.write('# hgfs_remote map as of {0}\n'.format(timestamp))
for repo in repos:
fp_.write('{0} = {1}\n'.format(repo['hash'], repo['url']))
except OSError:
pass
else:
log.info('Wrote new hgfs_remote map to {0}'.format(remote_map))
return repos
def init():
'''
Return a list of hglib objects for the various hgfs remotes
'''
bp_ = os.path.join(__opts__['cachedir'], 'hgfs')
new_remote = False
repos = []
hgfs_remotes = salt.utils.repack_dictlist(__opts__['hgfs_remotes'])
for repo_uri, repo_conf_params in hgfs_remotes.iteritems():
# Validate and compile per-remote configuration parameters, if present
repo_conf = dict([(x, None) for x in PER_REMOTE_PARAMS])
if repo_conf_params is not None:
repo_conf_params = salt.utils.repack_dictlist(repo_conf_params)
if not repo_conf_params:
log.error(
'Invalid per-remote configuration for remote {0!r}'
.format(repo_uri)
)
else:
for param, value in repo_conf_params.iteritems():
if param in PER_REMOTE_PARAMS:
repo_conf[param] = value
else:
log.error(
'Invalid configuration parameter {0!r} in remote '
'{1!r}. Valid parameters are: {2}. See the '
'documentation for further information.'
.format(
param, repo_uri, ', '.join(PER_REMOTE_PARAMS)
)
)
try:
repo_conf['mountpoint'] = salt.utils.strip_proto(
repo_conf['mountpoint']
)
except TypeError:
# mountpoint not specified
pass
repo_hash = hashlib.md5(repo_uri).hexdigest()
rp_ = os.path.join(bp_, repo_hash)
if not os.path.isdir(rp_):
os.makedirs(rp_)
if not os.listdir(rp_):
# Only init if the directory is empty.
hglib.init(rp_)
new_remote = True
try:
repo = hglib.open(rp_)
except hglib.error.ServerError:
log.error(
'Cache path {0} (corresponding remote: {1}) exists but is not '
'a valid mercurial repository. You will need to manually '
'delete this directory on the master to continue to use this '
'hgfs remote.'.format(rp_, repo_uri)
)
continue
refs = repo.config(names='paths')
if not refs:
# Write an hgrc defining the remote URI
hgconfpath = os.path.join(rp_, '.hg', 'hgrc')
with salt.utils.fopen(hgconfpath, 'w+') as hgconfig:
hgconfig.write('[paths]\n')
hgconfig.write('default = {0}\n'.format(repo_uri))
repo_conf.update({
'repo': repo,
'uri': repo_uri,
'hash': repo_hash,
'cachedir': rp_
})
repos.append(repo_conf)
repo.close()
if new_remote:
remote_map = os.path.join(__opts__['cachedir'], 'hgfs/remote_map.txt')
try:
with salt.utils.fopen(remote_map, 'w+') as fp_:
timestamp = datetime.now().strftime('%d %b %Y %H:%M:%S.%f')
fp_.write('# hgfs_remote map as of {0}\n'.format(timestamp))
for repo_conf in repos:
fp_.write(
'{0} = {1}\n'.format(
repo_conf['hash'], repo_conf['uri']
)
)
except OSError:
pass
else:
log.info('Wrote new hgfs_remote map to {0}'.format(remote_map))
return repos
def init():
'''
Return a list of hglib objects for the various hgfs remotes
'''
bp_ = os.path.join(__opts__['cachedir'], 'hgfs')
new_remote = False
repos = []
per_remote_defaults = {}
for param in PER_REMOTE_PARAMS:
per_remote_defaults[param] = \
_text_type(__opts__['hgfs_{0}'.format(param)])
for remote in __opts__['hgfs_remotes']:
repo_conf = copy.deepcopy(per_remote_defaults)
if isinstance(remote, dict):
repo_url = next(iter(remote))
per_remote_conf = dict(
[(key, _text_type(val)) for key, val in
salt.utils.repack_dictlist(remote[repo_url]).items()]
)
if not per_remote_conf:
log.error(
'Invalid per-remote configuration for remote {0}. If no '
'per-remote parameters are being specified, there may be '
'a trailing colon after the URI, which should be removed. '
'Check the master configuration file.'.format(repo_url)
)
branch_method = \
per_remote_conf.get('branch_method',
per_remote_defaults['branch_method'])
if branch_method not in VALID_BRANCH_METHODS:
log.error(
'Invalid branch_method {0!r} for remote {1}. Valid '
'branch methods are: {2}. This remote will be ignored.'
.format(branch_method, repo_url,
', '.join(VALID_BRANCH_METHODS))
)
continue
for param in (x for x in per_remote_conf
if x not in PER_REMOTE_PARAMS):
log.error(
'Invalid configuration parameter {0!r} for remote {1}. '
'Valid parameters are: {2}. See the documentation for '
'further information.'.format(
param, repo_url, ', '.join(PER_REMOTE_PARAMS)
)
)
per_remote_conf.pop(param)
repo_conf.update(per_remote_conf)
else:
repo_url = remote
if not isinstance(repo_url, string_types):
log.error(
'Invalid gitfs remote {0}. Remotes must be strings, you may '
'need to enclose the URI in quotes'.format(repo_url)
)
continue
try:
repo_conf['mountpoint'] = salt.utils.strip_proto(
repo_conf['mountpoint']
)
except TypeError:
# mountpoint not specified
pass
hash_type = getattr(hashlib, __opts__.get('hash_type', 'md5'))
repo_hash = hash_type(repo_url).hexdigest()
rp_ = os.path.join(bp_, repo_hash)
if not os.path.isdir(rp_):
os.makedirs(rp_)
if not os.listdir(rp_):
# Only init if the directory is empty.
hglib.init(rp_)
new_remote = True
try:
repo = hglib.open(rp_)
except hglib.error.ServerError:
log.error(
'Cache path {0} (corresponding remote: {1}) exists but is not '
'a valid mercurial repository. You will need to manually '
'delete this directory on the master to continue to use this '
'hgfs remote.'.format(rp_, repo_url)
)
continue
refs = repo.config(names='paths')
if not refs:
# Write an hgrc defining the remote URI
hgconfpath = os.path.join(rp_, '.hg', 'hgrc')
with salt.utils.fopen(hgconfpath, 'w+') as hgconfig:
hgconfig.write('[paths]\n')
hgconfig.write('default = {0}\n'.format(repo_url))
repo_conf.update({
'repo': repo,
'url': repo_url,
'hash': repo_hash,
'cachedir': rp_
})
repos.append(repo_conf)
repo.close()
if new_remote:
remote_map = os.path.join(__opts__['cachedir'], 'hgfs/remote_map.txt')
try:
with salt.utils.fopen(remote_map, 'w+') as fp_:
timestamp = datetime.now().strftime('%d %b %Y %H:%M:%S.%f')
fp_.write('# hgfs_remote map as of {0}\n'.format(timestamp))
for repo in repos:
fp_.write('{0} = {1}\n'.format(repo['hash'], repo['url']))
except OSError:
pass
else:
log.info('Wrote new hgfs_remote map to {0}'.format(remote_map))
return repos
def setUp(self):
super(TestMercurialRevisionFinder, self).setUp()
hglib.init(self.tmpdir)
def hgInit( wc_path ):
hglib.init( str(wc_path).encode('utf-8') )
