def clone_repositories(self, repository_set: set) -> None:
# Get path to mercurial repositories.
directory = self.__settings["analysis_workspace"] + self.__settings[
"repository_folder"] + "hg/"
# Create path for mercurial repositories.
if not os.path.exists(directory):
os.makedirs(directory)
# Iterate over every url and clone repositories.
for url in repository_set:
# Get repo name
regex_result = re.search(REGEX_REPO_NAME, url)
repo_name = regex_result.group(REGEX_REPO_NAME_GROUP)
# Notify user.
logging.info("[MercurialRepositoryCloner]: Cloning repository " +
repo_name + " from " + url + "...")
try:
# Create repo directory.
repo_directory = directory + repo_name
if not os.path.exists(repo_directory):
os.makedirs(repo_directory)
# Clone repository.
hglib.clone(url, repo_directory)
except hglib.error.CommandError:
logging.warning(
"[MercurialRepositoryCloner]: Could not clone repository "
+ repo_name)
def __init__(self, url, mirror_path):
# TODO: shared repositories in Mercurial are only possible
# through an extension, and it's not clear how to use those in
# this context. So here, we always make full clones for
# each of the environments.
self._path = os.path.abspath(mirror_path)
self._pulled = False
if hglib is None:
raise ImportError("hglib")
if self.is_local_repo(url):
# Local repository, no need for mirror
self._path = os.path.abspath(url)
self._pulled = True
elif not self.is_local_repo(self._path):
if os.path.exists(self._path):
self._raise_bad_mirror_error(self._path)
# Clone is missing
log.info("Cloning project")
if url.startswith("hg+"):
url = url[3:]
# Mercurial branches are global, so there is no need for
# an analog of git --mirror
hglib.clone(url, dest=self._path, noupdate=True)
self._repo = hglib.open(self._path)
def __init__(self, url, mirror_path):
# TODO: shared repositories in Mercurial are only possible
# through an extension, and it's not clear how to use those in
# this context. So here, we always make full clones for
# each of the environments.
self._path = os.path.abspath(mirror_path)
self._pulled = False
if hglib is None:
raise ImportError("hglib")
if self.is_local_repo(url):
# Local repository, no need for mirror
self._path = os.path.abspath(url)
self._pulled = True
elif not self.is_local_repo(self._path):
if os.path.exists(self._path):
self._raise_bad_mirror_error(self._path)
# Clone is missing
log.info("Cloning project")
if url.startswith("hg+"):
url = url[3:]
# Mercurial branches are global, so there is no need for
# an analog of git --mirror
hglib.clone(url, dest=self._path, noupdate=True)
self._repo = hglib.open(self._path)
def fork(self, repo):
try:
hglib.clone(repo['origin_url'],
repo['dir'],
updaterev='null',
pull=True)
except Exception, e:
raise TracError(_("Failed to clone repository: ") + str(e))
def tmp_hg(url):
from django.conf import settings
tmp_repo_path = os.path.join(settings.POOTLE_FS_PATH, "__tmp_hg_src__")
if os.path.exists(tmp_repo_path):
shutil.rmtree(tmp_repo_path)
hglib.clone(url, tmp_repo_path)
yield tmp_repo_path, hglib.open(tmp_repo_path)
shutil.rmtree(tmp_repo_path)
def ensure_updated_clone(path, remote):
"""Ensure path contains an up to date clone of the repo at remote."""
# TODO: hg robustcheckout extension.
if not os.path.exists(path):
hglib.clone(source=remote, dest=path, noupdate=True)
with hglib.open(path=path) as client:
assert client.pull()
def _cloneRepository(self, clone_url, clone_dir):
"""Clone repository
:param clone_url: repository clone url
:type clone_url: string
:param clone_dir: directory to clone repository to
:type clone_dir: string
"""
hglib.clone(b(clone_url), b(clone_dir))
def tmp_hg(url):
from django.conf import settings
tmp_repo_path = os.path.join(
settings.POOTLE_FS_PATH, "__tmp_hg_src__")
if os.path.exists(tmp_repo_path):
shutil.rmtree(tmp_repo_path)
hglib.clone(url, tmp_repo_path)
yield tmp_repo_path, hglib.open(tmp_repo_path)
shutil.rmtree(tmp_repo_path)
def handle(self, **options):
update = options.get('update', False)
all = options.get('all', False)
pull_args = {}
if update:
pull_args['update'] = True
from life.models import Repository, Changeset
import hglib
import os.path
from django.conf import settings
def resolve(path):
return os.path.join(settings.REPOSITORY_BASE, *path.split('/'))
# check for last push helper file
if not all and os.path.isfile(resolve('.latest_cs')):
latest_cs = int(open(resolve('.latest_cs')).read())
repos = (Repository.objects
.filter(changesets__id__gt=latest_cs)
.distinct())
else:
repos = Repository.objects.all()
latest_cs = Changeset.objects.order_by('-pk')[0].id
for repo in repos:
repopath = str(repo.local_path())
self.stdout.write(repo.name + '\n')
if not os.path.isdir(os.path.join(repopath, '.hg')):
# new repo, need to clone
if os.path.isdir(repopath):
self.stdout.write((
"\n\nCannot clone %s, "
"existing directory in the way\n\n") % repo.name)
continue
_parent = os.path.dirname(repopath)
if not os.path.isdir(_parent):
try:
os.makedirs(_parent)
except Exception as e:
self.stdout.write(
("\n\nFailed to prepare for clone, %s\n\n"
% str(e))
)
continue
try:
hglib.clone(str(repo.url), repopath, noupdate=not update)
except hglib.error.CommandError as e:
self.stdout.write('Clone problems, %s' % str(e))
else:
with hglib.open(repopath) as client:
try:
client.pull(**pull_args)
except hglib.error.CommandError as e:
self.stdout.write('Pull problems, %s' % str(e))
open(resolve('.latest_cs'), 'w').write('%i\n' % latest_cs)
def handle(self, **options):
update = options.get('update', False)
all = options.get('all', False)
pull_args = {}
if update:
pull_args['update'] = True
from life.models import Repository, Changeset
import hglib
import os.path
from django.conf import settings
def resolve(path):
return os.path.join(settings.REPOSITORY_BASE, *path.split('/'))
# check for last push helper file
if not all and os.path.isfile(resolve('.latest_cs')):
latest_cs = int(open(resolve('.latest_cs')).read())
repos = (Repository.objects.filter(
changesets__id__gt=latest_cs).distinct())
else:
repos = Repository.objects.all()
latest_cs = Changeset.objects.order_by('-pk')[0].id
for repo in repos:
repopath = str(repo.local_path())
self.stdout.write(repo.name + '\n')
if not os.path.isdir(os.path.join(repopath, '.hg')):
# new repo, need to clone
if os.path.isdir(repopath):
self.stdout.write(
("\n\nCannot clone %s, "
"existing directory in the way\n\n") % repo.name)
continue
_parent = os.path.dirname(repopath)
if not os.path.isdir(_parent):
try:
os.makedirs(_parent)
except Exception as e:
self.stdout.write(
("\n\nFailed to prepare for clone, %s\n\n" %
str(e)))
continue
try:
hglib.clone(str(repo.url), repopath, noupdate=not update)
except hglib.error.CommandError as e:
self.stdout.write('Clone problems, %s' % str(e))
else:
with hglib.open(repopath) as client:
try:
client.pull(**pull_args)
except hglib.error.CommandError as e:
self.stdout.write('Pull problems, %s' % str(e))
open(resolve('.latest_cs'), 'w').write('%i\n' % latest_cs)
def pull(self):
if not self.is_cloned:
logger.info(
"Cloning hg repository(%s): %s"
% (self.project.code, self.fs_url))
hglib.clone(self.fs_url, self.project.local_fs_path)
else:
logger.info(
"Pulling hg repository(%s): %s"
% (self.project.code, self.fs_url))
self.repo.pull()
def clone(self, source):
# Use of robustcheckout here would work, but is probably not worth
# the hassle as most of the benefits come from repeated working
# directory creation. Since this is a one-time clone and is unlikely
# to happen very often, we can get away with a standard clone.
hglib.clone(
source=source,
dest=self.path,
encoding=self.ENCODING,
configs=self._config_to_list(),
)
def _mercurial_clone(repos_path, repo, remote):
LOGGER.info('Cloning {0}..'.format(repo))
try:
hglib.clone('{0}/{1}'.format(remote, repo),
'{0}/{1}'.format(repos_path, repo))
except Exception as exc:
err = 'Failed to clone {0} repository. ' \
'Description: {1}'.format(repo, exc.args)
return {'repo': repo, 'status': 1, 'description': err}
return {'repo': repo, 'status': 0, 'description': None}
def _scm_clone(repo_url, dst_path):
try:
GitCmd.git_clone_repo(repo_url, dst_path)
return REPO_TYPE_GIT
except:
git_exc_info = sys.exc_info()
try:
hglib.clone(repo_url, dst_path)
return REPO_TYPE_MERCURIAL
except:
raise ValueError("Failed to clone repo '%s' to '%s':\nGIT:\n%s\nMercurial:\n%s\n",
repo_url, dst_path, str(git_exc_info), str(sys.exc_info()))
def clone(self):
logger.info("Checking out tip", repo=self.url, mode=self.checkout_mode)
if self.checkout_mode == "batch":
batch_checkout(self.url, self.dir, b"tip", self.batch_size)
elif self.checkout_mode == "robust":
robust_checkout(self.url, self.dir, b"tip")
else:
hglib.clone(self.url, self.dir)
logger.info("Full checkout finished")
# Setup repo in main process
self.repo.setcbout(lambda msg: logger.info("Mercurial", stdout=msg))
self.repo.setcberr(lambda msg: logger.info("Mercurial", stderr=msg))
def clone(self):
logger.info('Checking out tip', repo=self.url, mode=self.checkout_mode)
if self.checkout_mode == 'batch':
batch_checkout(self.url, self.dir, b'tip', self.batch_size)
elif self.checkout_mode == 'robust':
robust_checkout(self.url, self.dir, b'tip')
else:
hglib.clone(self.url, self.dir)
logger.info('Full checkout finished')
# Setup repo in main process
self.repo = hglib.open(self.dir)
self.repo.setcbout(lambda msg: logger.info('Mercurial', stdout=msg))
self.repo.setcberr(lambda msg: logger.info('Mercurial', stderr=msg))
def get_yt_supp():
import hglib
supp_path = os.path.join(os.environ["YT_DEST"], "src", "yt-supplemental")
# Now we check that the supplemental repository is checked out.
if not os.path.isdir(supp_path):
print()
print("*** The yt-supplemental repository is not checked ***")
print("*** out. I can do this for you, but because this ***")
print("*** is a delicate act, I require you to respond ***")
print("*** to the prompt with the word 'yes'. ***")
print()
response = input("Do you want me to try to check it out? ")
if response != "yes":
print()
print("Okay, I understand. You can check it out yourself.")
print("This command will do it:")
print()
print(
"$ hg clone http://bitbucket.org/yt_analysis/yt-supplemental/ ", end=" "
)
print(f"{supp_path}")
print()
sys.exit(1)
rv = hglib.clone("http://bitbucket.org/yt_analysis/yt-supplemental/", supp_path)
if rv:
print("Something has gone wrong. Quitting.")
sys.exit(1)
# Now we think we have our supplemental repository.
return supp_path
def hg_get(self, port):
if not os.path.exists(port.sources_root() + '/.hg'):
logging.debug('Cloning Mercurial repository {0} into {1}'.format(
port.portname, port.sources_root()))
hglib.clone(port.source.get('hg'), port.sources_root())
else:
logging.debug('Using existing repository in {0}'.format(
port.sources_root()))
client = hglib.open(port.sources_root())
version = ask_version(client.tags(), port)
if version is None:
raise Exception('No version selected and no default version found')
logging.debug('Version {0} was selected'.format(version))
client.update(version.encode())
logging.debug('Source is now at Version {0}'.format(version))
port.version = version
def __init__(self, remote_versions_root_path, versions_root_path):
VersionControlBackend.__init__(
self, remote_versions_root_path, versions_root_path
)
client = hglib.init(
dest=self.remote_versions_root_path
)
client.close()
self._mercurial_backend = hglib.clone(
'{0}'.format(self.remote_versions_root_path),
'{0}'.format(self.versions_root_path)
)
self._mercurial_backend.close()
with open(os.path.join(versions_root_path, 'init.txt'), 'w+') as f:
f.write('# init\n')
self._mercurial_backend = hglib.open(
'{0}'.format(self.versions_root_path)
)
path = os.path.join(versions_root_path, 'init.txt').encode()
self._mercurial_backend.add(path)
self._mercurial_backend.commit(
message='first commit', user='******', include=path
)
self._mercurial_backend.push()
def _clone(self):
self._log("clone %s" % (self.basedir), category="clone")
self.client = hglib.clone(self.remoteUrl,
self.basedir,
branch=self.branchname,
configs=self._configs)
self.pull()
self.verify()
def clone_new_repo(source=None):
"""Clones a new copy of yt_analysis/yt and returns a path to it"""
path = tempfile.mkdtemp()
dest_repo_path = path + '/yt-backport'
if source is None:
source = YT_REPO
hglib.clone(source=source, dest=dest_repo_path)
with hglib.open(dest_repo_path) as client:
# Changesets that are on the yt branch but aren't topological ancestors
# of whichever changeset the experimental bookmark is pointing at
bookmarks, _ = client.bookmarks()
bookmark_names = [b[0] for b in bookmarks]
if 'experimental' in bookmark_names:
client.update('heads(branch(yt) - ::bookmark(experimental))')
else:
client.update('heads(branch(yt))')
return dest_repo_path
def sync_hg(store, path):
"""Syncs the local documents via hg."""
storedir, _ = os.path.split(path)
# get or update the storage
if os.path.isdir(os.path.join(storedir, ".hg")):
client = hglib.open(storedir)
client.pull(update=True, force=True)
else:
# Strip off three characters for hg+
client = hglib.clone(store['url'][3:], storedir)
def sync_hg(store, path):
"""Syncs the local documents via hg."""
storedir, _ = os.path.split(path)
# get or update the storage
if os.path.isdir(os.path.join(storedir, ".hg")):
client = hglib.open(storedir)
client.pull(update=True, force=True)
else:
# Strip off three characters for hg+
client = hglib.clone(store["url"][3:], storedir)
def hg_tmp_clone(hg_repo):
"""Clone repository in a temporary path"""
try:
hg_tmp_path = u''
hg_repo_path = hg_repo
# clone remote repositories into a temporal directory
if hg_repo.startswith(u'http') or hg_repo.startswith(u'ssh'):
hg_tmp_path = hg_repo_path = tempfile.mkdtemp(prefix='dicto-hg')
try:
hglib.clone(hg_repo, hg_repo_path)
except hglib.error.ServerError as e:
error(u'hg: could not clone {} into {}: {}'
.format(hg_repo, hg_tmp_path, six.text_type(e)))
yield hg_repo_path
finally:
# remove temporal path if created
if os.path.isdir(hg_tmp_path):
shutil.rmtree(hg_tmp_path)
def checkout(self, path, commit_hash):
# Need to pull -- the copy is not updated automatically, since
# the repository data is not shared
def checkout_existing():
subrepo = hglib.open(path)
subrepo.pull()
subrepo.update(commit_hash, clean=True)
# TODO: Implement purge manually or call it on the command line
if os.path.isdir(path):
try:
checkout_existing()
except (hglib.error.CommandError, hglib.error.ServerError):
# Remove and re-clone
util.long_path_rmtree(path)
if not os.path.isdir(path):
hglib.clone(self._path, dest=path)
checkout_existing()
def checkout(self, path, commit_hash):
# Need to pull -- the copy is not updated automatically, since
# the repository data is not shared
def checkout_existing():
with hglib.open(path) as subrepo:
subrepo.pull()
subrepo.update(commit_hash, clean=True)
# TODO: Implement purge manually or call it on the command line
if os.path.isdir(path):
try:
checkout_existing()
except (hglib.error.CommandError, hglib.error.ServerError):
# Remove and re-clone
util.long_path_rmtree(path)
if not os.path.isdir(path):
hglib.clone(self._path, dest=path)
checkout_existing()
def create_working_copy(self):
"""
Create mercurial working copy.
"""
# try to get a copy of
try:
client = hglib.clone(self.repo, self.wc)
self.log.debug("repo %s cloned in %s" % (self.repo, self.wc))
except (HgCommandError, OSError), err:
# it might already have existed
self.log.warning("Mercurial local repo initialization failed, it might already exist: %s" % err)
def __init__(self, repo_uri):
''' Initialize a hg repo (or open it if it already exists) '''
self.repo_uri = repo_uri
cachedir = os.path.join(__opts__['cachedir'], 'hg_pillar')
hash_type = getattr(hashlib, __opts__.get('hash_type', 'md5'))
repo_hash = hash_type(repo_uri).hexdigest()
self.working_dir = os.path.join(cachedir, repo_hash)
if not os.path.isdir(self.working_dir):
self.repo = hglib.clone(repo_uri, self.working_dir)
self.repo.open()
else:
self.repo = hglib.open(self.working_dir)
def __init__(self, repo_uri):
"""Initialize a hg repo (or open it if it already exists)"""
self.repo_uri = repo_uri
cachedir = os.path.join(__opts__["cachedir"], "hg_pillar")
hash_type = getattr(hashlib, __opts__.get("hash_type", "md5"))
repo_hash = hash_type(salt.utils.stringutils.to_bytes(repo_uri)).hexdigest()
self.working_dir = os.path.join(cachedir, repo_hash)
if not os.path.isdir(self.working_dir):
self.repo = hglib.clone(repo_uri, self.working_dir)
self.repo.open()
else:
self.repo = hglib.open(self.working_dir)
def __init__(self, repo_uri):
''' Initialize a hg repo (or open it if it already exists) '''
self.repo_uri = repo_uri
cachedir = os.path.join(__opts__['cachedir'], 'hg_pillar')
hash_type = getattr(hashlib, __opts__.get('hash_type', 'md5'))
repo_hash = hash_type(repo_uri).hexdigest()
self.working_dir = os.path.join(cachedir, repo_hash)
if not os.path.isdir(self.working_dir):
self.repo = hglib.clone(repo_uri, self.working_dir)
self.repo.open()
else:
self.repo = hglib.open(self.working_dir)
def checkout(self, path, commit_hash):
# Need to pull -- the copy is not updated automatically, since
# the repository data is not shared
def checkout_existing():
with hglib.open(path) as subrepo:
subrepo.pull()
subrepo.update(commit_hash, clean=True)
subrepo.rawcommand(["--config", "extensions.purge=",
"purge", "--all"])
if os.path.isdir(path):
try:
checkout_existing()
except (hglib.error.CommandError, hglib.error.ServerError):
# Remove and re-clone
util.long_path_rmtree(path)
if not os.path.isdir(path):
hglib.clone(self._path, dest=path)
checkout_existing()
def deploy_hg(rc, name, url, src='html', dst=None):
"""Loads an hg database"""
if hglib is None:
raise ImportError('hglib')
targetdir = os.path.join(rc.deploydir, name)
# get or update the database
if os.path.isdir(targetdir):
client = hglib.open(targetdir)
client.pull(update=True)
else:
# Strip off hg+
hglib.clone(url[3:], targetdir)
client = hglib.open(targetdir)
# copy the files over
srcdir = os.path.join(rc.builddir, src)
dstdir = os.path.join(targetdir, dst) if dst else targetdir
copy_tree(srcdir, dstdir, verbose=1)
# commit everything
client.commit(message='regolith auto-deploy at {0}'.format(time.time()),
addremove=True)
client.push()
def checkout(self, path, commit_hash):
# Need to pull -- the copy is not updated automatically, since
# the repository data is not shared
def checkout_existing():
with hglib.open(path) as subrepo:
subrepo.pull()
subrepo.update(commit_hash, clean=True)
subrepo.rawcommand(
["--config", "extensions.purge=", "purge", "--all"])
if os.path.isdir(path):
try:
checkout_existing()
except (hglib.error.CommandError, hglib.error.ServerError):
# Remove and re-clone
util.long_path_rmtree(path)
if not os.path.isdir(path):
hglib.clone(self._path, dest=path)
checkout_existing()
def create_working_copy(self):
"""
Create mercurial working copy.
"""
# try to get a copy of
try:
client = hglib.clone(self.repo, self.wc)
self.log.debug("repo %s cloned in %s" % (self.repo, self.wc))
except (HgCommandError, OSError), err:
# it might already have existed
self.log.warning("Mercurial local repo initialization failed, it might already exist: %s" % err)
def __init__(self, versions, remote_versions):
self.versions_root_path = versions.strpath
self.remote_versions_root_path = remote_versions.strpath
self.base_versions_dir_path = versions.dirname
client = hglib.init(dest=self.remote_versions_root_path)
client.close()
self._mercurial_backend = hglib.clone(
'{0}'.format(self.remote_versions_root_path),
'{0}'.format(self.versions_root_path))
self._repos = {}
def sync_repos():
try:
gh_repo = git.Repo(LOCAL_GIT_REPO_PATH)
except (git.exc.NoSuchPathError, git.exc.InvalidGitRepositoryError):
logging.info('initializing %s' % LOCAL_GIT_REPO_PATH)
gh_repo = git.Repo.init(LOCAL_GIT_REPO_PATH)
gh_repo.create_remote('origin', GH_REPO)
finally:
gh_repo.close()
configs = ['extensions.hggit=']
try:
repo = hglib.open(LOCAL_HG_REPO_PATH, configs=configs)
repo.close()
except hglib.error.ServerError:
logging.info('cloning %s to %s' % (HG_REPO, LOCAL_HG_REPO_PATH))
hglib.clone(source=HG_REPO, dest=LOCAL_HG_REPO_PATH)
# need to do this to ensure the correct hashes in the
# converted git repo since two-way conversion is lossy see
# e.g. https://groups.google.com/forum/#!topic/hg-git/1r1LBrqLeXc
with hglib.open(LOCAL_HG_REPO_PATH, configs=configs) as repo:
logging.info('pushing %s to %s' %
(LOCAL_HG_REPO_PATH, LOCAL_GIT_REPO_PATH))
repo.push(LOCAL_GIT_REPO_PATH)
with git.Repo(LOCAL_GIT_REPO_PATH) as repo:
logging.info('pull from %s on branch master' % GH_REPO)
repo.remotes.origin.pull('master')
with hglib.open(LOCAL_HG_REPO_PATH, configs=configs) as repo:
logging.info('pull from %s to %s on branch master' %
(LOCAL_GIT_REPO_PATH, LOCAL_HG_REPO_PATH))
repo.pull(LOCAL_GIT_REPO_PATH)
repo.update('master', check=True)
logging.info('push from %s to %s on bookmark master' %
(LOCAL_HG_REPO_PATH, HG_REPO))
repo.push(HG_REPO, bookmark='master', force=True)
logging.info('Done!')
def clone(url, dest):
"""
Clone a repository, taking advantage of taskcluster caches
:param url: URL to the remote repository
:param dest: Folder to save the repository to
:return: true is successful, false otherwise
"""
alias = get_alias(url)
namespace = '{}.{}'.format(TC_NAMESPACE, hashlib.md5(alias).hexdigest())
if not os.path.exists(dest):
# check if we can use a cache
if not clone_from_cache(alias, namespace, dest):
# check out a clone
logging.info("cloning the repository without cache")
hglib.clone(url, dest)
if path_is_hg_repo(dest, alias):
log.debug("pulling latest revisions to repository")
client = hglib.open(dest)
return client.pull()
else:
log.error("{} exists but is not a known vcs type".format(dest))
return False
def load_hg_database(db, client, rc):
"""Loads an hg database"""
if hglib is None:
raise ImportError('hglib')
dbdir = dbdirname(db, rc)
# get or update the database
if os.path.isdir(dbdir):
client = hglib.open(dbdir)
client.pull(update=True, force=True)
else:
# Strip off three characters for hg+
client = hglib.clone(db['url'][3:], dbdir)
# import all of the data
client.load_database(db)
def load_hg_database(db, client, rc):
"""Loads an hg database"""
if hglib is None:
raise ImportError('hglib')
dbdir = dbdirname(db, rc)
# get or update the database
if os.path.isdir(dbdir):
client = hglib.open(dbdir)
client.pull(update=True, force=True)
else:
# Strip off three characters for hg+
client = hglib.clone(db['url'][3:], dbdir)
# import all of the data
client.load_database(db)
def __init__(self, url, path, _checkout_copy=False):
# TODO: shared repositories in Mercurial are only possible
# through an extension, and it's not clear how to use those in
# this context. So here, we always make full clones for
# each of the environments.
self._path = os.path.abspath(path)
self._pulled = False
if hglib is None:
raise ImportError("hglib")
if not os.path.exists(self._path):
if not _checkout_copy:
log.info("Cloning project")
if url.startswith("hg+"):
url = url[3:]
# Mercurial branches are global, so there is no need for
# an analog of git --mirror
hglib.clone(url, dest=self._path,
noupdate=(not _checkout_copy))
self._repo = hglib.open(self._path)
def create_working_copy(self):
"""
Create mercurial working copy.
"""
# try to get a copy of
try:
hglib.clone(self.repo, self.wc)
self.log.debug("repo %s cloned in %s" % (self.repo, self.wc))
except (HgCommandError, OSError) as err:
# it might already have existed
self.log.warning(
"Mercurial local repo initialization failed, it might already exist: %s"
% err)
# local repo should now exist, let's connect to it again
try:
self.log.debug("connection to mercurial repo in %s" % self.wc)
self.client = hglib.open(self.wc)
except HgServerError as err:
raise EasyBuildError(
"Could not connect to local mercurial repo: %s", err)
except (HgCapabilityError, HgResponseError) as err:
raise EasyBuildError("Server response: %s", err)
except (OSError, ValueError) as err:
raise EasyBuildError(
"Could not create a local mercurial repo in wc %s: %s",
self.wc, err)
# try to get the remote data in the local repo
try:
self.client.pull()
self.log.debug("pulled succesfully in %s" % self.wc)
except (HgCommandError, HgServerError, HgResponseError, OSError,
ValueError) as err:
raise EasyBuildError("pull in working copy %s went wrong: %s",
self.wc, err)
def _hg_repository_sync(repopath, url, do_update=False):
configpath = os.path.join(repopath, '.hg', 'hgrc')
if not os.path.isfile(configpath):
if not os.path.isdir(os.path.dirname(repopath)):
os.makedirs(os.path.dirname(repopath))
hgrepo = hglib.clone(source=str(url), dest=str(repopath))
cfg = open(configpath, 'a')
cfg.write('default-push = ssh%s\n' % str(url)[4:])
cfg.close()
hgrepo.open()
else:
hgrepo = hglib.open(repopath)
hgrepo.pull(source=hglib.util.b(url))
if do_update:
hgrepo.update()
return hgrepo
def __init__(self, path = '', src = None, uname = '', passwd = ''): """Initialize two repositories. Load local repository if exists. If not exists, clone from remote, or create new. :param path: :param src: :param uname: :param passwd: """ global hgUsefull self.User = uname self.Password = passwd self.AddRemove = True self._client = None self._local = True self._repoPath = os.path.join(path, '.hg') self._repoPathLocal = os.path.join(path, '.hg-local') self._repoPathRemote = os.path.join(path, '.hg-remote') if hgUsefull: conf = ['auth.def.prefix=*', '--config', 'auth.def.schemes=http https', '--config', 'auth.def.username=%s' % self.User, '--config', 'auth.def.password=%s' % self.Password] try: try: self._client = hglib.open(path, configs=conf) except hglib.error.ServerError: if src: self._client = hglib.clone(src, path, configs=conf) shutil.copytree(self._repoPath, self._repoPathRemote) self._client.open() else: self._client = hglib.init(path, configs=conf) shutil.copytree(self._repoPath, self._repoPathRemote) shutil.copytree(self._repoPath, self._repoPathLocal) self._client.open() except Exception: # not found HG? hgUsefull = False
def convert(repo_path, source, target, shas, force=False):
source_repo_p = os.path.join(source, repo_path)
target_repo_p = os.path.join(target, repo_path)
try:
client = hglib.open(source_repo_p)
except hglib.error.ServerError:
client = hglib.clone('https://hg.mozilla.org/integration/' + repo_path,
source_repo_p,
noupdate=True, rev='tip')
if not client.incoming(revrange='tip'):
if not force:
print repo_path, 'is cool'
return
else:
client.pull(rev='tip')
print 'converting', repo_path
client.close()
cmd = ['hg', 'gaiaconv', source_repo_p, target_repo_p,
os.path.join(shas, repo_path)]
print cmd
subprocess.call(cmd)
def hgClone( url, wc_path ):
hglib.clone( str(url).encode('utf-8'), str(wc_path).encode('utf-8') )
def clone(cls, url, folder):
'''Clone the repository'''
hglib.clone(url, folder)
def clone(self):
kwargs = {}
if self.branch:
kwargs['branch'] = self.branch
self._client = hglib.clone(self.remote, str(self.directory), **kwargs)
self._client.open()
def _clone(self):
self._log("clone %s" % (self.basedir),category="clone")
self.client = hglib.clone(self.remoteUrl, self.basedir, branch=self.branchname, configs=self._configs)
self.pull()
self.verify()
def main():
url = find_fw_url(options.url).replace("sw", "shortfw")
print url
target = fetch_model(options.url)
print "Fetching URL: " + url
print "Fetch directory: " + target
mysock = urllib.urlopen(url)
memfile = io.BytesIO(mysock.read())
content = ""
with ZipFile(memfile, 'r') as myzip:
filename = get_filename(myzip.namelist())
if filename is None:
if options.file is None:
print "Need --file options to parse"
filename = options.file
print filename
f = myzip.open(filename)
content = f.read() # or other file-like commands
hg_repos = dict()
for c in content.split("\n"):
entry = c.split("=")
reponame = entry[0].replace("/", "")
if len(entry) != 2:
print "Not 2 list entries:" + c
continue
if reponame in IGNORE_REPO:
continue
hg_repos[reponame] = entry[1]
for repo in hg_repos:
hg_info = hg_repos[repo].split()
if len(hg_info) == 2:
branch = hg_info[-1]
revision = hg_info[0].replace("+", "")
# if exists then update
# if not then clone
hgpath = os.path.join(options.output, target, repo)
hgrepo = ""
if target_exists(hgpath):
# TODO fix issue when wrong repo is present
print "Updating repo: " + repo
hgrepo = hglib.open(hgpath)
hgrepo.pull()
else:
variants = [x.replace('_T', target) for x in ['_T','fc_T', 'fc', ""]]
for v in variants:
hgurl = "ssh://[email protected]:2222/repos/{0}/{1}".format(repo, v)
print "Clonning repo: " + hgurl
try:
hglib.clone(hgurl, hgpath)
except hglib.error.CommandError:
pass
else:
break
hgrepo = hglib.open(hgpath)
print "Updateing to revision " + revision
hgrepo.update(rev = revision, clean = True)
def clone(self):
self._client = hglib.clone(self.remote, str(self.directory))
self._client.open()
