def is_git_url(url_string):
try:
giturlparse.parse(url_string)
except giturlparse.parser.ParserError:
raise argparse.ArgumentTypeError("%r is not a valid git url." %
url_string)
return url_string
def get_repo_url_info(location=os.getcwd(), repo_url=None):
"""
Returns the parsed URL.
Parameters
----------
location: str
Absolute location of the current directory.
repo_url: str
URL of the repository.
Returns
-------
parse: giturlparse.parser.Parsed
A named tuple.
"""
try:
if repo_url is None:
config = configparser.ConfigParser()
config.read(location + os.sep + '.git' + os.sep + 'config')
if 'remote "origin"' in config.sections():
return parse(config['remote "origin"']['url'])
else:
raise ParserError(
'Git config file does not exist please provide the repository url by using --url.'
)
else:
return parse(repo_url + '.git')
except configparser.DuplicateSectionError:
raise configparser.DuplicateSectionError(
'There seems to be a duplicate section in your config. Try giving the repository URL by using --url.'
)
def system_log(self, event):
rospack = rospkg.RosPack()
pkg_path = rospack.get_path('scheduler')
repo = git.Repo(pkg_path, search_parent_directories=True)
modules = {}
url = repo.remote().url
p = giturlparse.parse(url)
target = '{}/{}'.format(p.owner, p.name)
message = '{}'.format(repo.head.object.hexsha)
modules[target] = message
for sm in repo.submodules:
p = giturlparse.parse(sm.url)
target = '{}/{}'.format(p.owner, p.name)
message = '{}'.format(sm.hexsha)
modules[target] = message
for target in modules.keys():
self.casas.publish(
package_type='ROS',
sensor_type='Module_Version',
serial=self.mac_address,
target=target,
message=modules[target],
category='system'
)
def parse(cls, repo_or_url: str):
url_parse = giturlparse.parse(repo_or_url)
if not (hasattr(url_parse, "owner") and hasattr(url_parse, "repo")):
url = "https://github.com/" + repo_or_url
url_parse = giturlparse.parse(url)
if not (hasattr(url_parse, "owner") and hasattr(url_parse, "repo")):
raise ValueError(f"Unable to parse: {repo_or_url}")
return cls(owner=url_parse.owner, repo=url_parse.repo)
def get_repo(linter, check_github=True):
if linter.linter_url:
parse_res = parse(linter.linter_url)
if parse_res is not None and (
(check_github is True and parse_res.github is True)
or check_github is False):
return parse_res
if hasattr(linter, "linter_repo"):
parse_res = parse(linter.linter_repo)
if parse_res is not None and (
(check_github is True and parse_res.github is True)
or check_github is False):
return parse_res
return None
def main(args):
remo = repo.remotes[args.remote]
url = giturlparse.parse(remo.url)
tokens = []
if args.token != '':
tokens.append(args.token)
tokens.extend(
list(
map(
lambda t: t.value,
filter(lambda h: h.name == 'hub.' + url.resource + '.token',
repo.config.__iter__()))))
url_http = 'https://{}/{}'.format(url.resource, url.pathname)
url_api = 'https://{}/api/v3'.format(url.resource)
gh = None
for t in tokens:
# Github Enterprise with custom hostname
gh = Github(base_url=url_api, login_or_token=t)
if gh:
break
assert gh, "could not access remote repo at {}".format(url_http)
gh_repo = ghGetRepo(gh, url.owner, url.name)
assert gh_repo, "could retrieve remote repo at {}".format(url_http)
action = None
action_name = re.sub(r'\W+', '', args.action)
if hasattr(sys.modules[__name__], action_name):
action = getattr(sys.modules[__name__], action_name)
assert action, "{} is not a defined action".format(args.action)
action(args, gh_repo)
def _get_git_url(cwd):
cmd = "git config --get remote.origin.url"
ret, out = utils.execute(cmd, cwd=cwd, tracing=False)
if ret == 0:
# prepare url for getting file
git_url = out.strip()
git_url = giturlparse.parse(git_url)
git_url = git_url.url2https
if git_url.endswith('.git'):
git_url = git_url[:-4]
# get remote branch
branch = 'master'
#cmd = 'git rev-parse --abbrev-ref --symbolic-full-name @{u}'
cmd = 'git branch -a -r --contains HEAD'
ret, out = utils.execute(cmd, cwd=cwd, tracing=False)
if ret == 0:
for l in out.splitlines():
l = l.strip()
if 'HEAD' in l:
continue
branch = l.split('/')[1]
break
git_url = '%s/blob/%s' % (git_url, branch)
return git_url
def create_github_pull_request(repo):
ctx = click.get_current_context()
summary, message = get_merge_request_text(repo)
gh = Github(ctx.params["github_token"])
giturl = giturlparse.parse(get_remote_url(repo))
gh_project = gh.get_repo(f"{giturl.owner}/{giturl.repo}" # pylint: disable=no-member
)
try:
pull_request = gh_project.create_pull(
base="master",
head=repo.active_branch.name,
title=summary,
body=message,
)
except GithubException as ex:
if ex.status != 422:
raise ex
pull_request = list(
gh_project.get_pulls(state="open",
base="master",
head=repo.active_branch.name))[0]
reviewers = ctx.params["reviewers"]
if reviewers:
pull_request.create_review_request(reviewers=reviewers)
return pull_request.html_url
def get_new_component(project,
repository,
branch,
addon_name,
tmpl_component_slug,
addons_subdirectory=None):
po_file_mask = '{}/i18n/*.po'.format(addon_name)
pot_filepath = '{addon_name}/i18n/{addon_name}.pot'.format(
addon_name=addon_name)
if addons_subdirectory:
po_file_mask = os.path.join(addons_subdirectory, po_file_mask)
pot_filepath = os.path.join(addons_subdirectory, pot_filepath)
tmpl_component = Component.objects.get(slug=tmpl_component_slug)
tmpl_component_pk = tmpl_component.pk
parsed_repository_uri = giturlparse.parse(repository)
new_component = tmpl_component
new_component.pk = None
new_component.project = project
new_component.name = get_component_name(project, addon_name)
new_component.slug = get_component_slug(project, addon_name)
new_component.repo = parsed_repository_uri.url2ssh
new_component.push = parsed_repository_uri.url2ssh
new_component.branch = branch
new_component.filemask = po_file_mask
new_component.new_base = pot_filepath
new_component.file_format = 'po'
new_component.locked = False
new_component.save(force_insert=True)
copy_installed_addons(tmpl_component_pk, new_component)
return new_component
def get_pull_request_repo(
self, upstream_url: str, github_token: str, organization_name: str = ""
):
"""
:param upstream_url:
:param github_token:
:param organization_name:
:return: List of url if success, else False
"""
gh = GitHub(token=github_token)
parsed_url = parse(upstream_url)
# Fork the repo
status, user = gh.user.get()
user_name = (
user["login"] if not organization_name else organization_name
)
status, lst_pull = gh.repos[user_name][parsed_url.repo].pulls.get()
if type(lst_pull) is dict:
print(f"For url {upstream_url}, got {lst_pull.get('message')}")
return False
else:
for pull in lst_pull:
print(pull.get("html_url"))
return lst_pull
def batch():
# pylint:disable-msg=E1101
urls = request.form.get('urls', '') + "\n" # this makes m
n = 0
for l in urls.split("\n"):
l = l.strip()
if not l:
continue
if not parse(l).valid:
flash(_("Url %(url)s isn't accepted, parse error", url=l))
else:
dup = Project.query.filter_by(login=g.user.login,
is_github=False,
clone=l).first()
if dup:
flash(_("Url %(url)s is duplicate", url=l))
else:
project = Project(login=g.user.login, clone=l, is_github=False)
if project:
db.session.add(project)
db.session.commit()
db.session.refresh(project)
project.sync()
n = n + 1
flash(_("%(num)s repositories successfuly added", num=n))
return redirect(url_for('settings.repos'))
def __build_url_patterns(self):
# https://[email protected]/caltech-imss-ads/exeter_api/src/0.10.2/
#
if not self.url_patterns:
p = parse(self.repo.remote().url)
host = p.host
owner = p.owner
if host.startswith('codestar-connections'):
# We're in a CodePipeline, and our true origin is masked. Assume bitbucket.
host = "bitbucket.org"
owner = os.environ.get('CI_BUILDBOT_GIT_OWNER',
"caltech-imss-ads")
origin_url = f"https://{host}/{owner}/{p.repo}"
if origin_url.endswith('.git'):
origin_url = origin_url[:-4]
if p.github:
self.url_patterns[
'commit'] = f"<{origin_url}/commit/" + "{sha}|{sha}>"
self.url_patterns[
'project'] = f"<{origin_url}/tree/" + "{version}|{name}>"
self.url_patterns[
'diff'] = f"{origin_url}/compare/" + "{from_sha}..{to_sha}"
else:
# Assume bitbucket
self.url_patterns[
'commit'] = f"<{origin_url}/commits/" + "{sha}|{sha}>"
self.url_patterns[
'project'] = f"<{origin_url}/src/" + "{version}/|{name}>"
self.url_patterns[
'diff'] = f"{origin_url}/branches/compare/" + "{from_sha}..{to_sha}#diff"
self.url_patterns['repo'] = origin_url
def _get_config_generator(filename):
"""
A generator which populates and return a dict.
:parse filename: A string containing the path to YAML file.
:return: dict
"""
for d in _get_config(filename):
repo = d['git']
parsedrepo = giturlparse.parse(repo)
name = '{}.{}'.format(parsedrepo.owner, parsedrepo.repo)
src_dir = os.path.join(_get_clone_dir(), name)
files = d.get('files')
dst_dir = None
if not files:
dst_dir = _get_dst_dir(d['dst'])
yield {
'git': repo,
'lock_file': _get_lock_file(name),
'version': d['version'],
'name': name,
'src': src_dir,
'dst': dst_dir,
'files': _get_files_config(src_dir, files)
}
def load_repo_data():
# TODO: Allow to override the current working directory, e.g. via --dir
cwd = os.getcwd()
repo = Repo(cwd)
tracking_branch = repo.active_branch.tracking_branch()
if tracking_branch:
repo_branch = tracking_branch.remote_head
remote_name = tracking_branch.remote_name
repo_hash = tracking_branch.commit.hexsha
repo_url = repo.remote(remote_name).url
repo_url_parsed = giturlparse.parse(repo_url)
if repo_url_parsed.protocol == "ssh":
ssh_config_path = os.path.expanduser('~/.ssh/config')
if os.path.exists(ssh_config_path):
fp = open(ssh_config_path, 'r')
config = SSHConfig()
config.parse(fp)
repo_url = repo_url.replace(
repo_url_parsed.resource,
config.lookup(repo_url_parsed.resource)['hostname'])
# TODO: Detect and pass private key too
fp.close()
# TODO: Doesn't support unstaged changes
repo_diff = repo.git.diff(repo_hash)
return repo_url, repo_branch, repo_hash, repo_diff
else:
sys.exit(
f"No tracked branch configured for branch {repo.active_branch.name}"
)
def init(self):
self.repo_url = self.get_config("repo_url")
self.checkout_path = self.get_config("checkout_path")
branch = self.get_config("branch")
if not self.checkout_path:
parsed_url = giturlparse.parse(self.repo_url)
# print("pathname", parsed_url.pathname)
# print("href", parsed_url.href)
# print("user", parsed_url.user)
# print("name", parsed_url.name)
# print("owner", parsed_url.owner)
# print("resource", parsed_url.resource)
self.checkout_path = os.path.join(self.ctl.cachedir, "repo",
parsed_url.resource,
parsed_url.pathname)
# while checkout patch can be relative in the config, we want
# it to be absolute from here on out
self.checkout_path = os.path.abspath(self.checkout_path)
self.clone()
if branch != self.branch:
self.checkout(branch)
def __check_if_content_present(self):
files = [
os.path.join(self.content_folder, f)
for f in os.listdir(self.content_folder)
if os.path.isfile(os.path.join(self.content_folder, f))
]
matched_extension = next(
(ext for ext in PackageManager.SUPPORTED_EXTENSIONS
if self.source_url.endswith(ext)),
None) if not giturlparse.parse(self.source_url).valid else 'zip'
if matched_extension:
possible_files = [
self.__build_file_content_folder_path(matched_extension)
]
else:
possible_files = [
self.__build_file_content_folder_path(extension)
for extension in PackageManager.SUPPORTED_EXTENSIONS
]
matching_files = list(set(files) & set(possible_files))
if matching_files and len(matching_files) > 1:
raise PackageManagerException(
f'Package {self.package_name} content seems to be duplicated')
return matching_files[0] if matching_files else None
def fork_repo(self, upstream_url: str, github_token: str,
organization_name: str = ""):
# https://developer.github.com/apps/building-integrations/setting-up-and-registering-oauth-apps/about-scopes-for-oauth-apps/
gh = GitHub(token=github_token)
parsed_url = parse(upstream_url)
# Fork the repo
status, user = gh.user.get()
user_name = user['login'] if not organization_name else organization_name
status, forked_repo = gh.repos[user_name][parsed_url.repo].get()
if status == 404:
status, upstream_repo = (
gh.repos[parsed_url.owner][parsed_url.repo].get())
if status == 404:
print("Unable to find repo %s" % upstream_url)
exit(1)
args = {}
if organization_name:
args["organization"] = organization_name
status, forked_repo = (
gh.repos[parsed_url.owner][parsed_url.repo].forks.post(**args))
if status == 404:
print("Error when forking repo %s" % forked_repo)
exit(1)
else:
print("Forked %s to %s" % (upstream_url, forked_repo['ssh_url']))
elif status == 202:
print("Forked repo %s already exists" % forked_repo['full_name'])
elif status != 200:
print("Status not supported: %s - %s" % (status, forked_repo))
exit(1)
return forked_repo
def loadfromgit():
logger.debug('JSON : {0:s}'.format(str(request.json)))
if request.method == 'POST':
try:
git_url, git_branch = request.json['git_repository'].split('*')
parsed_git_url = giturlparse.parse(git_url)
template_git_dir = os.path.abspath(
os.path.join(bp.static_folder, 'templates', 'git'))
if not os.path.exists(template_git_dir):
os.makedirs(template_git_dir, exist_ok=True)
git_repo_dir = os.path.abspath(
os.path.join(template_git_dir, parsed_git_url.name))
if os.path.exists(git_repo_dir):
repo = Repo(git_repo_dir)
repo.remotes.origin.pull()
else:
repo = Repo.clone_from(git_url,
git_repo_dir,
branch=git_branch,
no_single_branch=True)
repo.remotes.origin.pull()
files = list(glob.iglob(os.path.join(git_repo_dir, "*.json")))
logger.info(files)
files_list = [f.replace("okit/okitweb/", "") for f in files]
logger.debug('JSON : {0:s}'.format(str(request.json)))
logger.debug('Files Walk : {0!s:s}'.format(files_list))
result = {"fileslist": files_list}
logger.info(result)
return json.dumps(result)
except Exception as e:
logger.exception(e)
return str(e), 500
def add(git_remote):
"""Add a new repository to look for Dockerfiles"""
git_remote = parse(git_remote)
if git_remote.valid:
repo_name = f'{git_remote.owner}_{git_remote.repo}'
destination = settings.REPO_DIR.joinpath(repo_name)
if destination.exists():
if click.confirm(
crayons.cyan(
'That repo is already added, do you want me to add it again?'
)):
click.echo(
crayons.red('Removing existing repo' + settings.LOADING))
shutil.rmtree(destination)
else:
raise click.Abort
click.echo('{0} {1}{2}'.format(
crayons.white('Adding repository', bold=True),
crayons.green(repo_name), crayons.white(settings.LOADING,
bold=True)))
click.echo(f'repo dir - {str(destination)}')
with Halo():
git.Repo.clone_from(git_remote.url2git, str(destination))
click.echo(
crayons.cyan('Successfully added repository ', bold=True) +
crayons.green(repo_name))
else:
click.echo(
crayons.red("I don't think that's a valid git remote url, sorry."))
def job_hook_to_check_action(
job_hook: jobs.JobHook,
checks_for_commit: List[checks.RunDetails],
) -> Union[checks.CreateRun, checks.UpdateRun]:
check_details = job_to_run_details(job_hook.job)
repo = giturlparse.parse(job_hook.pipeline.repository)
current_checks_by_id = {
check.external_id: check
for check in checks_for_commit
}
if check_details.external_id not in current_checks_by_id:
action = checks.CreateRun(
owner=repo.owner,
repo=repo.repo,
run=check_details,
)
action.run.head_sha = job_hook.build.commit
action.run.head_branch = job_hook.build.branch
else:
action = checks.UpdateRun(
owner=repo.owner,
repo=repo.repo,
id = current_checks_by_id[check_details.external_id].id,
run=check_details,
)
return action
def get_repo_name(repo_url, repo_name):
g = giturlparse.parse(repo_url)
if not repo_name:
repo_name = g.name
return repo_name
def links_to_csv(filename=None,lines=None):
if lines==None and filename!=None:
with open(filename,'r') as f:
lines = list(set(f.readlines()))
csv_dict = {
"name":[],
"repo":[],
"owner":[],
"link":[],
#"stars":[]
}
for i in lines:
data = parse(i)
csv_dict["name"].append(data.name)
csv_dict["repo"].append(data.repo)
csv_dict["owner"].append(data.owner)
#csv_dict["user"].append(data.user)
csv_dict["link"].append(i)
#csv_dict["stars"].append(data.stars)
print(csv_dict)
df = pd.DataFrame(data=csv_dict)
# filename = str(time())+str(".csv")
open(filename,'w+').close()
df.to_csv(filename)
async def _repo_change_url(self, args, session):
repository_id = args[0]
url = args[1]
repo = session.query(SourceRepository).filter(
SourceRepository.id == repository_id).first()
if not repo:
logger.error("repo_change_url: repo %d not found", repository_id)
return
if repo.state != "ready" and repo.state != "error":
logger.info("worker: repo %d not ready, requeueing", repository_id)
await enqueue_task({"repo_change_url": args})
await asyncio.sleep(2)
return
try:
repoinfo = giturlparse.parse(url)
except giturlparse.parser.ParserError:
logger.error("repo_change_url: invalid git url %s", url)
return
old_path = repo.src_path
repo.url = url
repo.name = repoinfo.name
session.commit()
await GitChangeUrl(old_path, repo.name, repo.url)
def __init__(self, package_name, package_metadata, temporal_fs_manager,
generator_dir):
if not package_name:
raise ValueError('package_name cannot be null or empty')
if not package_metadata:
raise ValueError('package_metadata cannot be null or empty')
self.temporal_fs_manager = temporal_fs_manager
self.package_name = package_name
self.content_folder = os.path.join(generator_dir, 'content')
self.package_content_path = None
self.reusable_content = None
self.source_url = package_metadata.get('source-url')
if not validators.url(self.source_url) and not giturlparse.parse(
self.source_url).valid:
raise ConfigurationException(
f'{self.package_name} source-url is not a valid URL')
self.package_version = package_metadata.get('version')
if not self.package_version:
raise ConfigurationException(
f'{self.package_name} version is empty')
if not os.path.exists(generator_dir):
os.mkdir(self.files_folder)
if not os.path.exists(self.content_folder):
os.mkdir(self.content_folder)
def batch():
# pylint:disable-msg=E1101
urls = request.form.get('urls', '') + "\n" # this makes m
n = 0
for l in urls.split("\n"):
l = l.strip()
if not l:
continue
if not parse(l).valid:
flash(_("Url %(url)s isn't accepted, parse error", url=l))
else:
dup = Project.query.filter_by(login=g.user.login,
is_github=False, clone=l).first()
if dup:
flash(_("Url %(url)s is duplicate", url=l))
else:
project = Project(login=g.user.login, clone=l, is_github=False)
if project:
db.session.add(project)
db.session.commit()
db.session.refresh(project)
project.sync()
n = n + 1
flash(_("%(num)s repositories successfuly added", num=n))
return redirect(url_for('settings.repos'))
def fetch_addons(options, addon):
parsed = parse(addon['url'])
url = urlparse(parsed.url2https)
if addon['auth'] and options.username and options.password:
url = url._replace(
netloc="{}:{}@{}".format(
options.username,
options.password,
url.netloc
)
)
repo_path = Path.cwd() / options.output_directory / parsed.repo
repo_path.mkdir(exist_ok=True)
with cd(repo_path):
run(['git', 'init'])
run(['git', 'remote', 'add', 'origin', url.geturl()])
ref = addon.get('commit') or addon.get('branch') or options.branch
if ref:
run(['git', 'fetch', 'origin', ref])
else:
run(['git', 'fetch', 'origin'])
run(['git', 'checkout', 'FETCH_HEAD'])
run(['git', 'remote', 'remove', 'origin'])
def get_protocol(name):
"""Get the service URL protocol.
Inspects the value of ``issue.<service>.https`` for ``0`` or ``false``
to assertain the URL schema to use, defaults to ``'https'`` if the
config setting is not found, ``'http'`` is returned otherwise.
Arguments:
:name: Name of the service.
Returns:
:str: ``'https'`` or ``'http'`` depending on the value of
``issue.<service>.https`` config setting.
"""
url = get_url(name)
if url:
_check_service_url_(name, url)
return parse(url).protocol
try:
use_https = get_config('issue.%s.https' % name)
if use_https == '0' or use_https == 'false':
return 'http'
except CalledProcessError:
pass
return 'https'
def __init__(self,
settings,
cache_directory,
public_url,
default_private_key=None):
if isinstance(settings, str):
url = settings
name = None
self.private_key = None
elif isinstance(settings, dict):
url = settings.get("url", None)
if url is None:
raise RuntimeError(
"Repo is an object but has no `url` property")
name = settings.get("name", None)
self.private_key = settings.get("ssh_key", default_private_key)
else:
raise RuntimeError("Repo should be a string or a object")
self.repo_url = url
self.parsed_url = giturlparse.parse(url)
if not self.parsed_url.valid:
raise RuntimeError("Repo {0} is not valid".format(url))
self._name = name or self.parsed_url.repo
self.cache_directory = os.path.join(cache_directory, self.name())
self.cache_public_url = public_url + "/cache/" + self.name()
self.public_url = public_url
def clone_git_repo(repo, checkout_dir):
parsedrepo = giturlparse.parse(repo, False)
directory = os.path.join(checkout_dir, parsedrepo.repo)
if not os.path.isdir(directory):
Repo.clone_from(repo, directory)
return directory
def get_remote_url():
"""Obtain a parsed remote URL.
Uses CI environment variables or git remotes.
"""
# Netlify doesnt have any git remotes
# It only sets the REPOSITORY_URL
url = os.environ.get('REPOSITORY_URL')
if not url:
remote = get_config_remote()
url = remote[0][1]
try:
url = giturlparse.parse(url)
except giturlparse.parser.ParserError:
url = giturlparse.parse(url + '.git')
return url
def get_list_fork_repo(upstream_url, github_token):
gh = GitHub(token=github_token)
parsed_url = parse(upstream_url)
status, user = gh.user.get()
# response = gh.repos[user['login']][parsed_url.repo].forks.get(sort="newest")
response = gh.repos["odoo"][parsed_url.repo].forks.get(sort="newest")
print(response)
def current_repo():
res = subprocess.run(['git', 'remote', 'get-url', 'origin'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if res.stderr:
raise Exception('Invalid repository url.')
return parse(res.stdout.decode('utf-8'))
def url(self, url):
"""Setter to the repository's URL."""
if validate(url):
self._url = url
git_parse = parse(url)
self._git_platform = git_parse.platform
self._git_host = git_parse.host
self._git_owner = git_parse.owner
self._git_repository = git_parse.repo
def fetch(self):
if self._path: return
log('info', 'Fetching app {} from {} to {}'.format(self.name, self.remote_url, self.path), show_header=True)
git_remote_host = giturlparse.parse(self.remote_url).host
ssh.remove_from_known_hosts(git_remote_host)
ssh.add_to_known_hosts(git_remote_host)
if not isdir(self.path):
local('mkdir -p {}'.format(self.path))
git.clone_repo(self.remote_url, self.path, self.local_branch)
else:
git.ensure_is_repo(self.path)
with lcd(self.path):
local_branch = self.local_branch if self.local_branch is not None else 'master'
local('git reset --hard HEAD')
local('git pull origin {}'.format(local_branch))
local('git fetch')
local('git checkout {}'.format(local_branch))
def __init__(self, docker, role,
ansible_paths=None, ansible_version='latest'):
self.ansible = None
self.docker = docker
self.role = role
self.work_dir = mktmpdir()
self.res = {'success': 0, 'skip': 0, 'failed': 0}
self.ansible_version = ansible_version
self.environment = {}
# check the role type
self.role_name = self.role
self.role_path = '/etc/ansible/roles/{0}'.format(role)
self.bindings = {
self.work_dir: {'bind': '/work', 'ro': False},
}
self.type = TestFramework.TYPE_GALAXY
self.ansible_paths = {
'roles': None,
'library': None,
'plugins': {
'action': None
}
}
if ansible_paths:
self.ansible_paths.update(ansible_paths)
self.setup_bindings()
if os.path.isdir(role):
# role is a folder name, use that
role = os.path.realpath(role)
self.role_name = os.path.basename(role)
self.role_path = '/etc/ansible/roles/{0}'.format(self.role_name)
self.bindings[os.path.realpath(self.role)] = {
'bind': self.role_path,
'ro': True
}
self.type = TestFramework.TYPE_LOCAL
elif role.endswith('.git') or '.git#' in role:
# role is a git repository
p = giturlparse.parse(role)
self.role_name = p.repo
self.role_path = '/etc/ansible/roles/{0}'.format(self.role_name)
self.type = TestFramework.TYPE_GIT
def read_repo(url, data_path):
"""Read articles from a git repo, save repo in data_path"""
p = giturlparse.parse(url)
assert p.valid, "git url %s invalid" % url
repo_path = os.path.abspath(os.path.join(data_path, "%s.%s" % (p.owner, p.host)))
if is_git_dir(os.path.join(repo_path, ".git")):
repo = git.Repo.init(repo_path)
assert not repo.bare
try:
repo.git(git_dir=repo_path).pull()
except git.exc.GitCommandError as ex:
logger.warning(str(ex))
if not ex.status == 128:
raise
else:
repo = git.Repo.clone_from(url, repo_path)
for x in read_articles(repo_path):
yield x
def addclone():
# According url schemes here http://git-scm.com/docs/git-push it is
# near impossibru to validate url, so just check if its length is > 10
# (let it be 10)
#
# TODO in the future, use http://schacon.github.io/git/git-ls-remote.html to validate the URL string
# http://stackoverflow.com/questions/9610131/how-to-check-the-validity-of-a-remote-git-repository-url
#
# TODO we could do very light validation that the form field is not 0 in JS
# TODO we could do very light validation of the URL in JS
clone = request.form.get('clone')
if len(clone) == 0:
flash(_('Enter a URL.'))
return redirect(url_for('frontend.splash'))
if not clone.lower().endswith('.git'):
clone += '.git' # quick fix to validate clone git url
# pylint:disable-msg=E1101
# user = ProjectCache.get_user_cache(g.user.login)
dup = Project.query.filter_by(login=g.user.login,
is_github=False, clone=clone).first()
if dup:
flash(_("This repository is a duplicate"))
if not parse(clone).valid:
flash(_("Problem parsing git url"))
return redirect(url_for('settings.repos') + "#tab_massgithub")
project = Project(login=g.user.login, clone=clone, is_github=False)
# if (clone in map(lambda x: x['clone_url'], user.data)
# or clone in map(lambda x: x['git_url'], user.data)):
# pass
if project:
db.session.add(project)
db.session.commit()
flash(_("Repository successfully added"))
project.sync()
return redirect(url_for('frontend.splash'))
def _set_pr_status(build_info):
if build_info['phase'] == 'PENDING':
state, description = 'pending', 'Build pending'
elif build_info['phase'] in ('FINALIZED', 'COMPLETED'):
if build_info['status'] == 'FAILURE':
state, description = 'failure', 'Build failed'
else:
state, description = 'success', 'Build succeeded'
else:
app.log.warning('Unknown phase received from jenkins %s', build_info['phase'])
return
job_path = build_info['url']
# FIXME: Will we get all repo that we are building or what ?
# detailled_build_info = jenkins.get_build_info(build_info['name'], build_info['build']['number'])
# FIXME: Hard coded dependency scm.
repo = giturlparse.parse(build_info['scm']['url'])
sha = build_info['scm']['commit']
statuses_url = 'https://api.github.com/repos/{owner}/{name}/statuses/{sha}'.format(sha=sha, name=repo.repo, owner=repo.owner)
app.logger.info('send notification to %s', statuses_url)
handlers._set_github_status(state, description, job_path, statuses_url)
def _fetch_git_repo(repo, ref):
home = os.path.expanduser("~")
reponimousdir = os.path.join(home, ".reponimous")
if not os.path.isdir(reponimousdir):
os.makedirs(reponimousdir)
cleanref = re.sub('/', '-', ref)
parsedrepo = giturlparse.parse(repo, False)
repodirname = os.path.join(reponimousdir, "%s-%s-%s" %
(parsedrepo.owner, parsedrepo.repo, cleanref))
if not os.path.isdir(repodirname):
git.Git().clone(repo, repodirname)
gitrepo = git.Repo(repodirname)
gitrepo.git.checkout(ref)
if not gitrepo.head.is_detached:
gitrepo.git.pull()
return repodirname
#!/usr/bin/python import sys from giturlparse import parse print parse(sys.argv[1]).host
def _test_valid(self, url, results):
p = parse(url)
self.failUnless(p.valid, "%s is not a valid URL" % url)
for k,v in results.items():
attr_v = getattr(p, k)
self.assertEqual(attr_v, v, "[%s] Property '%s' should be '%s' but is '%s'" % (url, k, attr_v, v))
def _test_invalid(self, url):
p = parse(url)
self.failIf(p.valid)
# Validate input and store objects
jjb_projects = {}
for repository_definition in repositories:
repository_url = None
project_name = None
# Check if definition is a simple string or a dict
if type(repository_definition) is dict:
repository_url = repository_definition.keys()[0]
project_name = repository_definition[repository_url]['project_name']
else:
repository_url = repository_definition
# Check if repository_url is valid
repository_parsed = parse(repository_url)
if not repository_parsed.valid:
raise Exception("The url '%s' doesn't appear to be a valid git url." % repository_url)
# Create a default project_name when not defined
project_name = project_name or repository_parsed.repo
# Find project name duplicates
if project_name in jjb_projects:
raise Exception("The '%s' project name is duplicated." % project_name)
# Store projects in a dict
jjb_projects[project_name] = {
"giturl": repository_url
}
def _test_rewrite(self, source, protocol, dest):
parsed = parse(source)
self.assertTrue(parsed.valid, "Invalid Url: %s" % source)
return self.assertEqual(parse(source).format(protocol), dest)
def _try_parse_git_url(self, url):
if url.startswith('git@'):
parsed_repo = parse(url)
return True, parsed_repo.owner, parsed_repo.repo
else:
return False, None, None
def parsed_repo(self):
with local.cwd(local.cwd / self.module_name):
return parse(git('config --get remote.origin.url'.split(' ')))
def download_collection(self, id, register_name):
"""Download a collection given its ID.
For remote git repositories, we will clone the repository first (or
pull if the repo is already cloned before) and copy the collection to
collections dir.
:param id: The ID of the collection.
:type id: str
:param register_name: The register name of the collection (the
section name of the collection)
:type register_name: unicode
"""
# Clone or pull the repositories first
local_repo_dir = os.path.join(
QgsApplication.qgisSettingsDirPath(),
'resource_sharing',
'repositories',
self.git_host, self.git_owner, self.git_repository
)
if not os.path.exists(os.path.join(local_repo_dir, '.git')):
os.makedirs(local_repo_dir)
try:
repo = porcelain.clone(
self.url, local_repo_dir,
errstream=writeOut
)
except Exception as e:
# Try to clone with https if it's ssh url
git_parsed = parse(self.url)
if self.url == git_parsed.url2ssh:
try:
repo = porcelain.clone(
git_parsed.url2https, local_repo_dir,
errstream=writeOut)
except Exception as e:
error_message = 'Error: %s' % str(e)
LOGGER.exception(traceback.format_exc())
return False, error_message
else:
error_message = 'Error: %s' % str(e)
LOGGER.exception(traceback.format_exc())
return False, error_message
if not repo:
error_message = ('Error: Cloning the repository of the '
'collection failed.')
return False, error_message
else:
try:
porcelain.pull(
local_repo_dir,
self.url,
b'refs/heads/master',
errstream=writeOut
)
except Exception as e:
# Try to pull with https if it's ssh url
git_parsed = parse(self.url)
if self.url == git_parsed.url2ssh:
try:
porcelain.pull(
local_repo_dir,
git_parsed.url2https,
b'refs/heads/master',
errstream=writeOut
)
except Exception as e:
error_message = 'Error: %s' % str(e)
LOGGER.exception(traceback.format_exc())
return False, error_message
else:
error_message = 'Error: %s' % str(e)
LOGGER.exception(traceback.format_exc())
return False, error_message
# Copy the specific downloaded collection to collections dir
src_dir = os.path.join(local_repo_dir, 'collections', register_name)
if not os.path.exists(src_dir):
error_message = ('Error: The collection does not exist in the '
'repository.')
return False, error_message
dest_dir = local_collection_path(id)
if os.path.exists(dest_dir):
shutil.rmtree(dest_dir)
shutil.copytree(src_dir, dest_dir)
return True, None
def parsed_repo(self):
return giturlparse.parse(self.remote_url)