def main():
# create the top-level parser
parser = argparse.ArgumentParser()
parser.add_argument("-v",
"--version",
help="print the version and exit",
action='store_true')
subparsers = parser.add_subparsers(title='commands',
description='qgis-plugin-ci command',
dest='command')
# package
package_parser = subparsers.add_parser(
'package', help='creates an archive of the plugin')
package_parser.add_argument('release_version',
help='The version to be released')
package_parser.add_argument(
'--transifex-token',
help=
'The Transifex API token. If specified translations will be pulled and compiled.'
)
package_parser.add_argument(
'--plugin-repo-url',
help=
'If specified, a XML repository file will be created in the current directory, the zip URL will use this parameter.'
)
package_parser.add_argument(
'--allow-uncommitted-changes',
action='store_true',
help=
'If omitted, uncommitted changes are not allowed before packaging. If specified and some changes are '
'detected, a hard reset on a stash create will be used to revert changes made by qgis-plugin-ci.'
)
package_parser.add_argument(
'--disable-submodule-update',
action='store_true',
help=
'If omitted, a git submodule is updated. If specified, git submodules will not be updated/initialized before packaging.'
)
# changelog
changelog_parser = subparsers.add_parser('changelog',
help='gets the changelog content')
changelog_parser.add_argument('release_version',
help='The version to be released')
# release
release_parser = subparsers.add_parser('release',
help='release the plugin')
release_parser.add_argument('release_version',
help='The version to be released')
release_parser.add_argument(
'--transifex-token',
help=
'The Transifex API token. If specified translations will be pulled and compiled.'
)
release_parser.add_argument(
'--github-token',
help=
'The Github API token. If specified, the archive will be pushed to an already existing release.'
)
release_parser.add_argument(
'--create-plugin-repo',
action='store_true',
help=
'Will create a XML repo as a Github release asset. Github token is required.'
)
release_parser.add_argument(
'--allow-uncommitted-changes',
action='store_true',
help=
'If omitted, uncommitted changes are not allowed before releasing. If specified and some changes are '
'detected, a hard reset on a stash create will be used to revert changes made by qgis-plugin-ci.'
)
release_parser.add_argument(
'--disable-submodule-update',
action='store_true',
help=
'If omitted, a git submodule is updated. If specified, git submodules will not be updated/initialized before packaging.'
)
release_parser.add_argument(
'--osgeo-username', help='The Osgeo user name to publish the plugin.')
release_parser.add_argument(
'--osgeo-password', help='The Osgeo password to publish the plugin.')
# pull-translation
pull_tr_parser = subparsers.add_parser(
'pull-translation', help='pull translations from Transifex')
pull_tr_parser.add_argument('transifex_token',
help='The Transifex API token')
pull_tr_parser.add_argument('--compile',
action='store_true',
help='Will compile TS files into QM files')
# push-translation
push_tr_parser = subparsers.add_parser(
'push-translation', help='update strings and push translations')
push_tr_parser.add_argument('transifex_token',
help='The Transifex API token')
args = parser.parse_args()
# print the version and exit
if args.version:
import pkg_resources
print('qgis-plugin-ci version: {}'.format(
pkg_resources.get_distribution('qgis-plugin-ci').version))
parser.exit()
# if no command is passed, print the help and exit
if not args.command:
parser.print_help()
parser.exit()
exit_val = 0
if os.path.isfile(".qgis-plugin-ci"):
arg_dict = yaml.safe_load(open(".qgis-plugin-ci"))
else:
config = configparser.ConfigParser()
config.read("setup.cfg")
if "qgis-plugin-ci" not in config.sections():
raise ConfigurationNotFound(
".qgis-plugin-ci or setup.cfg with a 'qgis-plugin-ci' section have not been found."
)
arg_dict = dict(config.items("qgis-plugin-ci"))
parameters = Parameters(arg_dict)
# PACKAGE
if args.command == 'package':
release(
parameters,
release_version=args.release_version,
transifex_token=args.transifex_token,
allow_uncommitted_changes=args.allow_uncommitted_changes,
plugin_repo_url=args.plugin_repo_url,
disable_submodule_update=args.disable_submodule_update,
)
# RELEASE
elif args.command == 'release':
release(
parameters,
release_version=args.release_version,
transifex_token=args.transifex_token,
github_token=args.github_token,
upload_plugin_repo_github=args.create_plugin_repo,
osgeo_username=args.osgeo_username,
osgeo_password=args.osgeo_password,
allow_uncommitted_changes=args.allow_uncommitted_changes,
disable_submodule_update=args.disable_submodule_update,
)
# CHANGELOG
elif args.command == 'changelog':
try:
c = ChangelogParser(parameters.changelog_regexp)
content = c.content(args.release_version)
if content:
print(content)
except Exception:
# Better to be safe
pass
# TRANSLATION PULL
elif args.command == 'pull-translation':
t = Translation(parameters, args.transifex_token)
t.pull()
if args.compile:
t.compile_strings()
# TRANSLATION PUSH
elif args.command == 'push-translation':
t = Translation(parameters, args.transifex_token)
t.update_strings()
t.push()
return exit_val
class TestRelease(unittest.TestCase):
def setUp(self):
arg_dict = yaml.safe_load(open(".qgis-plugin-ci"))
self.parameters = Parameters(arg_dict)
self.transifex_token = os.getenv('transifex_token')
self.github_token = os.getenv('github_token')
self.repo = None
if self.github_token:
print('init Github')
self.repo = Github(
self.github_token).get_repo('opengisch/qgis-plugin-ci')
self.clean_assets()
def tearDown(self):
self.clean_assets()
def clean_assets(self):
if self.repo:
rel = None
try:
rel = self.repo.get_release(id=RELEASE_VERSION_TEST)
except GithubException as e:
raise GithubReleaseNotFound(
'Release {} not found'.format(RELEASE_VERSION_TEST))
if rel:
print('deleting release assets')
for asset in rel.get_assets():
print(' delete {}'.format(asset.name))
asset.delete_asset()
def test_release(self):
release(self.parameters, RELEASE_VERSION_TEST)
def test_release_with_transifex(self):
assert self.transifex_token is not None
t = Translation(self.parameters, transifex_token=self.transifex_token)
release(self.parameters,
RELEASE_VERSION_TEST,
transifex_token=self.transifex_token)
def test_release_upload_github(self):
release(self.parameters,
RELEASE_VERSION_TEST,
github_token=self.github_token,
upload_plugin_repo_github=True)
# check the custom plugin repo
_, xml_repo = mkstemp(suffix='.xml')
url = 'https://github.com/opengisch/qgis-plugin-ci/releases/download/{}/plugins.xml'.format(
RELEASE_VERSION_TEST)
urllib.request.urlretrieve(url, xml_repo)
replace_in_file(xml_repo, r'<update_date>[\w-]+<\/update_date>',
'<update_date>__TODAY__</update_date>')
if not filecmp.cmp(
'test/plugins.xml.expected', xml_repo, shallow=False):
import difflib
text1 = open('test/plugins.xml.expected').readlines()
text2 = open(xml_repo).readlines()
self.assertFalse(True,
'\n'.join(difflib.unified_diff(text1, text2)))
# compare archive file size
gh_release = self.repo.get_release(id=RELEASE_VERSION_TEST)
archive_name = self.parameters.archive_name(RELEASE_VERSION_TEST)
fs = os.path.getsize(archive_name)
print('size: ', fs)
self.assertGreater(fs, 0, 'archive file size must be > 0')
found = False
for a in gh_release.get_assets():
if a.name == archive_name:
found = True
self.assertEqual(fs, a.size,
'asset size doesn\'t march archive size.')
break
self.assertTrue(found, 'asset not found')
def release(
parameters: Parameters,
release_version: str,
release_tag: str = None,
github_token: str = None,
upload_plugin_repo_github: str = False,
transifex_token: str = None,
osgeo_username: str = None,
osgeo_password: str = None,
allow_uncommitted_changes: bool = False,
plugin_repo_url: str = None,
disable_submodule_update: bool = False,
):
"""
Parameters
----------
parameters
The configuration parameters
release_version:
The release version (x.y.z)
release_tag:
The release tag (vx.y.z).
If not given, the release version will be used
github_token
The Github token
upload_plugin_repo_github
If true, a custom repo will be created as a release asset on Github and could later be used in QGIS as a custom plugin repository.
plugin_repo_url
If set, this URL will be used to create the ZIP URL in the XML file
transifex_token
The Transifex token
osgeo_username
osgeo username to upload the plugin to official QGIS repository
osgeo_password
osgeo password to upload the plugin to official QGIS repository
allow_uncommitted_changes
If False, uncommitted changes are not allowed before packaging/releasing.
If True and some changes are detected, a hard reset on a stash create will be used to revert changes made by qgis-plugin-ci.
disable_submodule_update
If omitted, a git submodule is updated. If specified, git submodules will not be updated/initialized before packaging.
"""
if transifex_token is not None:
tr = Translation(parameters,
create_project=False,
transifex_token=transifex_token)
tr.pull()
tr.compile_strings()
archive_name = parameters.archive_name(release_version)
is_prerelease = False
if github_token is not None:
is_prerelease = release_is_prerelease(parameters,
release_tag=release_version,
github_token=github_token)
print("*** is pre-release: {}".format("YES" if is_prerelease else "NO"))
create_archive(
parameters,
release_version,
archive_name,
add_translations=transifex_token is not None,
allow_uncommitted_changes=allow_uncommitted_changes,
is_prerelease=is_prerelease,
disable_submodule_update=disable_submodule_update,
)
# if pushing to QGIS repo and pre-release, create an extra package with qgisMinVersion to 3.14
# since only QGIS 3.14+ supports the beta/experimental plugins trial
experimental_archive_name = None
if osgeo_username is not None and is_prerelease:
experimental_archive_name = parameters.archive_name(
release_version, True)
create_archive(
parameters,
release_version,
experimental_archive_name,
add_translations=transifex_token is not None,
allow_uncommitted_changes=allow_uncommitted_changes,
is_prerelease=True,
raise_min_version='3.14',
disable_submodule_update=disable_submodule_update,
)
if github_token is not None:
upload_asset_to_github_release(parameters,
asset_path=archive_name,
release_tag=release_version,
github_token=github_token)
if upload_plugin_repo_github:
xml_repo = create_plugin_repo(parameters=parameters,
release_version=release_version,
release_tag=release_tag,
archive=archive_name,
osgeo_username=osgeo_username)
upload_asset_to_github_release(parameters,
asset_path=xml_repo,
release_tag=release_version,
github_token=github_token,
asset_name='plugins.xml')
if plugin_repo_url:
xml_repo = create_plugin_repo(
parameters=parameters,
release_version=release_version,
release_tag=release_tag,
archive=archive_name,
osgeo_username=osgeo_username,
plugin_repo_url=plugin_repo_url,
)
print('Local XML repo file created : {}'.format(xml_repo))
if osgeo_username is not None:
assert osgeo_password is not None
if is_prerelease:
assert experimental_archive_name is not None
upload_plugin_to_osgeo(username=osgeo_username,
password=osgeo_password,
archive=experimental_archive_name)
else:
upload_plugin_to_osgeo(username=osgeo_username,
password=osgeo_password,
archive=archive_name)
class TestRelease(unittest.TestCase):
def setUp(self):
with open(".qgis-plugin-ci", "r", encoding="utf8") as f:
arg_dict = yaml.safe_load(f)
self.parameters = Parameters(arg_dict)
self.transifex_token = os.getenv("transifex_token")
self.github_token = os.getenv("github_token")
self.repo = None
self.t = None
if self.github_token:
print("init Github")
self.repo = Github(
self.github_token).get_repo("opengisch/qgis-plugin-ci")
self.clean_assets()
def tearDown(self):
self.clean_assets()
def clean_assets(self):
if self.repo:
rel = None
try:
rel = self.repo.get_release(id=RELEASE_VERSION_TEST)
except GithubException:
raise GithubReleaseNotFound(
"Release {} not found".format(RELEASE_VERSION_TEST))
if rel:
print("deleting release assets")
for asset in rel.get_assets():
print(" delete {}".format(asset.name))
asset.delete_asset()
if self.t:
try:
self.t._t.delete_project(self.parameters.project_slug)
except PyTransifexException:
pass
try:
self.t._t.delete_team("{}-team".format(
self.parameters.project_slug))
except PyTransifexException:
pass
def test_release(self):
release(self.parameters, RELEASE_VERSION_TEST)
@unittest.skipIf(can_skip_test(), "Missing transifex_token")
def test_release_with_transifex(self):
Translation(self.parameters, transifex_token=self.transifex_token)
release(self.parameters,
RELEASE_VERSION_TEST,
transifex_token=self.transifex_token)
def test_zipname(self):
"""Tests about the zipname for the QGIS plugin manager.
See #22 about dash
and also capital letters
"""
self.assertEqual(
"my_plugin-experimental.0.0.0.zip",
Parameters.archive_name("my_plugin", "0.0.0", True),
)
self.assertEqual("My_Plugin.0.0.0.zip",
Parameters.archive_name("My_Plugin", "0.0.0", False))
with self.assertWarnsRegex(Warning, DASH_WARNING):
Parameters.archive_name("my-plugin", "0.0.0")
@unittest.skipIf(can_skip_test(), "Missing github_token")
def test_release_upload_github(self):
release(
self.parameters,
RELEASE_VERSION_TEST,
github_token=self.github_token,
upload_plugin_repo_github=True,
)
# check the custom plugin repo
_, xml_repo = mkstemp(suffix=".xml")
url = "https://github.com/opengisch/qgis-plugin-ci/releases/download/{}/plugins.xml".format(
RELEASE_VERSION_TEST)
print("retrieve repo from {}".format(url))
urllib.request.urlretrieve(url, xml_repo)
replace_in_file(
xml_repo,
r"<update_date>[\w-]+<\/update_date>",
"<update_date>__TODAY__</update_date>",
)
if not filecmp.cmp(
"test/plugins.xml.expected", xml_repo, shallow=False):
import difflib
with open("test/plugins.xml.expected") as f:
text1 = f.readlines()
with open(xml_repo) as f:
text2 = f.readlines()
self.assertFalse(True,
"\n".join(difflib.unified_diff(text1, text2)))
# compare archive file size
gh_release = self.repo.get_release(id=RELEASE_VERSION_TEST)
archive_name = self.parameters.archive_name(
self.parameters.plugin_path, RELEASE_VERSION_TEST)
fs = os.path.getsize(archive_name)
print("size: ", fs)
self.assertGreater(fs, 0, "archive file size must be > 0")
found = False
for a in gh_release.get_assets():
if a.name == archive_name:
found = True
self.assertEqual(fs, a.size,
"asset size doesn't march archive size.")
break
self.assertTrue(found, "asset not found")
def test_release_changelog(self):
"""Test if changelog in metadata.txt inside zipped plugin after release command."""
# variables
cli_config_changelog = Path(
"test/fixtures/.qgis-plugin-ci-test-changelog.yaml")
version_to_release = "0.1.2"
# load specific parameters
with cli_config_changelog.open() as in_cfg:
arg_dict = yaml.safe_load(in_cfg)
parameters = Parameters(arg_dict)
self.assertIsInstance(parameters, Parameters)
# get output zip path
archive_name = parameters.archive_name(
plugin_name=parameters.plugin_path,
release_version=version_to_release)
# extract last items from changelog
parser = ChangelogParser()
self.assertTrue(parser.has_changelog())
changelog_lastitems = parser.last_items(
count=parameters.changelog_number_of_entries)
# Include a changelog
release(
parameters=parameters,
release_version=version_to_release,
allow_uncommitted_changes=True,
)
# open archive and compare
with ZipFile(archive_name, "r") as zip_file:
data = zip_file.read(f"{parameters.plugin_path}/metadata.txt")
# Changelog
self.assertGreater(
data.find(bytes(changelog_lastitems, "utf8")),
0,
f"changelog detection failed in release: {data}",
)
# Commit number
self.assertEqual(1, len(re.findall(r"commitNumber=\d+", str(data))))
# Commit sha1 not in the metadata.txt
self.assertEqual(0, len(re.findall(r"commitSha1=\d+", str(data))))
def setUp(self):
arg_dict = yaml.safe_load(open(".qgis-plugin-ci"))
self.parameters = Parameters(arg_dict)
self.transifex_token = os.getenv('transifex_token')
assert self.transifex_token is not None
self.t = Translation(self.parameters, transifex_token=self.transifex_token)
class TestRelease(unittest.TestCase):
def setUp(self):
arg_dict = yaml.safe_load(open(".qgis-plugin-ci"))
self.parameters = Parameters(arg_dict)
self.transifex_token = os.getenv('transifex_token')
self.github_token = os.getenv('github_token')
self.repo = None
self.t = None
if self.github_token:
print('init Github')
self.repo = Github(self.github_token).get_repo('opengisch/qgis-plugin-ci')
self.clean_assets()
def tearDown(self):
self.clean_assets()
def clean_assets(self):
if self.repo:
rel = None
try:
rel = self.repo.get_release(id=RELEASE_VERSION_TEST)
except GithubException as e:
raise GithubReleaseNotFound('Release {} not found'.format(RELEASE_VERSION_TEST))
if rel:
print('deleting release assets')
for asset in rel.get_assets():
print(' delete {}'.format(asset.name))
asset.delete_asset()
if self.t:
try:
self.t._t.delete_project(self.parameters.project_slug)
except PyTransifexException:
pass
try:
self.t._t.delete_team('{}-team'.format(self.parameters.project_slug))
except PyTransifexException:
pass
def test_release(self):
release(self.parameters, RELEASE_VERSION_TEST)
def test_release_with_transifex(self):
assert self.transifex_token is not None
t = Translation(self.parameters, transifex_token=self.transifex_token)
release(self.parameters, RELEASE_VERSION_TEST, transifex_token=self.transifex_token)
def test_release_upload_github(self):
release(self.parameters, RELEASE_VERSION_TEST, github_token=self.github_token, upload_plugin_repo_github=True)
# check the custom plugin repo
_, xml_repo = mkstemp(suffix='.xml')
url = 'https://github.com/opengisch/qgis-plugin-ci/releases/download/{}/plugins.xml'.format(RELEASE_VERSION_TEST)
print('retrieve repo from {}'.format(url))
urllib.request.urlretrieve(url, xml_repo)
replace_in_file(xml_repo, r'<update_date>[\w-]+<\/update_date>', '<update_date>__TODAY__</update_date>')
if not filecmp.cmp('test/plugins.xml.expected', xml_repo, shallow=False):
import difflib
text1 = open('test/plugins.xml.expected').readlines()
text2 = open(xml_repo).readlines()
self.assertFalse(True, '\n'.join(difflib.unified_diff(text1, text2)))
# compare archive file size
gh_release = self.repo.get_release(id=RELEASE_VERSION_TEST)
archive_name = self.parameters.archive_name(RELEASE_VERSION_TEST)
fs = os.path.getsize(archive_name)
print('size: ', fs)
self.assertGreater(fs, 0, 'archive file size must be > 0')
found = False
for a in gh_release.get_assets():
if a.name == archive_name:
found = True
self.assertEqual(fs, a.size, 'asset size doesn\'t march archive size.')
break
self.assertTrue(found, 'asset not found')
def test_release_changelog(self):
""" Test about the changelog in the metadata.txt. """
expected = b'changelog=\n Version 0.1.2:\n * Add a CHANGELOG.md file for testing\n'
# Include a changelog
release(self.parameters, RELEASE_VERSION_TEST)
archive_name = self.parameters.archive_name(RELEASE_VERSION_TEST)
with ZipFile(archive_name, 'r') as zip_file:
data = zip_file.read('qgis_plugin_ci_testing/metadata.txt')
self.assertGreater(data.find(expected), 0)
class TestRelease(unittest.TestCase):
def setUp(self):
arg_dict = yaml.safe_load(open(".qgis-plugin-ci"))
self.parameters = Parameters(arg_dict)
self.transifex_token = os.getenv("transifex_token")
self.github_token = os.getenv("github_token")
self.repo = None
self.t = None
if self.github_token:
print("init Github")
self.repo = Github(self.github_token).get_repo("opengisch/qgis-plugin-ci")
self.clean_assets()
def tearDown(self):
self.clean_assets()
def clean_assets(self):
if self.repo:
rel = None
try:
rel = self.repo.get_release(id=RELEASE_VERSION_TEST)
except GithubException as e:
raise GithubReleaseNotFound(
"Release {} not found".format(RELEASE_VERSION_TEST)
)
if rel:
print("deleting release assets")
for asset in rel.get_assets():
print(" delete {}".format(asset.name))
asset.delete_asset()
if self.t:
try:
self.t._t.delete_project(self.parameters.project_slug)
except PyTransifexException:
pass
try:
self.t._t.delete_team("{}-team".format(self.parameters.project_slug))
except PyTransifexException:
pass
def test_release(self):
release(self.parameters, RELEASE_VERSION_TEST)
def test_release_with_transifex(self):
assert self.transifex_token is not None
t = Translation(self.parameters, transifex_token=self.transifex_token)
release(
self.parameters, RELEASE_VERSION_TEST, transifex_token=self.transifex_token
)
def test_zipname(self):
"""Tests about the zipname for the QGIS plugin manager.
See #22 about dash
and also capital letters
"""
self.assertEqual(
"my_plugin-experimental.0.0.0.zip",
Parameters.archive_name("my_plugin", "0.0.0", True),
)
self.assertEqual(
"My_Plugin.0.0.0.zip", Parameters.archive_name("My_Plugin", "0.0.0", False)
)
with self.assertWarnsRegex(Warning, DASH_WARNING):
Parameters.archive_name("my-plugin", "0.0.0")
def test_release_upload_github(self):
release(
self.parameters,
RELEASE_VERSION_TEST,
github_token=self.github_token,
upload_plugin_repo_github=True,
)
# check the custom plugin repo
_, xml_repo = mkstemp(suffix=".xml")
url = "https://github.com/opengisch/qgis-plugin-ci/releases/download/{}/plugins.xml".format(
RELEASE_VERSION_TEST
)
print("retrieve repo from {}".format(url))
urllib.request.urlretrieve(url, xml_repo)
replace_in_file(
xml_repo,
r"<update_date>[\w-]+<\/update_date>",
"<update_date>__TODAY__</update_date>",
)
if not filecmp.cmp("test/plugins.xml.expected", xml_repo, shallow=False):
import difflib
text1 = open("test/plugins.xml.expected").readlines()
text2 = open(xml_repo).readlines()
self.assertFalse(True, "\n".join(difflib.unified_diff(text1, text2)))
# compare archive file size
gh_release = self.repo.get_release(id=RELEASE_VERSION_TEST)
archive_name = self.parameters.archive_name(
self.parameters.plugin_path, RELEASE_VERSION_TEST
)
fs = os.path.getsize(archive_name)
print("size: ", fs)
self.assertGreater(fs, 0, "archive file size must be > 0")
found = False
for a in gh_release.get_assets():
if a.name == archive_name:
found = True
self.assertEqual(fs, a.size, "asset size doesn't march archive size.")
break
self.assertTrue(found, "asset not found")
def test_release_changelog(self):
""" Test about the changelog in the metadata.txt. """
expected = (
b"changelog=\n "
b"Version 0.1.2 :\n "
b"* Tag using a wrong format DD/MM/YYYY according to Keep A Changelog\n "
b'* Tag without "v" prefix\n '
b"* Add a CHANGELOG.md file for testing"
)
# Include a changelog
release(self.parameters, RELEASE_VERSION_TEST)
archive_name = self.parameters.archive_name(
self.parameters.plugin_path, RELEASE_VERSION_TEST
)
with ZipFile(archive_name, "r") as zip_file:
data = zip_file.read("qgis_plugin_CI_testing/metadata.txt")
self.assertGreater(data.find(expected), 0)