def parse_metadata(metadatapath, check_vcs=False):
'''parse metadata file, optionally checking the git repo for metadata first'''
_ignored, ext = fdroidserver.common.get_extension(metadatapath)
accepted = fdroidserver.common.config['accepted_formats']
if ext not in accepted:
warn_or_exception(
_('"{path}" is not an accepted format, convert to: {formats}').
format(path=metadatapath, formats=', '.join(accepted)))
app = App()
app.metadatapath = metadatapath
name, _ignored = fdroidserver.common.get_extension(
os.path.basename(metadatapath))
if name == '.fdroid':
check_vcs = False
else:
app.id = name
with open(metadatapath, 'r', encoding='utf-8') as mf:
if ext == 'txt':
parse_txt_metadata(mf, app)
elif ext == 'json':
parse_json_metadata(mf, app)
elif ext == 'yml':
parse_yaml_metadata(mf, app)
else:
warn_or_exception(
_('Unknown metadata format: {path}').format(path=metadatapath))
if check_vcs and app.Repo:
build_dir = fdroidserver.common.get_build_dir(app)
metadata_in_repo = os.path.join(build_dir, '.fdroid.yml')
if not os.path.isfile(metadata_in_repo):
vcs, build_dir = fdroidserver.common.setup_vcs(app)
if isinstance(vcs, fdroidserver.common.vcs_git):
vcs.gotorevision(
'HEAD') # HEAD since we can't know where else to go
if os.path.isfile(metadata_in_repo):
logging.debug('Including metadata from ' + metadata_in_repo)
# do not include fields already provided by main metadata file
app_in_repo = parse_metadata(metadata_in_repo)
for k, v in app_in_repo.items():
if k not in app:
app[k] = v
post_metadata_parse(app)
if not app.id:
if app.builds:
build = app.builds[-1]
if build.subdir:
root_dir = build.subdir
else:
root_dir = '.'
paths = fdroidserver.common.manifest_paths(root_dir, build.gradle)
_ignored, _ignored, app.id = fdroidserver.common.parse_androidmanifests(
paths, app)
return app
def print_help():
print(_("usage: ") + _("fdroid [<command>] [-h|--help|--version|<args>]"))
print("")
print(_("Valid commands are:"))
for cmd, summary in commands.items():
print(" " + cmd + ' ' * (15 - len(cmd)) + summary)
print("")
def parse_metadata(metadatapath):
"""parse metadata file, also checking the source repo for .fdroid.yml
If this is a metadata file from fdroiddata, it will first load the
source repo type and URL from fdroiddata, then read .fdroid.yml if
it exists, then include the rest of the metadata as specified in
fdroiddata, so that fdroiddata has precedence over the metadata in
the source code.
"""
app = App()
app.metadatapath = metadatapath
metadata_file = os.path.basename(metadatapath)
name, _ignored = fdroidserver.common.get_extension(metadata_file)
if name != '.fdroid':
app.id = name
if metadatapath.endswith('.yml'):
with open(metadatapath, 'r') as mf:
parse_yaml_metadata(mf, app)
else:
_warn_or_exception(
_('Unknown metadata format: {path} (use: *.yml)').format(
path=metadatapath))
if metadata_file != '.fdroid.yml' and app.Repo:
build_dir = fdroidserver.common.get_build_dir(app)
metadata_in_repo = os.path.join(build_dir, '.fdroid.yml')
if os.path.isfile(metadata_in_repo):
try:
commit_id = fdroidserver.common.get_head_commit_id(
git.repo.Repo(build_dir))
logging.debug(
_('Including metadata from %s@%s') %
(metadata_in_repo, commit_id))
except git.exc.InvalidGitRepositoryError:
logging.debug(
_('Including metadata from {path}').format(
metadata_in_repo))
app_in_repo = parse_metadata(metadata_in_repo)
for k, v in app_in_repo.items():
if k not in app:
app[k] = v
post_metadata_parse(app)
if not app.id:
if app.get('Builds'):
build = app['Builds'][-1]
if build.subdir:
root_dir = build.subdir
else:
root_dir = '.'
paths = fdroidserver.common.manifest_paths(root_dir, build.gradle)
_ignored, _ignored, app.id = fdroidserver.common.parse_androidmanifests(
paths, app)
return app
def parse_yaml_metadata(mf, app):
"""Parse the .yml file and post-process it
Clean metadata .yml files can be used directly, but in order to
make a better user experience for people editing .yml files, there
is post processing. .fdroid.yml is embedded in the app's source
repo, so it is "user-generated". That means that it can have
weird things in it that need to be removed so they don't break the
overall process.
"""
try:
yamldata = yaml.load(mf, Loader=SafeLoader)
except yaml.YAMLError as e:
_warn_or_exception(_("could not parse '{path}'").format(path=mf.name) +
'\n' +
fdroidserver.common.run_yamllint(mf.name, indent=4),
cause=e)
deprecated_in_yaml = ['Provides']
if yamldata:
for field in tuple(yamldata.keys()):
if field not in yaml_app_fields + deprecated_in_yaml:
msg = (_(
"Unrecognised app field '{fieldname}' in '{path}'").format(
fieldname=field, path=mf.name))
if os.path.basename(mf.name) == '.fdroid.yml':
logging.error(msg)
del yamldata[field]
else:
_warn_or_exception(msg)
for deprecated_field in deprecated_in_yaml:
if deprecated_field in yamldata:
logging.warning(
_("Ignoring '{field}' in '{metapath}' "
"metadata because it is deprecated.").format(
field=deprecated_field, metapath=mf.name))
del (yamldata[deprecated_field])
if yamldata.get('Builds', None):
for build in yamldata.get('Builds', []):
# put all build flag keywords into a set to avoid
# excessive looping action
build_flag_set = set()
for build_flag in build.keys():
build_flag_set.add(build_flag)
for build_flag in build_flag_set:
if build_flag not in build_flags:
_warn_or_exception(
_("Unrecognised build flag '{build_flag}' "
"in '{path}'").format(build_flag=build_flag,
path=mf.name))
post_parse_yaml_metadata(yamldata)
app.update(yamldata)
return app
def write_metadata(metadatapath, app):
if metadatapath.endswith('.yml'):
if importlib.util.find_spec('ruamel.yaml'):
with open(metadatapath, 'w') as mf:
return write_yaml(mf, app)
else:
raise FDroidException(_('ruamel.yaml not installed, can not write metadata.'))
_warn_or_exception(_('Unknown metadata format: %s') % metadatapath)
def read_metadata(appids={}, sort_by_time=False):
"""Return a list of App instances sorted newest first
This reads all of the metadata files in a 'data' repository, then
builds a list of App instances from those files. The list is
sorted based on creation time, newest first. Most of the time,
the newer files are the most interesting.
appids is a dict with appids a keys and versionCodes as values.
"""
# Always read the srclibs before the apps, since they can use a srlib as
# their source repository.
read_srclibs()
apps = OrderedDict()
for basedir in ('metadata', 'tmp'):
if not os.path.exists(basedir):
os.makedirs(basedir)
if appids:
vercodes = fdroidserver.common.read_pkg_args(appids)
metadatafiles = fdroidserver.common.get_metadata_files(vercodes)
else:
metadatafiles = (glob.glob(os.path.join('metadata', '*.yml')) +
glob.glob('.fdroid.yml'))
if sort_by_time:
entries = ((os.stat(path).st_mtime, path) for path in metadatafiles)
metadatafiles = []
for _ignored, path in sorted(entries, reverse=True):
metadatafiles.append(path)
else:
# most things want the index alpha sorted for stability
metadatafiles = sorted(metadatafiles)
for metadatapath in metadatafiles:
appid, _ignored = fdroidserver.common.get_extension(
os.path.basename(metadatapath))
if appid != '.fdroid' and not fdroidserver.common.is_valid_package_name(
appid):
_warn_or_exception(
_("{appid} from {path} is not a valid Java Package Name!").
format(appid=appid, path=metadatapath))
if appid in apps:
_warn_or_exception(
_("Found multiple metadata files for {appid}").format(
appid=appid))
app = parse_metadata(metadatapath)
check_metadata(app)
apps[app.id] = app
return apps
def print_help(available_plugins=None):
print(_("usage: ") + _("fdroid [<command>] [-h|--help|--version|<args>]"))
print("")
print(_("Valid commands are:"))
for cmd, summary in COMMANDS.items():
print(" " + cmd + ' ' * (15 - len(cmd)) + summary)
if available_plugins:
print(_('commands from plugin modules:'))
for command in sorted(available_plugins.keys()):
print(' {:15}{}'.format(command,
available_plugins[command]['summary']))
print("")
def parse_yaml_srclib(metadatapath):
thisinfo = {'RepoType': '',
'Repo': '',
'Subdir': None,
'Prepare': None}
if not os.path.exists(metadatapath):
_warn_or_exception(_("Invalid scrlib metadata: '{file}' "
"does not exist"
.format(file=metadatapath)))
return thisinfo
with open(metadatapath, "r", encoding="utf-8") as f:
try:
data = yaml.load(f, Loader=SafeLoader)
if type(data) is not dict:
raise yaml.error.YAMLError(_('{file} is blank or corrupt!')
.format(file=metadatapath))
except yaml.error.YAMLError as e:
_warn_or_exception(_("Invalid srclib metadata: could not "
"parse '{file}'")
.format(file=metadatapath) + '\n'
+ fdroidserver.common.run_yamllint(metadatapath,
indent=4),
cause=e)
return thisinfo
for key in data.keys():
if key not in thisinfo.keys():
_warn_or_exception(_("Invalid srclib metadata: unknown key "
"'{key}' in '{file}'")
.format(key=key, file=metadatapath))
return thisinfo
else:
if key == 'Subdir':
if isinstance(data[key], str):
thisinfo[key] = data[key].split(',')
elif isinstance(data[key], list):
thisinfo[key] = data[key]
elif data[key] is None:
thisinfo[key] = ['']
elif key == 'Prepare' and isinstance(data[key], list):
thisinfo[key] = ' && '.join(data[key])
else:
thisinfo[key] = str(data[key] or '')
return thisinfo
def check_versionCode(versionCode):
try:
int(versionCode)
except ValueError:
warn_or_exception(
_('Invalid versionCode: "{versionCode}" is not an integer!').
format(versionCode=versionCode))
def parse_srclib(metadatapath):
thisinfo = {}
# Defaults for fields that come from metadata
thisinfo['Repo Type'] = ''
thisinfo['Repo'] = ''
thisinfo['Subdir'] = None
thisinfo['Prepare'] = None
if not os.path.exists(metadatapath):
return thisinfo
metafile = open(metadatapath, "r", encoding='utf-8')
n = 0
for line in metafile:
n += 1
line = line.rstrip('\r\n')
if not line or line.startswith("#"):
continue
try:
f, v = line.split(':', 1)
except ValueError:
warn_or_exception(_("Invalid metadata in %s:%d") % (line, n))
if f == "Subdir":
thisinfo[f] = v.split(',')
else:
thisinfo[f] = v
metafile.close()
return thisinfo
def main():
common.config = {
'accepted_formats': 'yml',
'sdk_path': os.getenv('ANDROID_HOME'),
}
common.fill_config_defaults(common.config)
parser = argparse.ArgumentParser(
usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
common.setup_global_opts(parser)
parser.add_argument(
"appid",
nargs='*',
help=
_("applicationId with optional versionCode in the form APPID[:VERCODE]"
))
metadata.add_metadata_arguments(parser)
options = parser.parse_args()
common.options = options
pkgs = common.read_pkg_args(options.appid, True)
allapps = metadata.read_metadata(pkgs)
apps = common.read_app_args(options.appid, allapps, True)
srclib_dir = os.path.join('build', 'srclib')
os.makedirs(srclib_dir, exist_ok=True)
srclibpaths = []
for appid, app in apps.items():
for build in app.get('Builds', []):
for lib in build.srclibs:
srclibpaths.append(
common.getsrclib(lib, srclib_dir, build=build))
print('Set up srclibs:')
pprint.pprint(srclibpaths)
def parse_buildline(lines):
v = "".join(lines)
parts = [p.replace("\\,", ",") for p in re.split(build_line_sep, v)]
if len(parts) < 3:
warn_or_exception(
_("Invalid build format: {value} in {name}").format(
value=v, name=mf.name))
build = Build()
build.versionName = parts[0]
build.versionCode = parts[1]
check_versionCode(build.versionCode)
if parts[2].startswith('!'):
# For backwards compatibility, handle old-style disabling,
# including attempting to extract the commit from the message
build.disable = parts[2][1:]
commit = 'unknown - see disabled'
index = parts[2].rfind('at ')
if index != -1:
commit = parts[2][index + 3:]
if commit.endswith(')'):
commit = commit[:-1]
build.commit = commit
else:
build.commit = parts[2]
for p in parts[3:]:
add_buildflag(p, build)
return build
def main():
parser = argparse.ArgumentParser(
usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
common.setup_global_opts(parser)
parser.add_argument(
"appid",
nargs='*',
help=
_("applicationId with optional versionCode in the form APPID[:VERCODE]"
))
metadata.add_metadata_arguments(parser)
options = parser.parse_args()
common.options = options
pkgs = common.read_pkg_args(options.appid, True)
allapps = metadata.read_metadata(pkgs)
apps = common.read_app_args(options.appid, allapps, True)
common.read_config(options)
srclib_dir = os.path.join('build', 'srclib')
os.makedirs(srclib_dir, exist_ok=True)
srclibpaths = []
for appid, app in apps.items():
vcs, _ignored = common.setup_vcs(app)
vcs.gotorevision('HEAD', refresh=False)
for build in app.get('Builds', []):
for lib in build.srclibs:
srclibpaths.append(
common.getsrclib(lib,
srclib_dir,
prepare=False,
build=build))
print('Set up srclibs:')
pprint.pprint(srclibpaths)
def linkify(self, txt):
res_plain = ''
res_html = ''
while True:
index = txt.find("[")
if index == -1:
return (res_plain + self.formatted(txt, False),
res_html + self.formatted(txt, True))
res_plain += self.formatted(txt[:index], False)
res_html += self.formatted(txt[:index], True)
txt = txt[index:]
if txt.startswith("[["):
index = txt.find("]]")
if index == -1:
_warn_or_exception(_("Unterminated ]]"))
url = txt[2:index]
if self.linkResolver:
url, urltext = self.linkResolver.resolve_description_link(
url)
else:
urltext = url
res_html += '<a href="' + url + '">' + html.escape(
urltext, quote=False) + '</a>'
res_plain += urltext
txt = txt[index + 2:]
else:
index = txt.find("]")
if index == -1:
_warn_or_exception(_("Unterminated ]"))
url = txt[1:index]
index2 = url.find(' ')
if index2 == -1:
urltxt = url
else:
urltxt = url[index2 + 1:]
url = url[:index2]
if url == urltxt:
_warn_or_exception(
_("URL title is just the URL, use brackets: [URL]")
)
res_html += '<a href="' + url + '">' + html.escape(
urltxt, quote=False) + '</a>'
res_plain += urltxt
if urltxt != url:
res_plain += ' (' + url + ')'
txt = txt[index + 1:]
def parse_yaml_metadata(mf, app):
try:
yamldata = yaml.load(mf, Loader=SafeLoader)
except yaml.YAMLError as e:
_warn_or_exception(_("could not parse '{path}'")
.format(path=mf.name) + '\n'
+ fdroidserver.common.run_yamllint(mf.name,
indent=4),
cause=e)
deprecated_in_yaml = ['Provides']
if yamldata:
for field in yamldata:
if field not in yaml_app_fields:
if field not in deprecated_in_yaml:
_warn_or_exception(_("Unrecognised app field "
"'{fieldname}' in '{path}'")
.format(fieldname=field,
path=mf.name))
for deprecated_field in deprecated_in_yaml:
if deprecated_field in yamldata:
logging.warning(_("Ignoring '{field}' in '{metapath}' "
"metadata because it is deprecated.")
.format(field=deprecated_field,
metapath=mf.name))
del(yamldata[deprecated_field])
if yamldata.get('Builds', None):
for build in yamldata.get('Builds', []):
# put all build flag keywords into a set to avoid
# excessive looping action
build_flag_set = set()
for build_flag in build.keys():
build_flag_set.add(build_flag)
for build_flag in build_flag_set:
if build_flag not in build_flags:
_warn_or_exception(
_("Unrecognised build flag '{build_flag}' "
"in '{path}'").format(build_flag=build_flag,
path=mf.name))
post_parse_yaml_metadata(yamldata)
app.update(yamldata)
return app
def add_metadata_arguments(parser):
'''add common command line flags related to metadata processing'''
parser.add_argument(
"-W",
choices=['error', 'warn', 'ignore'],
default='error',
help=_(
"force metadata errors (default) to be warnings, or to be ignored."
))
def write_metadata(metadatapath, app):
_ignored, ext = fdroidserver.common.get_extension(metadatapath)
accepted = fdroidserver.common.config['accepted_formats']
if ext not in accepted:
warn_or_exception(
_('Cannot write "{path}", not an accepted format, use: {formats}').
format(path=metadatapath, formats=', '.join(accepted)))
try:
with open(metadatapath, 'w', encoding='utf8') as mf:
if ext == 'txt':
return write_txt(mf, app)
elif ext == 'yml':
return write_yaml(mf, app)
except FDroidException as e:
os.remove(metadatapath)
raise e
warn_or_exception(_('Unknown metadata format: %s') % metadatapath)
def check(self, v, appid):
if not v:
return
if type(v) == list:
values = v
else:
values = [v]
for v in values:
if not self.compiled.match(v):
_warn_or_exception(_("'{value}' is not a valid {field} in {appid}. Regex pattern: {pattern}")
.format(value=v, field=self.name, appid=appid, pattern=self.matching))
def add_buildflag(p, build):
if not p.strip():
warn_or_exception(
_("Empty build flag at {linedesc}").format(linedesc=linedesc))
bv = p.split('=', 1)
if len(bv) != 2:
warn_or_exception(
_("Invalid build flag at {line} in {linedesc}").format(
line=buildlines[0], linedesc=linedesc))
pk, pv = bv
pk = pk.lstrip()
if pk == 'update':
pk = 'androidupdate' # avoid conflicting with Build(dict).update()
t = flagtype(pk)
if t == TYPE_LIST:
pv = split_list_values(pv)
build[pk] = pv
elif t == TYPE_STRING or t == TYPE_SCRIPT:
build[pk] = pv
elif t == TYPE_BOOL:
build[pk] = _decode_bool(pv)
def parse_metadata(metadatapath, check_vcs=False, refresh=True):
'''parse metadata file, optionally checking the git repo for metadata first'''
app = App()
app.metadatapath = metadatapath
name, _ignored = fdroidserver.common.get_extension(
os.path.basename(metadatapath))
if name == '.fdroid':
check_vcs = False
else:
app.id = name
if metadatapath.endswith('.yml'):
with open(metadatapath, 'r') as mf:
parse_yaml_metadata(mf, app)
else:
_warn_or_exception(
_('Unknown metadata format: {path} (use: *.yml)').format(
path=metadatapath))
if check_vcs and app.Repo:
build_dir = fdroidserver.common.get_build_dir(app)
metadata_in_repo = os.path.join(build_dir, '.fdroid.yml')
if not os.path.isfile(metadata_in_repo):
vcs, build_dir = fdroidserver.common.setup_vcs(app)
if isinstance(vcs, fdroidserver.common.vcs_git):
vcs.gotorevision(
'HEAD',
refresh) # HEAD since we can't know where else to go
if os.path.isfile(metadata_in_repo):
logging.debug('Including metadata from ' + metadata_in_repo)
# do not include fields already provided by main metadata file
app_in_repo = parse_metadata(metadata_in_repo)
for k, v in app_in_repo.items():
if k not in app:
app[k] = v
post_metadata_parse(app)
if not app.id:
if app.builds:
build = app.builds[-1]
if build.subdir:
root_dir = build.subdir
else:
root_dir = '.'
paths = fdroidserver.common.manifest_paths(root_dir, build.gradle)
_ignored, _ignored, app.id = fdroidserver.common.parse_androidmanifests(
paths, app)
return app
def post_parse_yaml_metadata(yamldata):
"""transform yaml metadata to our internal data format"""
for build in yamldata.get('Builds', []):
for flag in build.keys():
_flagtype = flagtype(flag)
if _flagtype is TYPE_SCRIPT:
# concatenate script flags into a single string if they are stored as list
if isinstance(build[flag], list):
build[flag] = ' && '.join(build[flag])
elif _flagtype is TYPE_STRING:
# things like versionNames are strings, but without quotes can be numbers
if isinstance(build[flag], float) or isinstance(build[flag], int):
build[flag] = str(build[flag])
elif _flagtype is TYPE_INT:
# versionCode must be int
if not isinstance(build[flag], int):
_warn_or_exception(_('{build_flag} must be an integer, found: {value}')
.format(build_flag=flag, value=build[flag]))
def get_default_app_info(metadatapath=None):
if metadatapath is None:
appid = None
else:
appid, _ignored = fdroidserver.common.get_extension(
os.path.basename(metadatapath))
if appid == '.fdroid': # we have local metadata in the app's source
if os.path.exists('AndroidManifest.xml'):
manifestroot = fdroidserver.common.parse_xml('AndroidManifest.xml')
else:
pattern = re.compile(
r""".*manifest\.srcFile\s+'AndroidManifest\.xml'.*""")
for root, dirs, files in os.walk(os.getcwd()):
if 'build.gradle' in files:
p = os.path.join(root, 'build.gradle')
with open(p, 'rb') as f:
data = f.read()
m = pattern.search(data)
if m:
logging.debug(
'Using: ' +
os.path.join(root, 'AndroidManifest.xml'))
manifestroot = fdroidserver.common.parse_xml(
os.path.join(root, 'AndroidManifest.xml'))
break
if manifestroot is None:
warn_or_exception(
_("Cannot find an appid for {path}!").format(
path=metadatapath))
appid = manifestroot.attrib['package']
app = App()
app.metadatapath = metadatapath
if appid is not None:
app.id = appid
return app
def main():
available_plugins = find_plugins()
if len(sys.argv) <= 1:
print_help(available_plugins=available_plugins)
sys.exit(0)
command = sys.argv[1]
if command not in COMMANDS and command not in available_plugins.keys():
if command in ('-h', '--help'):
print_help(available_plugins=available_plugins)
sys.exit(0)
elif command == '--version':
output = _('no version info found!')
cmddir = os.path.realpath(
os.path.dirname(os.path.dirname(__file__)))
moduledir = os.path.realpath(
os.path.dirname(fdroidserver.common.__file__) + '/..')
if cmddir == moduledir:
# running from git
os.chdir(cmddir)
if os.path.isdir('.git'):
import subprocess
try:
output = subprocess.check_output(
['git', 'describe'],
stderr=subprocess.STDOUT,
universal_newlines=True)
except subprocess.CalledProcessError:
output = 'git commit ' + subprocess.check_output(
['git', 'rev-parse', 'HEAD'],
universal_newlines=True)
elif os.path.exists('setup.py'):
import re
m = re.search(
r'''.*[\s,\(]+version\s*=\s*["']([0-9a-z.]+)["'].*''',
open('setup.py').read(),
flags=re.MULTILINE)
if m:
output = m.group(1) + '\n'
else:
from pkg_resources import get_distribution
output = get_distribution('fdroidserver').version + '\n'
print(output)
sys.exit(0)
else:
print(_("Command '%s' not recognised.\n" % command))
print_help(available_plugins=available_plugins)
sys.exit(1)
verbose = any(s in sys.argv for s in ['-v', '--verbose'])
quiet = any(s in sys.argv for s in ['-q', '--quiet'])
# Helpful to differentiate warnings from errors even when on quiet
logformat = '%(levelname)s: %(message)s'
loglevel = logging.INFO
if verbose:
loglevel = logging.DEBUG
elif quiet:
loglevel = logging.WARN
logging.basicConfig(format=logformat, level=loglevel)
if verbose and quiet:
logging.critical(
_("Conflicting arguments: '--verbose' and '--quiet' "
"can not be specified at the same time."))
sys.exit(1)
# temporary workaround until server.py becomes deploy.py
if command == 'deploy':
command = 'server'
sys.argv.insert(2, 'update')
# Trick optparse into displaying the right usage when --help is used.
sys.argv[0] += ' ' + command
del sys.argv[1]
if command in COMMANDS.keys():
mod = __import__('fdroidserver.' + command, None, None, [command])
else:
mod = __import__(available_plugins[command]['name'], None, None,
[command])
system_langcode, system_encoding = locale.getdefaultlocale()
if system_encoding is None or system_encoding.lower() not in ('utf-8',
'utf8'):
logging.warning(
_("Encoding is set to '{enc}' fdroid might run "
"into encoding issues. Please set it to 'UTF-8' "
"for best results.".format(enc=system_encoding)))
try:
mod.main()
# These are ours, contain a proper message and are "expected"
except (fdroidserver.common.FDroidException,
fdroidserver.metadata.MetaDataException) as e:
if verbose:
raise
else:
logging.critical(str(e))
sys.exit(1)
except ArgumentError as e:
logging.critical(str(e))
sys.exit(1)
except KeyboardInterrupt:
print('')
fdroidserver.common.force_exit(1)
# These should only be unexpected crashes due to bugs in the code
# str(e) often doesn't contain a reason, so just show the backtrace
except Exception as e:
logging.critical(_("Unknown exception found!"))
raise e
sys.exit(0)
import re
import sys
import os
import locale
import pkgutil
import logging
import fdroidserver.common
import fdroidserver.metadata
from fdroidserver import _
from argparse import ArgumentError
from collections import OrderedDict
COMMANDS = OrderedDict([
("build", _("Build a package from source")),
("init", _("Quickly start a new repository")),
("publish", _("Sign and place packages in the repo")),
("gpgsign", _("Add PGP signatures using GnuPG for packages in repo")),
("update", _("Update repo information for new packages")),
("deploy", _("Interact with the repo HTTP server")),
("verify", _("Verify the integrity of downloaded packages")),
("checkupdates", _("Check for updates to applications")),
("import", _("Add a new application from its source code")),
("install", _("Install built packages on devices")),
("readmeta", _("Read all the metadata files and exit")),
("rewritemeta", _("Rewrite all the metadata files")),
("lint", _("Warn about possible metadata errors")),
("scanner", _("Scan the source code of a package")),
("stats", _("Update the stats of the repo")),
("server", _("Old, deprecated name for fdroid deploy")),
def read_metadata(xref=True, check_vcs=[], refresh=True, sort_by_time=False):
"""Return a list of App instances sorted newest first
This reads all of the metadata files in a 'data' repository, then
builds a list of App instances from those files. The list is
sorted based on creation time, newest first. Most of the time,
the newer files are the most interesting.
check_vcs is the list of appids to check for .fdroid.yml in source
"""
# Always read the srclibs before the apps, since they can use a srlib as
# their source repository.
read_srclibs()
apps = OrderedDict()
for basedir in ('metadata', 'tmp'):
if not os.path.exists(basedir):
os.makedirs(basedir)
metadatafiles = (glob.glob(os.path.join('metadata', '*.yml')) +
glob.glob('.fdroid.yml'))
if sort_by_time:
entries = ((os.stat(path).st_mtime, path) for path in metadatafiles)
metadatafiles = []
for _ignored, path in sorted(entries, reverse=True):
metadatafiles.append(path)
else:
# most things want the index alpha sorted for stability
metadatafiles = sorted(metadatafiles)
for metadatapath in metadatafiles:
appid, _ignored = fdroidserver.common.get_extension(
os.path.basename(metadatapath))
if appid != '.fdroid' and not fdroidserver.common.is_valid_package_name(
appid):
_warn_or_exception(
_("{appid} from {path} is not a valid Java Package Name!").
format(appid=appid, path=metadatapath))
if appid in apps:
_warn_or_exception(
_("Found multiple metadata files for {appid}").format(
appid=appid))
app = parse_metadata(metadatapath, appid in check_vcs, refresh)
check_metadata(app)
apps[app.id] = app
if xref:
# Parse all descriptions at load time, just to ensure cross-referencing
# errors are caught early rather than when they hit the build server.
for appid, app in apps.items():
try:
description_html(app.Description,
DummyDescriptionResolver(apps))
except MetaDataException as e:
_warn_or_exception(
_("Problem with description of {appid}: {error}").format(
appid=appid, error=str(e)))
return apps
def _decode_bool(s):
if bool_true.match(s):
return True
if bool_false.match(s):
return False
warn_or_exception(_("Invalid boolean '%s'") % s)
def linkres(appid):
if appid in apps:
return ("fdroid.app:" + appid, "Dummy name - don't know yet")
warn_or_exception(
_("Cannot resolve app id {appid}").format(appid=appid))
def read_metadata(xref=True, check_vcs=[], refresh=True, sort_by_time=False):
"""Return a list of App instances sorted newest first
This reads all of the metadata files in a 'data' repository, then
builds a list of App instances from those files. The list is
sorted based on creation time, newest first. Most of the time,
the newer files are the most interesting.
If there are multiple metadata files for a single appid, then the first
file that is parsed wins over all the others, and the rest throw an
exception. So the original .txt format is parsed first, at least until
newer formats stabilize.
check_vcs is the list of appids to check for .fdroid.yml in source
"""
# Always read the srclibs before the apps, since they can use a srlib as
# their source repository.
read_srclibs()
apps = OrderedDict()
for basedir in ('metadata', 'tmp'):
if not os.path.exists(basedir):
os.makedirs(basedir)
metadatafiles = (glob.glob(os.path.join('metadata', '*.txt')) +
glob.glob(os.path.join('metadata', '*.json')) +
glob.glob(os.path.join('metadata', '*.yml')) +
glob.glob('.fdroid.txt') + glob.glob('.fdroid.json') +
glob.glob('.fdroid.yml'))
if sort_by_time:
entries = ((os.stat(path).st_mtime, path) for path in metadatafiles)
metadatafiles = []
for _ignored, path in sorted(entries, reverse=True):
metadatafiles.append(path)
else:
# most things want the index alpha sorted for stability
metadatafiles = sorted(metadatafiles)
for metadatapath in metadatafiles:
if metadatapath == '.fdroid.txt':
warn_or_exception(
_('.fdroid.txt is not supported! Convert to .fdroid.yml or .fdroid.json.'
))
appid, _ignored = fdroidserver.common.get_extension(
os.path.basename(metadatapath))
if appid in apps:
warn_or_exception(
_("Found multiple metadata files for {appid}").format(
appid=appid))
app = parse_metadata(metadatapath, appid in check_vcs, refresh)
check_metadata(app)
apps[app.id] = app
if xref:
# Parse all descriptions at load time, just to ensure cross-referencing
# errors are caught early rather than when they hit the build server.
def linkres(appid):
if appid in apps:
return ("fdroid.app:" + appid, "Dummy name - don't know yet")
warn_or_exception(
_("Cannot resolve app id {appid}").format(appid=appid))
for appid, app in apps.items():
try:
description_html(app.Description, linkres)
except MetaDataException as e:
warn_or_exception(
_("Problem with description of {appid}: {error}").format(
appid=appid, error=str(e)))
return apps
def package(self, output=None, keep_box_file=False):
if not output:
output = "buildserver.box"
logging.debug(
"no output name set for packaging '%s', "
"defaulting to %s", self.srvname, output)
storagePool = self.conn.storagePoolLookupByName('default')
domainInfo = self.conn.lookupByName(self.srvname).info()
if storagePool:
if isfile('metadata.json'):
os.remove('metadata.json')
if isfile('Vagrantfile'):
os.remove('Vagrantfile')
if isfile('box.img'):
os.remove('box.img')
logging.debug('preparing box.img for box %s', output)
vol = storagePool.storageVolLookupByName(self.srvname + '.img')
imagepath = vol.path()
# TODO use a libvirt storage pool to ensure the img file is readable
if not os.access(imagepath, os.R_OK):
logging.warning(
_('Cannot read "{path}"!').format(path=imagepath))
_check_call([
'sudo', '/bin/chmod', '-R', 'a+rX',
'/var/lib/libvirt/images'
])
shutil.copy2(imagepath, 'box.img')
_check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img'])
img_info_raw = _check_output(
['qemu-img', 'info', '--output=json', 'box.img'])
img_info = json.loads(img_info_raw.decode('utf-8'))
metadata = {
"provider": "libvirt",
"format": img_info['format'],
"virtual_size":
math.ceil(img_info['virtual-size'] / (1024.**3)),
}
logging.debug('preparing metadata.json for box %s', output)
with open('metadata.json', 'w') as fp:
fp.write(json.dumps(metadata))
logging.debug('preparing Vagrantfile for box %s', output)
vagrantfile = textwrap.dedent("""\
Vagrant.configure("2") do |config|
config.ssh.username = "******"
config.ssh.password = "******"
config.vm.provider :libvirt do |libvirt|
libvirt.driver = "kvm"
libvirt.host = ""
libvirt.connect_via_ssh = false
libvirt.storage_pool_name = "default"
libvirt.cpus = {cpus}
libvirt.memory = {memory}
end
end""".format_map({
'memory': str(int(domainInfo[1] / 1024)),
'cpus': str(domainInfo[3])
}))
with open('Vagrantfile', 'w') as fp:
fp.write(vagrantfile)
with tarfile.open(output, 'w:gz') as tar:
logging.debug('adding metadata.json to box %s ...', output)
tar.add('metadata.json')
logging.debug('adding Vagrantfile to box %s ...', output)
tar.add('Vagrantfile')
logging.debug('adding box.img to box %s ...', output)
tar.add('box.img')
if not keep_box_file:
logging.debug(
'box packaging complete, removing temporary files.')
os.remove('metadata.json')
os.remove('Vagrantfile')
os.remove('box.img')
else:
logging.warn("could not connect to storage-pool 'default', "
"skip packaging buildserver box")
def package(self, output=None, keep_box_file=False):
if not output:
output = "buildserver.box"
logging.debug("no output name set for packaging '%s', "
"defaulting to %s", self.srvname, output)
storagePool = self.conn.storagePoolLookupByName('default')
domainInfo = self.conn.lookupByName(self.srvname).info()
if storagePool:
if isfile('metadata.json'):
os.remove('metadata.json')
if isfile('Vagrantfile'):
os.remove('Vagrantfile')
if isfile('box.img'):
os.remove('box.img')
logging.debug('preparing box.img for box %s', output)
vol = storagePool.storageVolLookupByName(self.srvname + '.img')
imagepath = vol.path()
# TODO use a libvirt storage pool to ensure the img file is readable
if not os.access(imagepath, os.R_OK):
logging.warning(_('Cannot read "{path}"!').format(path=imagepath))
_check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images'])
shutil.copy2(imagepath, 'box.img')
_check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img'])
img_info_raw = _check_output(['qemu-img', 'info', '--output=json', 'box.img'])
img_info = json.loads(img_info_raw.decode('utf-8'))
metadata = {"provider": "libvirt",
"format": img_info['format'],
"virtual_size": math.ceil(img_info['virtual-size'] / (1024. ** 3)),
}
logging.debug('preparing metadata.json for box %s', output)
with open('metadata.json', 'w') as fp:
fp.write(json.dumps(metadata))
logging.debug('preparing Vagrantfile for box %s', output)
vagrantfile = textwrap.dedent("""\
Vagrant.configure("2") do |config|
config.ssh.username = "******"
config.ssh.password = "******"
config.vm.provider :libvirt do |libvirt|
libvirt.driver = "kvm"
libvirt.host = ""
libvirt.connect_via_ssh = false
libvirt.storage_pool_name = "default"
libvirt.cpus = {cpus}
libvirt.memory = {memory}
end
end""".format_map({'memory': str(int(domainInfo[1] / 1024)), 'cpus': str(domainInfo[3])}))
with open('Vagrantfile', 'w') as fp:
fp.write(vagrantfile)
with tarfile.open(output, 'w:gz') as tar:
logging.debug('adding metadata.json to box %s ...', output)
tar.add('metadata.json')
logging.debug('adding Vagrantfile to box %s ...', output)
tar.add('Vagrantfile')
logging.debug('adding box.img to box %s ...', output)
tar.add('box.img')
if not keep_box_file:
logging.debug('box packaging complete, removing temporary files.')
os.remove('metadata.json')
os.remove('Vagrantfile')
os.remove('box.img')
else:
logging.warn("could not connect to storage-pool 'default', "
"skip packaging buildserver box")
def parse_txt_metadata(mf, app):
linedesc = None
def add_buildflag(p, build):
if not p.strip():
warn_or_exception(
_("Empty build flag at {linedesc}").format(linedesc=linedesc))
bv = p.split('=', 1)
if len(bv) != 2:
warn_or_exception(
_("Invalid build flag at {line} in {linedesc}").format(
line=buildlines[0], linedesc=linedesc))
pk, pv = bv
pk = pk.lstrip()
if pk == 'update':
pk = 'androidupdate' # avoid conflicting with Build(dict).update()
t = flagtype(pk)
if t == TYPE_LIST:
pv = split_list_values(pv)
build[pk] = pv
elif t == TYPE_STRING or t == TYPE_SCRIPT:
build[pk] = pv
elif t == TYPE_BOOL:
build[pk] = _decode_bool(pv)
elif t == TYPE_INT:
build[pk] = int(pv)
def parse_buildline(lines):
v = "".join(lines)
parts = [p.replace("\\,", ",") for p in re.split(build_line_sep, v)]
if len(parts) < 3:
warn_or_exception(
_("Invalid build format: {value} in {name}").format(
value=v, name=mf.name))
build = Build()
build.versionName = parts[0]
build.versionCode = parts[1]
check_versionCode(build.versionCode)
if parts[2].startswith('!'):
# For backwards compatibility, handle old-style disabling,
# including attempting to extract the commit from the message
build.disable = parts[2][1:]
commit = 'unknown - see disabled'
index = parts[2].rfind('at ')
if index != -1:
commit = parts[2][index + 3:]
if commit.endswith(')'):
commit = commit[:-1]
build.commit = commit
else:
build.commit = parts[2]
for p in parts[3:]:
add_buildflag(p, build)
return build
def check_versionCode(versionCode):
try:
int(versionCode)
except ValueError:
warn_or_exception(
_('Invalid versionCode: "{versionCode}" is not an integer!').
format(versionCode=versionCode))
def add_comments(key):
if not curcomments:
return
app.comments[key] = list(curcomments)
del curcomments[:]
mode = 0
buildlines = []
multiline_lines = []
curcomments = []
build = None
vc_seen = set()
app.builds = []
c = 0
for line in mf:
c += 1
linedesc = "%s:%d" % (mf.name, c)
line = line.rstrip('\r\n')
if mode == 3:
if build_cont.match(line):
if line.endswith('\\'):
buildlines.append(line[:-1].lstrip())
else:
buildlines.append(line.lstrip())
bl = ''.join(buildlines)
add_buildflag(bl, build)
del buildlines[:]
else:
if not build.commit and not build.disable:
warn_or_exception(
_("No commit specified for {versionName} in {linedesc}"
).format(versionName=build.versionName,
linedesc=linedesc))
app.builds.append(build)
add_comments('build:' + build.versionCode)
mode = 0
if mode == 0:
if not line:
continue
if line.startswith("#"):
curcomments.append(line[1:].strip())
continue
try:
f, v = line.split(':', 1)
except ValueError:
warn_or_exception(_("Invalid metadata in: ") + linedesc)
if f not in app_fields:
warn_or_exception(_('Unrecognised app field: ') + f)
# Translate obsolete fields...
if f == 'Market Version':
f = 'Current Version'
if f == 'Market Version Code':
f = 'Current Version Code'
f = f.replace(' ', '')
ftype = fieldtype(f)
if ftype not in [TYPE_BUILD, TYPE_BUILD_V2]:
add_comments(f)
if ftype == TYPE_MULTILINE:
mode = 1
if v:
warn_or_exception(
_("Unexpected text on same line as {field} in {linedesc}"
).format(field=f, linedesc=linedesc))
elif ftype == TYPE_STRING:
app[f] = v
elif ftype == TYPE_LIST:
app[f] = split_list_values(v)
elif ftype == TYPE_BUILD:
if v.endswith("\\"):
mode = 2
del buildlines[:]
buildlines.append(v[:-1])
else:
build = parse_buildline([v])
app.builds.append(build)
add_comments('build:' + app.builds[-1].versionCode)
elif ftype == TYPE_BUILD_V2:
vv = v.split(',')
if len(vv) != 2:
warn_or_exception(
_('Build should have comma-separated '
'versionName and versionCode, '
'not "{value}", in {linedesc}').format(
value=v, linedesc=linedesc))
build = Build()
build.versionName = vv[0]
build.versionCode = vv[1]
check_versionCode(build.versionCode)
if build.versionCode in vc_seen:
warn_or_exception(
_('Duplicate build recipe found for versionCode {versionCode} in {linedesc}'
).format(versionCode=build.versionCode,
linedesc=linedesc))
vc_seen.add(build.versionCode)
del buildlines[:]
mode = 3
elif ftype == TYPE_OBSOLETE:
pass # Just throw it away!
else:
warn_or_exception(
_("Unrecognised field '{field}' in {linedesc}").format(
field=f, linedesc=linedesc))
elif mode == 1: # Multiline field
if line == '.':
mode = 0
app[f] = '\n'.join(multiline_lines)
del multiline_lines[:]
else:
multiline_lines.append(line)
elif mode == 2: # Line continuation mode in Build Version
if line.endswith("\\"):
buildlines.append(line[:-1])
else:
buildlines.append(line)
build = parse_buildline(buildlines)
app.builds.append(build)
add_comments('build:' + app.builds[-1].versionCode)
mode = 0
add_comments(None)
# Mode at end of file should always be 0
if mode == 1:
warn_or_exception(
_("{field} not terminated in {name}").format(field=f,
name=mf.name))
if mode == 2:
warn_or_exception(
_("Unterminated continuation in {name}").format(name=mf.name))
if mode == 3:
warn_or_exception(
_("Unterminated build in {name}").format(name=mf.name))
return app
