def parse_metadata(metadatapath, check_vcs=False): '''parse metadata file, optionally checking the git repo for metadata first''' _ignored, ext = fdroidserver.common.get_extension(metadatapath) accepted = fdroidserver.common.config['accepted_formats'] if ext not in accepted: warn_or_exception( _('"{path}" is not an accepted format, convert to: {formats}'). format(path=metadatapath, formats=', '.join(accepted))) app = App() app.metadatapath = metadatapath name, _ignored = fdroidserver.common.get_extension( os.path.basename(metadatapath)) if name == '.fdroid': check_vcs = False else: app.id = name with open(metadatapath, 'r', encoding='utf-8') as mf: if ext == 'txt': parse_txt_metadata(mf, app) elif ext == 'json': parse_json_metadata(mf, app) elif ext == 'yml': parse_yaml_metadata(mf, app) else: warn_or_exception( _('Unknown metadata format: {path}').format(path=metadatapath)) if check_vcs and app.Repo: build_dir = fdroidserver.common.get_build_dir(app) metadata_in_repo = os.path.join(build_dir, '.fdroid.yml') if not os.path.isfile(metadata_in_repo): vcs, build_dir = fdroidserver.common.setup_vcs(app) if isinstance(vcs, fdroidserver.common.vcs_git): vcs.gotorevision( 'HEAD') # HEAD since we can't know where else to go if os.path.isfile(metadata_in_repo): logging.debug('Including metadata from ' + metadata_in_repo) # do not include fields already provided by main metadata file app_in_repo = parse_metadata(metadata_in_repo) for k, v in app_in_repo.items(): if k not in app: app[k] = v post_metadata_parse(app) if not app.id: if app.builds: build = app.builds[-1] if build.subdir: root_dir = build.subdir else: root_dir = '.' paths = fdroidserver.common.manifest_paths(root_dir, build.gradle) _ignored, _ignored, app.id = fdroidserver.common.parse_androidmanifests( paths, app) return app
def print_help(): print(_("usage: ") + _("fdroid [<command>] [-h|--help|--version|<args>]")) print("") print(_("Valid commands are:")) for cmd, summary in commands.items(): print(" " + cmd + ' ' * (15 - len(cmd)) + summary) print("")
def parse_metadata(metadatapath): """parse metadata file, also checking the source repo for .fdroid.yml If this is a metadata file from fdroiddata, it will first load the source repo type and URL from fdroiddata, then read .fdroid.yml if it exists, then include the rest of the metadata as specified in fdroiddata, so that fdroiddata has precedence over the metadata in the source code. """ app = App() app.metadatapath = metadatapath metadata_file = os.path.basename(metadatapath) name, _ignored = fdroidserver.common.get_extension(metadata_file) if name != '.fdroid': app.id = name if metadatapath.endswith('.yml'): with open(metadatapath, 'r') as mf: parse_yaml_metadata(mf, app) else: _warn_or_exception( _('Unknown metadata format: {path} (use: *.yml)').format( path=metadatapath)) if metadata_file != '.fdroid.yml' and app.Repo: build_dir = fdroidserver.common.get_build_dir(app) metadata_in_repo = os.path.join(build_dir, '.fdroid.yml') if os.path.isfile(metadata_in_repo): try: commit_id = fdroidserver.common.get_head_commit_id( git.repo.Repo(build_dir)) logging.debug( _('Including metadata from %s@%s') % (metadata_in_repo, commit_id)) except git.exc.InvalidGitRepositoryError: logging.debug( _('Including metadata from {path}').format( metadata_in_repo)) app_in_repo = parse_metadata(metadata_in_repo) for k, v in app_in_repo.items(): if k not in app: app[k] = v post_metadata_parse(app) if not app.id: if app.get('Builds'): build = app['Builds'][-1] if build.subdir: root_dir = build.subdir else: root_dir = '.' paths = fdroidserver.common.manifest_paths(root_dir, build.gradle) _ignored, _ignored, app.id = fdroidserver.common.parse_androidmanifests( paths, app) return app
def parse_yaml_metadata(mf, app): """Parse the .yml file and post-process it Clean metadata .yml files can be used directly, but in order to make a better user experience for people editing .yml files, there is post processing. .fdroid.yml is embedded in the app's source repo, so it is "user-generated". That means that it can have weird things in it that need to be removed so they don't break the overall process. """ try: yamldata = yaml.load(mf, Loader=SafeLoader) except yaml.YAMLError as e: _warn_or_exception(_("could not parse '{path}'").format(path=mf.name) + '\n' + fdroidserver.common.run_yamllint(mf.name, indent=4), cause=e) deprecated_in_yaml = ['Provides'] if yamldata: for field in tuple(yamldata.keys()): if field not in yaml_app_fields + deprecated_in_yaml: msg = (_( "Unrecognised app field '{fieldname}' in '{path}'").format( fieldname=field, path=mf.name)) if os.path.basename(mf.name) == '.fdroid.yml': logging.error(msg) del yamldata[field] else: _warn_or_exception(msg) for deprecated_field in deprecated_in_yaml: if deprecated_field in yamldata: logging.warning( _("Ignoring '{field}' in '{metapath}' " "metadata because it is deprecated.").format( field=deprecated_field, metapath=mf.name)) del (yamldata[deprecated_field]) if yamldata.get('Builds', None): for build in yamldata.get('Builds', []): # put all build flag keywords into a set to avoid # excessive looping action build_flag_set = set() for build_flag in build.keys(): build_flag_set.add(build_flag) for build_flag in build_flag_set: if build_flag not in build_flags: _warn_or_exception( _("Unrecognised build flag '{build_flag}' " "in '{path}'").format(build_flag=build_flag, path=mf.name)) post_parse_yaml_metadata(yamldata) app.update(yamldata) return app
def write_metadata(metadatapath, app): if metadatapath.endswith('.yml'): if importlib.util.find_spec('ruamel.yaml'): with open(metadatapath, 'w') as mf: return write_yaml(mf, app) else: raise FDroidException(_('ruamel.yaml not installed, can not write metadata.')) _warn_or_exception(_('Unknown metadata format: %s') % metadatapath)
def read_metadata(appids={}, sort_by_time=False): """Return a list of App instances sorted newest first This reads all of the metadata files in a 'data' repository, then builds a list of App instances from those files. The list is sorted based on creation time, newest first. Most of the time, the newer files are the most interesting. appids is a dict with appids a keys and versionCodes as values. """ # Always read the srclibs before the apps, since they can use a srlib as # their source repository. read_srclibs() apps = OrderedDict() for basedir in ('metadata', 'tmp'): if not os.path.exists(basedir): os.makedirs(basedir) if appids: vercodes = fdroidserver.common.read_pkg_args(appids) metadatafiles = fdroidserver.common.get_metadata_files(vercodes) else: metadatafiles = (glob.glob(os.path.join('metadata', '*.yml')) + glob.glob('.fdroid.yml')) if sort_by_time: entries = ((os.stat(path).st_mtime, path) for path in metadatafiles) metadatafiles = [] for _ignored, path in sorted(entries, reverse=True): metadatafiles.append(path) else: # most things want the index alpha sorted for stability metadatafiles = sorted(metadatafiles) for metadatapath in metadatafiles: appid, _ignored = fdroidserver.common.get_extension( os.path.basename(metadatapath)) if appid != '.fdroid' and not fdroidserver.common.is_valid_package_name( appid): _warn_or_exception( _("{appid} from {path} is not a valid Java Package Name!"). format(appid=appid, path=metadatapath)) if appid in apps: _warn_or_exception( _("Found multiple metadata files for {appid}").format( appid=appid)) app = parse_metadata(metadatapath) check_metadata(app) apps[app.id] = app return apps
def print_help(available_plugins=None): print(_("usage: ") + _("fdroid [<command>] [-h|--help|--version|<args>]")) print("") print(_("Valid commands are:")) for cmd, summary in COMMANDS.items(): print(" " + cmd + ' ' * (15 - len(cmd)) + summary) if available_plugins: print(_('commands from plugin modules:')) for command in sorted(available_plugins.keys()): print(' {:15}{}'.format(command, available_plugins[command]['summary'])) print("")
def parse_yaml_srclib(metadatapath): thisinfo = {'RepoType': '', 'Repo': '', 'Subdir': None, 'Prepare': None} if not os.path.exists(metadatapath): _warn_or_exception(_("Invalid scrlib metadata: '{file}' " "does not exist" .format(file=metadatapath))) return thisinfo with open(metadatapath, "r", encoding="utf-8") as f: try: data = yaml.load(f, Loader=SafeLoader) if type(data) is not dict: raise yaml.error.YAMLError(_('{file} is blank or corrupt!') .format(file=metadatapath)) except yaml.error.YAMLError as e: _warn_or_exception(_("Invalid srclib metadata: could not " "parse '{file}'") .format(file=metadatapath) + '\n' + fdroidserver.common.run_yamllint(metadatapath, indent=4), cause=e) return thisinfo for key in data.keys(): if key not in thisinfo.keys(): _warn_or_exception(_("Invalid srclib metadata: unknown key " "'{key}' in '{file}'") .format(key=key, file=metadatapath)) return thisinfo else: if key == 'Subdir': if isinstance(data[key], str): thisinfo[key] = data[key].split(',') elif isinstance(data[key], list): thisinfo[key] = data[key] elif data[key] is None: thisinfo[key] = [''] elif key == 'Prepare' and isinstance(data[key], list): thisinfo[key] = ' && '.join(data[key]) else: thisinfo[key] = str(data[key] or '') return thisinfo
def check_versionCode(versionCode): try: int(versionCode) except ValueError: warn_or_exception( _('Invalid versionCode: "{versionCode}" is not an integer!'). format(versionCode=versionCode))
def parse_srclib(metadatapath): thisinfo = {} # Defaults for fields that come from metadata thisinfo['Repo Type'] = '' thisinfo['Repo'] = '' thisinfo['Subdir'] = None thisinfo['Prepare'] = None if not os.path.exists(metadatapath): return thisinfo metafile = open(metadatapath, "r", encoding='utf-8') n = 0 for line in metafile: n += 1 line = line.rstrip('\r\n') if not line or line.startswith("#"): continue try: f, v = line.split(':', 1) except ValueError: warn_or_exception(_("Invalid metadata in %s:%d") % (line, n)) if f == "Subdir": thisinfo[f] = v.split(',') else: thisinfo[f] = v metafile.close() return thisinfo
def main(): common.config = { 'accepted_formats': 'yml', 'sdk_path': os.getenv('ANDROID_HOME'), } common.fill_config_defaults(common.config) parser = argparse.ArgumentParser( usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]") common.setup_global_opts(parser) parser.add_argument( "appid", nargs='*', help= _("applicationId with optional versionCode in the form APPID[:VERCODE]" )) metadata.add_metadata_arguments(parser) options = parser.parse_args() common.options = options pkgs = common.read_pkg_args(options.appid, True) allapps = metadata.read_metadata(pkgs) apps = common.read_app_args(options.appid, allapps, True) srclib_dir = os.path.join('build', 'srclib') os.makedirs(srclib_dir, exist_ok=True) srclibpaths = [] for appid, app in apps.items(): for build in app.get('Builds', []): for lib in build.srclibs: srclibpaths.append( common.getsrclib(lib, srclib_dir, build=build)) print('Set up srclibs:') pprint.pprint(srclibpaths)
def parse_buildline(lines): v = "".join(lines) parts = [p.replace("\\,", ",") for p in re.split(build_line_sep, v)] if len(parts) < 3: warn_or_exception( _("Invalid build format: {value} in {name}").format( value=v, name=mf.name)) build = Build() build.versionName = parts[0] build.versionCode = parts[1] check_versionCode(build.versionCode) if parts[2].startswith('!'): # For backwards compatibility, handle old-style disabling, # including attempting to extract the commit from the message build.disable = parts[2][1:] commit = 'unknown - see disabled' index = parts[2].rfind('at ') if index != -1: commit = parts[2][index + 3:] if commit.endswith(')'): commit = commit[:-1] build.commit = commit else: build.commit = parts[2] for p in parts[3:]: add_buildflag(p, build) return build
def main(): parser = argparse.ArgumentParser( usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]") common.setup_global_opts(parser) parser.add_argument( "appid", nargs='*', help= _("applicationId with optional versionCode in the form APPID[:VERCODE]" )) metadata.add_metadata_arguments(parser) options = parser.parse_args() common.options = options pkgs = common.read_pkg_args(options.appid, True) allapps = metadata.read_metadata(pkgs) apps = common.read_app_args(options.appid, allapps, True) common.read_config(options) srclib_dir = os.path.join('build', 'srclib') os.makedirs(srclib_dir, exist_ok=True) srclibpaths = [] for appid, app in apps.items(): vcs, _ignored = common.setup_vcs(app) vcs.gotorevision('HEAD', refresh=False) for build in app.get('Builds', []): for lib in build.srclibs: srclibpaths.append( common.getsrclib(lib, srclib_dir, prepare=False, build=build)) print('Set up srclibs:') pprint.pprint(srclibpaths)
def linkify(self, txt): res_plain = '' res_html = '' while True: index = txt.find("[") if index == -1: return (res_plain + self.formatted(txt, False), res_html + self.formatted(txt, True)) res_plain += self.formatted(txt[:index], False) res_html += self.formatted(txt[:index], True) txt = txt[index:] if txt.startswith("[["): index = txt.find("]]") if index == -1: _warn_or_exception(_("Unterminated ]]")) url = txt[2:index] if self.linkResolver: url, urltext = self.linkResolver.resolve_description_link( url) else: urltext = url res_html += '<a href="' + url + '">' + html.escape( urltext, quote=False) + '</a>' res_plain += urltext txt = txt[index + 2:] else: index = txt.find("]") if index == -1: _warn_or_exception(_("Unterminated ]")) url = txt[1:index] index2 = url.find(' ') if index2 == -1: urltxt = url else: urltxt = url[index2 + 1:] url = url[:index2] if url == urltxt: _warn_or_exception( _("URL title is just the URL, use brackets: [URL]") ) res_html += '<a href="' + url + '">' + html.escape( urltxt, quote=False) + '</a>' res_plain += urltxt if urltxt != url: res_plain += ' (' + url + ')' txt = txt[index + 1:]
def parse_yaml_metadata(mf, app): try: yamldata = yaml.load(mf, Loader=SafeLoader) except yaml.YAMLError as e: _warn_or_exception(_("could not parse '{path}'") .format(path=mf.name) + '\n' + fdroidserver.common.run_yamllint(mf.name, indent=4), cause=e) deprecated_in_yaml = ['Provides'] if yamldata: for field in yamldata: if field not in yaml_app_fields: if field not in deprecated_in_yaml: _warn_or_exception(_("Unrecognised app field " "'{fieldname}' in '{path}'") .format(fieldname=field, path=mf.name)) for deprecated_field in deprecated_in_yaml: if deprecated_field in yamldata: logging.warning(_("Ignoring '{field}' in '{metapath}' " "metadata because it is deprecated.") .format(field=deprecated_field, metapath=mf.name)) del(yamldata[deprecated_field]) if yamldata.get('Builds', None): for build in yamldata.get('Builds', []): # put all build flag keywords into a set to avoid # excessive looping action build_flag_set = set() for build_flag in build.keys(): build_flag_set.add(build_flag) for build_flag in build_flag_set: if build_flag not in build_flags: _warn_or_exception( _("Unrecognised build flag '{build_flag}' " "in '{path}'").format(build_flag=build_flag, path=mf.name)) post_parse_yaml_metadata(yamldata) app.update(yamldata) return app
def add_metadata_arguments(parser): '''add common command line flags related to metadata processing''' parser.add_argument( "-W", choices=['error', 'warn', 'ignore'], default='error', help=_( "force metadata errors (default) to be warnings, or to be ignored." ))
def write_metadata(metadatapath, app): _ignored, ext = fdroidserver.common.get_extension(metadatapath) accepted = fdroidserver.common.config['accepted_formats'] if ext not in accepted: warn_or_exception( _('Cannot write "{path}", not an accepted format, use: {formats}'). format(path=metadatapath, formats=', '.join(accepted))) try: with open(metadatapath, 'w', encoding='utf8') as mf: if ext == 'txt': return write_txt(mf, app) elif ext == 'yml': return write_yaml(mf, app) except FDroidException as e: os.remove(metadatapath) raise e warn_or_exception(_('Unknown metadata format: %s') % metadatapath)
def check(self, v, appid): if not v: return if type(v) == list: values = v else: values = [v] for v in values: if not self.compiled.match(v): _warn_or_exception(_("'{value}' is not a valid {field} in {appid}. Regex pattern: {pattern}") .format(value=v, field=self.name, appid=appid, pattern=self.matching))
def add_buildflag(p, build): if not p.strip(): warn_or_exception( _("Empty build flag at {linedesc}").format(linedesc=linedesc)) bv = p.split('=', 1) if len(bv) != 2: warn_or_exception( _("Invalid build flag at {line} in {linedesc}").format( line=buildlines[0], linedesc=linedesc)) pk, pv = bv pk = pk.lstrip() if pk == 'update': pk = 'androidupdate' # avoid conflicting with Build(dict).update() t = flagtype(pk) if t == TYPE_LIST: pv = split_list_values(pv) build[pk] = pv elif t == TYPE_STRING or t == TYPE_SCRIPT: build[pk] = pv elif t == TYPE_BOOL: build[pk] = _decode_bool(pv)
def parse_metadata(metadatapath, check_vcs=False, refresh=True): '''parse metadata file, optionally checking the git repo for metadata first''' app = App() app.metadatapath = metadatapath name, _ignored = fdroidserver.common.get_extension( os.path.basename(metadatapath)) if name == '.fdroid': check_vcs = False else: app.id = name if metadatapath.endswith('.yml'): with open(metadatapath, 'r') as mf: parse_yaml_metadata(mf, app) else: _warn_or_exception( _('Unknown metadata format: {path} (use: *.yml)').format( path=metadatapath)) if check_vcs and app.Repo: build_dir = fdroidserver.common.get_build_dir(app) metadata_in_repo = os.path.join(build_dir, '.fdroid.yml') if not os.path.isfile(metadata_in_repo): vcs, build_dir = fdroidserver.common.setup_vcs(app) if isinstance(vcs, fdroidserver.common.vcs_git): vcs.gotorevision( 'HEAD', refresh) # HEAD since we can't know where else to go if os.path.isfile(metadata_in_repo): logging.debug('Including metadata from ' + metadata_in_repo) # do not include fields already provided by main metadata file app_in_repo = parse_metadata(metadata_in_repo) for k, v in app_in_repo.items(): if k not in app: app[k] = v post_metadata_parse(app) if not app.id: if app.builds: build = app.builds[-1] if build.subdir: root_dir = build.subdir else: root_dir = '.' paths = fdroidserver.common.manifest_paths(root_dir, build.gradle) _ignored, _ignored, app.id = fdroidserver.common.parse_androidmanifests( paths, app) return app
def post_parse_yaml_metadata(yamldata): """transform yaml metadata to our internal data format""" for build in yamldata.get('Builds', []): for flag in build.keys(): _flagtype = flagtype(flag) if _flagtype is TYPE_SCRIPT: # concatenate script flags into a single string if they are stored as list if isinstance(build[flag], list): build[flag] = ' && '.join(build[flag]) elif _flagtype is TYPE_STRING: # things like versionNames are strings, but without quotes can be numbers if isinstance(build[flag], float) or isinstance(build[flag], int): build[flag] = str(build[flag]) elif _flagtype is TYPE_INT: # versionCode must be int if not isinstance(build[flag], int): _warn_or_exception(_('{build_flag} must be an integer, found: {value}') .format(build_flag=flag, value=build[flag]))
def get_default_app_info(metadatapath=None): if metadatapath is None: appid = None else: appid, _ignored = fdroidserver.common.get_extension( os.path.basename(metadatapath)) if appid == '.fdroid': # we have local metadata in the app's source if os.path.exists('AndroidManifest.xml'): manifestroot = fdroidserver.common.parse_xml('AndroidManifest.xml') else: pattern = re.compile( r""".*manifest\.srcFile\s+'AndroidManifest\.xml'.*""") for root, dirs, files in os.walk(os.getcwd()): if 'build.gradle' in files: p = os.path.join(root, 'build.gradle') with open(p, 'rb') as f: data = f.read() m = pattern.search(data) if m: logging.debug( 'Using: ' + os.path.join(root, 'AndroidManifest.xml')) manifestroot = fdroidserver.common.parse_xml( os.path.join(root, 'AndroidManifest.xml')) break if manifestroot is None: warn_or_exception( _("Cannot find an appid for {path}!").format( path=metadatapath)) appid = manifestroot.attrib['package'] app = App() app.metadatapath = metadatapath if appid is not None: app.id = appid return app
def main(): available_plugins = find_plugins() if len(sys.argv) <= 1: print_help(available_plugins=available_plugins) sys.exit(0) command = sys.argv[1] if command not in COMMANDS and command not in available_plugins.keys(): if command in ('-h', '--help'): print_help(available_plugins=available_plugins) sys.exit(0) elif command == '--version': output = _('no version info found!') cmddir = os.path.realpath( os.path.dirname(os.path.dirname(__file__))) moduledir = os.path.realpath( os.path.dirname(fdroidserver.common.__file__) + '/..') if cmddir == moduledir: # running from git os.chdir(cmddir) if os.path.isdir('.git'): import subprocess try: output = subprocess.check_output( ['git', 'describe'], stderr=subprocess.STDOUT, universal_newlines=True) except subprocess.CalledProcessError: output = 'git commit ' + subprocess.check_output( ['git', 'rev-parse', 'HEAD'], universal_newlines=True) elif os.path.exists('setup.py'): import re m = re.search( r'''.*[\s,\(]+version\s*=\s*["']([0-9a-z.]+)["'].*''', open('setup.py').read(), flags=re.MULTILINE) if m: output = m.group(1) + '\n' else: from pkg_resources import get_distribution output = get_distribution('fdroidserver').version + '\n' print(output) sys.exit(0) else: print(_("Command '%s' not recognised.\n" % command)) print_help(available_plugins=available_plugins) sys.exit(1) verbose = any(s in sys.argv for s in ['-v', '--verbose']) quiet = any(s in sys.argv for s in ['-q', '--quiet']) # Helpful to differentiate warnings from errors even when on quiet logformat = '%(levelname)s: %(message)s' loglevel = logging.INFO if verbose: loglevel = logging.DEBUG elif quiet: loglevel = logging.WARN logging.basicConfig(format=logformat, level=loglevel) if verbose and quiet: logging.critical( _("Conflicting arguments: '--verbose' and '--quiet' " "can not be specified at the same time.")) sys.exit(1) # temporary workaround until server.py becomes deploy.py if command == 'deploy': command = 'server' sys.argv.insert(2, 'update') # Trick optparse into displaying the right usage when --help is used. sys.argv[0] += ' ' + command del sys.argv[1] if command in COMMANDS.keys(): mod = __import__('fdroidserver.' + command, None, None, [command]) else: mod = __import__(available_plugins[command]['name'], None, None, [command]) system_langcode, system_encoding = locale.getdefaultlocale() if system_encoding is None or system_encoding.lower() not in ('utf-8', 'utf8'): logging.warning( _("Encoding is set to '{enc}' fdroid might run " "into encoding issues. Please set it to 'UTF-8' " "for best results.".format(enc=system_encoding))) try: mod.main() # These are ours, contain a proper message and are "expected" except (fdroidserver.common.FDroidException, fdroidserver.metadata.MetaDataException) as e: if verbose: raise else: logging.critical(str(e)) sys.exit(1) except ArgumentError as e: logging.critical(str(e)) sys.exit(1) except KeyboardInterrupt: print('') fdroidserver.common.force_exit(1) # These should only be unexpected crashes due to bugs in the code # str(e) often doesn't contain a reason, so just show the backtrace except Exception as e: logging.critical(_("Unknown exception found!")) raise e sys.exit(0)
import re import sys import os import locale import pkgutil import logging import fdroidserver.common import fdroidserver.metadata from fdroidserver import _ from argparse import ArgumentError from collections import OrderedDict COMMANDS = OrderedDict([ ("build", _("Build a package from source")), ("init", _("Quickly start a new repository")), ("publish", _("Sign and place packages in the repo")), ("gpgsign", _("Add PGP signatures using GnuPG for packages in repo")), ("update", _("Update repo information for new packages")), ("deploy", _("Interact with the repo HTTP server")), ("verify", _("Verify the integrity of downloaded packages")), ("checkupdates", _("Check for updates to applications")), ("import", _("Add a new application from its source code")), ("install", _("Install built packages on devices")), ("readmeta", _("Read all the metadata files and exit")), ("rewritemeta", _("Rewrite all the metadata files")), ("lint", _("Warn about possible metadata errors")), ("scanner", _("Scan the source code of a package")), ("stats", _("Update the stats of the repo")), ("server", _("Old, deprecated name for fdroid deploy")),
def read_metadata(xref=True, check_vcs=[], refresh=True, sort_by_time=False): """Return a list of App instances sorted newest first This reads all of the metadata files in a 'data' repository, then builds a list of App instances from those files. The list is sorted based on creation time, newest first. Most of the time, the newer files are the most interesting. check_vcs is the list of appids to check for .fdroid.yml in source """ # Always read the srclibs before the apps, since they can use a srlib as # their source repository. read_srclibs() apps = OrderedDict() for basedir in ('metadata', 'tmp'): if not os.path.exists(basedir): os.makedirs(basedir) metadatafiles = (glob.glob(os.path.join('metadata', '*.yml')) + glob.glob('.fdroid.yml')) if sort_by_time: entries = ((os.stat(path).st_mtime, path) for path in metadatafiles) metadatafiles = [] for _ignored, path in sorted(entries, reverse=True): metadatafiles.append(path) else: # most things want the index alpha sorted for stability metadatafiles = sorted(metadatafiles) for metadatapath in metadatafiles: appid, _ignored = fdroidserver.common.get_extension( os.path.basename(metadatapath)) if appid != '.fdroid' and not fdroidserver.common.is_valid_package_name( appid): _warn_or_exception( _("{appid} from {path} is not a valid Java Package Name!"). format(appid=appid, path=metadatapath)) if appid in apps: _warn_or_exception( _("Found multiple metadata files for {appid}").format( appid=appid)) app = parse_metadata(metadatapath, appid in check_vcs, refresh) check_metadata(app) apps[app.id] = app if xref: # Parse all descriptions at load time, just to ensure cross-referencing # errors are caught early rather than when they hit the build server. for appid, app in apps.items(): try: description_html(app.Description, DummyDescriptionResolver(apps)) except MetaDataException as e: _warn_or_exception( _("Problem with description of {appid}: {error}").format( appid=appid, error=str(e))) return apps
def _decode_bool(s): if bool_true.match(s): return True if bool_false.match(s): return False warn_or_exception(_("Invalid boolean '%s'") % s)
def linkres(appid): if appid in apps: return ("fdroid.app:" + appid, "Dummy name - don't know yet") warn_or_exception( _("Cannot resolve app id {appid}").format(appid=appid))
def read_metadata(xref=True, check_vcs=[], refresh=True, sort_by_time=False): """Return a list of App instances sorted newest first This reads all of the metadata files in a 'data' repository, then builds a list of App instances from those files. The list is sorted based on creation time, newest first. Most of the time, the newer files are the most interesting. If there are multiple metadata files for a single appid, then the first file that is parsed wins over all the others, and the rest throw an exception. So the original .txt format is parsed first, at least until newer formats stabilize. check_vcs is the list of appids to check for .fdroid.yml in source """ # Always read the srclibs before the apps, since they can use a srlib as # their source repository. read_srclibs() apps = OrderedDict() for basedir in ('metadata', 'tmp'): if not os.path.exists(basedir): os.makedirs(basedir) metadatafiles = (glob.glob(os.path.join('metadata', '*.txt')) + glob.glob(os.path.join('metadata', '*.json')) + glob.glob(os.path.join('metadata', '*.yml')) + glob.glob('.fdroid.txt') + glob.glob('.fdroid.json') + glob.glob('.fdroid.yml')) if sort_by_time: entries = ((os.stat(path).st_mtime, path) for path in metadatafiles) metadatafiles = [] for _ignored, path in sorted(entries, reverse=True): metadatafiles.append(path) else: # most things want the index alpha sorted for stability metadatafiles = sorted(metadatafiles) for metadatapath in metadatafiles: if metadatapath == '.fdroid.txt': warn_or_exception( _('.fdroid.txt is not supported! Convert to .fdroid.yml or .fdroid.json.' )) appid, _ignored = fdroidserver.common.get_extension( os.path.basename(metadatapath)) if appid in apps: warn_or_exception( _("Found multiple metadata files for {appid}").format( appid=appid)) app = parse_metadata(metadatapath, appid in check_vcs, refresh) check_metadata(app) apps[app.id] = app if xref: # Parse all descriptions at load time, just to ensure cross-referencing # errors are caught early rather than when they hit the build server. def linkres(appid): if appid in apps: return ("fdroid.app:" + appid, "Dummy name - don't know yet") warn_or_exception( _("Cannot resolve app id {appid}").format(appid=appid)) for appid, app in apps.items(): try: description_html(app.Description, linkres) except MetaDataException as e: warn_or_exception( _("Problem with description of {appid}: {error}").format( appid=appid, error=str(e))) return apps
def package(self, output=None, keep_box_file=False): if not output: output = "buildserver.box" logging.debug( "no output name set for packaging '%s', " "defaulting to %s", self.srvname, output) storagePool = self.conn.storagePoolLookupByName('default') domainInfo = self.conn.lookupByName(self.srvname).info() if storagePool: if isfile('metadata.json'): os.remove('metadata.json') if isfile('Vagrantfile'): os.remove('Vagrantfile') if isfile('box.img'): os.remove('box.img') logging.debug('preparing box.img for box %s', output) vol = storagePool.storageVolLookupByName(self.srvname + '.img') imagepath = vol.path() # TODO use a libvirt storage pool to ensure the img file is readable if not os.access(imagepath, os.R_OK): logging.warning( _('Cannot read "{path}"!').format(path=imagepath)) _check_call([ 'sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images' ]) shutil.copy2(imagepath, 'box.img') _check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img']) img_info_raw = _check_output( ['qemu-img', 'info', '--output=json', 'box.img']) img_info = json.loads(img_info_raw.decode('utf-8')) metadata = { "provider": "libvirt", "format": img_info['format'], "virtual_size": math.ceil(img_info['virtual-size'] / (1024.**3)), } logging.debug('preparing metadata.json for box %s', output) with open('metadata.json', 'w') as fp: fp.write(json.dumps(metadata)) logging.debug('preparing Vagrantfile for box %s', output) vagrantfile = textwrap.dedent("""\ Vagrant.configure("2") do |config| config.ssh.username = "******" config.ssh.password = "******" config.vm.provider :libvirt do |libvirt| libvirt.driver = "kvm" libvirt.host = "" libvirt.connect_via_ssh = false libvirt.storage_pool_name = "default" libvirt.cpus = {cpus} libvirt.memory = {memory} end end""".format_map({ 'memory': str(int(domainInfo[1] / 1024)), 'cpus': str(domainInfo[3]) })) with open('Vagrantfile', 'w') as fp: fp.write(vagrantfile) with tarfile.open(output, 'w:gz') as tar: logging.debug('adding metadata.json to box %s ...', output) tar.add('metadata.json') logging.debug('adding Vagrantfile to box %s ...', output) tar.add('Vagrantfile') logging.debug('adding box.img to box %s ...', output) tar.add('box.img') if not keep_box_file: logging.debug( 'box packaging complete, removing temporary files.') os.remove('metadata.json') os.remove('Vagrantfile') os.remove('box.img') else: logging.warn("could not connect to storage-pool 'default', " "skip packaging buildserver box")
def package(self, output=None, keep_box_file=False): if not output: output = "buildserver.box" logging.debug("no output name set for packaging '%s', " "defaulting to %s", self.srvname, output) storagePool = self.conn.storagePoolLookupByName('default') domainInfo = self.conn.lookupByName(self.srvname).info() if storagePool: if isfile('metadata.json'): os.remove('metadata.json') if isfile('Vagrantfile'): os.remove('Vagrantfile') if isfile('box.img'): os.remove('box.img') logging.debug('preparing box.img for box %s', output) vol = storagePool.storageVolLookupByName(self.srvname + '.img') imagepath = vol.path() # TODO use a libvirt storage pool to ensure the img file is readable if not os.access(imagepath, os.R_OK): logging.warning(_('Cannot read "{path}"!').format(path=imagepath)) _check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images']) shutil.copy2(imagepath, 'box.img') _check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img']) img_info_raw = _check_output(['qemu-img', 'info', '--output=json', 'box.img']) img_info = json.loads(img_info_raw.decode('utf-8')) metadata = {"provider": "libvirt", "format": img_info['format'], "virtual_size": math.ceil(img_info['virtual-size'] / (1024. ** 3)), } logging.debug('preparing metadata.json for box %s', output) with open('metadata.json', 'w') as fp: fp.write(json.dumps(metadata)) logging.debug('preparing Vagrantfile for box %s', output) vagrantfile = textwrap.dedent("""\ Vagrant.configure("2") do |config| config.ssh.username = "******" config.ssh.password = "******" config.vm.provider :libvirt do |libvirt| libvirt.driver = "kvm" libvirt.host = "" libvirt.connect_via_ssh = false libvirt.storage_pool_name = "default" libvirt.cpus = {cpus} libvirt.memory = {memory} end end""".format_map({'memory': str(int(domainInfo[1] / 1024)), 'cpus': str(domainInfo[3])})) with open('Vagrantfile', 'w') as fp: fp.write(vagrantfile) with tarfile.open(output, 'w:gz') as tar: logging.debug('adding metadata.json to box %s ...', output) tar.add('metadata.json') logging.debug('adding Vagrantfile to box %s ...', output) tar.add('Vagrantfile') logging.debug('adding box.img to box %s ...', output) tar.add('box.img') if not keep_box_file: logging.debug('box packaging complete, removing temporary files.') os.remove('metadata.json') os.remove('Vagrantfile') os.remove('box.img') else: logging.warn("could not connect to storage-pool 'default', " "skip packaging buildserver box")
def parse_txt_metadata(mf, app): linedesc = None def add_buildflag(p, build): if not p.strip(): warn_or_exception( _("Empty build flag at {linedesc}").format(linedesc=linedesc)) bv = p.split('=', 1) if len(bv) != 2: warn_or_exception( _("Invalid build flag at {line} in {linedesc}").format( line=buildlines[0], linedesc=linedesc)) pk, pv = bv pk = pk.lstrip() if pk == 'update': pk = 'androidupdate' # avoid conflicting with Build(dict).update() t = flagtype(pk) if t == TYPE_LIST: pv = split_list_values(pv) build[pk] = pv elif t == TYPE_STRING or t == TYPE_SCRIPT: build[pk] = pv elif t == TYPE_BOOL: build[pk] = _decode_bool(pv) elif t == TYPE_INT: build[pk] = int(pv) def parse_buildline(lines): v = "".join(lines) parts = [p.replace("\\,", ",") for p in re.split(build_line_sep, v)] if len(parts) < 3: warn_or_exception( _("Invalid build format: {value} in {name}").format( value=v, name=mf.name)) build = Build() build.versionName = parts[0] build.versionCode = parts[1] check_versionCode(build.versionCode) if parts[2].startswith('!'): # For backwards compatibility, handle old-style disabling, # including attempting to extract the commit from the message build.disable = parts[2][1:] commit = 'unknown - see disabled' index = parts[2].rfind('at ') if index != -1: commit = parts[2][index + 3:] if commit.endswith(')'): commit = commit[:-1] build.commit = commit else: build.commit = parts[2] for p in parts[3:]: add_buildflag(p, build) return build def check_versionCode(versionCode): try: int(versionCode) except ValueError: warn_or_exception( _('Invalid versionCode: "{versionCode}" is not an integer!'). format(versionCode=versionCode)) def add_comments(key): if not curcomments: return app.comments[key] = list(curcomments) del curcomments[:] mode = 0 buildlines = [] multiline_lines = [] curcomments = [] build = None vc_seen = set() app.builds = [] c = 0 for line in mf: c += 1 linedesc = "%s:%d" % (mf.name, c) line = line.rstrip('\r\n') if mode == 3: if build_cont.match(line): if line.endswith('\\'): buildlines.append(line[:-1].lstrip()) else: buildlines.append(line.lstrip()) bl = ''.join(buildlines) add_buildflag(bl, build) del buildlines[:] else: if not build.commit and not build.disable: warn_or_exception( _("No commit specified for {versionName} in {linedesc}" ).format(versionName=build.versionName, linedesc=linedesc)) app.builds.append(build) add_comments('build:' + build.versionCode) mode = 0 if mode == 0: if not line: continue if line.startswith("#"): curcomments.append(line[1:].strip()) continue try: f, v = line.split(':', 1) except ValueError: warn_or_exception(_("Invalid metadata in: ") + linedesc) if f not in app_fields: warn_or_exception(_('Unrecognised app field: ') + f) # Translate obsolete fields... if f == 'Market Version': f = 'Current Version' if f == 'Market Version Code': f = 'Current Version Code' f = f.replace(' ', '') ftype = fieldtype(f) if ftype not in [TYPE_BUILD, TYPE_BUILD_V2]: add_comments(f) if ftype == TYPE_MULTILINE: mode = 1 if v: warn_or_exception( _("Unexpected text on same line as {field} in {linedesc}" ).format(field=f, linedesc=linedesc)) elif ftype == TYPE_STRING: app[f] = v elif ftype == TYPE_LIST: app[f] = split_list_values(v) elif ftype == TYPE_BUILD: if v.endswith("\\"): mode = 2 del buildlines[:] buildlines.append(v[:-1]) else: build = parse_buildline([v]) app.builds.append(build) add_comments('build:' + app.builds[-1].versionCode) elif ftype == TYPE_BUILD_V2: vv = v.split(',') if len(vv) != 2: warn_or_exception( _('Build should have comma-separated ' 'versionName and versionCode, ' 'not "{value}", in {linedesc}').format( value=v, linedesc=linedesc)) build = Build() build.versionName = vv[0] build.versionCode = vv[1] check_versionCode(build.versionCode) if build.versionCode in vc_seen: warn_or_exception( _('Duplicate build recipe found for versionCode {versionCode} in {linedesc}' ).format(versionCode=build.versionCode, linedesc=linedesc)) vc_seen.add(build.versionCode) del buildlines[:] mode = 3 elif ftype == TYPE_OBSOLETE: pass # Just throw it away! else: warn_or_exception( _("Unrecognised field '{field}' in {linedesc}").format( field=f, linedesc=linedesc)) elif mode == 1: # Multiline field if line == '.': mode = 0 app[f] = '\n'.join(multiline_lines) del multiline_lines[:] else: multiline_lines.append(line) elif mode == 2: # Line continuation mode in Build Version if line.endswith("\\"): buildlines.append(line[:-1]) else: buildlines.append(line) build = parse_buildline(buildlines) app.builds.append(build) add_comments('build:' + app.builds[-1].versionCode) mode = 0 add_comments(None) # Mode at end of file should always be 0 if mode == 1: warn_or_exception( _("{field} not terminated in {name}").format(field=f, name=mf.name)) if mode == 2: warn_or_exception( _("Unterminated continuation in {name}").format(name=mf.name)) if mode == 3: warn_or_exception( _("Unterminated build in {name}").format(name=mf.name)) return app