def extract_pubkey(): """ Extracts and returns the repository's public key from the keystore. :return: public key in hex, repository fingerprint """ if 'repo_pubkey' in common.config: pubkey = unhexlify(common.config['repo_pubkey']) else: env_vars = { 'LC_ALL': 'C.UTF-8', 'FDROID_KEY_STORE_PASS': common.config['keystorepass'] } p = FDroidPopenBytes([ common.config['keytool'], '-exportcert', '-alias', common.config['repo_keyalias'], '-keystore', common.config['keystore'], '-storepass:env', 'FDROID_KEY_STORE_PASS' ] + list(common.config['smartcardoptions']), envs=env_vars, output=False, stderr_to_stdout=False) if p.returncode != 0 or len(p.output) < 20: msg = "Failed to get repo pubkey!" if common.config['keystore'] == 'NONE': msg += ' Is your crypto smartcard plugged in?' raise FDroidException(msg) pubkey = p.output repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey) return hexlify(pubkey), repo_pubkey_fingerprint
def write_metadata(metadatapath, app): if metadatapath.endswith('.yml'): if importlib.util.find_spec('ruamel.yaml'): with open(metadatapath, 'w') as mf: return write_yaml(mf, app) else: raise FDroidException(_('ruamel.yaml not installed, can not write metadata.')) _warn_or_exception(_('Unknown metadata format: %s') % metadatapath)
def _copy_to_local_copy_dir(repodir, f): local_copy_dir = common.config.get('local_copy_dir', '') if os.path.exists(local_copy_dir): destdir = os.path.join(local_copy_dir, repodir) if not os.path.exists(destdir): os.mkdir(destdir) shutil.copy2(f, destdir, follow_symlinks=False) elif local_copy_dir: raise FDroidException( _('"local_copy_dir" {path} does not exist!').format( path=local_copy_dir))
def make(apps, sortedids, apks, repodir, archive): """Generate the repo index files. This requires properly initialized options and config objects. :param apps: fully populated apps list :param sortedids: app package IDs, sorted :param apks: full populated apks list :param repodir: the repo directory :param archive: True if this is the archive repo, False if it's the main one. """ from fdroidserver.update import METADATA_VERSION def _resolve_description_link(appid): if appid in apps: return "fdroid.app:" + appid, apps[appid].Name raise MetaDataException("Cannot resolve app id " + appid) if not common.options.nosign: common.assert_config_keystore(common.config) repodict = collections.OrderedDict() repodict['timestamp'] = datetime.utcnow() repodict['version'] = METADATA_VERSION if common.config['repo_maxage'] != 0: repodict['maxage'] = common.config['repo_maxage'] if archive: repodict['name'] = common.config['archive_name'] repodict['icon'] = os.path.basename(common.config['archive_icon']) repodict['address'] = common.config['archive_url'] repodict['description'] = common.config['archive_description'] urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['archive_url']).path) else: repodict['name'] = common.config['repo_name'] repodict['icon'] = os.path.basename(common.config['repo_icon']) repodict['address'] = common.config['repo_url'] repodict['description'] = common.config['repo_description'] urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['repo_url']).path) mirrorcheckfailed = False mirrors = [] for mirror in sorted(common.config.get('mirrors', [])): base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/')) if common.config.get('nonstandardwebroot') is not True and base != 'fdroid': logging.error(_("mirror '%s' does not end with 'fdroid'!") % mirror) mirrorcheckfailed = True # must end with / or urljoin strips a whole path segment if mirror.endswith('/'): mirrors.append(urllib.parse.urljoin(mirror, urlbasepath)) else: mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath)) for mirror in common.config.get('servergitmirrors', []): for url in get_mirror_service_urls(mirror): mirrors.append(url + '/' + repodir) if mirrorcheckfailed: raise FDroidException(_("Malformed repository mirrors.")) if mirrors: repodict['mirrors'] = mirrors appsWithPackages = collections.OrderedDict() for packageName in sortedids: app = apps[packageName] if app['Disabled']: continue # only include apps with packages for apk in apks: if apk['packageName'] == packageName: newapp = copy.copy(app) # update wiki needs unmodified description newapp['Description'] = metadata.description_html(app['Description'], _resolve_description_link) appsWithPackages[packageName] = newapp break requestsdict = collections.OrderedDict() for command in ('install', 'uninstall'): packageNames = [] key = command + '_list' if key in common.config: if isinstance(common.config[key], str): packageNames = [common.config[key]] elif all(isinstance(item, str) for item in common.config[key]): packageNames = common.config[key] else: raise TypeError(_('only accepts strings, lists, and tuples')) requestsdict[command] = packageNames fdroid_signing_key_fingerprints = load_stats_fdroid_signing_key_fingerprints() make_v0(appsWithPackages, apks, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints) make_v1(appsWithPackages, apks, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints)
def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints): """ aka index.jar aka index.xml """ doc = Document() def addElement(name, value, doc, parent): el = doc.createElement(name) el.appendChild(doc.createTextNode(value)) parent.appendChild(el) def addElementNonEmpty(name, value, doc, parent): if not value: return addElement(name, value, doc, parent) def addElementIfInApk(name, apk, key, doc, parent): if key not in apk: return value = str(apk[key]) addElement(name, value, doc, parent) def addElementCDATA(name, value, doc, parent): el = doc.createElement(name) el.appendChild(doc.createCDATASection(value)) parent.appendChild(el) def addElementCheckLocalized(name, app, key, doc, parent, default=''): '''Fill in field from metadata or localized block For name/summary/description, they can come only from the app source, or from a dir in fdroiddata. They can be entirely missing from the metadata file if there is localized versions. This will fetch those from the localized version if its not available in the metadata file. ''' el = doc.createElement(name) value = app.get(key) lkey = key[:1].lower() + key[1:] localized = app.get('localized') if not value and localized: for lang in ['en-US'] + [x for x in localized.keys()]: if not lang.startswith('en'): continue if lang in localized: value = localized[lang].get(lkey) if value: break if not value and localized and len(localized) > 1: lang = list(localized.keys())[0] value = localized[lang].get(lkey) if not value: value = default el.appendChild(doc.createTextNode(value)) parent.appendChild(el) root = doc.createElement("fdroid") doc.appendChild(root) repoel = doc.createElement("repo") repoel.setAttribute("name", repodict['name']) if 'maxage' in repodict: repoel.setAttribute("maxage", str(repodict['maxage'])) repoel.setAttribute("icon", os.path.basename(repodict['icon'])) repoel.setAttribute("url", repodict['address']) addElement('description', repodict['description'], doc, repoel) for mirror in repodict.get('mirrors', []): addElement('mirror', mirror, doc, repoel) repoel.setAttribute("version", str(repodict['version'])) repoel.setAttribute("timestamp", '%d' % repodict['timestamp'].timestamp()) pubkey, repo_pubkey_fingerprint = extract_pubkey() repoel.setAttribute("pubkey", pubkey.decode('utf-8')) root.appendChild(repoel) for command in ('install', 'uninstall'): for packageName in requestsdict[command]: element = doc.createElement(command) root.appendChild(element) element.setAttribute('packageName', packageName) for appid, appdict in apps.items(): app = metadata.App(appdict) if app.Disabled is not None: continue # Get a list of the apks for this app... apklist = [] apksbyversion = collections.defaultdict(lambda: []) for apk in apks: if apk.get('versionCode') and apk.get('packageName') == appid: apksbyversion[apk['versionCode']].append(apk) for versionCode, apksforver in apksbyversion.items(): fdroidsig = fdroid_signing_key_fingerprints.get(appid, {}).get('signer') fdroid_signed_apk = None name_match_apk = None for x in apksforver: if fdroidsig and x.get('signer', None) == fdroidsig: fdroid_signed_apk = x if common.apk_release_filename.match(x.get('apkName', '')): name_match_apk = x # choose which of the available versions is most # suiteable for index v0 if fdroid_signed_apk: apklist.append(fdroid_signed_apk) elif name_match_apk: apklist.append(name_match_apk) else: apklist.append(apksforver[0]) if len(apklist) == 0: continue apel = doc.createElement("application") apel.setAttribute("id", app.id) root.appendChild(apel) addElement('id', app.id, doc, apel) if app.added: addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel) if app.lastUpdated: addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel) addElementCheckLocalized('name', app, 'Name', doc, apel) addElementCheckLocalized('summary', app, 'Summary', doc, apel) if app.icon: addElement('icon', app.icon, doc, apel) addElementCheckLocalized('desc', app, 'Description', doc, apel, '<p>No description available</p>') addElement('license', app.License, doc, apel) if app.Categories: addElement('categories', ','.join(app.Categories), doc, apel) # We put the first (primary) category in LAST, which will have # the desired effect of making clients that only understand one # category see that one. addElement('category', app.Categories[0], doc, apel) addElement('web', app.WebSite, doc, apel) addElement('source', app.SourceCode, doc, apel) addElement('tracker', app.IssueTracker, doc, apel) addElementNonEmpty('changelog', app.Changelog, doc, apel) addElementNonEmpty('author', app.AuthorName, doc, apel) addElementNonEmpty('email', app.AuthorEmail, doc, apel) addElementNonEmpty('donate', app.Donate, doc, apel) addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel) addElementNonEmpty('litecoin', app.Litecoin, doc, apel) addElementNonEmpty('flattr', app.FlattrID, doc, apel) addElementNonEmpty('liberapay', app.LiberapayID, doc, apel) # These elements actually refer to the current version (i.e. which # one is recommended. They are historically mis-named, and need # changing, but stay like this for now to support existing clients. addElement('marketversion', app.CurrentVersion, doc, apel) addElement('marketvercode', app.CurrentVersionCode, doc, apel) if app.Provides: pv = app.Provides.split(',') addElementNonEmpty('provides', ','.join(pv), doc, apel) if app.RequiresRoot: addElement('requirements', 'root', doc, apel) # Sort the apk list into version order, just so the web site # doesn't have to do any work by default... apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True) if 'antiFeatures' in apklist[0]: app.AntiFeatures.extend(apklist[0]['antiFeatures']) if app.AntiFeatures: addElementNonEmpty('antifeatures', ','.join(app.AntiFeatures), doc, apel) # Check for duplicates - they will make the client unhappy... for i in range(len(apklist) - 1): first = apklist[i] second = apklist[i + 1] if first['versionCode'] == second['versionCode'] \ and first['sig'] == second['sig']: if first['hash'] == second['hash']: raise FDroidException('"{0}/{1}" and "{0}/{2}" are exact duplicates!'.format( repodir, first['apkName'], second['apkName'])) else: raise FDroidException('duplicates: "{0}/{1}" - "{0}/{2}"'.format( repodir, first['apkName'], second['apkName'])) current_version_code = 0 current_version_file = None for apk in apklist: file_extension = common.get_file_extension(apk['apkName']) # find the APK for the "Current Version" if current_version_code < apk['versionCode']: current_version_code = apk['versionCode'] if current_version_code < int(app.CurrentVersionCode): current_version_file = apk['apkName'] apkel = doc.createElement("package") apel.appendChild(apkel) addElement('version', apk['versionName'], doc, apkel) addElement('versioncode', str(apk['versionCode']), doc, apkel) addElement('apkname', apk['apkName'], doc, apkel) addElementIfInApk('srcname', apk, 'srcname', doc, apkel) hashel = doc.createElement("hash") hashel.setAttribute('type', 'sha256') hashel.appendChild(doc.createTextNode(apk['hash'])) apkel.appendChild(hashel) addElement('size', str(apk['size']), doc, apkel) addElementIfInApk('sdkver', apk, 'minSdkVersion', doc, apkel) addElementIfInApk('targetSdkVersion', apk, 'targetSdkVersion', doc, apkel) addElementIfInApk('maxsdkver', apk, 'maxSdkVersion', doc, apkel) addElementIfInApk('obbMainFile', apk, 'obbMainFile', doc, apkel) addElementIfInApk('obbMainFileSha256', apk, 'obbMainFileSha256', doc, apkel) addElementIfInApk('obbPatchFile', apk, 'obbPatchFile', doc, apkel) addElementIfInApk('obbPatchFileSha256', apk, 'obbPatchFileSha256', doc, apkel) if 'added' in apk: addElement('added', apk['added'].strftime('%Y-%m-%d'), doc, apkel) if file_extension == 'apk': # sig is required for APKs, but only APKs addElement('sig', apk['sig'], doc, apkel) old_permissions = set() sorted_permissions = sorted(apk['uses-permission']) for perm in sorted_permissions: perm_name = perm.name if perm_name.startswith("android.permission."): perm_name = perm_name[19:] old_permissions.add(perm_name) addElementNonEmpty('permissions', ','.join(sorted(old_permissions)), doc, apkel) for permission in sorted_permissions: permel = doc.createElement('uses-permission') permel.setAttribute('name', permission.name) if permission.maxSdkVersion is not None: permel.setAttribute('maxSdkVersion', '%d' % permission.maxSdkVersion) apkel.appendChild(permel) for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']): permel = doc.createElement('uses-permission-sdk-23') permel.setAttribute('name', permission_sdk_23.name) if permission_sdk_23.maxSdkVersion is not None: permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23.maxSdkVersion) apkel.appendChild(permel) if 'nativecode' in apk: addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel) addElementNonEmpty('features', ','.join(sorted(apk['features'])), doc, apkel) if current_version_file is not None \ and common.config['make_current_version_link'] \ and repodir == 'repo': # only create these namefield = common.config['current_version_name_source'] sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8')) apklinkname = sanitized_name + os.path.splitext(current_version_file)[1].encode('utf-8') current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape') if os.path.islink(apklinkname): os.remove(apklinkname) os.symlink(current_version_path, apklinkname) # also symlink gpg signature, if it exists for extension in (b'.asc', b'.sig'): sigfile_path = current_version_path + extension if os.path.exists(sigfile_path): siglinkname = apklinkname + extension if os.path.islink(siglinkname): os.remove(siglinkname) os.symlink(sigfile_path, siglinkname) if common.options.pretty: output = doc.toprettyxml(encoding='utf-8') else: output = doc.toxml(encoding='utf-8') with open(os.path.join(repodir, 'index.xml'), 'wb') as f: f.write(output) if 'repo_keyalias' in common.config: if common.options.nosign: logging.info(_("Creating unsigned index in preparation for signing")) else: logging.info(_("Creating signed index with this key (SHA256):")) logging.info("%s" % repo_pubkey_fingerprint) # Create a jar of the index... jar_output = 'index_unsigned.jar' if common.options.nosign else 'index.jar' p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir) if p.returncode != 0: raise FDroidException("Failed to create {0}".format(jar_output)) # Sign the index... signed = os.path.join(repodir, 'index.jar') if common.options.nosign: # Remove old signed index if not signing if os.path.exists(signed): os.remove(signed) else: signindex.config = common.config signindex.sign_jar(signed) # Copy the repo icon into the repo directory... icon_dir = os.path.join(repodir, 'icons') iconfilename = os.path.join(icon_dir, os.path.basename(common.config['repo_icon'])) shutil.copyfile(common.config['repo_icon'], iconfilename)
def make(apps, apks, repodir, archive): """Generate the repo index files. This requires properly initialized options and config objects. :param apps: OrderedDict of apps to go into the index, each app should have at least one associated apk :param apks: list of apks to go into the index :param repodir: the repo directory :param archive: True if this is the archive repo, False if it's the main one. """ from fdroidserver.update import METADATA_VERSION if not common.options.nosign: common.assert_config_keystore(common.config) # Historically the index has been sorted by App Name, so we enforce this ordering here sortedids = sorted(apps, key=lambda appid: apps[appid]['Name'].upper()) sortedapps = collections.OrderedDict() for appid in sortedids: sortedapps[appid] = apps[appid] repodict = collections.OrderedDict() repodict['timestamp'] = datetime.utcnow().replace(tzinfo=timezone.utc) repodict['version'] = METADATA_VERSION if common.config['repo_maxage'] != 0: repodict['maxage'] = common.config['repo_maxage'] if archive: repodict['name'] = common.config['archive_name'] repodict['icon'] = os.path.basename(common.config['archive_icon']) repodict['address'] = common.config['archive_url'] repodict['description'] = common.config['archive_description'] urlbasepath = os.path.basename( urllib.parse.urlparse(common.config['archive_url']).path) else: repodict['name'] = common.config['repo_name'] repodict['icon'] = os.path.basename(common.config['repo_icon']) repodict['address'] = common.config['repo_url'] repodict['description'] = common.config['repo_description'] urlbasepath = os.path.basename( urllib.parse.urlparse(common.config['repo_url']).path) mirrorcheckfailed = False mirrors = [] for mirror in common.config.get('mirrors', []): base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/')) if common.config.get( 'nonstandardwebroot') is not True and base != 'fdroid': logging.error( _("mirror '%s' does not end with 'fdroid'!") % mirror) mirrorcheckfailed = True # must end with / or urljoin strips a whole path segment if mirror.endswith('/'): mirrors.append(urllib.parse.urljoin(mirror, urlbasepath)) else: mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath)) for mirror in common.config.get('servergitmirrors', []): for url in get_mirror_service_urls(mirror): mirrors.append(url + '/' + repodir) if mirrorcheckfailed: raise FDroidException(_("Malformed repository mirrors.")) if mirrors: repodict['mirrors'] = mirrors requestsdict = collections.OrderedDict() for command in ('install', 'uninstall'): packageNames = [] key = command + '_list' if key in common.config: if isinstance(common.config[key], str): packageNames = [common.config[key]] elif all(isinstance(item, str) for item in common.config[key]): packageNames = common.config[key] else: raise TypeError(_('only accepts strings, lists, and tuples')) requestsdict[command] = packageNames fdroid_signing_key_fingerprints = load_stats_fdroid_signing_key_fingerprints( ) make_v0(sortedapps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints) make_v1(sortedapps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints)
def write_yaml(mf, app): # import rumael.yaml and check version try: import ruamel.yaml except ImportError as e: raise FDroidException( 'ruamel.yaml not instlled, can not write metadata.') from e if not ruamel.yaml.__version__: raise FDroidException( 'ruamel.yaml.__version__ not accessible. Please make sure a ruamel.yaml >= 0.13 is installed..' ) m = re.match( r'(?P<major>[0-9]+)\.(?P<minor>[0-9]+)\.(?P<patch>[0-9]+)(-.+)?', ruamel.yaml.__version__) if not m: raise FDroidException( 'ruamel.yaml version malfored, please install an upstream version of ruamel.yaml' ) if int(m.group('major')) < 0 or int(m.group('minor')) < 13: raise FDroidException( 'currently installed version of ruamel.yaml ({}) is too old, >= 1.13 required.' .format(ruamel.yaml.__version__)) # suiteable version ruamel.yaml imported successfully _yaml_bools_true = ('y', 'Y', 'yes', 'Yes', 'YES', 'true', 'True', 'TRUE', 'on', 'On', 'ON') _yaml_bools_false = ('n', 'N', 'no', 'No', 'NO', 'false', 'False', 'FALSE', 'off', 'Off', 'OFF') _yaml_bools_plus_lists = [] _yaml_bools_plus_lists.extend(_yaml_bools_true) _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_true]) _yaml_bools_plus_lists.extend(_yaml_bools_false) _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_false]) def _class_as_dict_representer(dumper, data): '''Creates a YAML representation of a App/Build instance''' return dumper.represent_dict(data) def _field_to_yaml(typ, value): if typ is TYPE_STRING: if value in _yaml_bools_plus_lists: return ruamel.yaml.scalarstring.SingleQuotedScalarString( str(value)) return str(value) elif typ is TYPE_INT: return int(value) elif typ is TYPE_MULTILINE: if '\n' in value: return ruamel.yaml.scalarstring.preserve_literal(str(value)) else: return str(value) elif typ is TYPE_SCRIPT: if len(value) > 50: return ruamel.yaml.scalarstring.preserve_literal(value) else: return value else: return value def _app_to_yaml(app): cm = ruamel.yaml.comments.CommentedMap() insert_newline = False for field in yaml_app_field_order: if field is '\n': # next iteration will need to insert a newline insert_newline = True else: if app.get(field) or field is 'Builds': # .txt calls it 'builds' internally, everywhere else its 'Builds' if field is 'Builds': if app.get('builds'): cm.update({field: _builds_to_yaml(app)}) elif field is 'CurrentVersionCode': cm.update({ field: _field_to_yaml(TYPE_INT, getattr(app, field)) }) else: cm.update({ field: _field_to_yaml(fieldtype(field), getattr(app, field)) }) if insert_newline: # we need to prepend a newline in front of this field insert_newline = False # inserting empty lines is not supported so we add a # bogus comment and over-write its value cm.yaml_set_comment_before_after_key(field, 'bogus') cm.ca.items[field][1][-1].value = '\n' return cm def _builds_to_yaml(app): fields = ['versionName', 'versionCode'] fields.extend(build_flags_order) builds = ruamel.yaml.comments.CommentedSeq() for build in app.builds: b = ruamel.yaml.comments.CommentedMap() for field in fields: if hasattr(build, field) and getattr(build, field): value = getattr(build, field) if field == 'gradle' and value == ['off']: value = [ ruamel.yaml.scalarstring.SingleQuotedScalarString( 'off') ] if field in ('disable', 'maven', 'buildozer'): if value == 'no': continue elif value == 'yes': value = 'yes' b.update({field: _field_to_yaml(flagtype(field), value)}) builds.append(b) # insert extra empty lines between build entries for i in range(1, len(builds)): builds.yaml_set_comment_before_after_key(i, 'bogus') builds.ca.items[i][1][-1].value = '\n' return builds yaml_app_field_order = [ 'Disabled', 'AntiFeatures', 'Provides', 'Categories', 'License', 'AuthorName', 'AuthorEmail', 'AuthorWebSite', 'WebSite', 'SourceCode', 'IssueTracker', 'Translation', 'Changelog', 'Donate', 'FlattrID', 'LiberapayID', 'Bitcoin', 'Litecoin', '\n', 'Name', 'AutoName', 'Summary', 'Description', '\n', 'RequiresRoot', '\n', 'RepoType', 'Repo', 'Binaries', '\n', 'Builds', '\n', 'MaintainerNotes', '\n', 'ArchivePolicy', 'AutoUpdateMode', 'UpdateCheckMode', 'UpdateCheckIgnore', 'VercodeOperation', 'UpdateCheckName', 'UpdateCheckData', 'CurrentVersion', 'CurrentVersionCode', '\n', 'NoSourceSince', ] yaml_app = _app_to_yaml(app) ruamel.yaml.round_trip_dump(yaml_app, mf, indent=4, block_seq_indent=2)
def write_yaml(mf, app): """Write metadata in yaml format. :param mf: active file discriptor for writing :param app: app metadata to written to the yaml file """ # import rumael.yaml and check version try: import ruamel.yaml except ImportError as e: raise FDroidException('ruamel.yaml not installed, can not write metadata.') from e if not ruamel.yaml.__version__: raise FDroidException('ruamel.yaml.__version__ not accessible. Please make sure a ruamel.yaml >= 0.13 is installed..') m = re.match(r'(?P<major>[0-9]+)\.(?P<minor>[0-9]+)\.(?P<patch>[0-9]+)(-.+)?', ruamel.yaml.__version__) if not m: raise FDroidException('ruamel.yaml version malfored, please install an upstream version of ruamel.yaml') if int(m.group('major')) < 0 or int(m.group('minor')) < 13: raise FDroidException('currently installed version of ruamel.yaml ({}) is too old, >= 1.13 required.'.format(ruamel.yaml.__version__)) # suiteable version ruamel.yaml imported successfully _yaml_bools_true = ('y', 'Y', 'yes', 'Yes', 'YES', 'true', 'True', 'TRUE', 'on', 'On', 'ON') _yaml_bools_false = ('n', 'N', 'no', 'No', 'NO', 'false', 'False', 'FALSE', 'off', 'Off', 'OFF') _yaml_bools_plus_lists = [] _yaml_bools_plus_lists.extend(_yaml_bools_true) _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_true]) _yaml_bools_plus_lists.extend(_yaml_bools_false) _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_false]) def _class_as_dict_representer(dumper, data): '''Creates a YAML representation of a App/Build instance''' return dumper.represent_dict(data) def _field_to_yaml(typ, value): if typ is TYPE_STRING: if value in _yaml_bools_plus_lists: return ruamel.yaml.scalarstring.SingleQuotedScalarString(str(value)) return str(value) elif typ is TYPE_INT: return int(value) elif typ is TYPE_MULTILINE: if '\n' in value: return ruamel.yaml.scalarstring.preserve_literal(str(value)) else: return str(value) elif typ is TYPE_SCRIPT: if type(value) == list: if len(value) == 1: return value[0] else: return value else: script_lines = value.split(' && ') if len(script_lines) > 1: return script_lines else: return value else: return value def _app_to_yaml(app): cm = ruamel.yaml.comments.CommentedMap() insert_newline = False for field in yaml_app_field_order: if field == '\n': # next iteration will need to insert a newline insert_newline = True else: if app.get(field) or field == 'Builds': if field == 'Builds': if app.get('Builds'): cm.update({field: _builds_to_yaml(app)}) elif field == 'CurrentVersionCode': cm.update({field: _field_to_yaml(TYPE_INT, getattr(app, field))}) else: cm.update({field: _field_to_yaml(fieldtype(field), getattr(app, field))}) if insert_newline: # we need to prepend a newline in front of this field insert_newline = False # inserting empty lines is not supported so we add a # bogus comment and over-write its value cm.yaml_set_comment_before_after_key(field, 'bogus') cm.ca.items[field][1][-1].value = '\n' return cm def _builds_to_yaml(app): builds = ruamel.yaml.comments.CommentedSeq() for build in app.get('Builds', []): if not isinstance(build, Build): build = Build(build) b = ruamel.yaml.comments.CommentedMap() for field in build_flags: value = getattr(build, field) if hasattr(build, field) and value: if field == 'gradle' and value == ['off']: value = [ruamel.yaml.scalarstring.SingleQuotedScalarString('off')] if field in ('maven', 'buildozer'): if value == 'no': continue elif value == 'yes': value = 'yes' b.update({field: _field_to_yaml(flagtype(field), value)}) builds.append(b) # insert extra empty lines between build entries for i in range(1, len(builds)): builds.yaml_set_comment_before_after_key(i, 'bogus') builds.ca.items[i][1][-1].value = '\n' return builds yaml_app = _app_to_yaml(app) ruamel.yaml.round_trip_dump(yaml_app, mf, indent=4, block_seq_indent=2)
def read_metadata(appids={}, refresh=True, sort_by_time=False): """Return a list of App instances sorted newest first This reads all of the metadata files in a 'data' repository, then builds a list of App instances from those files. The list is sorted based on creation time, newest first. Most of the time, the newer files are the most interesting. appids is a dict with appids a keys and versionCodes as values. """ # Always read the srclibs before the apps, since they can use a srlib as # their source repository. read_srclibs() apps = OrderedDict() for basedir in ('metadata', 'tmp'): if not os.path.exists(basedir): os.makedirs(basedir) if appids: vercodes = fdroidserver.common.read_pkg_args(appids) found_invalid = False metadatafiles = [] for appid in vercodes.keys(): f = os.path.join('metadata', '%s.yml' % appid) if os.path.exists(f): metadatafiles.append(f) else: found_invalid = True logging.critical(_("No such package: %s") % appid) if found_invalid: raise FDroidException(_("Found invalid appids in arguments")) else: metadatafiles = (glob.glob(os.path.join('metadata', '*.yml')) + glob.glob('.fdroid.yml')) if sort_by_time: entries = ((os.stat(path).st_mtime, path) for path in metadatafiles) metadatafiles = [] for _ignored, path in sorted(entries, reverse=True): metadatafiles.append(path) else: # most things want the index alpha sorted for stability metadatafiles = sorted(metadatafiles) for metadatapath in metadatafiles: appid, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath)) if appid != '.fdroid' and not fdroidserver.common.is_valid_package_name(appid): _warn_or_exception(_("{appid} from {path} is not a valid Java Package Name!") .format(appid=appid, path=metadatapath)) if appid in apps: _warn_or_exception(_("Found multiple metadata files for {appid}") .format(appid=appid)) app = parse_metadata(metadatapath, appid in appids, refresh) check_metadata(app) apps[app.id] = app return apps