def extract_pubkey():
"""
Extracts and returns the repository's public key from the keystore.
:return: public key in hex, repository fingerprint
"""
if 'repo_pubkey' in common.config:
pubkey = unhexlify(common.config['repo_pubkey'])
else:
env_vars = {
'LC_ALL': 'C.UTF-8',
'FDROID_KEY_STORE_PASS': common.config['keystorepass']
}
p = FDroidPopenBytes([
common.config['keytool'], '-exportcert', '-alias',
common.config['repo_keyalias'], '-keystore',
common.config['keystore'], '-storepass:env',
'FDROID_KEY_STORE_PASS'
] + list(common.config['smartcardoptions']),
envs=env_vars,
output=False,
stderr_to_stdout=False)
if p.returncode != 0 or len(p.output) < 20:
msg = "Failed to get repo pubkey!"
if common.config['keystore'] == 'NONE':
msg += ' Is your crypto smartcard plugged in?'
raise FDroidException(msg)
pubkey = p.output
repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey)
return hexlify(pubkey), repo_pubkey_fingerprint
def write_metadata(metadatapath, app):
if metadatapath.endswith('.yml'):
if importlib.util.find_spec('ruamel.yaml'):
with open(metadatapath, 'w') as mf:
return write_yaml(mf, app)
else:
raise FDroidException(_('ruamel.yaml not installed, can not write metadata.'))
_warn_or_exception(_('Unknown metadata format: %s') % metadatapath)
def _copy_to_local_copy_dir(repodir, f):
local_copy_dir = common.config.get('local_copy_dir', '')
if os.path.exists(local_copy_dir):
destdir = os.path.join(local_copy_dir, repodir)
if not os.path.exists(destdir):
os.mkdir(destdir)
shutil.copy2(f, destdir, follow_symlinks=False)
elif local_copy_dir:
raise FDroidException(
_('"local_copy_dir" {path} does not exist!').format(
path=local_copy_dir))
def make(apps, sortedids, apks, repodir, archive):
"""Generate the repo index files.
This requires properly initialized options and config objects.
:param apps: fully populated apps list
:param sortedids: app package IDs, sorted
:param apks: full populated apks list
:param repodir: the repo directory
:param archive: True if this is the archive repo, False if it's the
main one.
"""
from fdroidserver.update import METADATA_VERSION
def _resolve_description_link(appid):
if appid in apps:
return "fdroid.app:" + appid, apps[appid].Name
raise MetaDataException("Cannot resolve app id " + appid)
if not common.options.nosign:
common.assert_config_keystore(common.config)
repodict = collections.OrderedDict()
repodict['timestamp'] = datetime.utcnow()
repodict['version'] = METADATA_VERSION
if common.config['repo_maxage'] != 0:
repodict['maxage'] = common.config['repo_maxage']
if archive:
repodict['name'] = common.config['archive_name']
repodict['icon'] = os.path.basename(common.config['archive_icon'])
repodict['address'] = common.config['archive_url']
repodict['description'] = common.config['archive_description']
urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['archive_url']).path)
else:
repodict['name'] = common.config['repo_name']
repodict['icon'] = os.path.basename(common.config['repo_icon'])
repodict['address'] = common.config['repo_url']
repodict['description'] = common.config['repo_description']
urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['repo_url']).path)
mirrorcheckfailed = False
mirrors = []
for mirror in sorted(common.config.get('mirrors', [])):
base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
if common.config.get('nonstandardwebroot') is not True and base != 'fdroid':
logging.error(_("mirror '%s' does not end with 'fdroid'!") % mirror)
mirrorcheckfailed = True
# must end with / or urljoin strips a whole path segment
if mirror.endswith('/'):
mirrors.append(urllib.parse.urljoin(mirror, urlbasepath))
else:
mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath))
for mirror in common.config.get('servergitmirrors', []):
for url in get_mirror_service_urls(mirror):
mirrors.append(url + '/' + repodir)
if mirrorcheckfailed:
raise FDroidException(_("Malformed repository mirrors."))
if mirrors:
repodict['mirrors'] = mirrors
appsWithPackages = collections.OrderedDict()
for packageName in sortedids:
app = apps[packageName]
if app['Disabled']:
continue
# only include apps with packages
for apk in apks:
if apk['packageName'] == packageName:
newapp = copy.copy(app) # update wiki needs unmodified description
newapp['Description'] = metadata.description_html(app['Description'],
_resolve_description_link)
appsWithPackages[packageName] = newapp
break
requestsdict = collections.OrderedDict()
for command in ('install', 'uninstall'):
packageNames = []
key = command + '_list'
if key in common.config:
if isinstance(common.config[key], str):
packageNames = [common.config[key]]
elif all(isinstance(item, str) for item in common.config[key]):
packageNames = common.config[key]
else:
raise TypeError(_('only accepts strings, lists, and tuples'))
requestsdict[command] = packageNames
fdroid_signing_key_fingerprints = load_stats_fdroid_signing_key_fingerprints()
make_v0(appsWithPackages, apks, repodir, repodict, requestsdict,
fdroid_signing_key_fingerprints)
make_v1(appsWithPackages, apks, repodir, repodict, requestsdict,
fdroid_signing_key_fingerprints)
def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
"""
aka index.jar aka index.xml
"""
doc = Document()
def addElement(name, value, doc, parent):
el = doc.createElement(name)
el.appendChild(doc.createTextNode(value))
parent.appendChild(el)
def addElementNonEmpty(name, value, doc, parent):
if not value:
return
addElement(name, value, doc, parent)
def addElementIfInApk(name, apk, key, doc, parent):
if key not in apk:
return
value = str(apk[key])
addElement(name, value, doc, parent)
def addElementCDATA(name, value, doc, parent):
el = doc.createElement(name)
el.appendChild(doc.createCDATASection(value))
parent.appendChild(el)
def addElementCheckLocalized(name, app, key, doc, parent, default=''):
'''Fill in field from metadata or localized block
For name/summary/description, they can come only from the app source,
or from a dir in fdroiddata. They can be entirely missing from the
metadata file if there is localized versions. This will fetch those
from the localized version if its not available in the metadata file.
'''
el = doc.createElement(name)
value = app.get(key)
lkey = key[:1].lower() + key[1:]
localized = app.get('localized')
if not value and localized:
for lang in ['en-US'] + [x for x in localized.keys()]:
if not lang.startswith('en'):
continue
if lang in localized:
value = localized[lang].get(lkey)
if value:
break
if not value and localized and len(localized) > 1:
lang = list(localized.keys())[0]
value = localized[lang].get(lkey)
if not value:
value = default
el.appendChild(doc.createTextNode(value))
parent.appendChild(el)
root = doc.createElement("fdroid")
doc.appendChild(root)
repoel = doc.createElement("repo")
repoel.setAttribute("name", repodict['name'])
if 'maxage' in repodict:
repoel.setAttribute("maxage", str(repodict['maxage']))
repoel.setAttribute("icon", os.path.basename(repodict['icon']))
repoel.setAttribute("url", repodict['address'])
addElement('description', repodict['description'], doc, repoel)
for mirror in repodict.get('mirrors', []):
addElement('mirror', mirror, doc, repoel)
repoel.setAttribute("version", str(repodict['version']))
repoel.setAttribute("timestamp", '%d' % repodict['timestamp'].timestamp())
pubkey, repo_pubkey_fingerprint = extract_pubkey()
repoel.setAttribute("pubkey", pubkey.decode('utf-8'))
root.appendChild(repoel)
for command in ('install', 'uninstall'):
for packageName in requestsdict[command]:
element = doc.createElement(command)
root.appendChild(element)
element.setAttribute('packageName', packageName)
for appid, appdict in apps.items():
app = metadata.App(appdict)
if app.Disabled is not None:
continue
# Get a list of the apks for this app...
apklist = []
apksbyversion = collections.defaultdict(lambda: [])
for apk in apks:
if apk.get('versionCode') and apk.get('packageName') == appid:
apksbyversion[apk['versionCode']].append(apk)
for versionCode, apksforver in apksbyversion.items():
fdroidsig = fdroid_signing_key_fingerprints.get(appid, {}).get('signer')
fdroid_signed_apk = None
name_match_apk = None
for x in apksforver:
if fdroidsig and x.get('signer', None) == fdroidsig:
fdroid_signed_apk = x
if common.apk_release_filename.match(x.get('apkName', '')):
name_match_apk = x
# choose which of the available versions is most
# suiteable for index v0
if fdroid_signed_apk:
apklist.append(fdroid_signed_apk)
elif name_match_apk:
apklist.append(name_match_apk)
else:
apklist.append(apksforver[0])
if len(apklist) == 0:
continue
apel = doc.createElement("application")
apel.setAttribute("id", app.id)
root.appendChild(apel)
addElement('id', app.id, doc, apel)
if app.added:
addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel)
if app.lastUpdated:
addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel)
addElementCheckLocalized('name', app, 'Name', doc, apel)
addElementCheckLocalized('summary', app, 'Summary', doc, apel)
if app.icon:
addElement('icon', app.icon, doc, apel)
addElementCheckLocalized('desc', app, 'Description', doc, apel,
'<p>No description available</p>')
addElement('license', app.License, doc, apel)
if app.Categories:
addElement('categories', ','.join(app.Categories), doc, apel)
# We put the first (primary) category in LAST, which will have
# the desired effect of making clients that only understand one
# category see that one.
addElement('category', app.Categories[0], doc, apel)
addElement('web', app.WebSite, doc, apel)
addElement('source', app.SourceCode, doc, apel)
addElement('tracker', app.IssueTracker, doc, apel)
addElementNonEmpty('changelog', app.Changelog, doc, apel)
addElementNonEmpty('author', app.AuthorName, doc, apel)
addElementNonEmpty('email', app.AuthorEmail, doc, apel)
addElementNonEmpty('donate', app.Donate, doc, apel)
addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
addElementNonEmpty('flattr', app.FlattrID, doc, apel)
addElementNonEmpty('liberapay', app.LiberapayID, doc, apel)
# These elements actually refer to the current version (i.e. which
# one is recommended. They are historically mis-named, and need
# changing, but stay like this for now to support existing clients.
addElement('marketversion', app.CurrentVersion, doc, apel)
addElement('marketvercode', app.CurrentVersionCode, doc, apel)
if app.Provides:
pv = app.Provides.split(',')
addElementNonEmpty('provides', ','.join(pv), doc, apel)
if app.RequiresRoot:
addElement('requirements', 'root', doc, apel)
# Sort the apk list into version order, just so the web site
# doesn't have to do any work by default...
apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
if 'antiFeatures' in apklist[0]:
app.AntiFeatures.extend(apklist[0]['antiFeatures'])
if app.AntiFeatures:
addElementNonEmpty('antifeatures', ','.join(app.AntiFeatures), doc, apel)
# Check for duplicates - they will make the client unhappy...
for i in range(len(apklist) - 1):
first = apklist[i]
second = apklist[i + 1]
if first['versionCode'] == second['versionCode'] \
and first['sig'] == second['sig']:
if first['hash'] == second['hash']:
raise FDroidException('"{0}/{1}" and "{0}/{2}" are exact duplicates!'.format(
repodir, first['apkName'], second['apkName']))
else:
raise FDroidException('duplicates: "{0}/{1}" - "{0}/{2}"'.format(
repodir, first['apkName'], second['apkName']))
current_version_code = 0
current_version_file = None
for apk in apklist:
file_extension = common.get_file_extension(apk['apkName'])
# find the APK for the "Current Version"
if current_version_code < apk['versionCode']:
current_version_code = apk['versionCode']
if current_version_code < int(app.CurrentVersionCode):
current_version_file = apk['apkName']
apkel = doc.createElement("package")
apel.appendChild(apkel)
addElement('version', apk['versionName'], doc, apkel)
addElement('versioncode', str(apk['versionCode']), doc, apkel)
addElement('apkname', apk['apkName'], doc, apkel)
addElementIfInApk('srcname', apk, 'srcname', doc, apkel)
hashel = doc.createElement("hash")
hashel.setAttribute('type', 'sha256')
hashel.appendChild(doc.createTextNode(apk['hash']))
apkel.appendChild(hashel)
addElement('size', str(apk['size']), doc, apkel)
addElementIfInApk('sdkver', apk,
'minSdkVersion', doc, apkel)
addElementIfInApk('targetSdkVersion', apk,
'targetSdkVersion', doc, apkel)
addElementIfInApk('maxsdkver', apk,
'maxSdkVersion', doc, apkel)
addElementIfInApk('obbMainFile', apk,
'obbMainFile', doc, apkel)
addElementIfInApk('obbMainFileSha256', apk,
'obbMainFileSha256', doc, apkel)
addElementIfInApk('obbPatchFile', apk,
'obbPatchFile', doc, apkel)
addElementIfInApk('obbPatchFileSha256', apk,
'obbPatchFileSha256', doc, apkel)
if 'added' in apk:
addElement('added', apk['added'].strftime('%Y-%m-%d'), doc, apkel)
if file_extension == 'apk': # sig is required for APKs, but only APKs
addElement('sig', apk['sig'], doc, apkel)
old_permissions = set()
sorted_permissions = sorted(apk['uses-permission'])
for perm in sorted_permissions:
perm_name = perm.name
if perm_name.startswith("android.permission."):
perm_name = perm_name[19:]
old_permissions.add(perm_name)
addElementNonEmpty('permissions', ','.join(sorted(old_permissions)), doc, apkel)
for permission in sorted_permissions:
permel = doc.createElement('uses-permission')
permel.setAttribute('name', permission.name)
if permission.maxSdkVersion is not None:
permel.setAttribute('maxSdkVersion', '%d' % permission.maxSdkVersion)
apkel.appendChild(permel)
for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']):
permel = doc.createElement('uses-permission-sdk-23')
permel.setAttribute('name', permission_sdk_23.name)
if permission_sdk_23.maxSdkVersion is not None:
permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23.maxSdkVersion)
apkel.appendChild(permel)
if 'nativecode' in apk:
addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel)
addElementNonEmpty('features', ','.join(sorted(apk['features'])), doc, apkel)
if current_version_file is not None \
and common.config['make_current_version_link'] \
and repodir == 'repo': # only create these
namefield = common.config['current_version_name_source']
sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8'))
apklinkname = sanitized_name + os.path.splitext(current_version_file)[1].encode('utf-8')
current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape')
if os.path.islink(apklinkname):
os.remove(apklinkname)
os.symlink(current_version_path, apklinkname)
# also symlink gpg signature, if it exists
for extension in (b'.asc', b'.sig'):
sigfile_path = current_version_path + extension
if os.path.exists(sigfile_path):
siglinkname = apklinkname + extension
if os.path.islink(siglinkname):
os.remove(siglinkname)
os.symlink(sigfile_path, siglinkname)
if common.options.pretty:
output = doc.toprettyxml(encoding='utf-8')
else:
output = doc.toxml(encoding='utf-8')
with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
f.write(output)
if 'repo_keyalias' in common.config:
if common.options.nosign:
logging.info(_("Creating unsigned index in preparation for signing"))
else:
logging.info(_("Creating signed index with this key (SHA256):"))
logging.info("%s" % repo_pubkey_fingerprint)
# Create a jar of the index...
jar_output = 'index_unsigned.jar' if common.options.nosign else 'index.jar'
p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
if p.returncode != 0:
raise FDroidException("Failed to create {0}".format(jar_output))
# Sign the index...
signed = os.path.join(repodir, 'index.jar')
if common.options.nosign:
# Remove old signed index if not signing
if os.path.exists(signed):
os.remove(signed)
else:
signindex.config = common.config
signindex.sign_jar(signed)
# Copy the repo icon into the repo directory...
icon_dir = os.path.join(repodir, 'icons')
iconfilename = os.path.join(icon_dir, os.path.basename(common.config['repo_icon']))
shutil.copyfile(common.config['repo_icon'], iconfilename)
def make(apps, apks, repodir, archive):
"""Generate the repo index files.
This requires properly initialized options and config objects.
:param apps: OrderedDict of apps to go into the index, each app should have
at least one associated apk
:param apks: list of apks to go into the index
:param repodir: the repo directory
:param archive: True if this is the archive repo, False if it's the
main one.
"""
from fdroidserver.update import METADATA_VERSION
if not common.options.nosign:
common.assert_config_keystore(common.config)
# Historically the index has been sorted by App Name, so we enforce this ordering here
sortedids = sorted(apps, key=lambda appid: apps[appid]['Name'].upper())
sortedapps = collections.OrderedDict()
for appid in sortedids:
sortedapps[appid] = apps[appid]
repodict = collections.OrderedDict()
repodict['timestamp'] = datetime.utcnow().replace(tzinfo=timezone.utc)
repodict['version'] = METADATA_VERSION
if common.config['repo_maxage'] != 0:
repodict['maxage'] = common.config['repo_maxage']
if archive:
repodict['name'] = common.config['archive_name']
repodict['icon'] = os.path.basename(common.config['archive_icon'])
repodict['address'] = common.config['archive_url']
repodict['description'] = common.config['archive_description']
urlbasepath = os.path.basename(
urllib.parse.urlparse(common.config['archive_url']).path)
else:
repodict['name'] = common.config['repo_name']
repodict['icon'] = os.path.basename(common.config['repo_icon'])
repodict['address'] = common.config['repo_url']
repodict['description'] = common.config['repo_description']
urlbasepath = os.path.basename(
urllib.parse.urlparse(common.config['repo_url']).path)
mirrorcheckfailed = False
mirrors = []
for mirror in common.config.get('mirrors', []):
base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
if common.config.get(
'nonstandardwebroot') is not True and base != 'fdroid':
logging.error(
_("mirror '%s' does not end with 'fdroid'!") % mirror)
mirrorcheckfailed = True
# must end with / or urljoin strips a whole path segment
if mirror.endswith('/'):
mirrors.append(urllib.parse.urljoin(mirror, urlbasepath))
else:
mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath))
for mirror in common.config.get('servergitmirrors', []):
for url in get_mirror_service_urls(mirror):
mirrors.append(url + '/' + repodir)
if mirrorcheckfailed:
raise FDroidException(_("Malformed repository mirrors."))
if mirrors:
repodict['mirrors'] = mirrors
requestsdict = collections.OrderedDict()
for command in ('install', 'uninstall'):
packageNames = []
key = command + '_list'
if key in common.config:
if isinstance(common.config[key], str):
packageNames = [common.config[key]]
elif all(isinstance(item, str) for item in common.config[key]):
packageNames = common.config[key]
else:
raise TypeError(_('only accepts strings, lists, and tuples'))
requestsdict[command] = packageNames
fdroid_signing_key_fingerprints = load_stats_fdroid_signing_key_fingerprints(
)
make_v0(sortedapps, apks, repodir, repodict, requestsdict,
fdroid_signing_key_fingerprints)
make_v1(sortedapps, apks, repodir, repodict, requestsdict,
fdroid_signing_key_fingerprints)
def write_yaml(mf, app):
# import rumael.yaml and check version
try:
import ruamel.yaml
except ImportError as e:
raise FDroidException(
'ruamel.yaml not instlled, can not write metadata.') from e
if not ruamel.yaml.__version__:
raise FDroidException(
'ruamel.yaml.__version__ not accessible. Please make sure a ruamel.yaml >= 0.13 is installed..'
)
m = re.match(
r'(?P<major>[0-9]+)\.(?P<minor>[0-9]+)\.(?P<patch>[0-9]+)(-.+)?',
ruamel.yaml.__version__)
if not m:
raise FDroidException(
'ruamel.yaml version malfored, please install an upstream version of ruamel.yaml'
)
if int(m.group('major')) < 0 or int(m.group('minor')) < 13:
raise FDroidException(
'currently installed version of ruamel.yaml ({}) is too old, >= 1.13 required.'
.format(ruamel.yaml.__version__))
# suiteable version ruamel.yaml imported successfully
_yaml_bools_true = ('y', 'Y', 'yes', 'Yes', 'YES', 'true', 'True', 'TRUE',
'on', 'On', 'ON')
_yaml_bools_false = ('n', 'N', 'no', 'No', 'NO', 'false', 'False', 'FALSE',
'off', 'Off', 'OFF')
_yaml_bools_plus_lists = []
_yaml_bools_plus_lists.extend(_yaml_bools_true)
_yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_true])
_yaml_bools_plus_lists.extend(_yaml_bools_false)
_yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_false])
def _class_as_dict_representer(dumper, data):
'''Creates a YAML representation of a App/Build instance'''
return dumper.represent_dict(data)
def _field_to_yaml(typ, value):
if typ is TYPE_STRING:
if value in _yaml_bools_plus_lists:
return ruamel.yaml.scalarstring.SingleQuotedScalarString(
str(value))
return str(value)
elif typ is TYPE_INT:
return int(value)
elif typ is TYPE_MULTILINE:
if '\n' in value:
return ruamel.yaml.scalarstring.preserve_literal(str(value))
else:
return str(value)
elif typ is TYPE_SCRIPT:
if len(value) > 50:
return ruamel.yaml.scalarstring.preserve_literal(value)
else:
return value
else:
return value
def _app_to_yaml(app):
cm = ruamel.yaml.comments.CommentedMap()
insert_newline = False
for field in yaml_app_field_order:
if field is '\n':
# next iteration will need to insert a newline
insert_newline = True
else:
if app.get(field) or field is 'Builds':
# .txt calls it 'builds' internally, everywhere else its 'Builds'
if field is 'Builds':
if app.get('builds'):
cm.update({field: _builds_to_yaml(app)})
elif field is 'CurrentVersionCode':
cm.update({
field:
_field_to_yaml(TYPE_INT, getattr(app, field))
})
else:
cm.update({
field:
_field_to_yaml(fieldtype(field),
getattr(app, field))
})
if insert_newline:
# we need to prepend a newline in front of this field
insert_newline = False
# inserting empty lines is not supported so we add a
# bogus comment and over-write its value
cm.yaml_set_comment_before_after_key(field, 'bogus')
cm.ca.items[field][1][-1].value = '\n'
return cm
def _builds_to_yaml(app):
fields = ['versionName', 'versionCode']
fields.extend(build_flags_order)
builds = ruamel.yaml.comments.CommentedSeq()
for build in app.builds:
b = ruamel.yaml.comments.CommentedMap()
for field in fields:
if hasattr(build, field) and getattr(build, field):
value = getattr(build, field)
if field == 'gradle' and value == ['off']:
value = [
ruamel.yaml.scalarstring.SingleQuotedScalarString(
'off')
]
if field in ('disable', 'maven', 'buildozer'):
if value == 'no':
continue
elif value == 'yes':
value = 'yes'
b.update({field: _field_to_yaml(flagtype(field), value)})
builds.append(b)
# insert extra empty lines between build entries
for i in range(1, len(builds)):
builds.yaml_set_comment_before_after_key(i, 'bogus')
builds.ca.items[i][1][-1].value = '\n'
return builds
yaml_app_field_order = [
'Disabled',
'AntiFeatures',
'Provides',
'Categories',
'License',
'AuthorName',
'AuthorEmail',
'AuthorWebSite',
'WebSite',
'SourceCode',
'IssueTracker',
'Translation',
'Changelog',
'Donate',
'FlattrID',
'LiberapayID',
'Bitcoin',
'Litecoin',
'\n',
'Name',
'AutoName',
'Summary',
'Description',
'\n',
'RequiresRoot',
'\n',
'RepoType',
'Repo',
'Binaries',
'\n',
'Builds',
'\n',
'MaintainerNotes',
'\n',
'ArchivePolicy',
'AutoUpdateMode',
'UpdateCheckMode',
'UpdateCheckIgnore',
'VercodeOperation',
'UpdateCheckName',
'UpdateCheckData',
'CurrentVersion',
'CurrentVersionCode',
'\n',
'NoSourceSince',
]
yaml_app = _app_to_yaml(app)
ruamel.yaml.round_trip_dump(yaml_app, mf, indent=4, block_seq_indent=2)
def write_yaml(mf, app):
"""Write metadata in yaml format.
:param mf: active file discriptor for writing
:param app: app metadata to written to the yaml file
"""
# import rumael.yaml and check version
try:
import ruamel.yaml
except ImportError as e:
raise FDroidException('ruamel.yaml not installed, can not write metadata.') from e
if not ruamel.yaml.__version__:
raise FDroidException('ruamel.yaml.__version__ not accessible. Please make sure a ruamel.yaml >= 0.13 is installed..')
m = re.match(r'(?P<major>[0-9]+)\.(?P<minor>[0-9]+)\.(?P<patch>[0-9]+)(-.+)?',
ruamel.yaml.__version__)
if not m:
raise FDroidException('ruamel.yaml version malfored, please install an upstream version of ruamel.yaml')
if int(m.group('major')) < 0 or int(m.group('minor')) < 13:
raise FDroidException('currently installed version of ruamel.yaml ({}) is too old, >= 1.13 required.'.format(ruamel.yaml.__version__))
# suiteable version ruamel.yaml imported successfully
_yaml_bools_true = ('y', 'Y', 'yes', 'Yes', 'YES',
'true', 'True', 'TRUE',
'on', 'On', 'ON')
_yaml_bools_false = ('n', 'N', 'no', 'No', 'NO',
'false', 'False', 'FALSE',
'off', 'Off', 'OFF')
_yaml_bools_plus_lists = []
_yaml_bools_plus_lists.extend(_yaml_bools_true)
_yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_true])
_yaml_bools_plus_lists.extend(_yaml_bools_false)
_yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_false])
def _class_as_dict_representer(dumper, data):
'''Creates a YAML representation of a App/Build instance'''
return dumper.represent_dict(data)
def _field_to_yaml(typ, value):
if typ is TYPE_STRING:
if value in _yaml_bools_plus_lists:
return ruamel.yaml.scalarstring.SingleQuotedScalarString(str(value))
return str(value)
elif typ is TYPE_INT:
return int(value)
elif typ is TYPE_MULTILINE:
if '\n' in value:
return ruamel.yaml.scalarstring.preserve_literal(str(value))
else:
return str(value)
elif typ is TYPE_SCRIPT:
if type(value) == list:
if len(value) == 1:
return value[0]
else:
return value
else:
script_lines = value.split(' && ')
if len(script_lines) > 1:
return script_lines
else:
return value
else:
return value
def _app_to_yaml(app):
cm = ruamel.yaml.comments.CommentedMap()
insert_newline = False
for field in yaml_app_field_order:
if field == '\n':
# next iteration will need to insert a newline
insert_newline = True
else:
if app.get(field) or field == 'Builds':
if field == 'Builds':
if app.get('Builds'):
cm.update({field: _builds_to_yaml(app)})
elif field == 'CurrentVersionCode':
cm.update({field: _field_to_yaml(TYPE_INT, getattr(app, field))})
else:
cm.update({field: _field_to_yaml(fieldtype(field), getattr(app, field))})
if insert_newline:
# we need to prepend a newline in front of this field
insert_newline = False
# inserting empty lines is not supported so we add a
# bogus comment and over-write its value
cm.yaml_set_comment_before_after_key(field, 'bogus')
cm.ca.items[field][1][-1].value = '\n'
return cm
def _builds_to_yaml(app):
builds = ruamel.yaml.comments.CommentedSeq()
for build in app.get('Builds', []):
if not isinstance(build, Build):
build = Build(build)
b = ruamel.yaml.comments.CommentedMap()
for field in build_flags:
value = getattr(build, field)
if hasattr(build, field) and value:
if field == 'gradle' and value == ['off']:
value = [ruamel.yaml.scalarstring.SingleQuotedScalarString('off')]
if field in ('maven', 'buildozer'):
if value == 'no':
continue
elif value == 'yes':
value = 'yes'
b.update({field: _field_to_yaml(flagtype(field), value)})
builds.append(b)
# insert extra empty lines between build entries
for i in range(1, len(builds)):
builds.yaml_set_comment_before_after_key(i, 'bogus')
builds.ca.items[i][1][-1].value = '\n'
return builds
yaml_app = _app_to_yaml(app)
ruamel.yaml.round_trip_dump(yaml_app, mf, indent=4, block_seq_indent=2)
def read_metadata(appids={}, refresh=True, sort_by_time=False):
"""Return a list of App instances sorted newest first
This reads all of the metadata files in a 'data' repository, then
builds a list of App instances from those files. The list is
sorted based on creation time, newest first. Most of the time,
the newer files are the most interesting.
appids is a dict with appids a keys and versionCodes as values.
"""
# Always read the srclibs before the apps, since they can use a srlib as
# their source repository.
read_srclibs()
apps = OrderedDict()
for basedir in ('metadata', 'tmp'):
if not os.path.exists(basedir):
os.makedirs(basedir)
if appids:
vercodes = fdroidserver.common.read_pkg_args(appids)
found_invalid = False
metadatafiles = []
for appid in vercodes.keys():
f = os.path.join('metadata', '%s.yml' % appid)
if os.path.exists(f):
metadatafiles.append(f)
else:
found_invalid = True
logging.critical(_("No such package: %s") % appid)
if found_invalid:
raise FDroidException(_("Found invalid appids in arguments"))
else:
metadatafiles = (glob.glob(os.path.join('metadata', '*.yml'))
+ glob.glob('.fdroid.yml'))
if sort_by_time:
entries = ((os.stat(path).st_mtime, path) for path in metadatafiles)
metadatafiles = []
for _ignored, path in sorted(entries, reverse=True):
metadatafiles.append(path)
else:
# most things want the index alpha sorted for stability
metadatafiles = sorted(metadatafiles)
for metadatapath in metadatafiles:
appid, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath))
if appid != '.fdroid' and not fdroidserver.common.is_valid_package_name(appid):
_warn_or_exception(_("{appid} from {path} is not a valid Java Package Name!")
.format(appid=appid, path=metadatapath))
if appid in apps:
_warn_or_exception(_("Found multiple metadata files for {appid}")
.format(appid=appid))
app = parse_metadata(metadatapath, appid in appids, refresh)
check_metadata(app)
apps[app.id] = app
return apps
