def main(): global config, options # Parse command line... parser = OptionParser(usage="Usage: %prog [options] [APPID [APPID ...]]") parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") parser.add_option("-q", "--quiet", action="store_true", default=False, help="Restrict output to warnings and errors") (options, args) = parser.parse_args() config = common.read_config(options) # Get all apps... allapps = metadata.read_metadata(xref=True) apps = common.read_app_args(args, allapps, False) for appid, app in apps.iteritems(): logging.info("Writing " + appid) metadata.write_metadata(os.path.join('metadata', appid) + '.txt', app) logging.info("Finished.")
def main(): global config, options anywarns = False # Parse command line... parser = ArgumentParser(usage="%(prog)s [options] [APPID [APPID ...]]") common.setup_global_opts(parser) parser.add_argument("-f", "--format", action="store_true", default=False, help="Also warn about formatting issues, like rewritemeta -l") parser.add_argument("appid", nargs='*', help="app-id in the form APPID") options = parser.parse_args() config = common.read_config(options) # Get all apps... allapps = metadata.read_metadata(xref=True) apps = common.read_app_args(options.appid, allapps, False) for appid, app in apps.iteritems(): if app.Disabled: continue warns = [] for check_func in [ check_regexes, check_ucm_tags, check_char_limits, check_old_links, check_checkupdates_ran, check_useless_fields, check_empty_fields, check_categories, check_duplicates, check_mediawiki_links, check_bulleted_lists, check_builds, ]: warns += check_func(app) if options.format: if not rewritemeta.proper_format(app): warns.append("Run rewritemeta to fix formatting") if warns: anywarns = True for warn in warns: print("%s: %s" % (appid, warn)) if anywarns: sys.exit(1)
def main(): global config, options # Parse command line... parser = ArgumentParser(usage="%(prog)s [options] [APPID [APPID ...]]") common.setup_global_opts(parser) parser.add_argument("-l", "--list", action="store_true", default=False, help="List files that would be reformatted") parser.add_argument("-t", "--to", default=None, help="Rewrite to a specific format") parser.add_argument("appid", nargs='*', help="app-id in the form APPID") options = parser.parse_args() config = common.read_config(options) # Get all apps... allapps = metadata.read_metadata(xref=True) apps = common.read_app_args(options.appid, allapps, False) if options.list and options.to is not None: parser.error("Cannot use --list and --to at the same time") supported = ['txt', 'yaml'] if options.to is not None and options.to not in supported: parser.error("Must give a valid format to --to") for appid, app in apps.iteritems(): base, ext = common.get_extension(app.metadatapath) if not options.to and ext not in supported: logging.info("Ignoring %s file at '%s'" % (ext, app.metadatapath)) continue to_ext = ext if options.to is not None: to_ext = options.to if options.list: if not proper_format(app): print app.metadatapath continue with open(base + '.' + to_ext, 'w') as f: metadata.write_metadata(to_ext, f, app) if ext != to_ext: os.remove(app.metadatapath) logging.debug("Finished.")
def main(): global config, options, curid, count curid = None count = Counter() def warn(message): global curid, count if curid: print "%s:" % curid curid = None count['app'] += 1 print ' %s' % message count['warn'] += 1 # Parse command line... parser = OptionParser(usage="Usage: %prog [options] [APPID [APPID ...]]") parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") parser.add_option("-q", "--quiet", action="store_true", default=False, help="Restrict output to warnings and errors") (options, args) = parser.parse_args() config = common.read_config(options) # Get all apps... allapps = metadata.read_metadata(xref=False) apps = common.read_app_args(args, allapps, False) filling_ucms = re.compile('^(Tags.*|RepoManifest.*)') for appid, app in apps.iteritems(): if app['Disabled']: continue curid = appid count['app_total'] += 1 # enabled_builds = 0 lowest_vercode = -1 curbuild = None for build in app['builds']: if not build['disable']: # enabled_builds += 1 vercode = int(build['vercode']) if lowest_vercode == -1 or vercode < lowest_vercode: lowest_vercode = vercode if not curbuild or int(build['vercode']) > int( curbuild['vercode']): curbuild = build # Incorrect UCM if (curbuild and curbuild['commit'] and app['Update Check Mode'] == 'RepoManifest' and not curbuild['commit'].startswith('unknown') and curbuild['vercode'] == app['Current Version Code'] and not curbuild['forcevercode'] and any(s in curbuild['commit'] for s in '.,_-/')): warn( "Last used commit '%s' looks like a tag, but Update Check Mode is '%s'" % (curbuild['commit'], app['Update Check Mode'])) # Summary size limit summ_chars = len(app['Summary']) if summ_chars > config['char_limits']['Summary']: warn("Summary of length %s is over the %i char limit" % (summ_chars, config['char_limits']['Summary'])) # Redundant info if app['Web Site'] and app['Source Code']: if app['Web Site'].lower() == app['Source Code'].lower(): warn("Website '%s' is just the app's source code link" % app['Web Site']) if filling_ucms.match(app['Update Check Mode']): if all(app[f] == metadata.app_defaults[f] for f in [ 'Auto Name', 'Current Version', 'Current Version Code', ]): warn( "UCM is set but it looks like checkupdates hasn't been run yet" ) if app['Update Check Name'] == appid: warn( "Update Check Name is set to the known app id - it can be removed" ) cvc = int(app['Current Version Code']) if cvc > 0 and cvc < lowest_vercode: warn("Current Version Code is lower than any enabled build") # Missing or incorrect categories if not app['Categories']: warn("Categories are not set") for categ in app['Categories']: if categ not in categories: warn("Category '%s' is not valid" % categ) if app['Name'] and app['Name'] == app['Auto Name']: warn("Name '%s' is just the auto name" % app['Name']) name = app['Name'] or app['Auto Name'] if app['Summary'] and name: if app['Summary'].lower() == name.lower(): warn("Summary '%s' is just the app's name" % app['Summary']) desc = app['Description'] if app['Summary'] and desc and len(desc) == 1: if app['Summary'].lower() == desc[0].lower(): warn("Description '%s' is just the app's summary" % app['Summary']) # Description size limit desc_charcount = sum(len(l) for l in desc) if desc_charcount > config['char_limits']['Description']: warn("Description of length %s is over the %i char limit" % (desc_charcount, config['char_limits']['Description'])) if (not desc[0] or not desc[-1] or any(not desc[l - 1] and not desc[l] for l in range(1, len(desc)))): warn("Description has an extra empty line") # Check for lists using the wrong characters validchars = ['*', '#'] lchar = '' lcount = 0 for l in app['Description']: if len(l) < 1: continue for um in desc_url.finditer(l): url = um.group(1) for m, r in http_warnings: if m.match(url): warn("URL '%s' in Description: %s" % (url, r)) c = l.decode('utf-8')[0] if c == lchar: lcount += 1 if lcount > 3 and lchar not in validchars: warn( "Description has a list (%s) but it isn't bulleted (*) nor numbered (#)" % lchar) break else: lchar = c lcount = 1 # Regex checks in all kinds of fields for f in regex_warnings: for m, r in regex_warnings[f]: v = app[f] if type(v) == str: if v is None: continue if m.match(v): warn("%s '%s': %s" % (f, v, r)) elif type(v) == list: for l in v: if m.match(l): warn("%s at line '%s': %s" % (f, l, r)) # Build warnings for build in app['builds']: if build['disable']: continue for s in ['master', 'origin', 'HEAD', 'default', 'trunk']: if build['commit'] and build['commit'].startswith(s): warn("Branch '%s' used as commit in build '%s'" % (s, build['version'])) for srclib in build['srclibs']: ref = srclib.split('@')[1].split('/')[0] if ref.startswith(s): warn("Branch '%s' used as commit in srclib '%s'" % (s, srclib)) if not curid: print logging.info("Found a total of %i warnings in %i apps out of %i total." % (count['warn'], count['app'], count['app_total'])) sys.exit(1 if count['warn'] > 0 else 0)
def main(): global config, options # Parse command line... parser = ArgumentParser(usage="%(prog)s [options] [APPID [APPID ...]]") common.setup_global_opts(parser) parser.add_argument("appid", nargs='*', help="app-id to check for updates") parser.add_argument("--auto", action="store_true", default=False, help="Process auto-updates") parser.add_argument("--autoonly", action="store_true", default=False, help="Only process apps with auto-updates") parser.add_argument("--commit", action="store_true", default=False, help="Commit changes") parser.add_argument("--gplay", action="store_true", default=False, help="Only print differences with the Play Store") options = parser.parse_args() config = common.read_config(options) # Get all apps... allapps = metadata.read_metadata() apps = common.read_app_args(options.appid, allapps, False) if options.gplay: for app in apps: version, reason = check_gplay(app) if version is None: if reason == '404': logging.info("{0} is not in the Play Store".format(common.getappname(app))) else: logging.info("{0} encountered a problem: {1}".format(common.getappname(app), reason)) if version is not None: stored = app.CurrentVersion if not stored: logging.info("{0} has no Current Version but has version {1} on the Play Store" .format(common.getappname(app), version)) elif LooseVersion(stored) < LooseVersion(version): logging.info("{0} has version {1} on the Play Store, which is bigger than {2}" .format(common.getappname(app), version, stored)) else: if stored != version: logging.info("{0} has version {1} on the Play Store, which differs from {2}" .format(common.getappname(app), version, stored)) else: logging.info("{0} has the same version {1} on the Play Store" .format(common.getappname(app), version)) return for appid, app in apps.iteritems(): if options.autoonly and app.AutoUpdateMode in ('None', 'Static'): logging.debug("Nothing to do for {0}...".format(appid)) continue logging.info("Processing " + appid + '...') checkupdates_app(app) logging.info("Finished.")
def main(): global config, options # Parse command line... parser = OptionParser(usage="Usage: %prog [options] [APPID [APPID ...]]") parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") parser.add_option("-q", "--quiet", action="store_true", default=False, help="Restrict output to warnings and errors") parser.add_option("--auto", action="store_true", default=False, help="Process auto-updates") parser.add_option("--autoonly", action="store_true", default=False, help="Only process apps with auto-updates") parser.add_option("--commit", action="store_true", default=False, help="Commit changes") parser.add_option("--gplay", action="store_true", default=False, help="Only print differences with the Play Store") (options, args) = parser.parse_args() config = common.read_config(options) # Get all apps... allapps = metadata.read_metadata() apps = common.read_app_args(args, allapps, False) if options.gplay: for app in apps: version, reason = check_gplay(app) if version is None: if reason == '404': logging.info("{0} is not in the Play Store".format(common.getappname(app))) else: logging.info("{0} encountered a problem: {1}".format(common.getappname(app), reason)) if version is not None: stored = app['Current Version'] if not stored: logging.info("{0} has no Current Version but has version {1} on the Play Store" .format(common.getappname(app), version)) elif LooseVersion(stored) < LooseVersion(version): logging.info("{0} has version {1} on the Play Store, which is bigger than {2}" .format(common.getappname(app), version, stored)) else: if stored != version: logging.info("{0} has version {1} on the Play Store, which differs from {2}" .format(common.getappname(app), version, stored)) else: logging.info("{0} has the same version {1} on the Play Store" .format(common.getappname(app), version)) return for appid, app in apps.iteritems(): if options.autoonly and app['Auto Update Mode'] in ('None', 'Static'): logging.debug("Nothing to do for {0}...".format(appid)) continue logging.info("Processing " + appid + '...') checkupdates_app(app) logging.info("Finished.")
def main(): global config, options, curid, count curid = None count = Counter() def warn(message): global curid, count if curid: print "%s:" % curid curid = None count['app'] += 1 print ' %s' % message count['warn'] += 1 def pwarn(message): if options.pedantic: warn(message) # Parse command line... parser = OptionParser(usage="Usage: %prog [options] [APPID [APPID ...]]") parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") parser.add_option("-q", "--quiet", action="store_true", default=False, help="Restrict output to warnings and errors") parser.add_option("-p", "--pedantic", action="store_true", default=False, help="Show pedantic warnings that might give false positives") (options, args) = parser.parse_args() config = common.read_config(options) # Get all apps... allapps = metadata.read_metadata(xref=False) apps = common.read_app_args(args, allapps, False) for appid, app in apps.iteritems(): if app['Disabled']: continue curid = appid count['app_total'] += 1 curbuild = None for build in app['builds']: if not curbuild or int(build['vercode']) > int(curbuild['vercode']): curbuild = build # Potentially incorrect UCM if (curbuild and curbuild['commit'] and app['Update Check Mode'] == 'RepoManifest' and any(s in curbuild['commit'] for s in '.,_-/')): pwarn("Last used commit '%s' looks like a tag, but Update Check Mode is '%s'" % ( curbuild['commit'], app['Update Check Mode'])) # Dangerous auto updates if curbuild and app['Auto Update Mode'] != 'None': for flag in ['target', 'srclibs', 'scanignore']: if curbuild[flag]: pwarn("Auto Update Mode is enabled but '%s' is manually set at '%s'" % (flag, curbuild[flag])) # Summary size limit summ_chars = len(app['Summary']) if summ_chars > config['char_limits']['Summary']: warn("Summary of length %s is over the %i char limit" % ( summ_chars, config['char_limits']['Summary'])) # Redundant info if app['Web Site'] and app['Source Code']: if app['Web Site'].lower() == app['Source Code'].lower(): warn("Website '%s' is just the app's source code link" % app['Web Site']) app['Web Site'] = '' name = app['Name'] or app['Auto Name'] if app['Summary'] and name: if app['Summary'].lower() == name.lower(): warn("Summary '%s' is just the app's name" % app['Summary']) if app['Summary'] and app['Description'] and len(app['Description']) == 1: if app['Summary'].lower() == app['Description'][0].lower(): warn("Description '%s' is just the app's summary" % app['Summary']) # Description size limit desc_chars = sum(len(l) for l in app['Description']) if desc_chars > config['char_limits']['Description']: warn("Description of length %s is over the %i char limit" % ( desc_chars, config['char_limits']['Description'])) # Regex checks in all kinds of fields for f in regex_warnings: for m, r in regex_warnings[f]: t = metadata.metafieldtype(f) if t == 'string': if m.match(app[f]): warn("%s '%s': %s" % (f, app[f], r)) elif t == 'multiline': for l in app[f]: if m.match(l): warn("%s at line '%s': %s" % (f, l, r)) # Regex pedantic checks in all kinds of fields if options.pedantic: for f in regex_pedantic: for m, r in regex_pedantic[f]: if m.match(app[f]): warn("%s '%s': %s" % (f, app[f], r)) # Build warnings for build in app['builds']: if build['disable']: continue for s in ['master', 'origin', 'HEAD', 'default', 'trunk']: if build['commit'] and build['commit'].startswith(s): warn("Branch '%s' used as commit in build '%s'" % ( s, build['version'])) for srclib in build['srclibs']: ref = srclib.split('@')[1].split('/')[0] if ref.startswith(s): warn("Branch '%s' used as commit in srclib '%s'" % ( s, srclib)) for s in ['git clone', 'git svn clone', 'svn checkout', 'svn co', 'hg clone']: for flag in ['init', 'prebuild', 'build']: if not build[flag]: continue if s in build[flag]: # TODO: This should not be pedantic! pwarn("'%s' used in %s '%s'" % (s, flag, build[flag])) if not curid: print logging.info("Found a total of %i warnings in %i apps out of %i total." % ( count['warn'], count['app'], count['app_total']))
def main(): global config, options # Parse command line... parser = OptionParser( usage="Usage: %prog [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]") parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") parser.add_option("-q", "--quiet", action="store_true", default=False, help="Restrict output to warnings and errors") (options, args) = parser.parse_args() config = common.read_config(options) # Read all app and srclib metadata allapps = metadata.read_metadata() apps = common.read_app_args(args, allapps, True) probcount = 0 build_dir = 'build' if not os.path.isdir(build_dir): logging.info("Creating build directory") os.makedirs(build_dir) srclib_dir = os.path.join(build_dir, 'srclib') extlib_dir = os.path.join(build_dir, 'extlib') for appid, app in apps.iteritems(): if app['Disabled']: logging.info("Skipping %s: disabled" % appid) continue if not app['builds']: logging.info("Skipping %s: no builds specified" % appid) continue logging.info("Processing " + appid) try: build_dir = 'build/' + appid # Set up vcs interface and make sure we have the latest code... vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir) for thisbuild in app['builds']: if thisbuild['disable']: logging.info( "...skipping version %s - %s" % (thisbuild['version'], thisbuild.get('disable', thisbuild['commit'][1:]))) else: logging.info("...scanning version " + thisbuild['version']) # Prepare the source code... root_dir, _ = common.prepare_source( vcs, app, thisbuild, build_dir, srclib_dir, extlib_dir, False) # Do the scan... count = common.scan_source(build_dir, root_dir, thisbuild) if count > 0: logging.warn('Scanner found %d problems in %s (%s)' % (count, appid, thisbuild['vercode'])) probcount += count except BuildException as be: logging.warn("Could not scan app %s due to BuildException: %s" % (appid, be)) probcount += 1 except VCSException as vcse: logging.warn("VCS error while scanning app %s: %s" % (appid, vcse)) probcount += 1 except Exception: logging.warn("Could not scan app %s due to unknown error: %s" % (appid, traceback.format_exc())) probcount += 1 logging.info("Finished:") print "%d app(s) with problems" % probcount
def main(): global options, config options, parser = parse_commandline() metadata_files = glob.glob('.fdroid.*[a-z]') # ignore files ending in ~ if os.path.isdir('metadata'): pass elif len(metadata_files) == 0: raise FDroidException("No app metadata found, nothing to process!") elif len(metadata_files) > 1: raise FDroidException("Only one local metadata file allowed! Found: " + " ".join(metadata_files)) if not options.appid and not options.all: parser.error("option %s: If you really want to build all the apps, use --all" % "all") config = common.read_config(options) if config['build_server_always']: options.server = True if options.resetserver and not options.server: parser.error("option %s: Using --resetserver without --server makes no sense" % "resetserver") log_dir = 'logs' if not os.path.isdir(log_dir): logging.info("Creating log directory") os.makedirs(log_dir) tmp_dir = 'tmp' if not os.path.isdir(tmp_dir): logging.info("Creating temporary directory") os.makedirs(tmp_dir) if options.test: output_dir = tmp_dir else: output_dir = 'unsigned' if not os.path.isdir(output_dir): logging.info("Creating output directory") os.makedirs(output_dir) if config['archive_older'] != 0: also_check_dir = 'archive' else: also_check_dir = None repo_dir = 'repo' build_dir = 'build' if not os.path.isdir(build_dir): logging.info("Creating build directory") os.makedirs(build_dir) srclib_dir = os.path.join(build_dir, 'srclib') extlib_dir = os.path.join(build_dir, 'extlib') # Read all app and srclib metadata allapps = metadata.read_metadata(xref=not options.onserver) apps = common.read_app_args(options.appid, allapps, True) for appid, app in apps.items(): if (app.Disabled and not options.force) or not app.RepoType or not app.builds: del apps[appid] if not apps: raise FDroidException("No apps to process.") if options.latest: for app in apps.itervalues(): for build in reversed(app.builds): if build.disable and not options.force: continue app.builds = [build] break if options.wiki: import mwclient site = mwclient.Site((config['wiki_protocol'], config['wiki_server']), path=config['wiki_path']) site.login(config['wiki_user'], config['wiki_password']) # Build applications... failed_apps = {} build_succeeded = [] for appid, app in apps.iteritems(): first = True for build in app.builds: wikilog = None try: # For the first build of a particular app, we need to set up # the source repo. We can reuse it on subsequent builds, if # there are any. if first: if app.RepoType == 'srclib': build_dir = os.path.join('build', 'srclib', app.Repo) else: build_dir = os.path.join('build', appid) # Set up vcs interface and make sure we have the latest code... logging.debug("Getting {0} vcs interface for {1}" .format(app.RepoType, app.Repo)) vcs = common.getvcs(app.RepoType, app.Repo, build_dir) first = False logging.debug("Checking " + build.version) if trybuild(app, build, build_dir, output_dir, also_check_dir, srclib_dir, extlib_dir, tmp_dir, repo_dir, vcs, options.test, options.server, options.force, options.onserver, options.refresh): if app.Binaries is not None: # This is an app where we build from source, and # verify the apk contents against a developer's # binary. We get that binary now, and save it # alongside our built one in the 'unsigend' # directory. url = app.Binaries url = url.replace('%v', build.version) url = url.replace('%c', str(build.vercode)) logging.info("...retrieving " + url) of = "{0}_{1}.apk.binary".format(app.id, build.vercode) of = os.path.join(output_dir, of) net.download_file(url, local_filename=of) build_succeeded.append(app) wikilog = "Build succeeded" except VCSException as vcse: reason = str(vcse).split('\n', 1)[0] if options.verbose else str(vcse) logging.error("VCS error while building app %s: %s" % ( appid, reason)) if options.stop: sys.exit(1) failed_apps[appid] = vcse wikilog = str(vcse) except FDroidException as e: with open(os.path.join(log_dir, appid + '.log'), 'a+') as f: f.write(str(e)) logging.error("Could not build app %s: %s" % (appid, e)) if options.stop: sys.exit(1) failed_apps[appid] = e wikilog = e.get_wikitext() except Exception as e: logging.error("Could not build app %s due to unknown error: %s" % ( appid, traceback.format_exc())) if options.stop: sys.exit(1) failed_apps[appid] = e wikilog = str(e) if options.wiki and wikilog: try: # Write a page with the last build log for this version code lastbuildpage = appid + '/lastbuild_' + build.vercode newpage = site.Pages[lastbuildpage] txt = "Build completed at " + time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) + "\n\n" + wikilog newpage.save(txt, summary='Build log') # Redirect from /lastbuild to the most recent build log newpage = site.Pages[appid + '/lastbuild'] newpage.save('#REDIRECT [[' + lastbuildpage + ']]', summary='Update redirect') except: logging.error("Error while attempting to publish build log") for app in build_succeeded: logging.info("success: %s" % (app.id)) if not options.verbose: for fa in failed_apps: logging.info("Build for app %s failed:\n%s" % (fa, failed_apps[fa])) logging.info("Finished.") if len(build_succeeded) > 0: logging.info(str(len(build_succeeded)) + ' builds succeeded') if len(failed_apps) > 0: logging.info(str(len(failed_apps)) + ' builds failed') sys.exit(0)
def main(): global config, options # Parse command line... parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]") common.setup_global_opts(parser) parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]") options = parser.parse_args() config = common.read_config(options) # Read all app and srclib metadata allapps = metadata.read_metadata() apps = common.read_app_args(options.appid, allapps, True) probcount = 0 build_dir = 'build' if not os.path.isdir(build_dir): logging.info("Creating build directory") os.makedirs(build_dir) srclib_dir = os.path.join(build_dir, 'srclib') extlib_dir = os.path.join(build_dir, 'extlib') for appid, app in apps.iteritems(): if app.Disabled: logging.info("Skipping %s: disabled" % appid) continue if not app.builds: logging.info("Skipping %s: no builds specified" % appid) continue logging.info("Processing " + appid) try: if app.RepoType == 'srclib': build_dir = os.path.join('build', 'srclib', app.Repo) else: build_dir = os.path.join('build', appid) # Set up vcs interface and make sure we have the latest code... vcs = common.getvcs(app.RepoType, app.Repo, build_dir) for build in app.builds: if build.disable: logging.info("...skipping version %s - %s" % ( build.version, build.get('disable', build.commit[1:]))) else: logging.info("...scanning version " + build.version) # Prepare the source code... root_dir, _ = common.prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, False) # Do the scan... count = scan_source(build_dir, root_dir, build) if count > 0: logging.warn('Scanner found %d problems in %s (%s)' % ( count, appid, build.vercode)) probcount += count except BuildException as be: logging.warn("Could not scan app %s due to BuildException: %s" % ( appid, be)) probcount += 1 except VCSException as vcse: logging.warn("VCS error while scanning app %s: %s" % (appid, vcse)) probcount += 1 except Exception: logging.warn("Could not scan app %s due to unknown error: %s" % ( appid, traceback.format_exc())) probcount += 1 logging.info("Finished:") print("%d problems found" % probcount)
def main(): global config, options, curid, count curid = None count = Counter() def warn(message): global curid, count if curid: print "%s:" % curid curid = None count['app'] += 1 print ' %s' % message count['warn'] += 1 def pwarn(message): if options.pedantic: warn(message) # Parse command line... parser = OptionParser(usage="Usage: %prog [options] [APPID [APPID ...]]") parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") parser.add_option("-q", "--quiet", action="store_true", default=False, help="Restrict output to warnings and errors") parser.add_option( "-p", "--pedantic", action="store_true", default=False, help="Show pedantic warnings that might give false positives") (options, args) = parser.parse_args() config = common.read_config(options) # Get all apps... allapps = metadata.read_metadata(xref=False) apps = common.read_app_args(args, allapps, False) for appid, app in apps.iteritems(): curid = appid lastcommit = '' if app['Disabled']: continue for build in app['builds']: if build['commit'] and not build['disable']: lastcommit = build['commit'] # Potentially incorrect UCM if (app['Update Check Mode'] == 'RepoManifest' and any(s in lastcommit for s in '.,_-/')): pwarn( "Last used commit '%s' looks like a tag, but Update Check Mode is '%s'" % (lastcommit, app['Update Check Mode'])) # Summary size limit summ_chars = len(app['Summary']) if summ_chars > config['char_limits']['Summary']: warn("Summary of length %s is over the %i char limit" % (summ_chars, config['char_limits']['Summary'])) # Redundant info if app['Web Site'] and app['Source Code']: if app['Web Site'].lower() == app['Source Code'].lower(): warn("Website '%s' is just the app's source code link" % app['Web Site']) app['Web Site'] = '' name = app['Name'] or app['Auto Name'] if app['Summary'] and name: if app['Summary'].lower() == name.lower(): warn("Summary '%s' is just the app's name" % app['Summary']) if app['Summary'] and app['Description']: if app['Summary'].lower() == app['Description'][0].lower(): warn("Description '%s' is just the app's summary" % app['Summary']) # Description size limit desc_chars = sum(len(l) for l in app['Description']) if desc_chars > config['char_limits']['Description']: warn("Description of length %s is over the %i char limit" % (desc_chars, config['char_limits']['Description'])) # Regex checks in all kinds of fields for f in regex_warnings: for m, r in regex_warnings[f]: t = metadata.metafieldtype(f) if t == 'string': if m.match(app[f]): warn("%s '%s': %s" % (f, app[f], r)) elif t == 'multiline': for l in app[f]: if m.match(l): warn("%s at line '%s': %s" % (f, l, r)) # Regex pedantic checks in all kinds of fields if options.pedantic: for f in regex_pedantic: for m, r in regex_pedantic[f]: if m.match(app[f]): warn("%s '%s': %s" % (f, app[f], r)) # Build warnings for build in app['builds']: if build['disable']: continue for s in ['master', 'origin', 'HEAD', 'default', 'trunk']: if build['commit'] and build['commit'].startswith(s): warn("Branch '%s' used as commit in build '%s'" % (s, build['version'])) for srclib in build['srclibs']: ref = srclib.split('@')[1].split('/')[0] if ref.startswith(s): warn("Branch '%s' used as commit in srclib '%s'" % (s, srclib)) for s in [ 'git clone', 'git svn clone', 'svn checkout', 'svn co', 'hg clone' ]: for flag in ['init', 'prebuild', 'build']: if not build[flag]: continue if s in build[flag]: # TODO: This should not be pedantic! pwarn("'%s' used in %s '%s'" % (s, flag, build[flag])) if not curid: print logging.info("Found a total of %i warnings in %i apps." % (count['warn'], count['app']))
def main(): global options, config options, args = parse_commandline() if not args and not options.all: raise OptionError("If you really want to build all the apps, use --all", "all") config = common.read_config(options) if config['build_server_always']: options.server = True if options.resetserver and not options.server: raise OptionError("Using --resetserver without --server makes no sense", "resetserver") log_dir = 'logs' if not os.path.isdir(log_dir): logging.info("Creating log directory") os.makedirs(log_dir) tmp_dir = 'tmp' if not os.path.isdir(tmp_dir): logging.info("Creating temporary directory") os.makedirs(tmp_dir) if options.test: output_dir = tmp_dir else: output_dir = 'unsigned' if not os.path.isdir(output_dir): logging.info("Creating output directory") os.makedirs(output_dir) if config['archive_older'] != 0: also_check_dir = 'archive' else: also_check_dir = None repo_dir = 'repo' build_dir = 'build' if not os.path.isdir(build_dir): logging.info("Creating build directory") os.makedirs(build_dir) srclib_dir = os.path.join(build_dir, 'srclib') extlib_dir = os.path.join(build_dir, 'extlib') # Read all app and srclib metadata allapps = metadata.read_metadata(xref=not options.onserver) apps = common.read_app_args(args, allapps, True) for appid, app in apps.items(): if (app['Disabled'] and not options.force) or not app['Repo Type'] or not app['builds']: del apps[appid] if not apps: raise FDroidException("No apps to process.") if options.latest: for app in apps.itervalues(): for build in reversed(app['builds']): if build['disable'] and not options.force: continue app['builds'] = [build] break if options.wiki: import mwclient site = mwclient.Site((config['wiki_protocol'], config['wiki_server']), path=config['wiki_path']) site.login(config['wiki_user'], config['wiki_password']) # Build applications... failed_apps = {} build_succeeded = [] for appid, app in apps.iteritems(): first = True for thisbuild in app['builds']: wikilog = None try: # For the first build of a particular app, we need to set up # the source repo. We can reuse it on subsequent builds, if # there are any. if first: if app['Repo Type'] == 'srclib': build_dir = os.path.join('build', 'srclib', app['Repo']) else: build_dir = os.path.join('build', appid) # Set up vcs interface and make sure we have the latest code... logging.debug("Getting {0} vcs interface for {1}" .format(app['Repo Type'], app['Repo'])) vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir) first = False logging.debug("Checking " + thisbuild['version']) if trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir, extlib_dir, tmp_dir, repo_dir, vcs, options.test, options.server, options.force, options.onserver): build_succeeded.append(app) wikilog = "Build succeeded" except BuildException as be: logfile = open(os.path.join(log_dir, appid + '.log'), 'a+') logfile.write(str(be)) logfile.close() print("Could not build app %s due to BuildException: %s" % (appid, be)) if options.stop: sys.exit(1) failed_apps[appid] = be wikilog = be.get_wikitext() except VCSException as vcse: reason = str(vcse).split('\n', 1)[0] if options.verbose else str(vcse) logging.error("VCS error while building app %s: %s" % ( appid, reason)) if options.stop: sys.exit(1) failed_apps[appid] = vcse wikilog = str(vcse) except Exception as e: logging.error("Could not build app %s due to unknown error: %s" % ( appid, traceback.format_exc())) if options.stop: sys.exit(1) failed_apps[appid] = e wikilog = str(e) if options.wiki and wikilog: try: # Write a page with the last build log for this version code lastbuildpage = appid + '/lastbuild_' + thisbuild['vercode'] newpage = site.Pages[lastbuildpage] txt = "Build completed at " + time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) + "\n\n" + wikilog newpage.save(txt, summary='Build log') # Redirect from /lastbuild to the most recent build log newpage = site.Pages[appid + '/lastbuild'] newpage.save('#REDIRECT [[' + lastbuildpage + ']]', summary='Update redirect') except: logging.error("Error while attempting to publish build log") for app in build_succeeded: logging.info("success: %s" % (app['id'])) if not options.verbose: for fa in failed_apps: logging.info("Build for app %s failed:\n%s" % (fa, failed_apps[fa])) logging.info("Finished.") if len(build_succeeded) > 0: logging.info(str(len(build_succeeded)) + ' builds succeeded') if len(failed_apps) > 0: logging.info(str(len(failed_apps)) + ' builds failed') sys.exit(0)
def main(): global options, config options, args = parse_commandline() if not args and not options.all: raise OptionError( "If you really want to build all the apps, use --all", "all") config = common.read_config(options) if config['build_server_always']: options.server = True if options.resetserver and not options.server: raise OptionError( "Using --resetserver without --server makes no sense", "resetserver") log_dir = 'logs' if not os.path.isdir(log_dir): logging.info("Creating log directory") os.makedirs(log_dir) tmp_dir = 'tmp' if not os.path.isdir(tmp_dir): logging.info("Creating temporary directory") os.makedirs(tmp_dir) if options.test: output_dir = tmp_dir else: output_dir = 'unsigned' if not os.path.isdir(output_dir): logging.info("Creating output directory") os.makedirs(output_dir) if config['archive_older'] != 0: also_check_dir = 'archive' else: also_check_dir = None repo_dir = 'repo' build_dir = 'build' if not os.path.isdir(build_dir): logging.info("Creating build directory") os.makedirs(build_dir) srclib_dir = os.path.join(build_dir, 'srclib') extlib_dir = os.path.join(build_dir, 'extlib') # Read all app and srclib metadata allapps = metadata.read_metadata(xref=not options.onserver) apps = common.read_app_args(args, allapps, True) for appid, app in apps.items(): if (app['Disabled'] and not options.force ) or not app['Repo Type'] or not app['builds']: del apps[appid] if not apps: raise FDroidException("No apps to process.") if options.latest: for app in apps.itervalues(): for build in reversed(app['builds']): if build['disable'] and not options.force: continue app['builds'] = [build] break if options.wiki: import mwclient site = mwclient.Site((config['wiki_protocol'], config['wiki_server']), path=config['wiki_path']) site.login(config['wiki_user'], config['wiki_password']) # Build applications... failed_apps = {} build_succeeded = [] for appid, app in apps.iteritems(): first = True for thisbuild in app['builds']: wikilog = None try: # For the first build of a particular app, we need to set up # the source repo. We can reuse it on subsequent builds, if # there are any. if first: if app['Repo Type'] == 'srclib': build_dir = os.path.join('build', 'srclib', app['Repo']) else: build_dir = os.path.join('build', appid) # Set up vcs interface and make sure we have the latest code... logging.debug("Getting {0} vcs interface for {1}".format( app['Repo Type'], app['Repo'])) vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir) first = False logging.debug("Checking " + thisbuild['version']) if trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir, extlib_dir, tmp_dir, repo_dir, vcs, options.test, options.server, options.force, options.onserver): build_succeeded.append(app) wikilog = "Build succeeded" except BuildException as be: logfile = open(os.path.join(log_dir, appid + '.log'), 'a+') logfile.write(str(be)) logfile.close() print("Could not build app %s due to BuildException: %s" % (appid, be)) if options.stop: sys.exit(1) failed_apps[appid] = be wikilog = be.get_wikitext() except VCSException as vcse: reason = str(vcse).split( '\n', 1)[0] if options.verbose else str(vcse) logging.error("VCS error while building app %s: %s" % (appid, reason)) if options.stop: sys.exit(1) failed_apps[appid] = vcse wikilog = str(vcse) except Exception as e: logging.error( "Could not build app %s due to unknown error: %s" % (appid, traceback.format_exc())) if options.stop: sys.exit(1) failed_apps[appid] = e wikilog = str(e) if options.wiki and wikilog: try: # Write a page with the last build log for this version code lastbuildpage = appid + '/lastbuild_' + thisbuild['vercode'] newpage = site.Pages[lastbuildpage] txt = "Build completed at " + time.strftime( "%Y-%m-%d %H:%M:%SZ", time.gmtime()) + "\n\n" + wikilog newpage.save(txt, summary='Build log') # Redirect from /lastbuild to the most recent build log newpage = site.Pages[appid + '/lastbuild'] newpage.save('#REDIRECT [[' + lastbuildpage + ']]', summary='Update redirect') except: logging.error( "Error while attempting to publish build log") for app in build_succeeded: logging.info("success: %s" % (app['id'])) if not options.verbose: for fa in failed_apps: logging.info("Build for app %s failed:\n%s" % (fa, failed_apps[fa])) logging.info("Finished.") if len(build_succeeded) > 0: logging.info(str(len(build_succeeded)) + ' builds succeeded') if len(failed_apps) > 0: logging.info(str(len(failed_apps)) + ' builds failed') sys.exit(0)
def main(): global config, options, curid, count curid = None count = Counter() def warn(message): global curid, count if curid: print "%s:" % curid curid = None count["app"] += 1 print " %s" % message count["warn"] += 1 # Parse command line... parser = OptionParser(usage="Usage: %prog [options] [APPID [APPID ...]]") parser.add_option( "-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal" ) parser.add_option( "-q", "--quiet", action="store_true", default=False, help="Restrict output to warnings and errors" ) (options, args) = parser.parse_args() config = common.read_config(options) # Get all apps... allapps = metadata.read_metadata(xref=False) apps = common.read_app_args(args, allapps, False) filling_ucms = re.compile("^(Tags.*|RepoManifest.*)") for appid, app in apps.iteritems(): if app["Disabled"]: continue curid = appid count["app_total"] += 1 # enabled_builds = 0 lowest_vercode = -1 curbuild = None for build in app["builds"]: if not build["disable"]: # enabled_builds += 1 vercode = int(build["vercode"]) if lowest_vercode == -1 or vercode < lowest_vercode: lowest_vercode = vercode if not curbuild or int(build["vercode"]) > int(curbuild["vercode"]): curbuild = build # Incorrect UCM if ( curbuild and curbuild["commit"] and app["Update Check Mode"] == "RepoManifest" and not curbuild["commit"].startswith("unknown") and curbuild["vercode"] == app["Current Version Code"] and not curbuild["forcevercode"] and any(s in curbuild["commit"] for s in ".,_-/") ): warn( "Last used commit '%s' looks like a tag, but Update Check Mode is '%s'" % (curbuild["commit"], app["Update Check Mode"]) ) # Summary size limit summ_chars = len(app["Summary"]) if summ_chars > config["char_limits"]["Summary"]: warn("Summary of length %s is over the %i char limit" % (summ_chars, config["char_limits"]["Summary"])) # Redundant info if app["Web Site"] and app["Source Code"]: if app["Web Site"].lower() == app["Source Code"].lower(): warn("Website '%s' is just the app's source code link" % app["Web Site"]) if filling_ucms.match(app["Update Check Mode"]): if all( app[f] == metadata.app_defaults[f] for f in ["Auto Name", "Current Version", "Current Version Code"] ): warn("UCM is set but it looks like checkupdates hasn't been run yet") if app["Update Check Name"] == appid: warn("Update Check Name is set to the known app id - it can be removed") cvc = int(app["Current Version Code"]) if cvc > 0 and cvc < lowest_vercode: warn("Current Version Code is lower than any enabled build") # Missing or incorrect categories if not app["Categories"]: warn("Categories are not set") for categ in app["Categories"]: if categ not in categories: warn("Category '%s' is not valid" % categ) if app["Name"] and app["Name"] == app["Auto Name"]: warn("Name '%s' is just the auto name" % app["Name"]) name = app["Name"] or app["Auto Name"] if app["Summary"] and name: if app["Summary"].lower() == name.lower(): warn("Summary '%s' is just the app's name" % app["Summary"]) desc = app["Description"] if app["Summary"] and desc and len(desc) == 1: if app["Summary"].lower() == desc[0].lower(): warn("Description '%s' is just the app's summary" % app["Summary"]) # Description size limit desc_charcount = sum(len(l) for l in desc) if desc_charcount > config["char_limits"]["Description"]: warn( "Description of length %s is over the %i char limit" % (desc_charcount, config["char_limits"]["Description"]) ) if not desc[0] or not desc[-1] or any(not desc[l - 1] and not desc[l] for l in range(1, len(desc))): warn("Description has an extra empty line") # Check for lists using the wrong characters validchars = ["*", "#"] lchar = "" lcount = 0 for l in app["Description"]: if len(l) < 1: continue for um in desc_url.finditer(l): url = um.group(1) for m, r in http_warnings: if m.match(url): warn("URL '%s' in Description: %s" % (url, r)) c = l.decode("utf-8")[0] if c == lchar: lcount += 1 if lcount > 3 and lchar not in validchars: warn("Description has a list (%s) but it isn't bulleted (*) nor numbered (#)" % lchar) break else: lchar = c lcount = 1 # Regex checks in all kinds of fields for f in regex_warnings: for m, r in regex_warnings[f]: v = app[f] if type(v) == str: if v is None: continue if m.match(v): warn("%s '%s': %s" % (f, v, r)) elif type(v) == list: for l in v: if m.match(l): warn("%s at line '%s': %s" % (f, l, r)) # Build warnings for build in app["builds"]: if build["disable"]: continue for s in ["master", "origin", "HEAD", "default", "trunk"]: if build["commit"] and build["commit"].startswith(s): warn("Branch '%s' used as commit in build '%s'" % (s, build["version"])) for srclib in build["srclibs"]: ref = srclib.split("@")[1].split("/")[0] if ref.startswith(s): warn("Branch '%s' used as commit in srclib '%s'" % (s, srclib)) if not curid: print logging.info( "Found a total of %i warnings in %i apps out of %i total." % (count["warn"], count["app"], count["app_total"]) ) sys.exit(1 if count["warn"] > 0 else 0)
def main(): global config, options # Parse command line... parser = OptionParser(usage="Usage: %prog [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]") parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") parser.add_option("-q", "--quiet", action="store_true", default=False, help="Restrict output to warnings and errors") (options, args) = parser.parse_args() config = common.read_config(options) # Read all app and srclib metadata allapps = metadata.read_metadata() apps = common.read_app_args(args, allapps, True) probcount = 0 build_dir = 'build' if not os.path.isdir(build_dir): logging.info("Creating build directory") os.makedirs(build_dir) srclib_dir = os.path.join(build_dir, 'srclib') extlib_dir = os.path.join(build_dir, 'extlib') for appid, app in apps.iteritems(): if app['Disabled']: logging.info("Skipping %s: disabled" % appid) continue if not app['builds']: logging.info("Skipping %s: no builds specified" % appid) continue logging.info("Processing " + appid) try: build_dir = 'build/' + appid # Set up vcs interface and make sure we have the latest code... vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir) for thisbuild in app['builds']: if thisbuild['disable']: logging.info("...skipping version %s - %s" % ( thisbuild['version'], thisbuild.get('disable', thisbuild['commit'][1:]))) else: logging.info("...scanning version " + thisbuild['version']) # Prepare the source code... root_dir, _ = common.prepare_source(vcs, app, thisbuild, build_dir, srclib_dir, extlib_dir, False) # Do the scan... count = common.scan_source(build_dir, root_dir, thisbuild) if count > 0: logging.warn('Scanner found %d problems in %s (%s)' % ( count, appid, thisbuild['vercode'])) probcount += count except BuildException as be: logging.warn("Could not scan app %s due to BuildException: %s" % ( appid, be)) probcount += 1 except VCSException as vcse: logging.warn("VCS error while scanning app %s: %s" % (appid, vcse)) probcount += 1 except Exception: logging.warn("Could not scan app %s due to unknown error: %s" % ( appid, traceback.format_exc())) probcount += 1 logging.info("Finished:") print "%d app(s) with problems" % probcount