def store_tvshow(api_tvshow_id, basepath): tvshow = net.get_tvshow(api_tvshow_id) tvshow_id = db.media_in_db(tvshow) if tvshow_id: print('Skipping {}'.format(tvshow.title)) return tvshow_id air_dates = [] for season_number in tvshow.season_numbers: air_dates += net.get_tvshow_season_air_dates(api_tvshow_id, season_number) air_dates = set(air_dates) # Drop duplicate air dates if basepath and tvshow.poster_path: net.download_file(net.get_poster_img_url(tvshow.poster_path), basepath) tvshow_credits = net.get_tvshow_credits(api_tvshow_id) creator_id = store_celebrity(tvshow.api_creator_id, basepath) tvshow_id = db.save_tvshow(tvshow, creator_id) db.save_tvshow_air_dates(tvshow_id, air_dates) db.save_media_genres(tvshow_id, tvshow.genres) store_cast(tvshow_credits.cast, tvshow_id, basepath) print('Saved tv show: {}'.format(tvshow.title)) return tvshow_id
def store_celebrity(api_celebrity_id, basepath): celebrity = net.get_celebrity(api_celebrity_id) if basepath and celebrity.profile_path: net.download_file(net.get_profile_img_url( celebrity.profile_path), basepath) celebrity_id = db.celebrity_in_db(celebrity) if celebrity_id: return celebrity_id return db.save_celebrity(celebrity)
def store_movie(api_movie_id, basepath): if not api_movie_id: return None movie = net.get_movie(api_movie_id) movie_id = db.media_in_db(movie) if movie_id: print('Skipping {}'.format(movie.title)) return movie_id if basepath and movie.poster_path: net.download_file(net.get_poster_img_url(movie.poster_path), basepath) movie_credits = net.get_movie_credits(api_movie_id) writer_id, director_id, producer_id = store_movie_crew( movie_credits, basepath) movie_id = db.save_movie(movie, writer_id, director_id, producer_id) db.save_media_genres(movie_id, movie.genres) store_cast(movie_credits.cast, movie_id, basepath) print('Saved movie: {}'.format(movie.title)) return movie_id
def year_handler(year): url = url_t.format(year) zip_path = zip_path_for_year(year) net.download_file(url, zip_path) extract_zip(zip_path, config.NIST_JSON_DIR)
def main(): global options, config options, parser = parse_commandline() metadata_files = glob.glob('.fdroid.*[a-z]') # ignore files ending in ~ if os.path.isdir('metadata'): pass elif len(metadata_files) == 0: raise FDroidException("No app metadata found, nothing to process!") elif len(metadata_files) > 1: raise FDroidException("Only one local metadata file allowed! Found: " + " ".join(metadata_files)) if not options.appid and not options.all: parser.error("option %s: If you really want to build all the apps, use --all" % "all") config = common.read_config(options) if config['build_server_always']: options.server = True if options.resetserver and not options.server: parser.error("option %s: Using --resetserver without --server makes no sense" % "resetserver") log_dir = 'logs' if not os.path.isdir(log_dir): logging.info("Creating log directory") os.makedirs(log_dir) tmp_dir = 'tmp' if not os.path.isdir(tmp_dir): logging.info("Creating temporary directory") os.makedirs(tmp_dir) if options.test: output_dir = tmp_dir else: output_dir = 'unsigned' if not os.path.isdir(output_dir): logging.info("Creating output directory") os.makedirs(output_dir) if config['archive_older'] != 0: also_check_dir = 'archive' else: also_check_dir = None repo_dir = 'repo' build_dir = 'build' if not os.path.isdir(build_dir): logging.info("Creating build directory") os.makedirs(build_dir) srclib_dir = os.path.join(build_dir, 'srclib') extlib_dir = os.path.join(build_dir, 'extlib') # Read all app and srclib metadata allapps = metadata.read_metadata(xref=not options.onserver) apps = common.read_app_args(options.appid, allapps, True) for appid, app in apps.items(): if (app.Disabled and not options.force) or not app.RepoType or not app.builds: del apps[appid] if not apps: raise FDroidException("No apps to process.") if options.latest: for app in apps.itervalues(): for build in reversed(app.builds): if build.disable and not options.force: continue app.builds = [build] break if options.wiki: import mwclient site = mwclient.Site((config['wiki_protocol'], config['wiki_server']), path=config['wiki_path']) site.login(config['wiki_user'], config['wiki_password']) # Build applications... failed_apps = {} build_succeeded = [] for appid, app in apps.iteritems(): first = True for build in app.builds: wikilog = None try: # For the first build of a particular app, we need to set up # the source repo. We can reuse it on subsequent builds, if # there are any. if first: if app.RepoType == 'srclib': build_dir = os.path.join('build', 'srclib', app.Repo) else: build_dir = os.path.join('build', appid) # Set up vcs interface and make sure we have the latest code... logging.debug("Getting {0} vcs interface for {1}" .format(app.RepoType, app.Repo)) vcs = common.getvcs(app.RepoType, app.Repo, build_dir) first = False logging.debug("Checking " + build.version) if trybuild(app, build, build_dir, output_dir, also_check_dir, srclib_dir, extlib_dir, tmp_dir, repo_dir, vcs, options.test, options.server, options.force, options.onserver, options.refresh): if app.Binaries is not None: # This is an app where we build from source, and # verify the apk contents against a developer's # binary. We get that binary now, and save it # alongside our built one in the 'unsigend' # directory. url = app.Binaries url = url.replace('%v', build.version) url = url.replace('%c', str(build.vercode)) logging.info("...retrieving " + url) of = "{0}_{1}.apk.binary".format(app.id, build.vercode) of = os.path.join(output_dir, of) net.download_file(url, local_filename=of) build_succeeded.append(app) wikilog = "Build succeeded" except VCSException as vcse: reason = str(vcse).split('\n', 1)[0] if options.verbose else str(vcse) logging.error("VCS error while building app %s: %s" % ( appid, reason)) if options.stop: sys.exit(1) failed_apps[appid] = vcse wikilog = str(vcse) except FDroidException as e: with open(os.path.join(log_dir, appid + '.log'), 'a+') as f: f.write(str(e)) logging.error("Could not build app %s: %s" % (appid, e)) if options.stop: sys.exit(1) failed_apps[appid] = e wikilog = e.get_wikitext() except Exception as e: logging.error("Could not build app %s due to unknown error: %s" % ( appid, traceback.format_exc())) if options.stop: sys.exit(1) failed_apps[appid] = e wikilog = str(e) if options.wiki and wikilog: try: # Write a page with the last build log for this version code lastbuildpage = appid + '/lastbuild_' + build.vercode newpage = site.Pages[lastbuildpage] txt = "Build completed at " + time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) + "\n\n" + wikilog newpage.save(txt, summary='Build log') # Redirect from /lastbuild to the most recent build log newpage = site.Pages[appid + '/lastbuild'] newpage.save('#REDIRECT [[' + lastbuildpage + ']]', summary='Update redirect') except: logging.error("Error while attempting to publish build log") for app in build_succeeded: logging.info("success: %s" % (app.id)) if not options.verbose: for fa in failed_apps: logging.info("Build for app %s failed:\n%s" % (fa, failed_apps[fa])) logging.info("Finished.") if len(build_succeeded) > 0: logging.info(str(len(build_succeeded)) + ' builds succeeded') if len(failed_apps) > 0: logging.info(str(len(failed_apps)) + ' builds failed') sys.exit(0)
def main(): global options, config # Parse command line... parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]") common.setup_global_opts(parser) parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]") options = parser.parse_args() config = common.read_config(options) tmp_dir = 'tmp' if not os.path.isdir(tmp_dir): logging.info("Creating temporary directory") os.makedirs(tmp_dir) unsigned_dir = 'unsigned' if not os.path.isdir(unsigned_dir): logging.error("No unsigned directory - nothing to do") sys.exit(0) verified = 0 notverified = 0 vercodes = common.read_pkg_args(options.appid, True) for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk'))): apkfilename = os.path.basename(apkfile) appid, vercode = common.apknameinfo(apkfile) if vercodes and appid not in vercodes: continue if vercodes[appid] and vercode not in vercodes[appid]: continue try: logging.info("Processing " + apkfilename) remoteapk = os.path.join(tmp_dir, apkfilename) if os.path.exists(remoteapk): os.remove(remoteapk) url = 'https://f-droid.org/repo/' + apkfilename logging.info("...retrieving " + url) net.download_file(url, dldir=tmp_dir) compare_result = common.compare_apks( os.path.join(unsigned_dir, apkfilename), remoteapk, tmp_dir) if compare_result: raise FDroidException(compare_result) logging.info("...successfully verified") verified += 1 except FDroidException as e: logging.info("...NOT verified - {0}".format(e)) notverified += 1 logging.info("Finished") logging.info("{0} successfully verified".format(verified)) logging.info("{0} NOT verified".format(notverified))