def store_tvshow(api_tvshow_id, basepath):
tvshow = net.get_tvshow(api_tvshow_id)
tvshow_id = db.media_in_db(tvshow)
if tvshow_id:
print('Skipping {}'.format(tvshow.title))
return tvshow_id
air_dates = []
for season_number in tvshow.season_numbers:
air_dates += net.get_tvshow_season_air_dates(api_tvshow_id, season_number)
air_dates = set(air_dates) # Drop duplicate air dates
if basepath and tvshow.poster_path:
net.download_file(net.get_poster_img_url(tvshow.poster_path), basepath)
tvshow_credits = net.get_tvshow_credits(api_tvshow_id)
creator_id = store_celebrity(tvshow.api_creator_id, basepath)
tvshow_id = db.save_tvshow(tvshow, creator_id)
db.save_tvshow_air_dates(tvshow_id, air_dates)
db.save_media_genres(tvshow_id, tvshow.genres)
store_cast(tvshow_credits.cast, tvshow_id, basepath)
print('Saved tv show: {}'.format(tvshow.title))
return tvshow_id
def store_celebrity(api_celebrity_id, basepath):
celebrity = net.get_celebrity(api_celebrity_id)
if basepath and celebrity.profile_path:
net.download_file(net.get_profile_img_url(
celebrity.profile_path), basepath)
celebrity_id = db.celebrity_in_db(celebrity)
if celebrity_id:
return celebrity_id
return db.save_celebrity(celebrity)
def store_movie(api_movie_id, basepath):
if not api_movie_id:
return None
movie = net.get_movie(api_movie_id)
movie_id = db.media_in_db(movie)
if movie_id:
print('Skipping {}'.format(movie.title))
return movie_id
if basepath and movie.poster_path:
net.download_file(net.get_poster_img_url(movie.poster_path), basepath)
movie_credits = net.get_movie_credits(api_movie_id)
writer_id, director_id, producer_id = store_movie_crew(
movie_credits, basepath)
movie_id = db.save_movie(movie, writer_id, director_id, producer_id)
db.save_media_genres(movie_id, movie.genres)
store_cast(movie_credits.cast, movie_id, basepath)
print('Saved movie: {}'.format(movie.title))
return movie_id
def year_handler(year):
url = url_t.format(year)
zip_path = zip_path_for_year(year)
net.download_file(url, zip_path)
extract_zip(zip_path, config.NIST_JSON_DIR)
def main():
global options, config
options, parser = parse_commandline()
metadata_files = glob.glob('.fdroid.*[a-z]') # ignore files ending in ~
if os.path.isdir('metadata'):
pass
elif len(metadata_files) == 0:
raise FDroidException("No app metadata found, nothing to process!")
elif len(metadata_files) > 1:
raise FDroidException("Only one local metadata file allowed! Found: "
+ " ".join(metadata_files))
if not options.appid and not options.all:
parser.error("option %s: If you really want to build all the apps, use --all" % "all")
config = common.read_config(options)
if config['build_server_always']:
options.server = True
if options.resetserver and not options.server:
parser.error("option %s: Using --resetserver without --server makes no sense" % "resetserver")
log_dir = 'logs'
if not os.path.isdir(log_dir):
logging.info("Creating log directory")
os.makedirs(log_dir)
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
logging.info("Creating temporary directory")
os.makedirs(tmp_dir)
if options.test:
output_dir = tmp_dir
else:
output_dir = 'unsigned'
if not os.path.isdir(output_dir):
logging.info("Creating output directory")
os.makedirs(output_dir)
if config['archive_older'] != 0:
also_check_dir = 'archive'
else:
also_check_dir = None
repo_dir = 'repo'
build_dir = 'build'
if not os.path.isdir(build_dir):
logging.info("Creating build directory")
os.makedirs(build_dir)
srclib_dir = os.path.join(build_dir, 'srclib')
extlib_dir = os.path.join(build_dir, 'extlib')
# Read all app and srclib metadata
allapps = metadata.read_metadata(xref=not options.onserver)
apps = common.read_app_args(options.appid, allapps, True)
for appid, app in apps.items():
if (app.Disabled and not options.force) or not app.RepoType or not app.builds:
del apps[appid]
if not apps:
raise FDroidException("No apps to process.")
if options.latest:
for app in apps.itervalues():
for build in reversed(app.builds):
if build.disable and not options.force:
continue
app.builds = [build]
break
if options.wiki:
import mwclient
site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
path=config['wiki_path'])
site.login(config['wiki_user'], config['wiki_password'])
# Build applications...
failed_apps = {}
build_succeeded = []
for appid, app in apps.iteritems():
first = True
for build in app.builds:
wikilog = None
try:
# For the first build of a particular app, we need to set up
# the source repo. We can reuse it on subsequent builds, if
# there are any.
if first:
if app.RepoType == 'srclib':
build_dir = os.path.join('build', 'srclib', app.Repo)
else:
build_dir = os.path.join('build', appid)
# Set up vcs interface and make sure we have the latest code...
logging.debug("Getting {0} vcs interface for {1}"
.format(app.RepoType, app.Repo))
vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
first = False
logging.debug("Checking " + build.version)
if trybuild(app, build, build_dir, output_dir,
also_check_dir, srclib_dir, extlib_dir,
tmp_dir, repo_dir, vcs, options.test,
options.server, options.force,
options.onserver, options.refresh):
if app.Binaries is not None:
# This is an app where we build from source, and
# verify the apk contents against a developer's
# binary. We get that binary now, and save it
# alongside our built one in the 'unsigend'
# directory.
url = app.Binaries
url = url.replace('%v', build.version)
url = url.replace('%c', str(build.vercode))
logging.info("...retrieving " + url)
of = "{0}_{1}.apk.binary".format(app.id, build.vercode)
of = os.path.join(output_dir, of)
net.download_file(url, local_filename=of)
build_succeeded.append(app)
wikilog = "Build succeeded"
except VCSException as vcse:
reason = str(vcse).split('\n', 1)[0] if options.verbose else str(vcse)
logging.error("VCS error while building app %s: %s" % (
appid, reason))
if options.stop:
sys.exit(1)
failed_apps[appid] = vcse
wikilog = str(vcse)
except FDroidException as e:
with open(os.path.join(log_dir, appid + '.log'), 'a+') as f:
f.write(str(e))
logging.error("Could not build app %s: %s" % (appid, e))
if options.stop:
sys.exit(1)
failed_apps[appid] = e
wikilog = e.get_wikitext()
except Exception as e:
logging.error("Could not build app %s due to unknown error: %s" % (
appid, traceback.format_exc()))
if options.stop:
sys.exit(1)
failed_apps[appid] = e
wikilog = str(e)
if options.wiki and wikilog:
try:
# Write a page with the last build log for this version code
lastbuildpage = appid + '/lastbuild_' + build.vercode
newpage = site.Pages[lastbuildpage]
txt = "Build completed at " + time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) + "\n\n" + wikilog
newpage.save(txt, summary='Build log')
# Redirect from /lastbuild to the most recent build log
newpage = site.Pages[appid + '/lastbuild']
newpage.save('#REDIRECT [[' + lastbuildpage + ']]', summary='Update redirect')
except:
logging.error("Error while attempting to publish build log")
for app in build_succeeded:
logging.info("success: %s" % (app.id))
if not options.verbose:
for fa in failed_apps:
logging.info("Build for app %s failed:\n%s" % (fa, failed_apps[fa]))
logging.info("Finished.")
if len(build_succeeded) > 0:
logging.info(str(len(build_succeeded)) + ' builds succeeded')
if len(failed_apps) > 0:
logging.info(str(len(failed_apps)) + ' builds failed')
sys.exit(0)
def main():
global options, config
# Parse command line...
parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
common.setup_global_opts(parser)
parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]")
options = parser.parse_args()
config = common.read_config(options)
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
logging.info("Creating temporary directory")
os.makedirs(tmp_dir)
unsigned_dir = 'unsigned'
if not os.path.isdir(unsigned_dir):
logging.error("No unsigned directory - nothing to do")
sys.exit(0)
verified = 0
notverified = 0
vercodes = common.read_pkg_args(options.appid, True)
for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk'))):
apkfilename = os.path.basename(apkfile)
appid, vercode = common.apknameinfo(apkfile)
if vercodes and appid not in vercodes:
continue
if vercodes[appid] and vercode not in vercodes[appid]:
continue
try:
logging.info("Processing " + apkfilename)
remoteapk = os.path.join(tmp_dir, apkfilename)
if os.path.exists(remoteapk):
os.remove(remoteapk)
url = 'https://f-droid.org/repo/' + apkfilename
logging.info("...retrieving " + url)
net.download_file(url, dldir=tmp_dir)
compare_result = common.compare_apks(
os.path.join(unsigned_dir, apkfilename),
remoteapk,
tmp_dir)
if compare_result:
raise FDroidException(compare_result)
logging.info("...successfully verified")
verified += 1
except FDroidException as e:
logging.info("...NOT verified - {0}".format(e))
notverified += 1
logging.info("Finished")
logging.info("{0} successfully verified".format(verified))
logging.info("{0} NOT verified".format(notverified))
