def check_repotrunk(app, branch=None):
try:
if app.RepoType == 'srclib':
build_dir = os.path.join('build', 'srclib', app.Repo)
repotype = common.getsrclibvcs(app.Repo)
else:
build_dir = os.path.join('build', app.id)
repotype = app.RepoType
if repotype not in ('git-svn', ):
return (None, 'RepoTrunk update mode only makes sense in git-svn repositories')
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
vcs.gotorevision(None)
ref = vcs.getref()
return (ref, ref)
except VCSException as vcse:
msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
return (None, msg)
except Exception:
msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
return (None, msg)
def check_repotrunk(app, branch=None):
try:
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib', app['Repo'])
repotype = common.getsrclibvcs(app['Repo'])
else:
build_dir = os.path.join('build/', app['id'])
repotype = app['Repo Type']
if repotype not in ('git-svn', ):
return (None, 'RepoTrunk update mode only makes sense in git-svn repositories')
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
vcs.gotorevision(None)
ref = vcs.getref()
return (ref, ref)
except VCSException as vcse:
msg = "VCS error while scanning app {0}: {1}".format(app['id'], vcse)
return (None, msg)
except Exception:
msg = "Could not scan app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
return (None, msg)
def check_repotrunk(app, branch=None):
try:
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib', app['Repo'])
repotype = common.getsrclibvcs(app['Repo'])
else:
build_dir = os.path.join('build/', app['id'])
repotype = app['Repo Type']
if repotype not in ('svn', 'git-svn'):
return (None, 'RepoTrunk update mode only makes sense in svn and git-svn repositories')
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
vcs.gotorevision(None)
ref = vcs.getref()
return (ref, ref)
except BuildException as be:
msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
return (None, msg)
except VCSException as vcse:
msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
return (None, msg)
except Exception:
msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
return (None, msg)
def check_tags(app):
try:
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib', app['Repo'])
repotype = common.getsrclibvcs(app['Repo'])
else:
build_dir = os.path.join('build/', app['id'])
repotype = app['Repo Type']
if repotype not in ('git', 'git-svn', 'hg', 'bzr'):
return (None, 'Tags update mode only works for git, hg, bzr and git-svn repositories currently', None)
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
vcs.gotorevision(None)
flavour = None
if len(app['builds']) > 0:
if 'subdir' in app['builds'][-1]:
build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
if 'gradle' in app['builds'][-1]:
flavour = app['builds'][-1]['gradle']
htag = None
hver = None
hcode = "0"
for tag in vcs.gettags():
vcs.gotorevision(tag)
# Only process tags where the manifest exists...
paths = common.manifest_paths(build_dir, flavour)
version, vercode, package = common.parse_androidmanifests(paths)
if package and package == app['id'] and version and vercode:
print "Manifest exists. Found version %s (%s)" % (
version, vercode)
if int(vercode) > int(hcode):
htag = tag
hcode = str(int(vercode))
hver = version
if hver:
return (hver, hcode, htag)
return (None, "Couldn't find any version information", None)
except BuildException as be:
msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
return (None, msg, None)
except VCSException as vcse:
msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
return (None, msg, None)
except Exception:
msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
return (None, msg, None)
def check_tags(app, sdk_path):
try:
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib')
repotype = common.getsrclibvcs(app['Repo'])
else:
build_dir = os.path.join('build/', app['id'])
repotype = app['Repo Type']
if repotype not in ('git', 'git-svn'):
return (None, 'Tags update mode only works for git and git-svn repositories currently')
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, sdk_path)
if app['Repo Type'] == 'srclib':
build_dir = os.path.join(build_dir, app['Repo'])
vcs.gotorevision(None)
if len(app['builds']) > 0:
if 'subdir' in app['builds'][-1]:
build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
hver = None
hcode = "0"
for tag in vcs.gettags():
vcs.gotorevision(tag)
# Only process tags where the manifest exists...
if os.path.exists(os.path.join(build_dir, 'AndroidManifest.xml')):
version, vercode, package = common.parse_androidmanifest(build_dir)
if package and package == app['id'] and version and vercode:
if int(vercode) > int(hcode):
hcode = str(int(vercode))
hver = version
if hver:
return (hver, hcode)
return (None, "Couldn't find any version information")
except BuildException as be:
msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
return (None, msg)
except VCSException as vcse:
msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
return (None, msg)
except Exception:
msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
return (None, msg)
def fetch_autoname(app, tag):
if not app["Repo Type"] or app['Update Check Mode'] in ('None', 'Static'):
return None
if app['Repo Type'] == 'srclib':
app_dir = os.path.join('build', 'srclib', app['Repo'])
else:
app_dir = os.path.join('build/', app['id'])
try:
vcs = common.getvcs(app["Repo Type"], app["Repo"], app_dir)
vcs.gotorevision(tag)
except VCSException:
return None
flavour = None
if len(app['builds']) > 0:
if app['builds'][-1]['subdir']:
app_dir = os.path.join(app_dir, app['builds'][-1]['subdir'])
if app['builds'][-1]['gradle']:
flavour = app['builds'][-1]['gradle']
if flavour == 'yes':
flavour = None
logging.debug("...fetch auto name from " + app_dir +
((" (flavour: %s)" % flavour) if flavour else ""))
new_name = common.fetch_real_name(app_dir, flavour)
commitmsg = None
if new_name:
logging.debug("...got autoname '" + new_name + "'")
if new_name != app['Auto Name']:
app['Auto Name'] = new_name
if not commitmsg:
commitmsg = "Set autoname of {0}".format(
common.getappname(app))
else:
logging.debug("...couldn't get autoname")
if app['Current Version'].startswith('@string/'):
cv = common.version_name(app['Current Version'], app_dir, flavour)
if app['Current Version'] != cv:
app['Current Version'] = cv
if not commitmsg:
commitmsg = "Fix CV of {0}".format(common.getappname(app))
return commitmsg
def fetch_autoname(app, tag):
if not app["Repo Type"] or app['Update Check Mode'] in ('None', 'Static'):
return None
if app['Repo Type'] == 'srclib':
app_dir = os.path.join('build', 'srclib', app['Repo'])
else:
app_dir = os.path.join('build/', app['id'])
try:
vcs = common.getvcs(app["Repo Type"], app["Repo"], app_dir)
vcs.gotorevision(tag)
except VCSException:
return None
flavours = []
if len(app['builds']) > 0:
if app['builds'][-1]['subdir']:
app_dir = os.path.join(app_dir, app['builds'][-1]['subdir'])
if app['builds'][-1]['gradle']:
flavours = app['builds'][-1]['gradle']
commitmsg = None
if not app['Auto Name']:
logging.debug("...fetch auto name from " + app_dir)
new_name = common.fetch_real_name(app_dir, flavours)
if new_name:
logging.debug("...got autoname '" + new_name + "'")
app['Auto Name'] = new_name
if not commitmsg:
commitmsg = "Set autoname of {0}".format(common.getappname(app))
else:
logging.debug("...couldn't get autoname")
if app['Current Version'].startswith('@string/'):
cv = common.version_name(app['Current Version'], app_dir, flavours)
if app['Current Version'] != cv:
app['Current Version'] = cv
if not commitmsg:
commitmsg = "Fix CV of {0}".format(common.getappname(app))
return commitmsg
def fetch_autoname(app, tag):
if not app.RepoType or app.UpdateCheckMode in ('None', 'Static'):
return None
if app.RepoType == 'srclib':
build_dir = os.path.join('build', 'srclib', app.Repo)
else:
build_dir = os.path.join('build', app.id)
try:
vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
vcs.gotorevision(tag)
except VCSException:
return None
last_build = metadata.Build()
if len(app.builds) > 0:
last_build = app.builds[-1]
logging.debug("...fetch auto name from " + build_dir)
new_name = None
for subdir in possible_subdirs(app):
if subdir == '.':
root_dir = build_dir
else:
root_dir = os.path.join(build_dir, subdir)
new_name = common.fetch_real_name(root_dir, last_build.gradle)
if new_name is not None:
break
commitmsg = None
if new_name:
logging.debug("...got autoname '" + new_name + "'")
if new_name != app.AutoName:
app.AutoName = new_name
if not commitmsg:
commitmsg = "Set autoname of {0}".format(common.getappname(app))
else:
logging.debug("...couldn't get autoname")
return commitmsg
def fetch_autoname(app, tag):
if not app["Repo Type"] or app['Update Check Mode'] in ('None', 'Static'):
return None
if app['Repo Type'] == 'srclib':
app_dir = os.path.join('build', 'srclib', app['Repo'])
else:
app_dir = os.path.join('build/', app['id'])
try:
vcs = common.getvcs(app["Repo Type"], app["Repo"], app_dir)
vcs.gotorevision(tag)
except VCSException:
return None
flavours = []
if len(app['builds']) > 0:
if app['builds'][-1]['subdir']:
app_dir = os.path.join(app_dir, app['builds'][-1]['subdir'])
if app['builds'][-1]['gradle']:
flavours = app['builds'][-1]['gradle']
logging.debug("...fetch auto name from " + app_dir)
new_name = common.fetch_real_name(app_dir, flavours)
commitmsg = None
if new_name:
logging.debug("...got autoname '" + new_name + "'")
if new_name != app['Auto Name']:
app['Auto Name'] = new_name
if not commitmsg:
commitmsg = "Set autoname of {0}".format(common.getappname(app))
else:
logging.debug("...couldn't get autoname")
return commitmsg
def main():
global options
# Read configuration...
globals()['build_server_always'] = False
globals()['mvn3'] = "mvn3"
globals()['archive_older'] = 0
execfile('config.py', globals())
options, args = parse_commandline()
if build_server_always:
options.server = True
if options.resetserver and not options.server:
print "Using --resetserver without --server makes no sense"
sys.exit(1)
# Get all apps...
apps = common.read_metadata(options.verbose, xref=not options.onserver)
log_dir = 'logs'
if not os.path.isdir(log_dir):
print "Creating log directory"
os.makedirs(log_dir)
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
print "Creating temporary directory"
os.makedirs(tmp_dir)
if options.test:
output_dir = tmp_dir
else:
output_dir = 'unsigned'
if not os.path.isdir(output_dir):
print "Creating output directory"
os.makedirs(output_dir)
if archive_older != 0:
also_check_dir = 'archive'
else:
also_check_dir = None
repo_dir = 'repo'
build_dir = 'build'
if not os.path.isdir(build_dir):
print "Creating build directory"
os.makedirs(build_dir)
srclib_dir = os.path.join(build_dir, 'srclib')
extlib_dir = os.path.join(build_dir, 'extlib')
# Filter apps and build versions according to command-line options, etc...
if options.package:
apps = [app for app in apps if app['id'] == options.package]
if len(apps) == 0:
print "No such package"
sys.exit(1)
apps = [
app for app in apps
if (options.force or not app['Disabled']) and app['builds']
and len(app['Repo Type']) > 0 and len(app['builds']) > 0
]
if len(apps) == 0:
print "Nothing to do - all apps are disabled or have no builds defined."
sys.exit(1)
if options.vercode:
for app in apps:
app['builds'] = [
b for b in app['builds']
if str(b['vercode']) == options.vercode
]
elif options.latest:
for app in apps:
m = max([i['vercode'] for i in app['builds']], key=int)
app['builds'] = [b for b in app['builds'] if b['vercode'] == m]
if options.wiki:
import mwclient
site = mwclient.Site(wiki_server, path=wiki_path)
site.login(wiki_user, wiki_password)
# Build applications...
failed_apps = {}
build_succeeded = []
for app in apps:
first = True
for thisbuild in app['builds']:
wikilog = None
try:
# For the first build of a particular app, we need to set up
# the source repo. We can reuse it on subsequent builds, if
# there are any.
if first:
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib',
app['Repo'])
else:
build_dir = os.path.join('build', app['id'])
# Set up vcs interface and make sure we have the latest code...
if options.verbose:
print "Getting {0} vcs interface for {1}".format(
app['Repo Type'], app['Repo'])
vcs = common.getvcs(app['Repo Type'], app['Repo'],
build_dir, sdk_path)
first = False
if options.verbose:
print "Checking " + thisbuild['version']
if trybuild(app, thisbuild, build_dir, output_dir,
also_check_dir, srclib_dir, extlib_dir, tmp_dir,
repo_dir, vcs, options.test, options.server,
options.install, options.force, options.verbose,
options.onserver):
build_succeeded.append(app)
wikilog = "Build succeeded"
except BuildException as be:
logfile = open(os.path.join(log_dir, app['id'] + '.log'), 'a+')
logfile.write(str(be))
logfile.close()
print "Could not build app %s due to BuildException: %s" % (
app['id'], be)
if options.stop:
sys.exit(1)
failed_apps[app['id']] = be
wikilog = be.get_wikitext()
except VCSException as vcse:
print "VCS error while building app %s: %s" % (app['id'], vcse)
if options.stop:
sys.exit(1)
failed_apps[app['id']] = vcse
wikilog = str(vcse)
except Exception as e:
print "Could not build app %s due to unknown error: %s" % (
app['id'], traceback.format_exc())
if options.stop:
sys.exit(1)
failed_apps[app['id']] = e
wikilog = str(e)
if options.wiki and wikilog:
try:
newpage = site.Pages[app['id'] + '/lastbuild']
txt = wikilog
if len(txt) > 8192:
txt = txt[-8192:]
txt = "Build completed at " + time.strftime(
"%Y-%m-%d %H:%M:%SZ", time.gmtime()) + "\n\n" + txt
newpage.save(wikilog, summary='Build log')
except:
print "Error while attempting to publish build log"
for app in build_succeeded:
print "success: %s" % (app['id'])
for fa in failed_apps:
print "Build for app %s failed:\n%s" % (fa, failed_apps[fa])
print "Finished."
if len(build_succeeded) > 0:
print str(len(build_succeeded)) + ' builds succeeded'
if len(failed_apps) > 0:
print str(len(failed_apps)) + ' builds failed'
sys.exit(0)
def main():
# Read configuration...
execfile('config.py', globals())
import common
# Parse command line...
parser = OptionParser()
parser.add_option("-u",
"--url",
default=None,
help="Project URL to import from.")
parser.add_option(
"-s",
"--subdir",
default=None,
help="Path to main android project subdirectory, if not in root.")
parser.add_option(
"-r",
"--repo",
default=None,
help=
"Allows a different repo to be specified for a multi-repo google code project"
)
(options, args) = parser.parse_args()
if not options.url:
print "Specify project url."
sys.exit(1)
url = options.url
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
print "Creating temporary directory"
os.makedirs(tmp_dir)
# Get all apps...
apps = common.read_metadata()
# Figure out what kind of project it is...
projecttype = None
issuetracker = None
license = None
website = url #by default, we might override it
if url.startswith('git://'):
projecttype = 'git'
repo = url
repotype = 'git'
sourcecode = ""
website = ""
if url.startswith('https://github.com'):
if url.endswith('/'):
url = url[:-1]
if url.endswith('.git'):
print "A github URL should point to the project, not the git repo"
sys.exit(1)
projecttype = 'github'
repo = url + '.git'
repotype = 'git'
sourcecode = url
elif url.startswith('https://gitorious.org/'):
projecttype = 'gitorious'
repo = 'https://git.gitorious.org/' + url[22:] + '.git'
repotype = 'git'
sourcecode = url
elif url.startswith('https://bitbucket.org/'):
if url.endswith('/'):
url = url[:-1]
projecttype = 'bitbucket'
sourcecode = url + '/src'
issuetracker = url + '/issues'
# Figure out the repo type and adddress...
repotype, repo = getrepofrompage(sourcecode)
if not repotype:
print "Unable to determine vcs type. " + repo
sys.exit(1)
elif url.startswith('http://code.google.com/p/'):
if not url.endswith('/'):
url += '/'
projecttype = 'googlecode'
sourcecode = url + 'source/checkout'
if options.repo:
sourcecode += "?repo=" + options.repo
issuetracker = url + 'issues/list'
# Figure out the repo type and adddress...
repotype, repo = getrepofrompage(sourcecode)
if not repotype:
print "Unable to determine vcs type. " + repo
sys.exit(1)
# Figure out the license...
req = urllib.urlopen(url)
if req.getcode() != 200:
print 'Unable to find project page at ' + sourcecode + ' - return code ' + str(
req.getcode())
sys.exit(1)
page = req.read()
index = page.find('Code license')
if index == -1:
print "Couldn't find license data"
sys.exit(1)
ltext = page[index:]
lprefix = 'rel="nofollow">'
index = ltext.find(lprefix)
if index == -1:
print "Couldn't find license text"
sys.exit(1)
ltext = ltext[index + len(lprefix):]
index = ltext.find('<')
if index == -1:
print "License text not formatted as expected"
sys.exit(1)
ltext = ltext[:index]
if ltext == 'GNU GPL v3':
license = 'GPLv3'
elif ltext == 'GNU GPL v2':
license = 'GPLv2'
elif ltext == 'Apache License 2.0':
license = 'Apache2'
elif ltext == 'MIT License':
license = 'MIT'
elif ltext == 'GNU Lesser GPL':
license = 'LGPL'
elif ltext == 'Mozilla Public License 1.1':
license = 'MPL'
elif ltext == 'New BSD License':
license = 'NewBSD'
else:
print "License " + ltext + " is not recognised"
sys.exit(1)
if not projecttype:
print "Unable to determine the project type."
print "The URL you supplied was not in one of the supported formats. Please consult"
print "the manual for a list of supported formats, and supply one of those."
sys.exit(1)
# Get a copy of the source so we can extract some info...
print 'Getting source from ' + repotype + ' repo at ' + repo
src_dir = os.path.join(tmp_dir, 'importer')
if os.path.exists(src_dir):
shutil.rmtree(src_dir)
vcs = common.getvcs(repotype, repo, src_dir, sdk_path)
vcs.gotorevision(None)
if options.subdir:
root_dir = os.path.join(src_dir, options.subdir)
else:
root_dir = src_dir
# Check AndroidManiifest.xml exists...
if not os.path.exists(root_dir + '/AndroidManifest.xml'):
print "AndroidManifest.xml did not exist in the expected location. Specify --subdir?"
sys.exit(1)
# Extract some information...
version, vercode, package = common.parse_androidmanifest(root_dir)
if not package:
print "Couldn't find package ID"
sys.exit(1)
if not version:
print "Couldn't find latest version name"
sys.exit(1)
if not vercode:
print "Couldn't find latest version code"
sys.exit(1)
# Make sure it's actually new...
for app in apps:
if app['id'] == package:
print "Package " + package + " already exists"
sys.exit(1)
# Construct the metadata...
app = common.parse_metadata(None)
app['id'] = package
app['Web Site'] = website
app['Source Code'] = sourcecode
if issuetracker:
app['Issue Tracker'] = issuetracker
if license:
app['License'] = license
app['Repo Type'] = repotype
app['Repo'] = repo
# Create a build line...
build = {}
build['version'] = version
build['vercode'] = vercode
build['commit'] = '?'
if options.subdir:
build['subdir'] = options.subdir
if os.path.exists(os.path.join(root_dir, 'jni')):
build['buildjni'] = 'yes'
app['builds'].append(build)
app['comments'].append((
'build:' + version,
"#Generated by import.py - check this is the right version, and find the right commit!"
))
# Keep the repo directory to save bandwidth...
if not os.path.exists('build'):
os.mkdir('build')
shutil.move(src_dir, os.path.join('build', package))
metafile = os.path.join('metadata', package + '.txt')
common.write_metadata(metafile, app)
print "Wrote " + metafile
def main():
global options
# Read configuration...
globals()['build_server_always'] = False
globals()['mvn3'] = "mvn3"
globals()['archive_older'] = 0
execfile('config.py', globals())
options, args = parse_commandline()
if build_server_always:
options.server = True
if options.resetserver and not options.server:
print "Using --resetserver without --server makes no sense"
sys.exit(1)
# Get all apps...
apps = common.read_metadata(options.verbose, xref=not options.onserver)
log_dir = 'logs'
if not os.path.isdir(log_dir):
print "Creating log directory"
os.makedirs(log_dir)
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
print "Creating temporary directory"
os.makedirs(tmp_dir)
if options.test:
output_dir = tmp_dir
else:
output_dir = 'unsigned'
if not os.path.isdir(output_dir):
print "Creating output directory"
os.makedirs(output_dir)
if archive_older != 0:
also_check_dir = 'archive'
else:
also_check_dir = None
repo_dir = 'repo'
build_dir = 'build'
if not os.path.isdir(build_dir):
print "Creating build directory"
os.makedirs(build_dir)
srclib_dir = os.path.join(build_dir, 'srclib')
extlib_dir = os.path.join(build_dir, 'extlib')
# Filter apps and build versions according to command-line options, etc...
if options.package:
apps = [app for app in apps if app['id'] == options.package]
if len(apps) == 0:
print "No such package"
sys.exit(1)
apps = [app for app in apps if (options.force or not app['Disabled']) and
app['builds'] and len(app['Repo Type']) > 0 and len(app['builds']) > 0]
if len(apps) == 0:
print "Nothing to do - all apps are disabled or have no builds defined."
sys.exit(1)
if options.vercode:
for app in apps:
app['builds'] = [b for b in app['builds']
if str(b['vercode']) == options.vercode]
elif options.latest:
for app in apps:
m = max([i['vercode'] for i in app['builds']], key=int)
app['builds'] = [b for b in app['builds'] if b['vercode'] == m]
if options.wiki:
import mwclient
site = mwclient.Site(wiki_server, path=wiki_path)
site.login(wiki_user, wiki_password)
# Build applications...
failed_apps = {}
build_succeeded = []
for app in apps:
first = True
for thisbuild in app['builds']:
wikilog = None
try:
# For the first build of a particular app, we need to set up
# the source repo. We can reuse it on subsequent builds, if
# there are any.
if first:
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib', app['Repo'])
else:
build_dir = os.path.join('build', app['id'])
# Set up vcs interface and make sure we have the latest code...
if options.verbose:
print "Getting {0} vcs interface for {1}".format(
app['Repo Type'], app['Repo'])
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, sdk_path)
first = False
if options.verbose:
print "Checking " + thisbuild['version']
if trybuild(app, thisbuild, build_dir, output_dir, also_check_dir,
srclib_dir, extlib_dir, tmp_dir, repo_dir, vcs, options.test,
options.server, options.install, options.force,
options.verbose, options.onserver):
build_succeeded.append(app)
wikilog = "Build succeeded"
except BuildException as be:
logfile = open(os.path.join(log_dir, app['id'] + '.log'), 'a+')
logfile.write(str(be))
logfile.close()
print "Could not build app %s due to BuildException: %s" % (app['id'], be)
if options.stop:
sys.exit(1)
failed_apps[app['id']] = be
wikilog = be.get_wikitext()
except VCSException as vcse:
print "VCS error while building app %s: %s" % (app['id'], vcse)
if options.stop:
sys.exit(1)
failed_apps[app['id']] = vcse
wikilog = str(vcse)
except Exception as e:
print "Could not build app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
if options.stop:
sys.exit(1)
failed_apps[app['id']] = e
wikilog = str(e)
if options.wiki and wikilog:
try:
newpage = site.Pages[app['id'] + '/lastbuild']
txt = wikilog
if len(txt) > 8192:
txt = txt[-8192:]
txt = "Build completed at " + time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) + "\n\n" + txt
newpage.save(wikilog, summary='Build log')
except:
print "Error while attempting to publish build log"
for app in build_succeeded:
print "success: %s" % (app['id'])
for fa in failed_apps:
print "Build for app %s failed:\n%s" % (fa, failed_apps[fa])
print "Finished."
if len(build_succeeded) > 0:
print str(len(build_succeeded)) + ' builds succeeded'
if len(failed_apps) > 0:
print str(len(failed_apps)) + ' builds failed'
sys.exit(0)
def check_tags(app, pattern):
try:
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib', app['Repo'])
repotype = common.getsrclibvcs(app['Repo'])
else:
build_dir = os.path.join('build/', app['id'])
repotype = app['Repo Type']
if repotype not in ('git', 'git-svn', 'hg', 'bzr'):
return (None, 'Tags update mode only works for git, hg, bzr and git-svn repositories currently', None)
if repotype == 'git-svn' and ';' not in app['Repo']:
return (None, 'Tags update mode used in git-svn, but the repo was not set up with tags', None)
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
vcs.gotorevision(None)
flavours = []
if len(app['builds']) > 0:
if app['builds'][-1]['subdir']:
build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
if app['builds'][-1]['gradle']:
flavours = app['builds'][-1]['gradle']
hpak = None
htag = None
hver = None
hcode = "0"
tags = vcs.gettags()
logging.debug("All tags: " + ','.join(tags))
if pattern:
pat = re.compile(pattern)
tags = [tag for tag in tags if pat.match(tag)]
logging.debug("Matching tags: " + ','.join(tags))
if repotype in ('git',):
tags = vcs.latesttags(tags, 5)
logging.debug("Latest tags: " + ','.join(tags))
for tag in tags:
logging.debug("Check tag: '{0}'".format(tag))
vcs.gotorevision(tag)
# Only process tags where the manifest exists...
paths = common.manifest_paths(build_dir, flavours)
version, vercode, package = \
common.parse_androidmanifests(paths, app['Update Check Ignore'])
if not app_matches_packagename(app, package) or not version or not vercode:
continue
logging.debug("Manifest exists. Found version {0} ({1})"
.format(version, vercode))
if int(vercode) > int(hcode):
hpak = package
htag = tag
hcode = str(int(vercode))
hver = version
if not hpak:
return (None, "Couldn't find package ID", None)
if hver:
return (hver, hcode, htag)
return (None, "Couldn't find any version information", None)
except VCSException as vcse:
msg = "VCS error while scanning app {0}: {1}".format(app['id'], vcse)
return (None, msg, None)
except Exception:
msg = "Could not scan app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
return (None, msg, None)
def check_repomanifest(app, branch=None):
try:
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib', app['Repo'])
repotype = common.getsrclibvcs(app['Repo'])
else:
build_dir = os.path.join('build/', app['id'])
repotype = app['Repo Type']
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
if repotype == 'git':
if branch:
branch = 'origin/'+branch
vcs.gotorevision(branch)
elif repotype == 'git-svn':
vcs.gotorevision(branch)
elif repotype == 'svn':
vcs.gotorevision(None)
elif repotype == 'hg':
vcs.gotorevision(branch)
elif repotype == 'bzr':
vcs.gotorevision(None)
flavour = None
if len(app['builds']) > 0:
if 'subdir' in app['builds'][-1]:
build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
if 'gradle' in app['builds'][-1]:
flavour = app['builds'][-1]['gradle']
if not os.path.isdir(build_dir):
return (None, "Subdir '" + app['builds'][-1]['subdir'] + "'is not a valid directory")
paths = common.manifest_paths(build_dir, flavour)
version, vercode, package = common.parse_androidmanifests(paths)
if not package:
return (None, "Couldn't find package ID")
if package != app['id']:
return (None, "Package ID mismatch")
if not version:
return (None,"Couldn't find latest version name")
if not vercode:
return (None,"Couldn't find latest version code")
return (version, str(int(vercode)))
except BuildException as be:
msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
return (None, msg)
except VCSException as vcse:
msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
return (None, msg)
except Exception:
msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
return (None, msg)
def main():
global config, options
# Parse command line...
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-p", "--package", default=None,
help="Scan only the specified package")
parser.add_option("--nosvn", action="store_true", default=False,
help="Skip svn repositories - for test purposes, because they are too slow.")
(options, args) = parser.parse_args()
config = common.read_config(options)
# Get all apps...
apps = metadata.read_metadata()
# Filter apps according to command-line options
if options.package:
apps = [app for app in apps if app['id'] == options.package]
if len(apps) == 0:
print "No such package"
sys.exit(1)
problems = []
build_dir = 'build'
if not os.path.isdir(build_dir):
print "Creating build directory"
os.makedirs(build_dir)
srclib_dir = os.path.join(build_dir, 'srclib')
extlib_dir = os.path.join(build_dir, 'extlib')
for app in apps:
skip = False
if app['Disabled']:
print "Skipping %s: disabled" % app['id']
skip = True
elif not app['builds']:
print "Skipping %s: no builds specified" % app['id']
skip = True
elif options.nosvn and app['Repo Type'] == 'svn':
skip = True
if not skip:
print "Processing " + app['id']
try:
build_dir = 'build/' + app['id']
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
for thisbuild in app['builds']:
if 'disable' in thisbuild:
print ("..skipping version " + thisbuild['version'] + " - " +
thisbuild.get('disable', thisbuild['commit'][1:]))
else:
print "..scanning version " + thisbuild['version']
# Prepare the source code...
root_dir, _ = common.prepare_source(vcs, app, thisbuild,
build_dir, srclib_dir, extlib_dir, False)
# Do the scan...
buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
for problem in buildprobs:
problems.append(problem +
' in ' + app['id'] + ' ' + thisbuild['version'])
except BuildException as be:
msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
problems.append(msg)
except VCSException as vcse:
msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
problems.append(msg)
except Exception:
msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
problems.append(msg)
print "Finished:"
for problem in problems:
print problem
print str(len(problems)) + ' problems.'
def main():
global config, options
# Parse command line...
parser = OptionParser()
parser.add_option("-u", "--url", default=None,
help="Project URL to import from.")
parser.add_option("-s", "--subdir", default=None,
help="Path to main android project subdirectory, if not in root.")
parser.add_option("-r", "--repo", default=None,
help="Allows a different repo to be specified for a multi-repo google code project")
parser.add_option("--rev", default=None,
help="Allows a different revision (or git branch) to be specified for the initial import")
(options, args) = parser.parse_args()
config = common.read_config(options)
if not options.url:
print "Specify project url."
sys.exit(1)
url = options.url
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
print "Creating temporary directory"
os.makedirs(tmp_dir)
# Get all apps...
apps = metadata.read_metadata()
# Figure out what kind of project it is...
projecttype = None
issuetracker = None
license = None
website = url #by default, we might override it
if url.startswith('git://'):
projecttype = 'git'
repo = url
repotype = 'git'
sourcecode = ""
website = ""
elif url.startswith('https://github.com'):
if url.endswith('/'):
url = url[:-1]
if url.endswith('.git'):
print "A github URL should point to the project, not the git repo"
sys.exit(1)
projecttype = 'github'
repo = url + '.git'
repotype = 'git'
sourcecode = url
issuetracker = url + '/issues'
elif url.startswith('https://gitorious.org/'):
projecttype = 'gitorious'
repo = 'https://git.gitorious.org/' + url[22:] + '.git'
repotype = 'git'
sourcecode = url
elif url.startswith('https://bitbucket.org/'):
if url.endswith('/'):
url = url[:-1]
projecttype = 'bitbucket'
sourcecode = url + '/src'
issuetracker = url + '/issues'
# Figure out the repo type and adddress...
repotype, repo = getrepofrompage(sourcecode)
if not repotype:
print "Unable to determine vcs type. " + repo
sys.exit(1)
elif url.startswith('http://code.google.com/p/'):
if not url.endswith('/'):
url += '/';
projecttype = 'googlecode'
sourcecode = url + 'source/checkout'
if options.repo:
sourcecode += "?repo=" + options.repo
issuetracker = url + 'issues/list'
# Figure out the repo type and adddress...
repotype, repo = getrepofrompage(sourcecode)
if not repotype:
print "Unable to determine vcs type. " + repo
sys.exit(1)
# Figure out the license...
req = urllib.urlopen(url)
if req.getcode() != 200:
print 'Unable to find project page at ' + sourcecode + ' - return code ' + str(req.getcode())
sys.exit(1)
page = req.read()
index = page.find('Code license')
if index == -1:
print "Couldn't find license data"
sys.exit(1)
ltext = page[index:]
lprefix = 'rel="nofollow">'
index = ltext.find(lprefix)
if index == -1:
print "Couldn't find license text"
sys.exit(1)
ltext = ltext[index + len(lprefix):]
index = ltext.find('<')
if index == -1:
print "License text not formatted as expected"
sys.exit(1)
ltext = ltext[:index]
if ltext == 'GNU GPL v3':
license = 'GPLv3'
elif ltext == 'GNU GPL v2':
license = 'GPLv2'
elif ltext == 'Apache License 2.0':
license = 'Apache2'
elif ltext == 'MIT License':
license = 'MIT'
elif ltext == 'GNU Lesser GPL':
license = 'LGPL'
elif ltext == 'Mozilla Public License 1.1':
license = 'MPL'
elif ltext == 'New BSD License':
license = 'NewBSD'
else:
print "License " + ltext + " is not recognised"
sys.exit(1)
if not projecttype:
print "Unable to determine the project type."
print "The URL you supplied was not in one of the supported formats. Please consult"
print "the manual for a list of supported formats, and supply one of those."
sys.exit(1)
# Get a copy of the source so we can extract some info...
print 'Getting source from ' + repotype + ' repo at ' + repo
src_dir = os.path.join(tmp_dir, 'importer')
if os.path.exists(src_dir):
shutil.rmtree(src_dir)
vcs = common.getvcs(repotype, repo, src_dir)
vcs.gotorevision(options.rev)
if options.subdir:
root_dir = os.path.join(src_dir, options.subdir)
else:
root_dir = src_dir
# Extract some information...
paths = common.manifest_paths(root_dir, None)
if paths:
version, vercode, package = common.parse_androidmanifests(paths)
if not package:
print "Couldn't find package ID"
sys.exit(1)
if not version:
print "WARNING: Couldn't find latest version name"
if not vercode:
print "WARNING: Couldn't find latest version code"
else:
spec = os.path.join(root_dir, 'buildozer.spec')
if os.path.exists(spec):
defaults = {'orientation': 'landscape', 'icon': '',
'permissions': '', 'android.api': "18"}
bconfig = ConfigParser(defaults, allow_no_value=True)
bconfig.read(spec)
package = bconfig.get('app', 'package.domain') + '.' + bconfig.get('app', 'package.name')
version = bconfig.get('app', 'version')
vercode = None
else:
print "No android or kivy project could be found. Specify --subdir?"
sys.exit(1)
# Make sure it's actually new...
for app in apps:
if app['id'] == package:
print "Package " + package + " already exists"
sys.exit(1)
# Construct the metadata...
app = metadata.parse_metadata(None)
app['id'] = package
app['Web Site'] = website
app['Source Code'] = sourcecode
if issuetracker:
app['Issue Tracker'] = issuetracker
if license:
app['License'] = license
app['Repo Type'] = repotype
app['Repo'] = repo
app['Update Check Mode'] = "Tags"
# Create a build line...
build = {}
build['version'] = version if version else '?'
build['vercode'] = vercode if vercode else '?'
build['commit'] = '?'
build['disable'] = 'Generated by import.py - check/set version fields and commit id'
if options.subdir:
build['subdir'] = options.subdir
if os.path.exists(os.path.join(root_dir, 'jni')):
build['buildjni'] = 'yes'
app['builds'].append(build)
# Keep the repo directory to save bandwidth...
if not os.path.exists('build'):
os.mkdir('build')
shutil.move(src_dir, os.path.join('build', package))
with open('build/.fdroidvcs-' + package, 'w') as f:
f.write(repotype + ' ' + repo)
metafile = os.path.join('metadata', package + '.txt')
metadata.write_metadata(metafile, app)
print "Wrote " + metafile
def check_repomanifest(app, branch=None):
try:
if app.RepoType == 'srclib':
build_dir = os.path.join('build', 'srclib', app.Repo)
repotype = common.getsrclibvcs(app.Repo)
else:
build_dir = os.path.join('build', app.id)
repotype = app.RepoType
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
if repotype == 'git':
if branch:
branch = 'origin/' + branch
vcs.gotorevision(branch)
elif repotype == 'git-svn':
vcs.gotorevision(branch)
elif repotype == 'hg':
vcs.gotorevision(branch)
elif repotype == 'bzr':
vcs.gotorevision(None)
last_build = metadata.Build()
if len(app.builds) > 0:
last_build = app.builds[-1]
if last_build.submodules:
vcs.initsubmodules()
hpak = None
hver = None
hcode = "0"
for subdir in possible_subdirs(app):
if subdir == '.':
root_dir = build_dir
else:
root_dir = os.path.join(build_dir, subdir)
paths = common.manifest_paths(root_dir, last_build.gradle)
version, vercode, package = common.parse_androidmanifests(paths, app)
if vercode:
logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
.format(subdir, version, vercode))
if int(vercode) > int(hcode):
hpak = package
hcode = str(int(vercode))
hver = version
if not hpak:
return (None, "Couldn't find package ID")
if hver:
return (hver, hcode)
return (None, "Couldn't find any version information")
except VCSException as vcse:
msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
return (None, msg)
except Exception:
msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
return (None, msg)
def main():
global config, options
# Parse command line...
parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
common.setup_global_opts(parser)
parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]")
options = parser.parse_args()
config = common.read_config(options)
# Read all app and srclib metadata
allapps = metadata.read_metadata()
apps = common.read_app_args(options.appid, allapps, True)
probcount = 0
build_dir = 'build'
if not os.path.isdir(build_dir):
logging.info("Creating build directory")
os.makedirs(build_dir)
srclib_dir = os.path.join(build_dir, 'srclib')
extlib_dir = os.path.join(build_dir, 'extlib')
for appid, app in apps.iteritems():
if app.Disabled:
logging.info("Skipping %s: disabled" % appid)
continue
if not app.builds:
logging.info("Skipping %s: no builds specified" % appid)
continue
logging.info("Processing " + appid)
try:
if app.RepoType == 'srclib':
build_dir = os.path.join('build', 'srclib', app.Repo)
else:
build_dir = os.path.join('build', appid)
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
for build in app.builds:
if build.disable:
logging.info("...skipping version %s - %s" % (
build.version, build.get('disable', build.commit[1:])))
else:
logging.info("...scanning version " + build.version)
# Prepare the source code...
root_dir, _ = common.prepare_source(vcs, app, build,
build_dir, srclib_dir,
extlib_dir, False)
# Do the scan...
count = scan_source(build_dir, root_dir, build)
if count > 0:
logging.warn('Scanner found %d problems in %s (%s)' % (
count, appid, build.vercode))
probcount += count
except BuildException as be:
logging.warn("Could not scan app %s due to BuildException: %s" % (
appid, be))
probcount += 1
except VCSException as vcse:
logging.warn("VCS error while scanning app %s: %s" % (appid, vcse))
probcount += 1
except Exception:
logging.warn("Could not scan app %s due to unknown error: %s" % (
appid, traceback.format_exc()))
probcount += 1
logging.info("Finished:")
print("%d problems found" % probcount)
def main():
global options, config
options, args = parse_commandline()
if not args and not options.all:
raise OptionError("If you really want to build all the apps, use --all", "all")
config = common.read_config(options)
if config['build_server_always']:
options.server = True
if options.resetserver and not options.server:
raise OptionError("Using --resetserver without --server makes no sense", "resetserver")
log_dir = 'logs'
if not os.path.isdir(log_dir):
logging.info("Creating log directory")
os.makedirs(log_dir)
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
logging.info("Creating temporary directory")
os.makedirs(tmp_dir)
if options.test:
output_dir = tmp_dir
else:
output_dir = 'unsigned'
if not os.path.isdir(output_dir):
logging.info("Creating output directory")
os.makedirs(output_dir)
if config['archive_older'] != 0:
also_check_dir = 'archive'
else:
also_check_dir = None
repo_dir = 'repo'
build_dir = 'build'
if not os.path.isdir(build_dir):
logging.info("Creating build directory")
os.makedirs(build_dir)
srclib_dir = os.path.join(build_dir, 'srclib')
extlib_dir = os.path.join(build_dir, 'extlib')
# Read all app and srclib metadata
allapps = metadata.read_metadata(xref=not options.onserver)
apps = common.read_app_args(args, allapps, True)
for appid, app in apps.items():
if (app['Disabled'] and not options.force) or not app['Repo Type'] or not app['builds']:
del apps[appid]
if not apps:
raise FDroidException("No apps to process.")
if options.latest:
for app in apps.itervalues():
for build in reversed(app['builds']):
if build['disable'] and not options.force:
continue
app['builds'] = [build]
break
if options.wiki:
import mwclient
site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
path=config['wiki_path'])
site.login(config['wiki_user'], config['wiki_password'])
# Build applications...
failed_apps = {}
build_succeeded = []
for appid, app in apps.iteritems():
first = True
for thisbuild in app['builds']:
wikilog = None
try:
# For the first build of a particular app, we need to set up
# the source repo. We can reuse it on subsequent builds, if
# there are any.
if first:
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib', app['Repo'])
else:
build_dir = os.path.join('build', appid)
# Set up vcs interface and make sure we have the latest code...
logging.debug("Getting {0} vcs interface for {1}"
.format(app['Repo Type'], app['Repo']))
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
first = False
logging.debug("Checking " + thisbuild['version'])
if trybuild(app, thisbuild, build_dir, output_dir,
also_check_dir, srclib_dir, extlib_dir,
tmp_dir, repo_dir, vcs, options.test,
options.server, options.force,
options.onserver):
build_succeeded.append(app)
wikilog = "Build succeeded"
except BuildException as be:
logfile = open(os.path.join(log_dir, appid + '.log'), 'a+')
logfile.write(str(be))
logfile.close()
print("Could not build app %s due to BuildException: %s" % (appid, be))
if options.stop:
sys.exit(1)
failed_apps[appid] = be
wikilog = be.get_wikitext()
except VCSException as vcse:
reason = str(vcse).split('\n', 1)[0] if options.verbose else str(vcse)
logging.error("VCS error while building app %s: %s" % (
appid, reason))
if options.stop:
sys.exit(1)
failed_apps[appid] = vcse
wikilog = str(vcse)
except Exception as e:
logging.error("Could not build app %s due to unknown error: %s" % (
appid, traceback.format_exc()))
if options.stop:
sys.exit(1)
failed_apps[appid] = e
wikilog = str(e)
if options.wiki and wikilog:
try:
# Write a page with the last build log for this version code
lastbuildpage = appid + '/lastbuild_' + thisbuild['vercode']
newpage = site.Pages[lastbuildpage]
txt = "Build completed at " + time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) + "\n\n" + wikilog
newpage.save(txt, summary='Build log')
# Redirect from /lastbuild to the most recent build log
newpage = site.Pages[appid + '/lastbuild']
newpage.save('#REDIRECT [[' + lastbuildpage + ']]', summary='Update redirect')
except:
logging.error("Error while attempting to publish build log")
for app in build_succeeded:
logging.info("success: %s" % (app['id']))
if not options.verbose:
for fa in failed_apps:
logging.info("Build for app %s failed:\n%s" % (fa, failed_apps[fa]))
logging.info("Finished.")
if len(build_succeeded) > 0:
logging.info(str(len(build_succeeded)) + ' builds succeeded')
if len(failed_apps) > 0:
logging.info(str(len(failed_apps)) + ' builds failed')
sys.exit(0)
def check_repomanifest(app, sdk_path, branch=None):
try:
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib')
repotype = common.getsrclibvcs(app['Repo'])
else:
build_dir = os.path.join('build/', app['id'])
repotype = app['Repo Type']
if repotype == 'bzr':
return (None, 'RepoManifest update mode has not been ported to bzr repositories yet')
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, sdk_path)
if app['Repo Type'] == 'srclib':
build_dir = os.path.join(build_dir, app['Repo'])
if vcs.repotype() == 'git':
if branch:
vcs.gotorevision('origin/'+branch)
else:
vcs.gotorevision('origin/master')
pass
elif vcs.repotype() == 'git-svn':
if branch:
vcs.gotorevision(branch)
else:
vcs.gotorevision(None)
elif vcs.repotype() == 'svn':
vcs.gotorevision(None)
elif vcs.repotype() == 'hg':
if branch:
vcs.gotorevision(branch)
else:
vcs.gotorevision('default')
if len(app['builds']) > 0:
if 'subdir' in app['builds'][-1]:
build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
version, vercode, package = common.parse_androidmanifest(build_dir)
if not package:
return (None, "Couldn't find package ID")
if package != app['id']:
return (None, "Package ID mismatch")
if not version:
return (None,"Couldn't find latest version name")
if not vercode:
return (None,"Couldn't find latest version code")
return (version, str(int(vercode)))
except BuildException as be:
msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
return (None, msg)
except VCSException as vcse:
msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
return (None, msg)
except Exception:
msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
return (None, msg)
def main():
global config, options
# Parse command line...
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-p", "--package", default=None,
help="Check only the specified package")
parser.add_option("--auto", action="store_true", default=False,
help="Process auto-updates")
parser.add_option("--autoonly", action="store_true", default=False,
help="Only process apps with auto-updates")
parser.add_option("--commit", action="store_true", default=False,
help="Commit changes")
parser.add_option("--gplay", action="store_true", default=False,
help="Only print differences with the Play Store")
(options, args) = parser.parse_args()
config = common.read_config(options)
# Get all apps...
apps = metadata.read_metadata(options.verbose)
# Filter apps according to command-line options
if options.package:
apps = [app for app in apps if app['id'] == options.package]
if len(apps) == 0:
print "No such package"
sys.exit(1)
if options.gplay:
for app in apps:
version, reason = check_gplay(app)
if version is None and options.verbose:
if reason == '404':
print "%s is not in the Play Store" % common.getappname(app)
else:
print "%s encountered a problem: %s" % common.getappname(app)
if version is not None:
stored = app['Current Version']
if LooseVersion(stored) < LooseVersion(version):
print "%s has version %s on the Play Store, which is bigger than %s" % (
common.getappname(app), version, stored)
elif options.verbose:
print "%s has the same version %s on the Play Store" % (
common.getappname(app), version)
return
for app in apps:
if options.autoonly and app['Auto Update Mode'] == 'None':
if options.verbose:
print "Nothing to do for %s..." % app['id']
continue
print "Processing " + app['id'] + '...'
writeit = False
logmsg = None
tag = None
msg = None
vercode = None
mode = app['Update Check Mode']
if mode == 'Tags':
(version, vercode, tag) = check_tags(app)
elif mode == 'RepoManifest':
(version, vercode) = check_repomanifest(app)
elif mode.startswith('RepoManifest/'):
tag = mode[13:]
(version, vercode) = check_repomanifest(app, tag)
elif mode == 'RepoTrunk':
(version, vercode) = check_repotrunk(app)
elif mode == 'HTTP':
(version, vercode) = check_http(app)
elif mode == 'Static':
version = None
msg = 'Checking disabled'
elif mode == 'None':
version = None
msg = 'Checking disabled'
else:
version = None
msg = 'Invalid update check method'
if vercode and app['Vercode Operation']:
op = app['Vercode Operation'].replace("%c", str(int(vercode)))
vercode = str(eval(op))
updating = False
if not version:
print "...%s" % msg
elif vercode == app['Current Version Code']:
print "...up to date"
else:
app['Current Version'] = version
app['Current Version Code'] = str(int(vercode))
updating = True
writeit = True
# Do the Auto Name thing as well as finding the CV real name
if len(app["Repo Type"]) > 0:
try:
if app['Repo Type'] == 'srclib':
app_dir = os.path.join('build', 'srclib', app['Repo'])
else:
app_dir = os.path.join('build/', app['id'])
vcs = common.getvcs(app["Repo Type"], app["Repo"], app_dir)
vcs.gotorevision(tag)
flavour = None
if len(app['builds']) > 0:
if 'subdir' in app['builds'][-1]:
app_dir = os.path.join(app_dir, app['builds'][-1]['subdir'])
if 'gradle' in app['builds'][-1]:
flavour = app['builds'][-1]['gradle']
new_name = common.fetch_real_name(app_dir, flavour)
if new_name != app['Auto Name']:
app['Auto Name'] = new_name
if app['Current Version'].startswith('@string/'):
cv = common.version_name(app['Current Version'], app_dir, flavour)
if app['Current Version'] != cv:
app['Current Version'] = cv
writeit = True
except Exception:
print "ERROR: Auto Name or Current Version failed for %s due to exception: %s" % (app['id'], traceback.format_exc())
if updating:
name = common.getappname(app)
ver = common.getcvname(app)
print '...updating to version %s' % ver
logmsg = 'Update CV of %s to %s' % (name, ver)
if options.auto:
mode = app['Auto Update Mode']
if mode == 'None':
pass
elif mode.startswith('Version '):
pattern = mode[8:]
if pattern.startswith('+'):
o = pattern.find(' ')
suffix = pattern[1:o]
pattern = pattern[o + 1:]
else:
suffix = ''
gotcur = False
latest = None
for build in app['builds']:
if build['vercode'] == app['Current Version Code']:
gotcur = True
if not latest or int(build['vercode']) > int(latest['vercode']):
latest = build
if not gotcur:
newbuild = latest.copy()
if 'origlines' in newbuild:
del newbuild['origlines']
newbuild['vercode'] = app['Current Version Code']
newbuild['version'] = app['Current Version'] + suffix
print "...auto-generating build for " + newbuild['version']
commit = pattern.replace('%v', newbuild['version'])
commit = commit.replace('%c', newbuild['vercode'])
newbuild['commit'] = commit
app['builds'].append(newbuild)
writeit = True
name = common.getappname(app)
ver = common.getcvname(app)
logmsg = "Update %s to %s" % (name, ver)
else:
print 'Invalid auto update mode "' + mode + '"'
if writeit:
metafile = os.path.join('metadata', app['id'] + '.txt')
metadata.write_metadata(metafile, app)
if options.commit and logmsg:
print "Commiting update for " + metafile
gitcmd = ["git", "commit", "-m",
logmsg]
if 'auto_author' in config:
gitcmd.extend(['--author', config['auto_author']])
gitcmd.extend(["--", metafile])
if subprocess.call(gitcmd) != 0:
print "Git commit failed"
sys.exit(1)
print "Finished."
def main():
global config, options
# Parse command line...
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-q", "--quiet", action="store_true", default=False,
help="Restrict output to warnings and errors")
parser.add_option("-u", "--url", default=None,
help="Project URL to import from.")
parser.add_option("-s", "--subdir", default=None,
help="Path to main android project subdirectory, if not in root.")
parser.add_option("--rev", default=None,
help="Allows a different revision (or git branch) to be specified for the initial import")
(options, args) = parser.parse_args()
config = common.read_config(options)
if not options.url:
logging.error("Specify project url.")
sys.exit(1)
url = options.url
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
logging.info("Creating temporary directory")
os.makedirs(tmp_dir)
# Get all apps...
apps = metadata.read_metadata()
# Figure out what kind of project it is...
projecttype = None
issuetracker = None
license = None
website = url # by default, we might override it
if url.startswith('git://'):
projecttype = 'git'
repo = url
repotype = 'git'
sourcecode = ""
website = ""
elif url.startswith('https://github.com'):
projecttype = 'github'
repo = url
repotype = 'git'
sourcecode = url
issuetracker = url + '/issues'
website = ""
elif url.startswith('https://gitlab.com/'):
projecttype = 'gitlab'
repo = url
repotype = 'git'
sourcecode = url + '/tree/HEAD'
issuetracker = url + '/issues'
elif url.startswith('https://bitbucket.org/'):
if url.endswith('/'):
url = url[:-1]
projecttype = 'bitbucket'
sourcecode = url + '/src'
issuetracker = url + '/issues'
# Figure out the repo type and adddress...
repotype, repo = getrepofrompage(sourcecode)
if not repotype:
logging.error("Unable to determine vcs type. " + repo)
sys.exit(1)
if not projecttype:
logging.error("Unable to determine the project type.")
logging.error("The URL you supplied was not in one of the supported formats. Please consult")
logging.error("the manual for a list of supported formats, and supply one of those.")
sys.exit(1)
# Ensure we have a sensible-looking repo address at this point. If not, we
# might have got a page format we weren't expecting. (Note that we
# specifically don't want git@...)
if ((repotype != 'bzr' and (not repo.startswith('http://') and
not repo.startswith('https://') and
not repo.startswith('git://'))) or
' ' in repo):
logging.error("Repo address '{0}' does not seem to be valid".format(repo))
sys.exit(1)
# Get a copy of the source so we can extract some info...
logging.info('Getting source from ' + repotype + ' repo at ' + repo)
src_dir = os.path.join(tmp_dir, 'importer')
if os.path.exists(src_dir):
shutil.rmtree(src_dir)
vcs = common.getvcs(repotype, repo, src_dir)
vcs.gotorevision(options.rev)
if options.subdir:
root_dir = os.path.join(src_dir, options.subdir)
else:
root_dir = src_dir
# Extract some information...
paths = common.manifest_paths(root_dir, [])
if paths:
version, vercode, package = common.parse_androidmanifests(paths)
if not package:
logging.error("Couldn't find package ID")
sys.exit(1)
if not version:
logging.warn("Couldn't find latest version name")
if not vercode:
logging.warn("Couldn't find latest version code")
else:
spec = os.path.join(root_dir, 'buildozer.spec')
if os.path.exists(spec):
defaults = {'orientation': 'landscape', 'icon': '',
'permissions': '', 'android.api': "18"}
bconfig = ConfigParser(defaults, allow_no_value=True)
bconfig.read(spec)
package = bconfig.get('app', 'package.domain') + '.' + bconfig.get('app', 'package.name')
version = bconfig.get('app', 'version')
vercode = None
else:
logging.error("No android or kivy project could be found. Specify --subdir?")
sys.exit(1)
# Make sure it's actually new...
if package in apps:
logging.error("Package " + package + " already exists")
sys.exit(1)
# Construct the metadata...
app = metadata.parse_metadata(None)[1]
app['Web Site'] = website
app['Source Code'] = sourcecode
if issuetracker:
app['Issue Tracker'] = issuetracker
if license:
app['License'] = license
app['Repo Type'] = repotype
app['Repo'] = repo
app['Update Check Mode'] = "Tags"
# Create a build line...
build = {}
build['version'] = version or '?'
build['vercode'] = vercode or '?'
build['commit'] = '?'
build['disable'] = 'Generated by import.py - check/set version fields and commit id'
if options.subdir:
build['subdir'] = options.subdir
if os.path.exists(os.path.join(root_dir, 'jni')):
build['buildjni'] = ['yes']
for flag, value in metadata.flag_defaults.iteritems():
if flag in build:
continue
build[flag] = value
app['builds'].append(build)
# Keep the repo directory to save bandwidth...
if not os.path.exists('build'):
os.mkdir('build')
shutil.move(src_dir, os.path.join('build', package))
with open('build/.fdroidvcs-' + package, 'w') as f:
f.write(repotype + ' ' + repo)
metafile = os.path.join('metadata', package + '.txt')
metadata.write_metadata(metafile, app)
logging.info("Wrote " + metafile)
def main():
global config, options
# Parse command line...
parser = OptionParser(
usage="Usage: %prog [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="Spew out even more information than normal")
parser.add_option("-q",
"--quiet",
action="store_true",
default=False,
help="Restrict output to warnings and errors")
(options, args) = parser.parse_args()
config = common.read_config(options)
# Read all app and srclib metadata
allapps = metadata.read_metadata()
apps = common.read_app_args(args, allapps, True)
probcount = 0
build_dir = 'build'
if not os.path.isdir(build_dir):
logging.info("Creating build directory")
os.makedirs(build_dir)
srclib_dir = os.path.join(build_dir, 'srclib')
extlib_dir = os.path.join(build_dir, 'extlib')
for appid, app in apps.iteritems():
if app['Disabled']:
logging.info("Skipping %s: disabled" % appid)
continue
if not app['builds']:
logging.info("Skipping %s: no builds specified" % appid)
continue
logging.info("Processing " + appid)
try:
build_dir = 'build/' + appid
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
for thisbuild in app['builds']:
if thisbuild['disable']:
logging.info(
"...skipping version %s - %s" %
(thisbuild['version'],
thisbuild.get('disable', thisbuild['commit'][1:])))
else:
logging.info("...scanning version " + thisbuild['version'])
# Prepare the source code...
root_dir, _ = common.prepare_source(
vcs, app, thisbuild, build_dir, srclib_dir, extlib_dir,
False)
# Do the scan...
count = common.scan_source(build_dir, root_dir, thisbuild)
if count > 0:
logging.warn('Scanner found %d problems in %s (%s)' %
(count, appid, thisbuild['vercode']))
probcount += count
except BuildException as be:
logging.warn("Could not scan app %s due to BuildException: %s" %
(appid, be))
probcount += 1
except VCSException as vcse:
logging.warn("VCS error while scanning app %s: %s" % (appid, vcse))
probcount += 1
except Exception:
logging.warn("Could not scan app %s due to unknown error: %s" %
(appid, traceback.format_exc()))
probcount += 1
logging.info("Finished:")
print "%d app(s) with problems" % probcount
def main():
global config, options
# Parse command line...
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-q", "--quiet", action="store_true", default=False,
help="Restrict output to warnings and errors")
parser.add_option("-u", "--url", default=None,
help="Project URL to import from.")
parser.add_option("-s", "--subdir", default=None,
help="Path to main android project subdirectory, if not in root.")
parser.add_option("-r", "--repo", default=None,
help="Allows a different repo to be specified for a multi-repo google code project")
parser.add_option("--rev", default=None,
help="Allows a different revision (or git branch) to be specified for the initial import")
(options, args) = parser.parse_args()
config = common.read_config(options)
if not options.url:
logging.error("Specify project url.")
sys.exit(1)
url = options.url
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
logging.info("Creating temporary directory")
os.makedirs(tmp_dir)
# Get all apps...
apps = metadata.read_metadata()
# Figure out what kind of project it is...
projecttype = None
issuetracker = None
license = None
website = url # by default, we might override it
if url.startswith('git://'):
projecttype = 'git'
repo = url
repotype = 'git'
sourcecode = ""
website = ""
elif url.startswith('https://github.com'):
projecttype = 'github'
repo = url
repotype = 'git'
sourcecode = url
issuetracker = url + '/issues'
elif url.startswith('https://gitlab.com/'):
projecttype = 'gitlab'
repo = url
repotype = 'git'
sourcecode = url
issuetracker = url + '/issues'
elif url.startswith('https://gitorious.org/'):
projecttype = 'gitorious'
repo = 'https://git.gitorious.org/' + url[22:] + '.git'
repotype = 'git'
sourcecode = url
elif url.startswith('https://bitbucket.org/'):
if url.endswith('/'):
url = url[:-1]
projecttype = 'bitbucket'
sourcecode = url + '/src'
issuetracker = url + '/issues'
# Figure out the repo type and adddress...
repotype, repo = getrepofrompage(sourcecode)
if not repotype:
logging.error("Unable to determine vcs type. " + repo)
sys.exit(1)
elif (url.startswith('http://code.google.com/p/') or
url.startswith('https://code.google.com/p/')):
if not url.endswith('/'):
url += '/'
projecttype = 'googlecode'
sourcecode = url + 'source/checkout'
if options.repo:
sourcecode += "?repo=" + options.repo
issuetracker = url + 'issues/list'
# Figure out the repo type and adddress...
repotype, repo = getrepofrompage(sourcecode)
if not repotype:
logging.error("Unable to determine vcs type. " + repo)
sys.exit(1)
# Figure out the license...
req = urllib.urlopen(url)
if req.getcode() != 200:
logging.error('Unable to find project page at ' + sourcecode + ' - return code ' + str(req.getcode()))
sys.exit(1)
page = req.read()
index = page.find('Code license')
if index == -1:
logging.error("Couldn't find license data")
sys.exit(1)
ltext = page[index:]
lprefix = 'rel="nofollow">'
index = ltext.find(lprefix)
if index == -1:
logging.error("Couldn't find license text")
sys.exit(1)
ltext = ltext[index + len(lprefix):]
index = ltext.find('<')
if index == -1:
logging.error("License text not formatted as expected")
sys.exit(1)
ltext = ltext[:index]
if ltext == 'GNU GPL v3':
license = 'GPLv3'
elif ltext == 'GNU GPL v2':
license = 'GPLv2'
elif ltext == 'Apache License 2.0':
license = 'Apache2'
elif ltext == 'MIT License':
license = 'MIT'
elif ltext == 'GNU Lesser GPL':
license = 'LGPL'
elif ltext == 'Mozilla Public License 1.1':
license = 'MPL'
elif ltext == 'New BSD License':
license = 'NewBSD'
else:
logging.error("License " + ltext + " is not recognised")
sys.exit(1)
if not projecttype:
logging.error("Unable to determine the project type.")
logging.error("The URL you supplied was not in one of the supported formats. Please consult")
logging.error("the manual for a list of supported formats, and supply one of those.")
sys.exit(1)
# Get a copy of the source so we can extract some info...
logging.info('Getting source from ' + repotype + ' repo at ' + repo)
src_dir = os.path.join(tmp_dir, 'importer')
if os.path.exists(src_dir):
shutil.rmtree(src_dir)
vcs = common.getvcs(repotype, repo, src_dir)
vcs.gotorevision(options.rev)
if options.subdir:
root_dir = os.path.join(src_dir, options.subdir)
else:
root_dir = src_dir
# Extract some information...
paths = common.manifest_paths(root_dir, None)
if paths:
version, vercode, package = common.parse_androidmanifests(paths)
if not package:
logging.error("Couldn't find package ID")
sys.exit(1)
if not version:
logging.warn("Couldn't find latest version name")
if not vercode:
logging.warn("Couldn't find latest version code")
else:
spec = os.path.join(root_dir, 'buildozer.spec')
if os.path.exists(spec):
defaults = {'orientation': 'landscape', 'icon': '',
'permissions': '', 'android.api': "18"}
bconfig = ConfigParser(defaults, allow_no_value=True)
bconfig.read(spec)
package = bconfig.get('app', 'package.domain') + '.' + bconfig.get('app', 'package.name')
version = bconfig.get('app', 'version')
vercode = None
else:
logging.error("No android or kivy project could be found. Specify --subdir?")
sys.exit(1)
# Make sure it's actually new...
if package in apps:
logging.error("Package " + package + " already exists")
sys.exit(1)
# Construct the metadata...
app = metadata.parse_metadata(None)[1]
app['Web Site'] = website
app['Source Code'] = sourcecode
if issuetracker:
app['Issue Tracker'] = issuetracker
if license:
app['License'] = license
app['Repo Type'] = repotype
app['Repo'] = repo
app['Update Check Mode'] = "Tags"
# Create a build line...
build = {}
build['version'] = version or '?'
build['vercode'] = vercode or '?'
build['commit'] = '?'
build['disable'] = 'Generated by import.py - check/set version fields and commit id'
if options.subdir:
build['subdir'] = options.subdir
if os.path.exists(os.path.join(root_dir, 'jni')):
build['buildjni'] = ['yes']
for flag, value in metadata.flag_defaults.iteritems():
if flag in build:
continue
build[flag] = value
app['builds'].append(build)
# Keep the repo directory to save bandwidth...
if not os.path.exists('build'):
os.mkdir('build')
shutil.move(src_dir, os.path.join('build', package))
with open('build/.fdroidvcs-' + package, 'w') as f:
f.write(repotype + ' ' + repo)
metafile = os.path.join('metadata', package + '.txt')
metadata.write_metadata(metafile, app)
logging.info("Wrote " + metafile)
def main():
global config, options
# Parse command line...
parser = OptionParser(usage="Usage: %prog [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-q", "--quiet", action="store_true", default=False,
help="Restrict output to warnings and errors")
(options, args) = parser.parse_args()
config = common.read_config(options)
# Read all app and srclib metadata
allapps = metadata.read_metadata()
apps = common.read_app_args(args, allapps, True)
probcount = 0
build_dir = 'build'
if not os.path.isdir(build_dir):
logging.info("Creating build directory")
os.makedirs(build_dir)
srclib_dir = os.path.join(build_dir, 'srclib')
extlib_dir = os.path.join(build_dir, 'extlib')
for appid, app in apps.iteritems():
if app['Disabled']:
logging.info("Skipping %s: disabled" % appid)
continue
if not app['builds']:
logging.info("Skipping %s: no builds specified" % appid)
continue
logging.info("Processing " + appid)
try:
build_dir = 'build/' + appid
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
for thisbuild in app['builds']:
if thisbuild['disable']:
logging.info("...skipping version %s - %s" % (
thisbuild['version'], thisbuild.get('disable', thisbuild['commit'][1:])))
else:
logging.info("...scanning version " + thisbuild['version'])
# Prepare the source code...
root_dir, _ = common.prepare_source(vcs, app, thisbuild,
build_dir, srclib_dir,
extlib_dir, False)
# Do the scan...
count = common.scan_source(build_dir, root_dir, thisbuild)
if count > 0:
logging.warn('Scanner found %d problems in %s (%s)' % (
count, appid, thisbuild['vercode']))
probcount += count
except BuildException as be:
logging.warn("Could not scan app %s due to BuildException: %s" % (
appid, be))
probcount += 1
except VCSException as vcse:
logging.warn("VCS error while scanning app %s: %s" % (appid, vcse))
probcount += 1
except Exception:
logging.warn("Could not scan app %s due to unknown error: %s" % (
appid, traceback.format_exc()))
probcount += 1
logging.info("Finished:")
print "%d app(s) with problems" % probcount
def check_repomanifest(app, branch=None):
try:
appid = app['Update Check Name'] or app['id']
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib', app['Repo'])
repotype = common.getsrclibvcs(app['Repo'])
else:
build_dir = os.path.join('build/', app['id'])
repotype = app['Repo Type']
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
if repotype == 'git':
if branch:
branch = 'origin/' + branch
vcs.gotorevision(branch)
elif repotype == 'git-svn':
vcs.gotorevision(branch)
elif repotype == 'hg':
vcs.gotorevision(branch)
elif repotype == 'bzr':
vcs.gotorevision(None)
flavours = []
if len(app['builds']) > 0:
if app['builds'][-1]['subdir']:
build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
if app['builds'][-1]['gradle']:
flavours = app['builds'][-1]['gradle']
if not os.path.isdir(build_dir):
return (None, "Subdir '" + app['builds'][-1]['subdir'] + "'is not a valid directory")
paths = common.manifest_paths(build_dir, flavours)
version, vercode, package = \
common.parse_androidmanifests(paths, app['Update Check Ignore'])
if not package:
return (None, "Couldn't find package ID")
if package != appid:
return (None, "Package ID mismatch - expected {0}, got {1}"
.format(appid, package))
if not version:
return (None, "Couldn't find latest version name")
if not vercode:
if "Ignore" == version:
return (None, "Latest version is ignored")
return (None, "Couldn't find latest version code")
vercode = str(int(vercode))
logging.debug("Manifest exists. Found version {0} ({1})".format(version, vercode))
return (version, vercode)
except VCSException as vcse:
msg = "VCS error while scanning app {0}: {1}".format(app['id'], vcse)
return (None, msg)
except Exception:
msg = "Could not scan app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
return (None, msg)
def main():
# Read configuration...
global build_server_always, mvn3
globals()['build_server_always'] = False
globals()['mvn3'] = "mvn3"
execfile('config.py', globals())
# Parse command line...
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-p", "--package", default=None,
help="Scan only the specified package")
parser.add_option("--nosvn", action="store_true", default=False,
help="Skip svn repositories - for test purposes, because they are too slow.")
(options, args) = parser.parse_args()
# Get all apps...
apps = common.read_metadata(options.verbose)
# Filter apps according to command-line options
if options.package:
apps = [app for app in apps if app['id'] == options.package]
if len(apps) == 0:
print "No such package"
sys.exit(1)
html_parser = HTMLParser.HTMLParser()
problems = []
extlib_dir = os.path.join('build', 'extlib')
for app in apps:
skip = False
if app['Disabled']:
print "Skipping %s: disabled" % app['id']
skip = True
elif not app['builds']:
print "Skipping %s: no builds specified" % app['id']
skip = True
elif options.nosvn and app['Repo Type'] == 'svn':
skip = True
if not skip:
print "Processing " + app['id']
try:
build_dir = 'build/' + app['id']
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, sdk_path)
for thisbuild in app['builds']:
if thisbuild['commit'].startswith('!'):
print ("..skipping version " + thisbuild['version'] + " - " +
thisbuild['commit'][1:])
else:
print "..scanning version " + thisbuild['version']
# Prepare the source code...
root_dir, _ = common.prepare_source(vcs, app, thisbuild,
build_dir, extlib_dir, sdk_path, ndk_path, javacc_path, mvn3, options.verbose)
# Do the scan...
buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
for problem in buildprobs:
problems.append(problem +
' in ' + app['id'] + ' ' + thisbuild['version'])
except BuildException as be:
msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
problems.append(msg)
except VCSException as vcse:
msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
problems.append(msg)
except Exception:
msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
problems.append(msg)
print "Finished:"
for problem in problems:
print problem
print str(len(problems)) + ' problems.'
def check_tags(app, pattern):
try:
appid = app['Update Check Name'] or app['id']
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib', app['Repo'])
repotype = common.getsrclibvcs(app['Repo'])
else:
build_dir = os.path.join('build/', app['id'])
repotype = app['Repo Type']
if repotype not in ('git', 'git-svn', 'hg', 'bzr'):
return (None, 'Tags update mode only works for git, hg, bzr and git-svn repositories currently', None)
if repotype == 'git-svn' and ';' not in app['Repo']:
return (None, 'Tags update mode used in git-svn, but the repo was not set up with tags', None)
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
vcs.gotorevision(None)
flavours = []
if len(app['builds']) > 0:
if app['builds'][-1]['subdir']:
build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
if app['builds'][-1]['gradle']:
flavours = app['builds'][-1]['gradle']
hpak = None
htag = None
hver = None
hcode = "0"
tags = vcs.gettags()
if pattern:
pat = re.compile(pattern)
tags = [tag for tag in tags if pat.match(tag)]
if repotype in ('git',):
tags = vcs.latesttags(tags, 5)
for tag in tags:
logging.debug("Check tag: '{0}'".format(tag))
vcs.gotorevision(tag)
# Only process tags where the manifest exists...
paths = common.manifest_paths(build_dir, flavours)
version, vercode, package = \
common.parse_androidmanifests(paths, app['Update Check Ignore'])
if not package or package != appid or not version or not vercode:
continue
logging.debug("Manifest exists. Found version {0} ({1})"
.format(version, vercode))
if int(vercode) > int(hcode):
hpak = package
htag = tag
hcode = str(int(vercode))
hver = version
if not hpak:
return (None, "Couldn't find package ID", None)
if hver:
return (hver, hcode, htag)
return (None, "Couldn't find any version information", None)
except VCSException as vcse:
msg = "VCS error while scanning app {0}: {1}".format(app['id'], vcse)
return (None, msg, None)
except Exception:
msg = "Could not scan app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
return (None, msg, None)
def get_metadata_from_url(app, url):
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
logging.info("Creating temporary directory")
os.makedirs(tmp_dir)
# Figure out what kind of project it is...
projecttype = None
app.WebSite = url # by default, we might override it
if url.startswith('git://'):
projecttype = 'git'
repo = url
repotype = 'git'
app.SourceCode = ""
app.WebSite = ""
elif url.startswith('https://github.com'):
projecttype = 'github'
repo = url
repotype = 'git'
app.SourceCode = url
app.IssueTracker = url + '/issues'
app.WebSite = ""
elif url.startswith('https://gitlab.com/'):
projecttype = 'gitlab'
# git can be fussy with gitlab URLs unless they end in .git
if url.endswith('.git'):
repo = url
else:
repo = url + '.git'
repotype = 'git'
app.SourceCode = url + '/tree/HEAD'
app.IssueTracker = url + '/issues'
elif url.startswith('https://bitbucket.org/'):
if url.endswith('/'):
url = url[:-1]
projecttype = 'bitbucket'
app.SourceCode = url + '/src'
app.IssueTracker = url + '/issues'
# Figure out the repo type and adddress...
repotype, repo = getrepofrompage(app.SourceCode)
if not repotype:
logging.error("Unable to determine vcs type. " + repo)
sys.exit(1)
if not projecttype:
logging.error("Unable to determine the project type.")
logging.error("The URL you supplied was not in one of the supported formats. Please consult")
logging.error("the manual for a list of supported formats, and supply one of those.")
sys.exit(1)
# Ensure we have a sensible-looking repo address at this point. If not, we
# might have got a page format we weren't expecting. (Note that we
# specifically don't want git@...)
if ((repotype != 'bzr' and (not repo.startswith('http://') and
not repo.startswith('https://') and
not repo.startswith('git://'))) or
' ' in repo):
logging.error("Repo address '{0}' does not seem to be valid".format(repo))
sys.exit(1)
# Get a copy of the source so we can extract some info...
logging.info('Getting source from ' + repotype + ' repo at ' + repo)
build_dir = os.path.join(tmp_dir, 'importer')
if os.path.exists(build_dir):
shutil.rmtree(build_dir)
vcs = common.getvcs(repotype, repo, build_dir)
vcs.gotorevision(options.rev)
root_dir = get_subdir(build_dir)
app.RepoType = repotype
app.Repo = repo
return root_dir, build_dir
def main():
global options, config
options, parser = parse_commandline()
metadata_files = glob.glob('.fdroid.*[a-z]') # ignore files ending in ~
if os.path.isdir('metadata'):
pass
elif len(metadata_files) == 0:
raise FDroidException("No app metadata found, nothing to process!")
elif len(metadata_files) > 1:
raise FDroidException("Only one local metadata file allowed! Found: "
+ " ".join(metadata_files))
if not options.appid and not options.all:
parser.error("option %s: If you really want to build all the apps, use --all" % "all")
config = common.read_config(options)
if config['build_server_always']:
options.server = True
if options.resetserver and not options.server:
parser.error("option %s: Using --resetserver without --server makes no sense" % "resetserver")
log_dir = 'logs'
if not os.path.isdir(log_dir):
logging.info("Creating log directory")
os.makedirs(log_dir)
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
logging.info("Creating temporary directory")
os.makedirs(tmp_dir)
if options.test:
output_dir = tmp_dir
else:
output_dir = 'unsigned'
if not os.path.isdir(output_dir):
logging.info("Creating output directory")
os.makedirs(output_dir)
if config['archive_older'] != 0:
also_check_dir = 'archive'
else:
also_check_dir = None
repo_dir = 'repo'
build_dir = 'build'
if not os.path.isdir(build_dir):
logging.info("Creating build directory")
os.makedirs(build_dir)
srclib_dir = os.path.join(build_dir, 'srclib')
extlib_dir = os.path.join(build_dir, 'extlib')
# Read all app and srclib metadata
allapps = metadata.read_metadata(xref=not options.onserver)
apps = common.read_app_args(options.appid, allapps, True)
for appid, app in apps.items():
if (app.Disabled and not options.force) or not app.RepoType or not app.builds:
del apps[appid]
if not apps:
raise FDroidException("No apps to process.")
if options.latest:
for app in apps.itervalues():
for build in reversed(app.builds):
if build.disable and not options.force:
continue
app.builds = [build]
break
if options.wiki:
import mwclient
site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
path=config['wiki_path'])
site.login(config['wiki_user'], config['wiki_password'])
# Build applications...
failed_apps = {}
build_succeeded = []
for appid, app in apps.iteritems():
first = True
for build in app.builds:
wikilog = None
try:
# For the first build of a particular app, we need to set up
# the source repo. We can reuse it on subsequent builds, if
# there are any.
if first:
if app.RepoType == 'srclib':
build_dir = os.path.join('build', 'srclib', app.Repo)
else:
build_dir = os.path.join('build', appid)
# Set up vcs interface and make sure we have the latest code...
logging.debug("Getting {0} vcs interface for {1}"
.format(app.RepoType, app.Repo))
vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
first = False
logging.debug("Checking " + build.version)
if trybuild(app, build, build_dir, output_dir,
also_check_dir, srclib_dir, extlib_dir,
tmp_dir, repo_dir, vcs, options.test,
options.server, options.force,
options.onserver, options.refresh):
if app.Binaries is not None:
# This is an app where we build from source, and
# verify the apk contents against a developer's
# binary. We get that binary now, and save it
# alongside our built one in the 'unsigend'
# directory.
url = app.Binaries
url = url.replace('%v', build.version)
url = url.replace('%c', str(build.vercode))
logging.info("...retrieving " + url)
of = "{0}_{1}.apk.binary".format(app.id, build.vercode)
of = os.path.join(output_dir, of)
net.download_file(url, local_filename=of)
build_succeeded.append(app)
wikilog = "Build succeeded"
except VCSException as vcse:
reason = str(vcse).split('\n', 1)[0] if options.verbose else str(vcse)
logging.error("VCS error while building app %s: %s" % (
appid, reason))
if options.stop:
sys.exit(1)
failed_apps[appid] = vcse
wikilog = str(vcse)
except FDroidException as e:
with open(os.path.join(log_dir, appid + '.log'), 'a+') as f:
f.write(str(e))
logging.error("Could not build app %s: %s" % (appid, e))
if options.stop:
sys.exit(1)
failed_apps[appid] = e
wikilog = e.get_wikitext()
except Exception as e:
logging.error("Could not build app %s due to unknown error: %s" % (
appid, traceback.format_exc()))
if options.stop:
sys.exit(1)
failed_apps[appid] = e
wikilog = str(e)
if options.wiki and wikilog:
try:
# Write a page with the last build log for this version code
lastbuildpage = appid + '/lastbuild_' + build.vercode
newpage = site.Pages[lastbuildpage]
txt = "Build completed at " + time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) + "\n\n" + wikilog
newpage.save(txt, summary='Build log')
# Redirect from /lastbuild to the most recent build log
newpage = site.Pages[appid + '/lastbuild']
newpage.save('#REDIRECT [[' + lastbuildpage + ']]', summary='Update redirect')
except:
logging.error("Error while attempting to publish build log")
for app in build_succeeded:
logging.info("success: %s" % (app.id))
if not options.verbose:
for fa in failed_apps:
logging.info("Build for app %s failed:\n%s" % (fa, failed_apps[fa]))
logging.info("Finished.")
if len(build_succeeded) > 0:
logging.info(str(len(build_succeeded)) + ' builds succeeded')
if len(failed_apps) > 0:
logging.info(str(len(failed_apps)) + ' builds failed')
sys.exit(0)
def main():
global options, config
options, args = parse_commandline()
if not args and not options.all:
raise OptionError(
"If you really want to build all the apps, use --all", "all")
config = common.read_config(options)
if config['build_server_always']:
options.server = True
if options.resetserver and not options.server:
raise OptionError(
"Using --resetserver without --server makes no sense",
"resetserver")
log_dir = 'logs'
if not os.path.isdir(log_dir):
logging.info("Creating log directory")
os.makedirs(log_dir)
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
logging.info("Creating temporary directory")
os.makedirs(tmp_dir)
if options.test:
output_dir = tmp_dir
else:
output_dir = 'unsigned'
if not os.path.isdir(output_dir):
logging.info("Creating output directory")
os.makedirs(output_dir)
if config['archive_older'] != 0:
also_check_dir = 'archive'
else:
also_check_dir = None
repo_dir = 'repo'
build_dir = 'build'
if not os.path.isdir(build_dir):
logging.info("Creating build directory")
os.makedirs(build_dir)
srclib_dir = os.path.join(build_dir, 'srclib')
extlib_dir = os.path.join(build_dir, 'extlib')
# Read all app and srclib metadata
allapps = metadata.read_metadata(xref=not options.onserver)
apps = common.read_app_args(args, allapps, True)
for appid, app in apps.items():
if (app['Disabled'] and not options.force
) or not app['Repo Type'] or not app['builds']:
del apps[appid]
if not apps:
raise FDroidException("No apps to process.")
if options.latest:
for app in apps.itervalues():
for build in reversed(app['builds']):
if build['disable'] and not options.force:
continue
app['builds'] = [build]
break
if options.wiki:
import mwclient
site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
path=config['wiki_path'])
site.login(config['wiki_user'], config['wiki_password'])
# Build applications...
failed_apps = {}
build_succeeded = []
for appid, app in apps.iteritems():
first = True
for thisbuild in app['builds']:
wikilog = None
try:
# For the first build of a particular app, we need to set up
# the source repo. We can reuse it on subsequent builds, if
# there are any.
if first:
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib',
app['Repo'])
else:
build_dir = os.path.join('build', appid)
# Set up vcs interface and make sure we have the latest code...
logging.debug("Getting {0} vcs interface for {1}".format(
app['Repo Type'], app['Repo']))
vcs = common.getvcs(app['Repo Type'], app['Repo'],
build_dir)
first = False
logging.debug("Checking " + thisbuild['version'])
if trybuild(app, thisbuild, build_dir, output_dir,
also_check_dir, srclib_dir, extlib_dir, tmp_dir,
repo_dir, vcs, options.test, options.server,
options.force, options.onserver):
build_succeeded.append(app)
wikilog = "Build succeeded"
except BuildException as be:
logfile = open(os.path.join(log_dir, appid + '.log'), 'a+')
logfile.write(str(be))
logfile.close()
print("Could not build app %s due to BuildException: %s" %
(appid, be))
if options.stop:
sys.exit(1)
failed_apps[appid] = be
wikilog = be.get_wikitext()
except VCSException as vcse:
reason = str(vcse).split(
'\n', 1)[0] if options.verbose else str(vcse)
logging.error("VCS error while building app %s: %s" %
(appid, reason))
if options.stop:
sys.exit(1)
failed_apps[appid] = vcse
wikilog = str(vcse)
except Exception as e:
logging.error(
"Could not build app %s due to unknown error: %s" %
(appid, traceback.format_exc()))
if options.stop:
sys.exit(1)
failed_apps[appid] = e
wikilog = str(e)
if options.wiki and wikilog:
try:
# Write a page with the last build log for this version code
lastbuildpage = appid + '/lastbuild_' + thisbuild['vercode']
newpage = site.Pages[lastbuildpage]
txt = "Build completed at " + time.strftime(
"%Y-%m-%d %H:%M:%SZ", time.gmtime()) + "\n\n" + wikilog
newpage.save(txt, summary='Build log')
# Redirect from /lastbuild to the most recent build log
newpage = site.Pages[appid + '/lastbuild']
newpage.save('#REDIRECT [[' + lastbuildpage + ']]',
summary='Update redirect')
except:
logging.error(
"Error while attempting to publish build log")
for app in build_succeeded:
logging.info("success: %s" % (app['id']))
if not options.verbose:
for fa in failed_apps:
logging.info("Build for app %s failed:\n%s" %
(fa, failed_apps[fa]))
logging.info("Finished.")
if len(build_succeeded) > 0:
logging.info(str(len(build_succeeded)) + ' builds succeeded')
if len(failed_apps) > 0:
logging.info(str(len(failed_apps)) + ' builds failed')
sys.exit(0)
def check_tags(app, pattern):
try:
if app.RepoType == 'srclib':
build_dir = os.path.join('build', 'srclib', app.Repo)
repotype = common.getsrclibvcs(app.Repo)
else:
build_dir = os.path.join('build', app.id)
repotype = app.RepoType
if repotype not in ('git', 'git-svn', 'hg', 'bzr'):
return (None, 'Tags update mode only works for git, hg, bzr and git-svn repositories currently', None)
if repotype == 'git-svn' and ';' not in app.Repo:
return (None, 'Tags update mode used in git-svn, but the repo was not set up with tags', None)
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
vcs.gotorevision(None)
last_build = metadata.Build()
if len(app.builds) > 0:
last_build = app.builds[-1]
if last_build.submodules:
vcs.initsubmodules()
hpak = None
htag = None
hver = None
hcode = "0"
tags = vcs.gettags()
if not tags:
return (None, "No tags found", None)
logging.debug("All tags: " + ','.join(tags))
if pattern:
pat = re.compile(pattern)
tags = [tag for tag in tags if pat.match(tag)]
if not tags:
return (None, "No matching tags found", None)
logging.debug("Matching tags: " + ','.join(tags))
if len(tags) > 5 and repotype in ('git',):
tags = vcs.latesttags(tags, 5)
logging.debug("Latest tags: " + ','.join(tags))
for tag in tags:
logging.debug("Check tag: '{0}'".format(tag))
vcs.gotorevision(tag)
for subdir in possible_subdirs(app):
if subdir == '.':
root_dir = build_dir
else:
root_dir = os.path.join(build_dir, subdir)
paths = common.manifest_paths(root_dir, last_build.gradle)
version, vercode, package = common.parse_androidmanifests(paths, app)
if vercode:
logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
.format(subdir, version, vercode))
if int(vercode) > int(hcode):
hpak = package
htag = tag
hcode = str(int(vercode))
hver = version
if not hpak:
return (None, "Couldn't find package ID", None)
if hver:
return (hver, hcode, htag)
return (None, "Couldn't find any version information", None)
except VCSException as vcse:
msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
return (None, msg, None)
except Exception:
msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
return (None, msg, None)
def check_repomanifest(app, branch=None):
try:
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib', app['Repo'])
repotype = common.getsrclibvcs(app['Repo'])
else:
build_dir = os.path.join('build/', app['id'])
repotype = app['Repo Type']
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
if repotype == 'git':
if branch:
branch = 'origin/' + branch
vcs.gotorevision(branch)
elif repotype == 'git-svn':
vcs.gotorevision(branch)
elif repotype == 'hg':
vcs.gotorevision(branch)
elif repotype == 'bzr':
vcs.gotorevision(None)
flavours = []
if len(app['builds']) > 0:
if app['builds'][-1]['subdir']:
build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
if app['builds'][-1]['gradle']:
flavours = app['builds'][-1]['gradle']
if not os.path.isdir(build_dir):
return (None, "Subdir '" + app['builds'][-1]['subdir'] + "'is not a valid directory")
paths = common.manifest_paths(build_dir, flavours)
version, vercode, package = \
common.parse_androidmanifests(paths, app['Update Check Ignore'])
if not package:
return (None, "Couldn't find package ID")
if not app_matches_packagename(app, package):
return (None, "Package ID mismatch - got {0}".format(package))
if not version:
return (None, "Couldn't find latest version name")
if not vercode:
if "Ignore" == version:
return (None, "Latest version is ignored")
return (None, "Couldn't find latest version code")
vercode = str(int(vercode))
logging.debug("Manifest exists. Found version {0} ({1})".format(version, vercode))
return (version, vercode)
except VCSException as vcse:
msg = "VCS error while scanning app {0}: {1}".format(app['id'], vcse)
return (None, msg)
except Exception:
msg = "Could not scan app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
return (None, msg)
def main():
# Read configuration...
execfile('config.py', globals())
import common
# Parse command line...
parser = OptionParser()
parser.add_option("-u", "--url", default=None,
help="Project URL to import from.")
parser.add_option("-s", "--subdir", default=None,
help="Path to main android project subdirectory, if not in root.")
parser.add_option("-r", "--repo", default=None,
help="Allows a different repo to be specified for a multi-repo google code project")
(options, args) = parser.parse_args()
if not options.url:
print "Specify project url."
sys.exit(1)
url = options.url
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
print "Creating temporary directory"
os.makedirs(tmp_dir)
# Get all apps...
apps = common.read_metadata()
# Figure out what kind of project it is...
projecttype = None
issuetracker = None
license = None
website = url #by default, we might override it
if url.startswith('git://'):
projecttype = 'git'
repo = url
repotype = 'git'
sourcecode = ""
website = ""
if url.startswith('https://github.com'):
if url.endswith('/'):
url = url[:-1]
if url.endswith('.git'):
print "A github URL should point to the project, not the git repo"
sys.exit(1)
projecttype = 'github'
repo = url + '.git'
repotype = 'git'
sourcecode = url
elif url.startswith('https://gitorious.org/'):
projecttype = 'gitorious'
repo = 'https://git.gitorious.org/' + url[22:] + '.git'
repotype = 'git'
sourcecode = url
elif url.startswith('https://bitbucket.org/'):
if url.endswith('/'):
url = url[:-1]
projecttype = 'bitbucket'
sourcecode = url + '/src'
issuetracker = url + '/issues'
# Figure out the repo type and adddress...
repotype, repo = getrepofrompage(sourcecode)
if not repotype:
print "Unable to determine vcs type. " + repo
sys.exit(1)
elif url.startswith('http://code.google.com/p/'):
if not url.endswith('/'):
url += '/';
projecttype = 'googlecode'
sourcecode = url + 'source/checkout'
if options.repo:
sourcecode += "?repo=" + options.repo
issuetracker = url + 'issues/list'
# Figure out the repo type and adddress...
repotype, repo = getrepofrompage(sourcecode)
if not repotype:
print "Unable to determine vcs type. " + repo
sys.exit(1)
# Figure out the license...
req = urllib.urlopen(url)
if req.getcode() != 200:
print 'Unable to find project page at ' + sourcecode + ' - return code ' + str(req.getcode())
sys.exit(1)
page = req.read()
index = page.find('Code license')
if index == -1:
print "Couldn't find license data"
sys.exit(1)
ltext = page[index:]
lprefix = 'rel="nofollow">'
index = ltext.find(lprefix)
if index == -1:
print "Couldn't find license text"
sys.exit(1)
ltext = ltext[index + len(lprefix):]
index = ltext.find('<')
if index == -1:
print "License text not formatted as expected"
sys.exit(1)
ltext = ltext[:index]
if ltext == 'GNU GPL v3':
license = 'GPLv3'
elif ltext == 'GNU GPL v2':
license = 'GPLv2'
elif ltext == 'Apache License 2.0':
license = 'Apache2'
elif ltext == 'MIT License':
license = 'MIT'
elif ltext == 'GNU Lesser GPL':
license = 'LGPL'
elif ltext == 'Mozilla Public License 1.1':
license = 'MPL'
elif ltext == 'New BSD License':
license = 'NewBSD'
else:
print "License " + ltext + " is not recognised"
sys.exit(1)
if not projecttype:
print "Unable to determine the project type."
print "The URL you supplied was not in one of the supported formats. Please consult"
print "the manual for a list of supported formats, and supply one of those."
sys.exit(1)
# Get a copy of the source so we can extract some info...
print 'Getting source from ' + repotype + ' repo at ' + repo
src_dir = os.path.join(tmp_dir, 'importer')
if os.path.exists(src_dir):
shutil.rmtree(src_dir)
vcs = common.getvcs(repotype, repo, src_dir, sdk_path)
vcs.gotorevision(None)
if options.subdir:
root_dir = os.path.join(src_dir, options.subdir)
else:
root_dir = src_dir
# Check AndroidManiifest.xml exists...
if not os.path.exists(root_dir + '/AndroidManifest.xml'):
print "AndroidManifest.xml did not exist in the expected location. Specify --subdir?"
sys.exit(1)
# Extract some information...
version, vercode, package = common.parse_androidmanifest(root_dir)
if not package:
print "Couldn't find package ID"
sys.exit(1)
if not version:
print "Couldn't find latest version name"
sys.exit(1)
if not vercode:
print "Couldn't find latest version code"
sys.exit(1)
# Make sure it's actually new...
for app in apps:
if app['id'] == package:
print "Package " + package + " already exists"
sys.exit(1)
# Construct the metadata...
app = common.parse_metadata(None)
app['id'] = package
app['Web Site'] = website
app['Source Code'] = sourcecode
if issuetracker:
app['Issue Tracker'] = issuetracker
if license:
app['License'] = license
app['Repo Type'] = repotype
app['Repo'] = repo
# Create a build line...
build = {}
build['version'] = version
build['vercode'] = vercode
build['commit'] = '?'
if options.subdir:
build['subdir'] = options.subdir
if os.path.exists(os.path.join(root_dir, 'jni')):
build['buildjni'] = 'yes'
app['builds'].append(build)
app['comments'].append(('build:' + version,
"#Generated by import.py - check this is the right version, and find the right commit!"))
# Keep the repo directory to save bandwidth...
if not os.path.exists('build'):
os.mkdir('build')
shutil.move(src_dir, os.path.join('build', package))
metafile = os.path.join('metadata', package + '.txt')
common.write_metadata(metafile, app)
print "Wrote " + metafile
def main():
global config, options
# Parse command line...
parser = OptionParser()
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="Spew out even more information than normal")
parser.add_option("-q",
"--quiet",
action="store_true",
default=False,
help="Restrict output to warnings and errors")
parser.add_option("-u",
"--url",
default=None,
help="Project URL to import from.")
parser.add_option(
"-s",
"--subdir",
default=None,
help="Path to main android project subdirectory, if not in root.")
parser.add_option(
"--rev",
default=None,
help=
"Allows a different revision (or git branch) to be specified for the initial import"
)
(options, args) = parser.parse_args()
config = common.read_config(options)
if not options.url:
logging.error("Specify project url.")
sys.exit(1)
url = options.url
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
logging.info("Creating temporary directory")
os.makedirs(tmp_dir)
# Get all apps...
apps = metadata.read_metadata()
# Figure out what kind of project it is...
projecttype = None
issuetracker = None
license = None
website = url # by default, we might override it
if url.startswith('git://'):
projecttype = 'git'
repo = url
repotype = 'git'
sourcecode = ""
website = ""
elif url.startswith('https://github.com'):
projecttype = 'github'
repo = url
repotype = 'git'
sourcecode = url
issuetracker = url + '/issues'
website = ""
elif url.startswith('https://gitlab.com/'):
projecttype = 'gitlab'
repo = url
repotype = 'git'
sourcecode = url + '/tree/HEAD'
issuetracker = url + '/issues'
elif url.startswith('https://bitbucket.org/'):
if url.endswith('/'):
url = url[:-1]
projecttype = 'bitbucket'
sourcecode = url + '/src'
issuetracker = url + '/issues'
# Figure out the repo type and adddress...
repotype, repo = getrepofrompage(sourcecode)
if not repotype:
logging.error("Unable to determine vcs type. " + repo)
sys.exit(1)
if not projecttype:
logging.error("Unable to determine the project type.")
logging.error(
"The URL you supplied was not in one of the supported formats. Please consult"
)
logging.error(
"the manual for a list of supported formats, and supply one of those."
)
sys.exit(1)
# Ensure we have a sensible-looking repo address at this point. If not, we
# might have got a page format we weren't expecting. (Note that we
# specifically don't want git@...)
if ((repotype != 'bzr' and
(not repo.startswith('http://') and not repo.startswith('https://')
and not repo.startswith('git://'))) or ' ' in repo):
logging.error(
"Repo address '{0}' does not seem to be valid".format(repo))
sys.exit(1)
# Get a copy of the source so we can extract some info...
logging.info('Getting source from ' + repotype + ' repo at ' + repo)
src_dir = os.path.join(tmp_dir, 'importer')
if os.path.exists(src_dir):
shutil.rmtree(src_dir)
vcs = common.getvcs(repotype, repo, src_dir)
vcs.gotorevision(options.rev)
if options.subdir:
root_dir = os.path.join(src_dir, options.subdir)
else:
root_dir = src_dir
# Extract some information...
paths = common.manifest_paths(root_dir, [])
if paths:
version, vercode, package = common.parse_androidmanifests(paths)
if not package:
logging.error("Couldn't find package ID")
sys.exit(1)
if not version:
logging.warn("Couldn't find latest version name")
if not vercode:
logging.warn("Couldn't find latest version code")
else:
spec = os.path.join(root_dir, 'buildozer.spec')
if os.path.exists(spec):
defaults = {
'orientation': 'landscape',
'icon': '',
'permissions': '',
'android.api': "18"
}
bconfig = ConfigParser(defaults, allow_no_value=True)
bconfig.read(spec)
package = bconfig.get('app', 'package.domain') + '.' + bconfig.get(
'app', 'package.name')
version = bconfig.get('app', 'version')
vercode = None
else:
logging.error(
"No android or kivy project could be found. Specify --subdir?")
sys.exit(1)
# Make sure it's actually new...
if package in apps:
logging.error("Package " + package + " already exists")
sys.exit(1)
# Construct the metadata...
app = metadata.parse_metadata(None)[1]
app['Web Site'] = website
app['Source Code'] = sourcecode
if issuetracker:
app['Issue Tracker'] = issuetracker
if license:
app['License'] = license
app['Repo Type'] = repotype
app['Repo'] = repo
app['Update Check Mode'] = "Tags"
# Create a build line...
build = {}
build['version'] = version or '?'
build['vercode'] = vercode or '?'
build['commit'] = '?'
build[
'disable'] = 'Generated by import.py - check/set version fields and commit id'
if options.subdir:
build['subdir'] = options.subdir
if os.path.exists(os.path.join(root_dir, 'jni')):
build['buildjni'] = ['yes']
for flag, value in metadata.flag_defaults.iteritems():
if flag in build:
continue
build[flag] = value
app['builds'].append(build)
# Keep the repo directory to save bandwidth...
if not os.path.exists('build'):
os.mkdir('build')
shutil.move(src_dir, os.path.join('build', package))
with open('build/.fdroidvcs-' + package, 'w') as f:
f.write(repotype + ' ' + repo)
metafile = os.path.join('metadata', package + '.txt')
metadata.write_metadata(metafile, app)
logging.info("Wrote " + metafile)
