def get_version(rel_url, ext_path, refs):
if 'tag' in refs:
return
bare_svn = False
if git_externals[rel_url]["vcs"] == "svn":
revision = command('svnversion', '-c').strip()
match = re_from_svnversion.search(revision)
if match:
revision = "svn:r" + match.group(2) # 565:56555 -> svn:r56555
bare_svn = True
else:
message = git("log", "--format=%b", "--grep", "git-svn-id:", "-1")
match = re_from_git_svn_id.search(message)
if match:
revision = "svn:r" + match.group(1)
else:
here = os.path.relpath(os.getcwd(), repo_root)
error("Unsupported external format, svn or git-svn repo expected:\n\t{}".format(here))
else:
branch_name = current_branch()
remote_name = git("config", "branch.%s.remote" % branch_name)
revision = git("log", "%s/%s" % (remote_name, branch_name), "-1", "--format=%H")
info("Freeze {0} at {1}".format(rel_url, revision))
if messages and not bare_svn:
old = resolve_revision(git_externals[rel_url]["ref"])
new = resolve_revision(revision)
git("log", "--format=- %h %s", "{}..{}".format(old, new), capture=False)
git_externals[rel_url]["ref"] = revision
def gitext_update(recursive, gitsvn, reset):
"""Update the working copy cloning externals if needed and create the desired layout using symlinks
"""
from git_externals import externals_sanity_check, root_path, is_workingtree_clean, foreach_externals, gitext_up
externals_sanity_check()
root = root_path()
if reset:
git('reset', '--hard')
# Aggregate in a list the `clean flags` of all working trees (root + externals)
clean = [is_workingtree_clean(root, fail_on_empty=False)]
foreach_externals(root,
lambda u, p, r: clean.append(
is_workingtree_clean(p, fail_on_empty=False)),
recursive=recursive)
if reset or all(clean):
# Proceed with update if everything is clean
try:
gitext_up(recursive, reset=reset, use_gitsvn=gitsvn)
except ProgError as e:
error(str(e), exitcode=e.errcode)
else:
echo(
"Cannot perform git externals update because one or more repositories contain some local modifications"
)
echo("Run:\tgit externals status\tto have more information")
def get_version(rel_url, ext_path, refs):
if 'tag' in refs:
return
if git_externals[rel_url]["vcs"] == "svn":
revision = command('svnversion', '-c').strip()
match = re_from_svnversion.search(revision)
if match:
revision = "svn:r" + match.group(2) # 565:56555 -> svn:r56555
else:
message = git("log", "--format=%b", "--grep", "git-svn-id:",
"-1")
match = re_from_git_svn_id.search(message)
if match:
revision = "svn:r" + match.group(1)
else:
error(
"Unsupported external format, should be svn or git-svn repo"
)
else:
branch_name = current_branch()
remote_name = git("config", "branch.%s.remote" % branch_name)
revision = git("log", "%s/%s" % (remote_name, branch_name), "-1",
"--format=%H")
info("Freeze {0} at {1}".format(rel_url, revision))
git_externals[rel_url]["ref"] = revision
def gitext_update(recursive, gitsvn, reset):
"""Update the working copy cloning externals if needed and create the desired layout using symlinks
"""
from git_externals import externals_sanity_check, root_path, is_workingtree_clean, foreach_externals, gitext_up
externals_sanity_check()
root = root_path()
if reset:
git('reset', '--hard')
# Aggregate in a list the `clean flags` of all working trees (root + externals)
clean = [is_workingtree_clean(root, fail_on_empty=False)]
foreach_externals(root,
lambda u, p, r: clean.append(is_workingtree_clean(p, fail_on_empty=False)),
recursive=recursive)
if reset or all(clean):
# Proceed with update if everything is clean
try:
gitext_up(recursive, reset=reset, use_gitsvn=gitsvn)
except ProgError as e:
error(str(e), exitcode=e.errcode)
else:
echo("Cannot perform git externals update because one or more repositories contain some local modifications")
echo("Run:\tgit externals status\tto have more information")
def build(self):
if self.args.target is None:
print('No default target expecting something like "x86_64-unknown-elf"')
return 1
self.args.installPrefixDir = '{}/{}'.format(
self.args.installPrefixDir, self.args.target)
dst_dir = os.path.join(self.args.installPrefixDir, 'bin')
os.makedirs(dst_dir, exist_ok=True)
retval = 0
self.args.app = '{}-{}'.format(self.args.target, self.args.app)
try:
dst = os.path.join(dst_dir, self.args.app)
output = subprocess.check_output([dst, '--version'],
stderr=subprocess.STDOUT)
if output is None:
output = b''
except BaseException as err:
output = b''
if not self.args.forceInstall and bytes(self.args.ver, 'utf-8') in output:
print('{app} {ver} is already installed'
.format(app=self.args.app, ver=self.args.ver))
else:
print('compiling {app} {ver}'
.format(app=self.args.app, ver=self.args.ver))
code_dir = os.path.join(self.args.codePrefixDir,
'{}/{}'.format(self.args.crossDir, self.args.target))
if self.args.forceInstall:
shutil.rmtree(code_dir, ignore_errors=True)
os.makedirs(code_dir)
if self.args.target == X86_64_TARGET or self.args.target == I386_TARGET:
# Get gcc from git which supports attribute(interrupt)
# gcc_path must match GCC_CUSTOM_LOCATION in config.x86_64-unknown-elf
gcc_path = os.path.expanduser(os.path.join('~/prgs', 'ct-ng-gcc'))
shutil.rmtree(gcc_path, ignore_errors=True)
print('gcc_install: gcc_path=', gcc_path)
utils.git('clone', [GCC_GIT_REPO_URL, gcc_path, '--depth', '1', '--single-branch',
'--branch', CHECKOUT_LABEL])
os.chdir(gcc_path)
thisDir = os.path.dirname(os.path.realpath(__file__))
src = os.path.abspath('{}/config.{}'.format(thisDir, self.args.target))
dst = os.path.abspath('{}/.config'.format(code_dir))
print('config src=', src)
print('config dst=', dst)
shutil.copy2(src, dst)
os.chdir(code_dir)
# Build and install app's
subprocess.check_call(['ct-ng', 'build.4'])
return retval
def get_status(rel_url, ext_path, targets):
try:
if porcelain:
echo(rel_url)
click.echo(git('status', '--porcelain'))
elif verbose or not is_workingtree_clean(ext_path):
info("External {}".format(get_repo_name(rel_url)))
echo(git('status', '--untracked-files=no' if not verbose else ''))
except CommandError as err:
error(str(err), exitcode=err.errcode)
def get_status(rel_url, ext_path, targets):
try:
if porcelain:
echo(rel_url)
click.echo(git('status', '--porcelain'))
elif verbose or not is_workingtree_clean(ext_path):
info("External {}".format(get_repo_name(rel_url)))
echo(
git('status',
'--untracked-files=no' if not verbose else ''))
except CommandError as err:
error(str(err), exitcode=err.errcode)
def install(self):
dst_dir = os.path.join(self.args.installPrefixDir, 'bin')
os.makedirs(dst_dir, exist_ok=True)
retval = 0
try:
theApp = AN_APP
if self.args.target != '':
theApp = '{}-{}'.format(self.args.target, theApp)
theApp = os.path.join(dst_dir, theApp)
output = subprocess.check_output([theApp, '--version'],
stderr=subprocess.STDOUT)
if output is None:
output = b''
except BaseException as err:
output = b''
if not self.args.forceInstall and bytes(self.args.ver, 'utf-8') in output:
print('{app} {ver} is already installed'
.format(app=self.args.app, ver=self.args.ver))
else:
print('compiling {app} {ver}'
.format(app=self.args.app, ver=self.args.ver))
code_dir = self.args.codePrefixDir
if self.args.target != '':
code_dir = os.path.join(code_dir, self.args.target)
code_dir = os.path.join(code_dir, self.args.app)
if self.args.forceInstall:
shutil.rmtree(code_dir, ignore_errors=True)
os.makedirs(code_dir)
utils.git('clone', [URL, code_dir])
os.chdir(code_dir)
version = self.args.ver.replace('.','_')
utils.git('checkout', ['binutils-{}'.format(version)])
os.mkdir('build')
os.chdir('build')
configureCmd = '../configure --prefix={} --disable-nls'.format(
self.args.installPrefixDir)
if self.args.target != '':
configureCmd += ' --target={}'.format(self.args.target)
subprocess.run(configureCmd,
shell=True,
stdout=subprocess.DEVNULL) # Too much logging overflows 4MB travis-ci log limit
subprocess.run('make all -j {}'.format(multiprocessing.cpu_count()),
shell=True,
stdout=subprocess.DEVNULL) # Too much logging overflows 4MB travis-ci log limit
utils.bash('make install')
return retval
def __init__(self):
print('---> saving to github')
git(['pull', '--allow-unrelated-history', 'origin', 'dev'])
git(['add', '.'])
git(['commit', '-m', 'octoblog-save'])
git(['push', 'origin', 'dev'])
print('---> done')
def install(self):
dst_dir = os.path.join(self.args.installPrefixDir, 'bin')
os.makedirs(dst_dir, exist_ok=True)
retval = 0
try:
dst = os.path.join(dst_dir, self.args.app)
output = subprocess.check_output([dst, 'version'],
stderr=subprocess.STDOUT)
if output is None:
output = b''
except BaseException as err:
output = b''
if not self.args.forceInstall and bytes(self.args.ver, 'utf-8') in output:
print('{app} {ver} is already installed'
.format(app=self.args.app, ver=self.args.ver))
else:
print('compiling {app} {ver}'
.format(app=self.args.app, ver=self.args.ver))
code_dir = os.path.join(self.args.codePrefixDir, self.args.app)
if self.args.forceInstall:
shutil.rmtree(code_dir, ignore_errors=True)
os.makedirs(code_dir)
utils.git('clone', [URL, code_dir])
os.chdir(code_dir)
if CHECKOUT_SHA1:
checkout_ver = self.args.ver
else:
checkout_ver = 'crosstool-ng-{}'.format(self.args.ver)
utils.git('checkout', [checkout_ver])
subprocess.check_call(['./bootstrap'])
subprocess.check_call(['./configure', '--prefix={}'.format(self.args.installPrefixDir)])
subprocess.check_call(['make'])
subprocess.check_call(['make', 'install'])
return retval
def get_version(rel_url, ext_path, refs):
if 'tag' in refs:
return
bare_svn = False
if git_externals[rel_url]["vcs"] == "svn":
revision = command('svnversion', '-c').strip()
match = re_from_svnversion.search(revision)
if match:
revision = "svn:r" + match.group(2) # 565:56555 -> svn:r56555
bare_svn = True
else:
message = git("log", "--format=%b", "--grep", "git-svn-id:",
"-1")
match = re_from_git_svn_id.search(message)
if match:
revision = "svn:r" + match.group(1)
else:
here = os.path.relpath(os.getcwd(), repo_root)
error(
"Unsupported external format, svn or git-svn repo expected:\n\t{}"
.format(here))
else:
branch_name = current_branch()
remote_name = git("config", "branch.%s.remote" % branch_name)
revision = git("log", "%s/%s" % (remote_name, branch_name), "-1",
"--format=%H")
info("Freeze {0} at {1}".format(rel_url, revision))
if messages and not bare_svn:
old = resolve_revision(git_externals[rel_url]["ref"])
new = resolve_revision(revision)
git("log",
"--format=- %h %s",
"{}..{}".format(old, new),
capture=False)
git_externals[rel_url]["ref"] = revision
print('gcc-install: mpfr_path=', mpfr_path)
mpc_path = os.path.join(os.path.dirname(args.o.src), 'mpc')
print('gcc-install: mpc_path=', mpc_path)
try:
utils.wget_extract(GMP_URL, dst_path=gmp_path)
utils.wget_extract(MPFR_URL, dst_path=mpfr_path)
utils.wget_extract(MPC_URL, dst_path=mpc_path)
if True:
# Use wget as its faster
utils.wget_extract(GCC_URL.format(args.o.ver), dst_path=args.o.src)
os.chdir(args.o.src)
else:
# Use git, but this was slower
os.makedirs(args.o.src, exist_ok=True)
utils.git('clone', [GCC_GIT_REPO_URL, args.o.src])
os.chdir(args.o.src)
utils.git('checkout', [CHECKOUT_LABEL])
# Create the build directory and cd into it
os.mkdir('build')
os.chdir('build')
except BaseException as ex:
traceback.print_exc()
exit(1)
# ls the directory with gcc, gmp, mpfr and mpc for debug purposes
#utils.bash('ls -al {}'.format(os.path.dirname(args.o.src)))
try:
print('gcc-install: configure')
output = subprocess.check_output([dst, '--version'])
if output is None:
output = b''
except BaseException as err:
output = b''
if bytes(args.o.ver, 'utf-8') in output:
print('{app} {ver} is already installed'.format(app=args.o.src, ver=VER))
exit(0)
else:
print('compiling {app} {ver}'.format(app=args.o.src, ver=VER))
os.makedirs(args.o.src, exist_ok=True)
try:
url = 'git://sourceware.org/git/binutils-gdb.git'
utils.git('clone', [url, args.o.src])
os.chdir(args.o.src)
version = VER.replace('.','_')
utils.git('checkout', ['binutils-{ver}'.format(ver=version)])
os.mkdir('build')
os.chdir('build')
except BaseException as ex:
traceback.print_exc()
exit(1)
print('configure')
utils.bash('../configure --prefix={0} --target={1} --disable-nls'
.format(args.o.prefix, TARGET))
utils.bash('make all -j {}'.format(multiprocessing.cpu_count()))
utils.bash('make install')
def enable_colored_output():
from git_externals import externals_root_path, get_entries
for entry in get_entries():
with chdir(os.path.join(externals_root_path(), entry)):
git('config', 'color.ui', 'always')
output = subprocess.check_output([dst, '--version'])
if output is None:
output = b''
except BaseException as err:
output = b''
if bytes(args.o.ver, 'utf-8') in output:
print('{app} {ver} is already installed'.format(app=args.o.src, ver=VER))
exit(0)
else:
print('compiling {app} {ver}'.format(app=args.o.src, ver=VER))
os.makedirs(args.o.src, exist_ok=True)
try:
url = 'git://git.qemu.org/qemu.git'
utils.git('clone', [url, args.o.src])
os.chdir(args.o.src)
utils.git('checkout', ['v{ver}'.format(ver=VER)])
utils.git('submodule', ['update', '--init', 'dtc'])
os.mkdir('build')
os.chdir('build')
except BaseException as ex:
traceback.print_exc()
exit(1)
print('configure')
utils.bashPython2(
'../configure --prefix={} --target-list=arm-softmmu,arm-linux-user'
.format(args.o.prefix))
utils.bashPython2('make -j {}'.format(multiprocessing.cpu_count()))
utils.bashPython2('make install')
def gitext_diff(external):
"""Call git diff on the given externals"""
from git_externals import iter_externals
for _ in iter_externals(external):
click.echo(git('diff'))
def __init__(self):
git(['checkout', 'master'])
git(['commit', '--allow-empty', '-m', 'octoblog-publish notification'])
git(['checkout', 'dev'])
print('---> generating blog to www/ directory')
clear_dir('www')
config = json.load(open('config.json', 'r'))
generate_html(config, 'www')
print('---> done')
SaveCommand()
print('---> pushing contents of www/ to master')
git(['checkout', 'master'])
clear_dir(os.getcwd())
git(['checkout', 'dev', '--', 'www'])
flatten('www')
git(['add', '.'])
git(['commit', '-m', '"octoblog-publish files"'])
git(['push', 'origin', 'master'])
git(['checkout', 'dev'])
print('---> done')
def _fresh_blog_push(self, remote):
git(['init'])
git(['add', '.gitignore'])
git(['commit', '-m', '"initial commit"'])
git(['remote', 'add', 'origin', remote])
git(['push', '-u', 'origin', 'master'])
git(['branch', 'dev'])
git(['checkout', 'dev'])
git(['add', '.'])
git(['commit', '-m', '"initial commit"'])
git(['push', '-u', 'origin', 'dev'])
def __init__(self, remote):
git(['clone', remote, '.'])
git(['checkout', 'dev'])
def gitext_diff(external):
"""Call git diff on the given externals"""
from git_externals import iter_externals
for _ in iter_externals(external):
click.echo(git('diff'))
try:
output = subprocess.check_output([dst, "--version"])
if output is None:
output = b""
except BaseException as err:
output = b""
if bytes(args.o.ver, "utf-8") in output:
print("{app} {ver} is already installed".format(app=APP, ver=args.o.ver))
exit(0)
else:
print("compiling {app} {ver}".format(app=APP, ver=args.o.ver))
url = "https://github.com/martine/ninja.git"
os.makedirs(args.o.src, exist_ok=True)
utils.git("clone", [url, args.o.src])
os.chdir(args.o.src)
try:
subprocess.check_call(["./configure.py", "--bootstrap"])
except:
traceback.print_exc()
exit(1)
dst = os.path.abspath(args.o.prefix + "/bin")
os.makedirs(dst, exist_ok=True)
dst = os.path.abspath(dst + "/{app}".format(app=APP))
try:
shutil.copy2("./{app}".format(app=APP), dst)
except OSError as err:
traceback.print_exc()
def enable_colored_output():
from git_externals import externals_root_path, get_entries
for entry in get_entries():
with chdir(os.path.join(externals_root_path(), entry)):
git('config', 'color.ui', 'always')
def install(self):
dst_dir = os.path.join(self.args.installPrefixDir, 'bin')
os.makedirs(dst_dir, exist_ok=True)
retval = 0
self.parseUnknownArgs()
try:
theApp = AN_APP
if self.args.target != '':
theApp = '{}-{}'.format(self.args.target, theApp)
theApp = os.path.join(dst_dir, theApp)
output = subprocess.check_output([theApp, '--version'],
stderr=subprocess.STDOUT)
if output is None:
output = b''
except BaseException as err:
output = b''
if not self.args.forceInstall and bytes(self.args.ver, 'utf-8') in output:
print('{app} {ver} is already installed'
.format(app=self.args.app, ver=self.args.ver))
else:
code_dir = self.args.codePrefixDir
if self.args.target != '':
code_dir = os.path.join(code_dir, self.args.target)
gmp_path = os.path.join(code_dir, 'gmp')
print('gcc_install: gmp_path=', gmp_path)
mpfr_path = os.path.join(code_dir, 'mpfr')
print('gcc_install: mpfr_path=', mpfr_path)
mpc_path = os.path.join(code_dir, 'mpc')
print('gcc_install: mpc_path=', mpc_path)
#isl does't compile and its optional so don't add
#isl_path = os.path.join(code_dir, 'isl')
#print('gcc_install: isl_path=', isl_path)
gcc_path = os.path.join(code_dir, 'gcc')
print('gcc_install: gcc_path=', gcc_path)
#envVars = ('LDFLAGS=-L/home/wink/opt/lib' +
# ' CPPFLAGS=-I/home/wink/opt/include')
envVars = ''
# If desired make false when testing so we don't download.
if True:
if self.args.forceInstall:
shutil.rmtree(gmp_path, ignore_errors=True)
shutil.rmtree(mpfr_path, ignore_errors=True)
shutil.rmtree(mpc_path, ignore_errors=True)
#shutil.rmtree(isl_path, ignore_errors=True)
shutil.rmtree(gcc_path, ignore_errors=True)
utils.wget_extract(GMP_URL, dst_path=gmp_path)
utils.wget_extract(MPFR_URL, dst_path=mpfr_path)
utils.wget_extract(MPC_URL, dst_path=mpc_path)
#utils.wget_extract(ISL_URL, dst_path=isl_path)
if True:
# Use wget as its faster
utils.wget_extract(GCC_URL.format(self.args.ver), dst_path=gcc_path)
os.chdir(gcc_path)
else:
# Use git, but its slower
os.makedirs(self.args.codePrefixDir, exist_ok=True)
utils.git('clone', [GCC_GIT_REPO_URL, gcc_path])
os.chdir(gcc_path)
utils.git('checkout', [CHECKOUT_LABEL])
# Make the prerequisites (isl is optional)
#self.makePrerequisiteLibrary(envVars, isl_path, os.path.dirname(self.args.installPrefixDir))
# Make the prerequisites (Not needed if I install existing versions??)
#self.makePrerequisiteLibrary(envVars, gmp_path, os.path.dirname(self.args.installPrefixDir))
#self.makePrerequisiteLibrary(envVars, mpfr_path, os.path.dirname(self.args.installPrefixDir))
#self.makePrerequisiteLibrary(envVars, mpc_path, os.path.dirname(self.args.installPrefixDir))
# Create the build directory and cd into it
os.chdir(gcc_path)
os.makedirs('build', exist_ok=True)
os.chdir('build')
# ls the directory with gcc, gmp, mpfr and mpc for debug purposes
#utils.bash('ls -al {}'.format(os.path.dirname(self.args.codePrefixDir)))
cmd = ('{env}' +
' ../configure --prefix={prefix}' +
' --with-gmp={gmp}' +
' --with-gmp-include={gmp}' +
' --with-mpfr={mpfr}' +
' --with-mpfr-include={mpfr}/src' +
' --with-mpc={mpc}' +
' --with-mpc-include={mpc}/src' +
#' --with-isl={isl}' +
#' --with-isl-include={isl}/include' +
' --disable-nls ' +
' --enable-languages=c,c++' +
' --disable-multilib' +
' --without-headers').format(
env=envVars, prefix=self.args.installPrefixDir, gmp=gmp_path, mpfr=mpfr_path,
mpc=mpc_path) #, isl=isl_path)
if self.args.target != '':
cmd += ' --target={}'.format(self.args.target)
# Add extraFlags added on the command line
for extraFlag in self.extraArgs.extraGccConfigFlags:
cmd += ' {}'.format(extraFlag)
# Add extraFlags passed to the constructor
for extraFlag in self.extraFlags:
cmd += ' {}'.format(extraFlag)
self.runCmd(cmd)
cpu_count = multiprocessing.cpu_count()
cci = os.environ.get('CIRCLECI')
if cci != None and cci == 'true':
cpu_count = 4;
self.runCmd('{env} make all-gcc -j {c}'
.format(env=envVars, c=cpu_count), verbose=True)
self.runCmd('{env} make install-gcc'
.format(env=envVars))
self.runCmd('{env} make all-target-libgcc -j {c}'
.format(env=envVars, c=cpu_count))
self.runCmd('{env} make install-target-libgcc'
.format(env=envVars))
return 0