def svn_remote(parentdir, scope='session'):
"""Create a git repo with 1 commit, used as a remote."""
server_dirname = 'server_dir'
server_dir = parentdir.join(server_dirname)
run(['svnadmin', 'create', str(server_dir)])
return str(server_dir)
def svn_dummy_repo_dir(tmpdir_repoparent, scope='session'):
"""Create a git repo with 1 commit, used as a remote."""
server_dirname = 'server_dir'
server_dir = tmpdir_repoparent.join(server_dirname)
run(['svnadmin', 'create', str(server_dir)])
return str(server_dir)
def svn_remote(parentdir, scope='session'):
"""Create a git repo with 1 commit, used as a remote."""
server_dirname = 'server_dir'
server_dir = parentdir.join(server_dirname)
run(['svnadmin', 'create', str(server_dir)])
return str(server_dir)
def fn(repo_name, testfile_filename='testfile.test'):
repo_path = str(tmpdir_repoparent.join(repo_name))
run(['git', 'init', repo_name], cwd=str(tmpdir_repoparent))
run(['touch', testfile_filename], cwd=repo_path)
run(['git', 'add', testfile_filename], cwd=repo_path)
run(['git', 'commit', '-m', 'test file for %s' % repo_name], cwd=repo_path)
return repo_path
def test_repo_git_obtain_initial_commit_repo(tmpdir):
"""initial commit repos return 'initial'.
note: this behaviors differently from git(1)'s use of the word "bare".
running `git rev-parse --is-bare-repository` would return false.
"""
repo_name = 'my_git_project'
run(['git', 'init', repo_name], cwd=str(tmpdir))
bare_repo_dir = tmpdir.join(repo_name)
git_repo = create_repo_from_pip_url(
**{
'pip_url': 'git+file://' + str(bare_repo_dir),
'repo_dir': str(tmpdir.join('obtaining a bare repo')),
})
git_repo.obtain()
assert git_repo.get_revision() == 'initial'
def test_repo_mercurial(tmpdir, parentdir, hg_remote):
repo_name = 'my_mercurial_project'
mercurial_repo = create_repo_from_pip_url(
**{
'pip_url': 'hg+file://' + hg_remote,
'repo_dir': str(parentdir.join(repo_name)),
})
run(['hg', 'init', mercurial_repo.name], cwd=str(tmpdir))
mercurial_repo.update_repo()
test_repo_revision = run(
['hg', 'parents', '--template={rev}'],
cwd=str(parentdir.join(repo_name)),
)
assert mercurial_repo.get_revision() == test_repo_revision
assert os.path.exists(str(tmpdir.join(repo_name)))
def test_progress_callback(tmpdir, git_remote, mocker):
def progress_callback_spy(output, timestamp):
assert isinstance(output, string_types)
assert isinstance(timestamp, datetime.datetime)
progress_callback = mocker.Mock(name='progress_callback_stub',
side_effect=progress_callback_spy)
run(['git', 'rev-parse', 'HEAD'], cwd=git_remote)
# create a new repo with the repo as a remote
git_repo = create_repo_from_pip_url(
**{
'pip_url': 'git+file://' + git_remote,
'repo_dir': str(tmpdir.join('myrepo')),
'progress_callback': progress_callback,
})
git_repo.obtain()
assert progress_callback.called
def test_repo_mercurial(tmpdir, parentdir, hg_remote):
repo_name = 'my_mercurial_project'
mercurial_repo = create_repo_from_pip_url(
**{
'pip_url': 'hg+file://' + hg_remote,
'repo_dir': str(parentdir.join(repo_name)),
}
)
run(['hg', 'init', mercurial_repo.name], cwd=str(tmpdir))
mercurial_repo.update_repo()
test_repo_revision = run(
['hg', 'parents', '--template={rev}'], cwd=str(parentdir.join(repo_name))
)
assert mercurial_repo.get_revision() == test_repo_revision
assert os.path.exists(str(tmpdir.join(repo_name)))
def test_repo_git_obtain_initial_commit_repo(tmpdir):
"""initial commit repos return 'initial'.
note: this behaviors differently from git(1)'s use of the word "bare".
running `git rev-parse --is-bare-repository` would return false.
"""
repo_name = 'my_git_project'
run(['git', 'init', repo_name], cwd=str(tmpdir))
bare_repo_dir = tmpdir.join(repo_name)
git_repo = create_repo_from_pip_url(
**{
'pip_url': 'git+file://' + str(bare_repo_dir),
'repo_dir': str(tmpdir.join('obtaining a bare repo')),
}
)
git_repo.obtain()
assert git_repo.get_revision() == 'initial'
def git_dummy_repo_dir(tmpdir_repoparent, scope='session'):
"""Create a git repo with 1 commit, used as a remote."""
name = 'dummyrepo'
repo_path = str(tmpdir_repoparent.join(name))
run(['git', 'init', name], cwd=str(tmpdir_repoparent))
testfile_filename = 'testfile.test'
run(['touch', testfile_filename], cwd=repo_path)
run(['git', 'add', testfile_filename], cwd=repo_path)
run(['git', 'commit', '-m', 'test file for %s' % name], cwd=repo_path)
return repo_path
def hg_remote(parentdir, scope='session'):
"""Create a git repo with 1 commit, used as a remote."""
name = 'dummyrepo'
repo_path = str(parentdir.join(name))
run(['hg', 'init', name], cwd=str(parentdir))
testfile_filename = 'testfile.test'
run(['touch', testfile_filename], cwd=repo_path)
run(['hg', 'add', testfile_filename], cwd=repo_path)
run(['hg', 'commit', '-m', 'test file for %s' % name], cwd=repo_path)
return repo_path
def hg_remote(parentdir, scope='session'):
"""Create a git repo with 1 commit, used as a remote."""
name = 'dummyrepo'
repo_path = str(parentdir.join(name))
run(['hg', 'init', name], cwd=str(parentdir))
testfile_filename = 'testfile.test'
run(['touch', testfile_filename], cwd=repo_path)
run(['hg', 'add', testfile_filename], cwd=repo_path)
run(['hg', 'commit', '-m', 'test file for %s' % name], cwd=repo_path)
return repo_path
def test_progress_callback(tmpdir, git_remote, mocker):
def progress_callback_spy(output, timestamp):
assert isinstance(output, string_types)
assert isinstance(timestamp, datetime.datetime)
progress_callback = mocker.Mock(
name='progress_callback_stub', side_effect=progress_callback_spy
)
run(['git', 'rev-parse', 'HEAD'], cwd=git_remote)
# create a new repo with the repo as a remote
git_repo = create_repo_from_pip_url(
**{
'pip_url': 'git+file://' + git_remote,
'repo_dir': str(tmpdir.join('myrepo')),
'progress_callback': progress_callback,
}
)
git_repo.obtain()
assert progress_callback.called
def test_repo_git_obtain_full(tmpdir, git_remote):
git_repo = create_repo_from_pip_url(
**{
'pip_url': 'git+file://' + git_remote,
'repo_dir': str(tmpdir.join('myrepo')),
})
git_repo.obtain()
test_repo_revision = run(['git', 'rev-parse', 'HEAD'], cwd=git_remote)
assert git_repo.get_revision() == test_repo_revision
assert os.path.exists(str(tmpdir.join('myrepo')))
def test_repo_git_obtain_full(tmpdir, git_remote):
git_repo = create_repo_from_pip_url(
**{
'pip_url': 'git+file://' + git_remote,
'repo_dir': str(tmpdir.join('myrepo')),
}
)
git_repo.obtain()
test_repo_revision = run(['git', 'rev-parse', 'HEAD'], cwd=git_remote)
assert git_repo.get_revision() == test_repo_revision
assert os.path.exists(str(tmpdir.join('myrepo')))
def buildCode(build_target, build_arch, nproc, force, dry, pack_sources, single, tar):
log_info_triple("Building ", build_target, " for {} architecture ...".format(build_arch))
version_string = datetime.now().strftime("%Y%m%d")
build_order = createBuildOrder(build_target, build_arch, getArchitecture(), True)
pos = 0
if single:
t = build_order[-1]
arch = t[0]
target = targets[t[1]]
deps = target.dependencies
if build_target == target.name and target.top_package:
res = set()
for d in build_order:
dep = targets[d[1]]
if (dep and dep.resources):
for r in dep.resources:
res.add(r)
deps += list(res)
target_build_order = []
target_build_order.append(tuple((build_arch,build_target)))
total_pos = len(target_build_order) + len(deps)
for d in deps:
pos += 1
dep = targets[d]
needed = True
if dep.arch and arch not in dep.arch:
needed = False
if needed:
build_info = ""
dep_arch = arch
if (dep.build_native and build_arch != getArchitecture()):
dep_arch = getArchitecture()
build_info = " [" + dep_arch + "]"
output_dir = os.path.join(OUTPUTS_ROOT, dep_arch, dep.name)
hash_file = os.path.join(output_dir, '.hash')
log_info_triple("Step [{:2d}/{:2d}] loading hash ".format(pos,total_pos), dep.name + build_info)
if dry:
continue
if (os.path.exists(hash_file)):
dep.hash = open(hash_file, 'r').read()
else:
log_error("Missing hash file for {} does not exist.".format(dep.name + build_info))
else:
target_build_order = build_order
total_pos = len(target_build_order)
for t in target_build_order:
pos += 1
arch = t[0]
target = targets[t[1]]
target.hash = calculateHash(target, arch, build_order)
build_info = ""
if (build_arch != arch):
build_info = " [" + arch + "]"
output_dir = os.path.join(OUTPUTS_ROOT, arch, target.name)
forceBuild = force
for dep in sorted(target.dependencies):
forceBuild = forceBuild or targets[dep].built
hash_file = os.path.join(output_dir, '.hash')
if (not forceBuild and os.path.exists(hash_file)):
if target.hash == open(hash_file, 'r').read():
log_info_triple("Step [{:2d}/{:2d}] skipping ".format(pos, total_pos), target.name + build_info)
continue
log_info_triple("Step [{:2d}/{:2d}] building ".format(pos, total_pos), target.name + build_info)
if dry:
continue
log_step("Remove old output dir ...")
if os.path.exists(output_dir):
shutil.rmtree(output_dir, onerror=removeError)
log_step("Creating output dir ...")
os.makedirs(output_dir)
build_dir = os.path.join(BUILDS_ROOT, arch, target.name)
if not target.top_package:
log_step("Remove old build dir ...")
if os.path.exists(build_dir):
shutil.rmtree(build_dir, onerror=removeError)
log_step("Creating build dir ...")
os.makedirs(build_dir)
for s in target.sources:
src_dir = os.path.join(SOURCES_ROOT, s)
log_step_triple("Copy '", s, "' source to build dir ...")
run(['rsync','-a', src_dir, build_dir])
deps = target.dependencies
if t[1] == target.name and target.top_package:
res = set()
for d in build_order:
dep = targets[d[1]]
if (dep and dep.resources):
for r in dep.resources:
res.add(r)
deps += list(res)
prefix = "/yosyshq"
packages = set()
for d in deps:
dep = targets[d]
needed = True
if dep.arch and arch not in dep.arch:
needed = False
if needed:
dep_build_info = ""
if (dep.build_native and build_arch != getArchitecture()):
dep_build_info = " [" + getArchitecture() + "]"
dep_dir = os.path.join(OUTPUTS_ROOT, getArchitecture(), d)
else:
dep_dir = os.path.join(OUTPUTS_ROOT, arch, d)
if not os.path.exists(dep_dir):
log_error("Dependency output directory for {} does not exist.".format(d + dep_build_info))
if not target.top_package:
log_step_triple("Copy '", d + dep_build_info, "' output to build dir ...")
run(['rsync','-a', dep_dir, build_dir])
else:
if (dep.package):
packages.add(dep.package)
log_step_triple("Copy '", d + dep_build_info, "' output to package dir ...")
run(['rsync','-a', dep_dir+"/", output_dir])
if target.top_package:
version_meta = dict({ 'branding': target.branding, 'product': target.release_name, 'arch': arch, 'version': version_string, 'package_name': target.release_name + "-" + arch + "-" + version_string})
package_meta = dict.fromkeys(sorted(list(packages)))
tools_meta = dict.fromkeys(sorted(list(deps)))
for key in package_meta:
package_meta[key] = dict({'size': 0, 'files' : [], 'installed' : True })
for key in tools_meta:
tools_meta[key] = dict({'files' : [], 'active' : True, 'package' : None })
for d in deps:
dep = targets[d]
needed = True
if dep.arch and arch not in dep.arch:
needed = False
if dep.tools is not None:
for key in dep.tools:
tools_meta[key] = dict({'files' : dep.tools[key], 'active' : True, 'package' : dep.package })
continue
if needed:
if (dep.build_native and build_arch != getArchitecture()):
dep_dir = os.path.join(OUTPUTS_ROOT, getArchitecture(), d)
else:
dep_dir = os.path.join(OUTPUTS_ROOT, arch, d)
if dep.package:
package_meta[dep.package]['size'] += get_size(dep_dir + prefix)
for root, _, files in sorted(os.walk(dep_dir)):
for filename in sorted(files):
fpath = os.path.join(root, filename).replace(dep_dir,"")
if fpath.startswith(prefix):
name = fpath.replace("/yosyshq/","")
if name.startswith("bin/"):
tools_meta[dep.name]['files'].append(name[4:])
tools_meta[dep.name]['package'] = dep.package
if dep.package:
package_meta[dep.package]['files'].append(name)
if name.startswith("bin/") and arch != 'windows-x64':
package_meta[dep.package]['files'].append("libexec" + name[3:])
metadata = dict({'version' : version_meta, 'packages' : package_meta, 'tools' : tools_meta })
with open(os.path.join(output_dir, "yosyshq", "share", "manifest.json"), "w") as manifest_file:
json.dump(metadata, manifest_file)
code = executeBuild(target, arch, prefix, build_dir if not target.top_package else output_dir, output_dir, nproc, pack_sources)
if code!=0:
log_error("Script returned error code {}.".format(code))
if target.license_file is not None or target.license_url is not None:
log_step("Generating license file ...")
license_dir = os.path.join(output_dir + prefix, "license")
os.makedirs(license_dir)
license_file = os.path.join(license_dir, "LICENSE." + target.name)
with open(license_file, 'w') as f:
f.write("YosysHQ embeds '{}' in its distribution bundle.\n".format(target.name))
f.write("\nBuild is based on folowing sources:\n")
f.write('=' * 80 + '\n')
for s in target.sources:
f.write("{} {} checkout revision {}\n".format(sources[s].vcs, sources[s].location, sources[s].hash))
f.write("\nFollowing files are included:\n")
f.write('=' * 80 + '\n')
for root, _, files in sorted(os.walk(output_dir)):
for filename in sorted(files):
fpath = os.path.join(root, filename).replace(output_dir,"")
if not fpath.startswith("/dev"):
f.write(fpath.replace("/yosyshq/","") + '\n')
f.write("\nSoftware is under following license :\n")
f.write('=' * 80 + '\n')
if target.license_url is not None:
log_step("Retrieving license file for {}...".format(target.name))
try:
with urllib.request.urlopen(target.license_url) as lf:
f.write(lf.read().decode('utf-8'))
except urllib.error.URLError as e:
log_error(str(e))
if target.license_file is not None:
with open(os.path.join(build_dir, target.license_file), 'r') as lf:
f.write(lf.read())
f.write('\n' + '=' * 80 + '\n')
if target.top_package:
if arch == 'windows-x64':
package_name = target.release_name + "-" + arch + "-" + version_string +".exe"
log_step("Packing {} ...".format(package_name))
os.replace(os.path.join(output_dir, "yosyshq"), os.path.join(output_dir, target.release_name))
create_exe(package_name, target.release_name, output_dir)
else:
package_name = target.release_name + "-" + arch + "-" + version_string +".tgz"
log_step("Packing {} ...".format(package_name))
os.replace(os.path.join(output_dir, "yosyshq"), os.path.join(output_dir, target.release_name))
create_tar(package_name, target.release_name, output_dir)
log_step("Marking build finished ...")
with open(hash_file, 'w') as f:
f.write(target.hash)
target.built = True
if tar:
package_name = arch + "-" + target.name +".tgz"
log_step("Packing {} ...".format(package_name))
create_tar(package_name, output_dir, ".")
if not target.top_package:
log_step("Remove build dir ...")
if os.path.exists(build_dir):
shutil.rmtree(build_dir, onerror=removeError)
