def build_jtharness(top_dir, tag=None):
work_dir = join(top_dir, 'jtharness_work')
hg_dir = join(work_dir, 'jtharness')
build_dir = join(hg_dir, 'build')
mkdir(work_dir)
chdir(work_dir)
# clone the jtharness mercurial repository
hg_clone(jtharness_repo)
chdir(hg_dir)
if tag is None:
# find the latest tag
tag = get_latest_hg_tag('jt')
hg_switch_tag(tag)
print(str.format('Using jtharness tag {0}', tag))
# download and extract dependencies
for jtharness_dependecy in jtharness_dependencies:
utils.download_artifact(jtharness_dependecy[0], jtharness_dependecy[1])
utils.extract_archive(jtharness_dependecy[1], build_dir)
move(join('build', 'jh2.0', 'javahelp', 'lib', 'jhall.jar'), build_dir)
move(join('build', 'jh2.0', 'javahelp', 'lib', 'jh.jar'), build_dir)
chdir(build_dir)
# create build properties
build_properties = 'local.properties'
with open(build_properties, 'w') as properties:
properties.write('jhalljar = ./build/jhall.jar\n')
properties.write('jhjar = ./build/jh.jar\n')
properties.write('jcommjar = ./build/comm.jar\n')
properties.write('servletjar = ./build/servlet-api.jar\n')
properties.write(
'bytecodelib = ./build/asm-3.1.jar:./build/asm-commons-3.1.jar\n')
properties.write('junitlib = ./build/junit-4.4.jar\n')
properties.write('BUILD_DIR = ./JTHarness-build\n')
# run the ant build
utils.run_cmd([
'ant', 'build', '-propertyfile', build_properties,
'-Djvmargs="-Xdoclint:none"', '-debug'
])
# copy the archive
bundles = os.listdir(join(hg_dir, 'JTHarness-build', 'bundles'))
bundle_pattern = re.compile('jtharness-([0-9]+\.[0-9]+)\.zip')
for bundle in bundles:
match = bundle_pattern.match(bundle)
if match is not None:
jtharness_version = match.group(1)
copy(join(hg_dir, 'JTHarness-build', 'bundles', bundle),
join(top_dir, 'jtharness.zip'))
return jtharness_version
def _extract(self, context):
context.try_skip(self.extract_path)
if os.path.exists(self.extract_path):
shutil.rmtree(self.extract_path)
extract_archive(pj(self.buildEnv.archive_dir, self.archive.name),
self.buildEnv.source_dir,
topdir=self.archive_top_dir,
name=self.source_dir)
def _download(self):
download_dir = get_download_dir()
zip_file_path = os.path.join(
download_dir, "{}.zip".format(self.ds_name))
# TODO move to dgl host _get_dgl_url
download(_url, path=zip_file_path)
extract_dir = os.path.join(
download_dir, "{}".format(self.ds_name))
extract_archive(zip_file_path, extract_dir)
return extract_dir
def download(self) -> None:
"""Download the EMNIST data if it doesn't exist in processed_folder already."""
import shutil
if self._check_exists():
return
os.makedirs(self.raw_folder, exist_ok=True)
os.makedirs(self.processed_folder, exist_ok=True)
# download files
print('Downloading and extracting zip archive')
download_and_extract_archive(self.url,
download_root=self.raw_folder,
filename="emnist.zip",
remove_finished=True,
md5=self.md5)
gzip_folder = os.path.join(self.raw_folder, 'gzip')
for gzip_file in os.listdir(gzip_folder):
if gzip_file.endswith('.gz'):
extract_archive(os.path.join(gzip_folder, gzip_file),
gzip_folder)
# process and save as torch files
for split in self.splits:
print('Processing ' + split)
training_set = (
read_image_file(
os.path.join(
gzip_folder,
'emnist-{}-train-images-idx3-ubyte'.format(split))),
read_label_file(
os.path.join(
gzip_folder,
'emnist-{}-train-labels-idx1-ubyte'.format(split))))
test_set = (
read_image_file(
os.path.join(
gzip_folder,
'emnist-{}-test-images-idx3-ubyte'.format(split))),
read_label_file(
os.path.join(
gzip_folder,
'emnist-{}-test-labels-idx1-ubyte'.format(split))))
with open(
os.path.join(self.processed_folder,
self._training_file(split)), 'wb') as f:
torch.save(training_set, f)
with open(
os.path.join(self.processed_folder,
self._test_file(split)), 'wb') as f:
torch.save(test_set, f)
shutil.rmtree(gzip_folder)
print('Done!')
def download_serv_config(self):
configs_fold = Path(cfg['configs_fold']).expanduser()
configs_fold.mkdir(parents=True, exist_ok=True)
config_archive_path: Path = configs_fold / self.server['config_name']
config_path = configs_fold / config_archive_path.stem
if not self.force_dl_config and config_path.exists():
return
print("Downloading server config.")
r = self.sess.get(self.server['config_url'])
with open(config_archive_path, 'wb') as f:
f.write(r.content)
extract_archive(config_archive_path)
print("Saved config to", configs_fold / config_path)
def test_extract_archive(self):
"""
Check if `extract_archive` function correctly extracts a given
.zip archive.
The function is expected to extract the contents of the
archive to a single folder located in the same directory as
the archive. The name of the folder is expected to be
identical to that of the archive without the `.zip` extension.
To test the function, this method creates a temporary file of
size 1KB, adds it to a .zip archive, and then runs the
function on the archive.
"""
with self.__create_temp_file('.txt', 1024) as temp_file:
test_zip = self.__create_test_zip(
self.file_prefix+'.zip', temp_file.name)
folder = extract_archive(os.path.join(self.upload_dir, test_zip))
self.assertTrue(os.path.exists(folder))
self.assertEquals(
folder, os.path.join(self.upload_dir, self.file_prefix))
def main(argv=None):
parser = argparse.ArgumentParser()
parser.add_argument('-m', '--major', help='the SapMachine major version to build', metavar='MAJOR', required=True)
parser.add_argument('-d', '--destination', help='the download destination', metavar='DIR', required=True)
args = parser.parse_args()
boot_jdk_major_max = int(args.major)
boot_jdk_major_min = boot_jdk_major_max - 1
destination = os.path.realpath(args.destination)
releases = utils.github_api_request('releases', per_page=100)
platform = str.format('{0}-{1}_bin', utils.get_system(), utils.get_arch())
for release in releases:
if release['prerelease']:
continue
version, version_part, major, build_number, sap_build_number, os_ext = utils.sapmachine_tag_components(release['name'])
if major is None:
continue
major = int(major)
if major <= boot_jdk_major_max and major >= boot_jdk_major_min:
assets = release['assets']
for asset in assets:
asset_name = asset['name']
asset_url = asset['browser_download_url']
if 'jdk' in asset_name and platform in asset_name and not asset_name.endswith('.txt'):
archive_path = join(destination, asset_name)
utils.remove_if_exists(archive_path)
utils.download_artifact(asset_url, archive_path)
boot_jdk_exploded = join(destination, 'boot_jdk')
utils.remove_if_exists(boot_jdk_exploded)
os.makedirs(boot_jdk_exploded)
utils.extract_archive(archive_path, boot_jdk_exploded)
sapmachine_folder = glob.glob(join(boot_jdk_exploded, 'sapmachine*'))
if sapmachine_folder is not None:
sapmachine_folder = sapmachine_folder[0]
files = os.listdir(sapmachine_folder)
for f in files:
shutil.move(join(sapmachine_folder, f), boot_jdk_exploded)
utils.remove_if_exists(sapmachine_folder)
if utils.get_system() == 'osx':
files = os.listdir(join(boot_jdk_exploded, 'Contents', 'Home'))
for f in files:
shutil.move(join(boot_jdk_exploded, 'Contents', 'Home', f), boot_jdk_exploded)
utils.remove_if_exists(join(boot_jdk_exploded, 'Contents'))
return 0
return 0
def main(argv=None):
parser = argparse.ArgumentParser()
parser.add_argument('-m',
'--major',
help='the SapMachine major version to build',
metavar='MAJOR',
required=True)
parser.add_argument('-d',
'--destination',
help='the download destination',
metavar='DIR',
required=True)
args = parser.parse_args()
boot_jdk_major_max = int(args.major)
boot_jdk_major_min = boot_jdk_major_max - 1
destination = os.path.realpath(args.destination)
releases = utils.get_github_releases()
platform = str.format('{0}-{1}_bin', utils.get_system(), utils.get_arch())
retries = 2
releases = extra_bootjdks + releases
while retries > 0:
for release in releases:
if release['prerelease']:
continue
tag = SapMachineTag.from_string(release['name'])
if tag is None:
print(
str.format("SapMachine release {0} not recognized",
release['name']))
continue
major = tag.get_major()
if major <= boot_jdk_major_max and major >= boot_jdk_major_min:
assets = release['assets']
for asset in assets:
asset_name = asset['name']
asset_url = asset['browser_download_url']
if 'jdk' in asset_name and platform in asset_name and (
asset_name.endswith('.tar.gz')
or asset_name.endswith('.zip')
) and 'symbols' not in asset_name:
archive_path = join(destination, asset_name)
utils.remove_if_exists(archive_path)
utils.download_artifact(asset_url, archive_path)
boot_jdk_exploded = join(destination, 'boot_jdk')
utils.remove_if_exists(boot_jdk_exploded)
os.makedirs(boot_jdk_exploded)
utils.extract_archive(archive_path, boot_jdk_exploded)
sapmachine_folder = [
f for f_ in [
glob.glob(join(boot_jdk_exploded, e))
for e in ('sapmachine*', 'jdk*')
] for f in f_
]
if sapmachine_folder is not None:
sapmachine_folder = sapmachine_folder[0]
files = os.listdir(sapmachine_folder)
for f in files:
shutil.move(join(sapmachine_folder, f),
boot_jdk_exploded)
utils.remove_if_exists(sapmachine_folder)
if utils.get_system() == 'osx':
files = os.listdir(
join(boot_jdk_exploded, 'Contents',
'Home'))
for f in files:
shutil.move(
join(boot_jdk_exploded, 'Contents',
'Home', f), boot_jdk_exploded)
utils.remove_if_exists(
join(boot_jdk_exploded, 'Contents'))
return 0
retries -= 1
if retries == 1:
boot_jdk_major_min = boot_jdk_major_max - 2
return 0
def main(argv=None):
parser = argparse.ArgumentParser()
parser.add_argument('-t', '--tag', help='the tag to create the debian packages from', metavar='TAG', required=True)
parser.add_argument('-d', '--templates-directory', help='specify the templates directory', metavar='DIR', required=True)
args = parser.parse_args()
templates_dir = realpath(args.templates_directory)
tag = args.tag
if tag.endswith('-alpine'):
# the "-alpine" tags do not contain any assets
tag = tag[:-len('-alpine')]
cwd = os.getcwd()
work_dir = join(cwd, 'deb_work')
version, version_part, major, build_number, sap_build_number, os_ext = utils.sapmachine_tag_components(tag)
version = version.replace('-', '.')
jdk_name = str.format('sapmachine-{0}-jdk-{1}', major, version)
jre_name = str.format('sapmachine-{0}-jre-{1}', major, version)
jdk_url, jre_url = utils.fetch_tag(tag, 'linux-x64', utils.get_github_api_accesstoken())
utils.remove_if_exists(work_dir)
mkdir(work_dir)
jdk_archive = join(work_dir, jdk_url.rsplit('/', 1)[-1])
jre_archive = join(work_dir, jre_url.rsplit('/', 1)[-1])
utils.download_artifact(jdk_url, jdk_archive)
utils.download_artifact(jre_url, jre_archive)
clone_sapmachine(join(work_dir, 'sapmachine_master'))
src_dir = join(work_dir, 'sapmachine_master')
jdk_dir = join(work_dir, jdk_name)
jre_dir = join(work_dir, jre_name)
mkdir(jdk_dir)
mkdir(jre_dir)
utils.extract_archive(jdk_archive, jdk_dir)
utils.extract_archive(jre_archive, jre_dir)
env = os.environ.copy()
env['DEBFULLNAME'] = 'SapMachine'
env['DEBEMAIL'] = '*****@*****.**'
utils.run_cmd(['dh_make', '-n', '-s', '-y'], cwd=jdk_dir, env=env)
utils.run_cmd(['dh_make', '-n', '-s', '-y'], cwd=jre_dir, env=env)
jre_exploded_image = glob.glob(join(jre_dir, 'sapmachine-*'))[0]
generate_configuration(
templates_dir=join(templates_dir, 'jre'),
major=major,
target_dir=join(jre_dir, 'debian'),
exploded_image=jre_exploded_image,
src_dir=src_dir,
download_url=jre_url)
jdk_exploded_image = glob.glob(join(jdk_dir, 'sapmachine-*'))[0]
generate_configuration(
templates_dir=join(templates_dir, 'jdk'),
major=major,
target_dir=join(jdk_dir, 'debian'),
exploded_image=jdk_exploded_image,
src_dir=src_dir,
download_url=jdk_url)
utils.run_cmd(['debuild', '-b', '-uc', '-us'], cwd=jre_dir, env=env)
utils.run_cmd(['debuild', '-b', '-uc', '-us'], cwd=jdk_dir, env=env)
deb_files = glob.glob(join(work_dir, '*.deb'))
for deb_file in deb_files:
copy(deb_file, cwd)
remove(deb_file)
def main(argv=None):
parser = argparse.ArgumentParser()
parser.add_argument('-a',
'--asset',
help='the SapMachine asset file',
metavar='ASSET',
required=True)
parser.add_argument('-j',
'--jre',
help='Build SapMachine JRE installer',
action='store_true',
default=False)
parser.add_argument('-s',
'--sapmachine-directory',
help='specify the SapMachine GIT directory',
metavar='DIR',
required=True)
args = parser.parse_args()
cwd = os.getcwd()
work_dir = join(cwd, 'msi_work')
asset = os.path.realpath(args.asset)
is_jre = args.jre
sapmachine_git_dir = args.sapmachine_directory
products = None
product_id = None
upgrade_code = None
utils.remove_if_exists(work_dir)
os.makedirs(work_dir)
utils.extract_archive(asset, work_dir, remove_archive=False)
sapmachine_folder = glob.glob(join(work_dir, 'sapmachine*'))
os.rename(sapmachine_folder[0], join(work_dir, 'SourceDir'))
_, _, version_output = utils.run_cmd(
[join(work_dir, 'SourceDir', 'bin', 'java.exe'), '-version'], std=True)
version, version_part, major, version_sap, build_number = utils.sapmachine_version_components(
version_output, multiline=True)
sapmachine_version = [e for e in version_part.split('.')]
if len(sapmachine_version) < 3:
sapmachine_version += [
'0' for sapmachine_version in range(0, 3 - len(sapmachine_version))
]
if len(sapmachine_version) == 4:
sapmachine_version[3] = str((int(sapmachine_version[3]) << 8) & 0xFF00)
if len(sapmachine_version) == 5:
merged_version = str((int(sapmachine_version[3]) << 8)
| (int(sapmachine_version[4]) & 0xFF))
del sapmachine_version[4]
sapmachine_version[3] = merged_version
sapmachine_version = '.'.join(sapmachine_version)
shutil.copyfile(
join(sapmachine_git_dir, 'src', 'java.base', 'windows', 'native',
'launcher', 'icons', 'awt.ico'), join(work_dir, 'sapmachine.ico'))
write_as_rtf(join(sapmachine_git_dir, 'LICENSE'),
join(work_dir, 'license.rtf'))
infrastructure_dir = join(work_dir, 'sapmachine_infrastructure')
templates_dir = join(infrastructure_dir, 'wix-templates')
utils.git_clone('github.com/SAP/SapMachine-infrastructure', 'master',
infrastructure_dir)
with open(join(templates_dir, 'products.yml'), 'r') as products_yml:
products = yaml.safe_load(products_yml.read())
image_type = 'jre' if is_jre else 'jdk'
if products[image_type] is None or major not in products[image_type]:
product_id = str(uuid.uuid4())
upgrade_code = str(uuid.uuid4())
if products[image_type] is None:
products[image_type] = {}
products[image_type][major] = {
'product_id': product_id,
'upgrade_code': upgrade_code
}
with open(join(templates_dir, 'products.yml'), 'w') as products_yml:
products_yml.write(yaml.dump(products, default_flow_style=False))
utils.git_commit(infrastructure_dir, 'Updated product codes.',
[join('wix-templates', 'products.yml')])
utils.git_push(infrastructure_dir)
else:
product_id = products[image_type][major]['product_id']
upgrade_code = products[image_type][major]['upgrade_code']
create_sapmachine_wxs(
join(
templates_dir, 'SapMachine.jre.wxs.template'
if is_jre else 'SapMachine.jdk.wxs.template'),
join(work_dir, 'SapMachine.wxs'), product_id, upgrade_code,
sapmachine_version, major)
shutil.copyfile(join(work_dir, 'SourceDir', 'release'),
join(work_dir, 'release'))
utils.remove_if_exists(join(work_dir, 'SourceDir', 'release'))
utils.run_cmd(
'heat dir SourceDir -swall -srd -gg -platform x64 -template:module -cg SapMachineGroup -out SapMachineModule.wxs'
.split(' '),
cwd=work_dir)
with open(join(work_dir, 'SapMachineModule.wxs'),
'r+') as sapmachine_module:
sapmachine_module_content = sapmachine_module.read()
sapmachine_module_content = sapmachine_module_content.replace(
'PUT-MODULE-NAME-HERE', 'SapMachineModule')
sapmachine_module_content = sapmachine_module_content.replace(
'PUT-COMPANY-NAME-HERE', 'SapMachine Team')
sapmachine_module.seek(0)
sapmachine_module.truncate()
sapmachine_module.write(sapmachine_module_content)
utils.run_cmd('candle -arch x64 SapMachineModule.wxs'.split(' '),
cwd=work_dir)
utils.run_cmd('light SapMachineModule.wixobj'.split(' '), cwd=work_dir)
utils.run_cmd('candle -arch x64 SapMachine.wxs'.split(' '), cwd=work_dir)
utils.run_cmd('light -ext WixUIExtension SapMachine.wixobj'.split(' '),
cwd=work_dir)
msi_name = os.path.basename(asset)
msi_name = os.path.splitext(msi_name)[0]
os.rename(join(work_dir, 'SapMachine.msi'),
join(cwd, str.format('{0}.msi', msi_name)))
return 0
def main(argv=None):
parser = argparse.ArgumentParser()
parser.add_argument('-t', '--tag', help='the tag to create the debian packages from', metavar='TAG', required=True)
args = parser.parse_args()
tag = args.tag
cwd = os.getcwd()
work_dir = join(cwd, 'rpm_work')
version, version_part, major, update, version_sap, build_number, os_ext = utils.sapmachine_tag_components(tag)
version = version.replace('-', '.')
jdk_name = str.format('sapmachine-jdk-{0}', version)
jdk_url, jre_url = utils.get_asset_url(tag, 'linux-x64')
utils.remove_if_exists(work_dir)
mkdir(work_dir)
jdk_archive = join(work_dir, jdk_url.rsplit('/', 1)[-1])
utils.download_artifact(jdk_url, jdk_archive)
utils.extract_archive(jdk_archive, work_dir)
bin_dir = join(work_dir, jdk_name, 'bin')
tools = [f for f in listdir(bin_dir) if isfile(join(bin_dir, f))]
alternatives = []
alternatives_t = Template(alternatives_template)
for tool in tools:
alternatives.append(alternatives_t.substitute(tool=tool, major=major))
alternatives = '\n'.join(alternatives)
specfile_t = Template(spec_template)
specfile_content = specfile_t.substitute(
version=version,
major=major,
alternatives=alternatives,
workdir=work_dir
)
with open(join(work_dir, 'sapmachine.spec'), 'w') as specfile:
specfile.write(specfile_content)
rpmbuild_dir = join(work_dir, 'rpmbuild')
mkdir(rpmbuild_dir)
rpmbuild_cmd = str.format('rpmbuild -bb -v --buildroot={0}/BUILD {0}/sapmachine.spec', work_dir)
rpmbuild_cmd = rpmbuild_cmd.split(' ')
rpmbuild_cmd.append('--define')
rpmbuild_cmd.append(str.format('_rpmdir {0}', work_dir))
rpmbuild_cmd.append('--define')
rpmbuild_cmd.append(str.format('_topdir {0}', rpmbuild_dir))
utils.run_cmd(rpmbuild_cmd, cwd=work_dir)
rpm_files = glob.glob(join(work_dir, 'x86_64', '*.rpm'))
for rpm_file in rpm_files:
copy(rpm_file, cwd)
remove(rpm_file)
return 0
def build_jtreg(top_dir, jtharness_version, tag=None, build_number=None):
work_dir = join(top_dir, 'jtreg_work')
hg_dir = join(work_dir, 'jtreg')
build_dir = join(hg_dir, 'build')
dependencies_dir = join(hg_dir, 'dependencies')
images_dir = join(hg_dir, 'build', 'images')
mkdir(work_dir)
chdir(work_dir)
# clone the jtreg mercurial repository
hg_clone(jtreg_repo)
chdir(hg_dir)
mkdir(dependencies_dir)
if tag is None:
# find the latest tag
tag = get_latest_hg_tag('jtreg')
if build_number is None:
build_number = tag.split('-')[1]
else:
if build_number is None:
build_number = 'b01'
hg_switch_tag(tag)
print(str.format('Using jtreg tag {0}', tag))
# download and extract dependencies
for jtreg_dependecy in jtreg_dependencies:
utils.download_artifact(jtreg_dependecy[0], jtreg_dependecy[1])
utils.extract_archive(jtreg_dependecy[1], dependencies_dir)
# workaround for jtreg.gmk JAVAHELP_JAR rule
with open('DUMMY.SF', 'w+') as dummy:
dummy.write('dummy')
with zipfile.ZipFile(
join(dependencies_dir, 'jh2.0', 'javahelp', 'lib', 'jh.jar'),
'a') as java_help:
java_help.write('DUMMY.SF', join('META-INF', 'DUMMY.SF'))
utils.extract_archive(join(top_dir, 'jtharness.zip'), dependencies_dir)
copytree(join(top_dir, 'asmtools-release'),
join(dependencies_dir, 'asmtools'))
# build configuration
javac = dirname(dirname(realpath(utils.which('javac'))))
ant = dirname(dirname(realpath(utils.which('ant'))))
make_build_env = os.environ.copy()
make_build_env['JDK17HOME'] = javac
make_build_env['JDK18HOME'] = javac
make_build_env['JDKHOME'] = javac
make_build_env['ANTHOME'] = ant
make_build_env['ASMTOOLS_HOME'] = join(dependencies_dir, 'asmtools')
make_build_env['JAVAHELP_HOME'] = join(dependencies_dir, 'jh2.0',
'javahelp')
make_build_env['JTHARNESS_HOME'] = join(dependencies_dir,
'jtharness-' + jtharness_version)
make_build_env['TESTNG_JAR'] = join(dependencies_dir, 'testng.jar')
make_build_env['JUNIT_JAR'] = join(dependencies_dir, 'junit.jar')
make_build_env['JCOV_JAR'] = join(dependencies_dir, 'JCOV_BUILD',
'jcov_3.0', 'jcov.jar')
make_build_env['JCOV_NETWORK_SAVER_JAR'] = join(dependencies_dir,
'JCOV_BUILD', 'jcov_3.0',
'jcov_network_saver.jar')
make_build_env['JCOMMANDER_JAR'] = join(dependencies_dir,
'jcommander-1.48.jar')
# run make
utils.run_cmd(['make', '-C', 'make', 'BUILD_NUMBER=' + build_number],
env=make_build_env)
# add additional libraries to the archive
# with zipfile.ZipFile(join(images_dir, 'jtreg.zip'), 'a') as jtreg_archive:
# jtreg_archive.write(join(dependencies_dir, 'jcommander-1.48.jar'), join('jtreg', 'lib', 'jcommander.jar'))
# jtreg_archive.write(join(dependencies_dir, 'testng.jar'), join('jtreg', 'lib', 'testng.jar'))
# copy the build result
copy(join(images_dir, 'jtreg.zip'), top_dir)