def generate() -> None:
authorities = get_targz_path(check_exists=False)
if os.path.exists(authorities):
command.fail("authorities.tgz already exists")
# tempfile.TemporaryDirectory() creates the directory with 0o600, which protects the private keys
with tempfile.TemporaryDirectory() as d:
certdir = os.path.join(d, "certdir")
keyserver_yaml = os.path.join(d, "keyserver.yaml")
util.writefile(keyserver_yaml, configuration.get_keyserver_yaml().encode())
os.mkdir(certdir)
print("generating authorities...")
try:
# TODO: avoid having these touch disk
subprocess.check_call(["keygen", keyserver_yaml, certdir, "supervisor-nodes"])
except FileNotFoundError as e:
if e.filename == "keygen":
command.fail("could not find keygen binary. is the homeworld-keyserver dependency installed?")
else:
raise e
print("encrypting authorities...")
cryptdir = os.path.join(d, "cryptdir")
os.mkdir(cryptdir)
for filename in os.listdir(certdir):
if filename.endswith(".pub") or filename.endswith(".pem"):
# public keys; copy over without encryption
util.copy(os.path.join(certdir, filename), os.path.join(cryptdir, filename))
else:
# private keys; encrypt when copying
keycrypt.gpg_encrypt_file(os.path.join(certdir, filename), os.path.join(cryptdir, filename))
subprocess.check_call(["shred", "--"] + os.listdir(certdir), cwd=certdir)
print("packing authorities...")
subprocess.check_call(["tar", "-C", cryptdir, "-czf", authorities, "."])
subprocess.check_call(["shred", "--"] + os.listdir(cryptdir), cwd=cryptdir)
def link(self, args):
"""
Actually deploy the configuration by linking the correct spot to the original file.
@param args: The parsed arguments from the command-line.
"""
if args.dry:
print(str(self.source) + " -> " + str(self.destination))
return
if not self.src_exists:
return
if self.dst_exists_before:
if args.replace:
if self.dst_is_link_before:
util.unlink(self.destination)
elif self.dst_is_file_before:
util.remove(self.destination)
elif self.dst_is_dir_before:
util.remove_dir(self.destination)
else:
#SUS should never get here.
raise Exception("WTF is this shit")
else:
# File already exists and isn't going to be replaced.
return
else:
#This is some weird nonsense conserning broken links
if self.dst_is_link_before:
util.unlink(self.destination)
if args.copy:
util.copy(self.source, self.destination)
else:
util.link(self.source, self.destination)
def run():
npcs = {}
npc_pages = api.query_category("Monsters")
for name, page in npc_pages.items():
if name.startswith("Category:"):
continue
try:
code = mw.parse(page, skip_style_tags=True)
for (vid, version) in util.each_version("Infobox Monster", code):
doc = util.get_doc_for_id_string(name + str(vid), version,
npcs)
if doc == None:
continue
util.copy("name", doc, version, lambda x: x)
for key in ["hitpoints", "combat"]:
try:
util.copy(key, doc, version, lambda x: int(x))
except ValueError:
print("NPC {} has an non integer {}".format(name, key))
except (KeyboardInterrupt, SystemExit):
raise
except:
print("NPC {} failed:".format(name))
traceback.print_exc()
for npcId in copy.copy(npcs):
npc = npcs[npcId]
if not 'combat' in npc:
del npcs[npcId]
util.write_json("npcs.json", "npcs.min.json", npcs)
def export_https(name, keyout, certout):
keypath = os.path.join(configuration.get_project(),
"https.%s.key.crypt" % name)
certpath = os.path.join(configuration.get_project(), "https.%s.pem" % name)
keycrypt.gpg_decrypt_file(keypath, keyout)
util.copy(certpath, certout)
def import_plugin(self):
import util
directory = QtGui.QFileDialog.getExistingDirectory(
self,
QtGui.QApplication.translate(
"MainWindow", "Choose folder with plugin", None, QtGui.QApplication.UnicodeUTF8
),
util.curr_directory(),
QtGui.QFileDialog.ShowDirsOnly | QtGui.QFileDialog.DontUseNativeDialog,
)
if directory:
src = directory + "/"
dest = curr_directory() + "/plugins/"
util.copy(src, dest)
msgBox = QtGui.QMessageBox()
msgBox.setWindowTitle(
QtGui.QApplication.translate("MainWindow", "Restart Toxygen", None, QtGui.QApplication.UnicodeUTF8)
)
msgBox.setText(
QtGui.QApplication.translate(
"MainWindow", "Plugin will be loaded after restart", None, QtGui.QApplication.UnicodeUTF8
)
)
msgBox.exec_()
def export_https(name, keyout, certout):
"decrypt and export the HTTPS keypair for a particular server"
keypath = os.path.join(configuration.get_project(),
"https.%s.key.crypt" % name)
certpath = os.path.join(configuration.get_project(), "https.%s.pem" % name)
keycrypt.gpg_decrypt_file(keypath, keyout)
util.copy(certpath, certout)
def export_https(name, keyout, certout):
if name != setup.REGISTRY_HOSTNAME:
command.fail("unexpected https host: %s" % name)
keypath = os.path.join(configuration.get_project(),
"https.%s.key.crypt" % name)
certpath = os.path.join(configuration.get_project(), "https.%s.pem" % name)
keycrypt.gpg_decrypt_file(keypath, keyout)
util.copy(certpath, certout)
def import_https(name, keyfile, certfile):
check_pem_type(certfile, "CERTIFICATE")
check_pem_type(keyfile, "RSA PRIVATE KEY")
keypath = os.path.join(configuration.get_project(),
"https.%s.key.crypt" % name)
certpath = os.path.join(configuration.get_project(), "https.%s.pem" % name)
keycrypt.gpg_encrypt_file(keyfile, keypath)
util.copy(certfile, certpath)
def copyTests(list, dst) :
# For list of test specified in the 'list' file copies test files, exams, models etc from the folder of lication 'list' file to 'dst' folder
import util
from pathlib import Path
dst = Path(dst)
src = Path(list).parents[0]
if not dst.exists() : dst.mkdir(parents = True)
for file in listTestFiles(list) :
util.copy(Path(src, file), dst)
pass
pass
def copyTests(list, dst):
# For list of test specified in the 'list' file copies test files, exams, models etc from the folder of lication 'list' file to 'dst' folder
import util
from pathlib import Path
dst = Path(dst)
src = Path(list).parents[0]
if not dst.exists(): dst.mkdir(parents=True)
for file in listTestFiles(list):
util.copy(Path(src, file), dst)
pass
pass
def export_profile(self, new_path, use_new_path):
path = new_path + os.path.basename(self._path)
with open(self._path, 'rb') as fin:
data = fin.read()
with open(path, 'wb') as fout:
fout.write(data)
print('Profile exported successfully')
copy(self._directory + 'avatars', new_path + 'avatars')
if use_new_path:
self._path = new_path + os.path.basename(self._path)
self._directory = new_path
Settings.get_instance().update_path()
def import_https(name, keyfile, certfile):
if name != setup.REGISTRY_HOSTNAME:
command.fail("unexpected https host: %s" % name)
check_pem_type(certfile, "CERTIFICATE")
check_pem_type(keyfile, "RSA PRIVATE KEY")
keypath = os.path.join(configuration.get_project(),
"https.%s.key.crypt" % name)
certpath = os.path.join(configuration.get_project(), "https.%s.pem" % name)
keycrypt.gpg_encrypt_file(keyfile, keypath)
util.copy(certfile, certpath)
def export_profile(self, new_path, use_new_path):
path = new_path + os.path.basename(self._path)
with open(self._path, 'rb') as fin:
data = fin.read()
with open(path, 'wb') as fout:
fout.write(data)
print('Profile exported successfully')
copy(self._directory + 'avatars', new_path + 'avatars')
if use_new_path:
self._path = new_path + os.path.basename(self._path)
self._directory = new_path
Settings.get_instance().update_path()
def import_sm(self):
directory = QtGui.QFileDialog.getExistingDirectory(self,
QtGui.QApplication.translate("MainWindow",
'Choose folder with smiley pack',
None,
QtGui.QApplication.UnicodeUTF8),
curr_directory(),
QtGui.QFileDialog.ShowDirsOnly | QtGui.QFileDialog.DontUseNativeDialog)
if directory:
src = directory + '/'
dest = curr_directory() + '/smileys/' + os.path.basename(directory) + '/'
copy(src, dest)
def __call__(self, *args, **kwargs):
# iterate label-path dict to split these image labeled NORMAL_ and LESION
for tag, path in self.source.items():
images_lst = os.listdir(path)
train_nums = int(self.percentage * len(images_lst))
# iterate images list and copy these to responding directory(train and val) respectively
for idx, name in enumerate(images_lst):
abs_path = os.path.join(path, name)
# if less than targeted training image nums: copy to train/NORMAL_ or train/LESION
# else: copy to val/NORMAL_ or val/LESION(depend on the variable 'tag')
target_path = os.path.join(self.train_path, tag, name) if idx < train_nums \
else os.path.join(self.val_path, tag, name)
copy(abs_path, target_path)
print('copy %s to %s' % (abs_path, target_path))
def create_download_tool_trigger_test():
version_dir = "D:\crane_cus\crane_r1_rc_0001_20200529_142934"
dist_dir = r'\\sh2-filer02\Data\FP_RLS\crane_r1_rc'
board = "crane_evb_z2"
download_controller = DownloadToolController()
download_tool_dir = os.path.join(version_dir, board, "download_tool")
download_controller.update_download_tool()
_root_dir = os.path.join(version_dir, board, "cp_images")
_images = [os.path.join(_root_dir,_file) for _file in os.listdir(_root_dir)]
download_controller.prepare_download_tool(_images)
download_controller.release_zip(os.path.dirname(_root_dir))
download_controller.release_download_tool(os.path.basename(version_dir),
board, download_tool_dir)
release_dist = os.path.join(dist_dir, os.path.basename(version_dir))
copy(version_dir, release_dist)
auto_cus_build_cls.trigger_auto_test(release_dist, "evb_customer")
def import_plugin(self):
import util
directory = QtGui.QFileDialog.getExistingDirectory(self,
QtGui.QApplication.translate("MainWindow", 'Choose folder with plugin',
None,
QtGui.QApplication.UnicodeUTF8),
util.curr_directory(),
QtGui.QFileDialog.ShowDirsOnly | QtGui.QFileDialog.DontUseNativeDialog)
if directory:
src = directory + '/'
dest = curr_directory() + '/plugins/'
util.copy(src, dest)
msgBox = QtGui.QMessageBox()
msgBox.setWindowTitle(
QtGui.QApplication.translate("MainWindow", "Restart Toxygen", None, QtGui.QApplication.UnicodeUTF8))
msgBox.setText(
QtGui.QApplication.translate("MainWindow", 'Plugin will be loaded after restart', None,
QtGui.QApplication.UnicodeUTF8))
msgBox.exec_()
def run():
npcs = {}
npc_pages = api.query_category("Monsters")
for name, page in npc_pages.items():
if name.startswith("Category:"):
continue
try:
code = mw.parse(page, skip_style_tags=True)
for (vid, version) in util.each_version("Infobox Monster", code):
if "removal" in version and not str(
version["removal"]).strip().lower() in ["", "no"]:
continue
doc = util.get_doc_for_id_string(name + str(vid), version,
npcs)
if doc == None:
continue
util.copy("name", doc, version)
if not "name" in doc:
doc["name"] = name
scaling = util.has_template("Chambers of Xeric",
code) or util.has_template(
"Theatre of Blood", code)
if not scaling:
for key in ["hitpoints"]:
try:
util.copy(key, doc, version, lambda x: int(x))
except ValueError:
print("NPC {} has an non integer {}".format(
name, key))
except (KeyboardInterrupt, SystemExit):
raise
except:
print("NPC {} failed:".format(name))
traceback.print_exc()
util.write_json("npcs.json", "npcs.min.json", npcs)
def create(run_folder,tag=''):
''' create unique experiment folder '''
rf = Folder(run_folder)
#basename = 'e'+str(int(time.time()))+'_'+rf.name()
dstr = datetime.now().strftime('%Y%m%d_%H%M_%S')
basename = dstr+'_'+rf.name()+'_'+tag
name = basename
i = 1
while name in ezex.exfolder:
name = basename + '_' + str(i)
i = i+1
if i > 100:
raise RuntimeError('Could not create unique experiment folder')
# ezex.exp_folder[name] = Folder(create=True)
# exp = ezex.exp_folder[name]
# exp.copy(rf)
path = ezex.config['exfolder']+'/'+name
os.mkdir(path)
util.copy(run_folder,path,symlinks=True,ignore='.*')
return path
def iterate():
global word, guessed, guessHistory, hint, remaining, sequence
if guessHistory:
guessHistory.sort()
print(f"You've guessed: {', '.join(guessHistory)}")
print(f"""{guessed}
{remaining} guess{"es" if remaining != 1 else ""} remaining""")
guess = input("⇒ ")
while not guess:
guess = input("(hint: {}) ⇒ ".format(hint(guessed)))
guessed = util.copy(word, guessed, guess)
guessCost = cost(word, guess, guessHistory)
if not guess in guessHistory:
guessHistory.append(guess)
remaining -= guessCost
if guessCost > 0:
frame = sequence[-1]
if remaining < len(sequence):
frame = sequence[remaining]
print(frame)
checkWinLose()
def run():
limits = getLimits()
stats = {}
item_pages = api.query_category("Items")
for name, page in item_pages.items():
if name.startswith("Category:"):
continue
try:
code = mw.parse(page, skip_style_tags=True)
equips = {}
for (vid, version) in util.each_version("Infobox Bonuses", code):
doc = {}
equips[vid] = doc
slotID = str(version["slot"]).strip().lower()
doc["slot"] = slotIDs[slotID]
if slotID == "2h":
doc["is2h"] = True
for key in [
"astab", "aslash", "acrush", "amagic", "arange", "dstab", "dslash", "dcrush", "dmagic", "drange", "str",
"rstr", "mdmg", "prayer", ("speed", "aspeed")
]:
try:
util.copy(key, doc, version, lambda x: int(x))
except ValueError:
print("Item {} has an non integer {}".format(name, key))
for (vid, version) in util.each_version("Infobox Item", code):
if "removal" in version and str(version["removal"]).strip():
continue
doc = util.get_doc_for_id_string(name + str(vid), version, stats)
if doc == None:
continue
util.copy("name", doc, version)
if not "name" in doc:
doc["name"] = name
util.copy("quest", doc, version, lambda x: x.lower() != "no")
equipable = "equipable" in version and "yes" in str(version["equipable"]).strip().lower()
if "weight" in version:
strval = str(version["weight"]).strip()
if strval.endswith("kg"):
strval = strval[:-2].strip()
if strval != "":
red = WEIGHT_REDUCTION_EXTRACTOR.match(strval)
if red:
strval = red.group(2)
floatval = float(strval)
if floatval != 0:
doc["weight"] = floatval
equipVid = vid if vid in equips else -1 if -1 in equips else None
if equipVid != None:
if equipable or not "broken" in version["name"].lower():
if not equipable:
print("Item {} has Infobox Bonuses but not equipable".format(name))
doc["equipable"] = True
doc["equipment"] = equips[equipVid]
elif equipable:
print("Item {} has equipable but not Infobox Bonuses".format(name))
doc["equipable"] = True
doc["equipment"] = {}
itemName = name
if "gemwname" in version:
itemName = str(version["gemwname"]).strip()
elif "name" in version:
itemName = str(version["name"]).strip()
if itemName in limits:
doc['ge_limit'] = limits[itemName]
except (KeyboardInterrupt, SystemExit):
raise
except:
print("Item {} failed:".format(name))
traceback.print_exc()
util.write_json("stats.json", "stats.ids.min.json", stats)
def preserve(self, prompt=lambda x, y: True):
if prompt(self.destination, self.reponame):
util.copy(self.destination, self.reponame)
def copySdkReverseImp(dst, sourcecode, sourcebinaries, testfolder):
from pathlib import Path
import util
# source files to be copied
source = {
"include": ["in_assert.h", "in_base.h", "in_ref.h"],
"kit": ["kt_assert.h", "kt_std.h"],
"3DTransVidiaReverse": ["tvr_reverse.h", "tvr_std.h"],
"math_ex": ["mx_reverse_def.h"]
}
# binaries to be copied
bin = [
"3DTransVidiaKit.exe", "3DTransVidiaReverse.dll", "ikernel.dll",
"ikernel_imp.dll", "kit.dll", "math.dll", "math_ap.dll", "math_ex.dll",
"math_kit.dll", "modeler.dll", "modeler_ex.dll", "tr_config.dll",
"tr_step.dll", "tr_xml.dll", "translator.dll", "util_lm.dll",
"win_kit.dll", "win_kit_mfc.dll", "opt_ui.dll", "glew32.dll"
]
# libraries (TODO: why for SDK reverse are copied ikernel.lib and ikernel_imp.lib but not kit.lib and math_ex.lib according to .h-files)
libs = ["3DTransVidiaReverse.lib", "ikernel.lib", "ikernel_imp.lib"]
# resources from 'Data' folder
resources = [
"3dv_string.csv", "3dv_string_repair.csv", "3dv_strings_translator.csv"
]
# configurations
configurations = ["x64", "win32"]
# make path to destination folder
dst = Path(dst)
if not dst.is_absolute(): dst = os.getcwd() / dst
# TODO: create destination folder by SVN latest revision number
#if dst.exists() : util.rmDir(dst)
if dst.exists(): return False
dst.mkdir()
# create folders for binary and libraries
for cfg in configurations:
Path(dst, "bin", cfg).mkdir(parents=True)
Path(dst, "lib", cfg).mkdir(parents=True)
# copy source code and corresponting .lib files
dirCodeDst = Path(dst, "include")
dirCodeDst.mkdir(parents=True)
for folder in source:
dirCodeSrc = Path(sourcecode, folder)
# copy source code files
for mask in source[folder]:
for file in list(dirCodeSrc.glob(mask)):
util.copy(file, dirCodeDst)
pass
# copy corresponding .lib file
'''
for cfg in configurations :
lib = Path(sourcebinaries, "bin/{0}_gold/{1}.lib".format(cfg, folder))
if lib.exists() : util.copy(lib, Path(dst, "lib", cfg))
pass
'''
for lib in libs:
for cfg in configurations:
file = Path(sourcebinaries, "bin/{0}_gold/{1}".format(cfg, lib))
if file.exists(): util.copy(file, Path(dst, "lib", cfg))
pass
# copy binaries and resource data
for cfg in configurations:
dirBinDst = Path(dst, r"bin", cfg)
dirBinSrc = Path(sourcebinaries, "bin/{0}_gold".format(cfg))
for file in bin:
fSrc = Path(dirBinSrc, file)
if fSrc.exists(): util.copy(fSrc, dirBinDst)
pass
dirDataSrc = Path(sourcecode, "data")
for res in resources:
fSrc = Path(dirDataSrc, res)
if fSrc.exists(): util.copy(fSrc, dirBinDst)
pass
# copy redist
util.copyDirs(Path(sourcecode, "install"), Path(dst, "prerequisites"),
["dotNetFx", "vcredist_*_2012"])
# copy sample tests
util.copyDir(Path(testfolder, r"reverse\sdk_reverse"), Path(dst, "sample"))
# copy readme
util.copy(
Path(sourcecode,
r"doc\sdk_reverse\3DTransVidiaReverseSDK_Installation.pdf"),
dst.joinpath("3DTransVidiaReverseSDK_Installation.pdf"))
pass
def gen_iso(iso_image, authorized_key, mode=None):
"generate ISO"
with tempfile.TemporaryDirectory() as d:
config = configuration.get_config()
inclusion = []
with open(os.path.join(d, "dns_bootstrap_lines"), "w") as outfile:
outfile.write(setup.dns_bootstrap_lines())
inclusion += ["dns_bootstrap_lines"]
util.copy(authorized_key, os.path.join(d, "authorized.pub"))
util.writefile(os.path.join(d, "keyservertls.pem"),
authority.get_pubkey_by_filename("./clusterca.pem"))
inclusion += ["authorized.pub", "keyservertls.pem"]
os.makedirs(os.path.join(d, "var/lib/dpkg/info"))
scripts = {
"//spire/resources:postinstall.sh":
"postinstall.sh",
"//spire/resources:prepartition.sh":
"prepartition.sh",
"//spire/resources:netcfg.postinst":
"var/lib/dpkg/info/netcfg.postinst",
}
for source, destination in sorted(scripts.items()):
resource.extract(source, os.path.join(d, destination))
os.chmod(os.path.join(d, destination), 0o755)
inclusion.append(destination)
util.writefile(os.path.join(d, "keyserver.domain"),
configuration.get_keyserver_domain().encode())
inclusion.append("keyserver.domain")
util.writefile(os.path.join(d, "vlan.txt"), b"%d\n" % config.vlan)
inclusion.append("vlan.txt")
resource.extract("//spire/resources:sshd_config",
os.path.join(d, "sshd_config.new"))
preseeded = resource.get("//spire/resources:preseed.cfg.in")
generated_password = util.pwgen(20)
creation_time = datetime.datetime.now().isoformat()
git_hash = metadata.get_git_version().encode()
add_password_to_log(generated_password, creation_time)
print("generated password added to log")
preseeded = preseeded.replace(b"{{HASH}}",
util.mkpasswd(generated_password))
preseeded = preseeded.replace(b"{{BUILDDATE}}", creation_time.encode())
preseeded = preseeded.replace(b"{{GITHASH}}", git_hash)
mirror = config.mirror
if mirror.count("/") < 1 or mirror.count(".") < 1:
command.fail(
"invalid mirror specification '%s'; must be of the form HOST.NAME/PATH"
)
mirror_host, mirror_dir = mirror.split("/", 1)
preseeded = preseeded.replace(b"{{MIRROR-HOST}}", mirror_host.encode())
preseeded = preseeded.replace(b"{{MIRROR-DIR}}",
("/" + mirror_dir).encode())
preseeded = preseeded.replace(b"{{KERBEROS-REALM}}",
config.realm.encode())
cidr_nodes = config.cidr_nodes
node_cidr_prefix = ".".join(
str(cidr_nodes.network_address).split(".")[:-1]) + "."
preseeded = preseeded.replace(b"{{IP-PREFIX}}",
node_cidr_prefix.encode())
node_cidr_gateway = next(cidr_nodes.hosts())
preseeded = preseeded.replace(b"{{GATEWAY}}",
str(node_cidr_gateway).encode())
preseeded = preseeded.replace(b"{{NETMASK}}",
str(cidr_nodes.netmask).encode())
preseeded = preseeded.replace(
b"{{NAMESERVERS}}",
" ".join(str(server_ip)
for server_ip in config.dns_upstreams).encode())
util.writefile(os.path.join(d, "preseed.cfg"), preseeded)
inclusion += ["sshd_config.new", "preseed.cfg"]
for package_name, (short_filename,
package_bytes) in packages.verified_download_full(
PACKAGES).items():
if ("/" in short_filename
or not short_filename.startswith(package_name + "_")
or not short_filename.endswith("_amd64.deb")):
raise ValueError("invalid package name: %s for %s" %
(short_filename, package_name))
util.writefile(os.path.join(d, short_filename), package_bytes)
inclusion.append(short_filename)
cddir = os.path.join(d, "cd")
os.mkdir(cddir)
subprocess.check_call(
["bsdtar", "-C", cddir, "-xzf", "/usr/share/homeworld/debian.iso"])
subprocess.check_call(["chmod", "+w", "--recursive", cddir])
if mode is not None:
if mode not in MODES:
command.fail("no such ISO mode: %s" % mode)
MODES[mode](d, cddir, inclusion)
with gzip.open(os.path.join(cddir, "initrd.gz"), "ab") as f:
f.write(
subprocess.check_output(
["cpio", "--create", "--format=newc"],
input="".join("%s\n" % filename
for filename in inclusion).encode(),
cwd=d))
files_for_md5sum = subprocess.check_output(
["find", ".", "-follow", "-type", "f", "-print0"],
cwd=cddir).decode().split("\0")
files_for_md5sum = [x for x in files_for_md5sum if x]
md5s = subprocess.check_output(["md5sum", "--"] + files_for_md5sum,
cwd=cddir)
util.writefile(os.path.join(cddir, "md5sum.txt"), md5s)
temp_iso = os.path.join(d, "temp.iso")
subprocess.check_call([
"xorriso", "-as", "mkisofs", "-quiet", "-o", temp_iso, "-r", "-J",
"-c", "boot.cat", "-b", "isolinux.bin", "-no-emul-boot",
"-boot-load-size", "4", "-boot-info-table", cddir
])
subprocess.check_call(["isohybrid", "-h", "64", "-s", "32", temp_iso])
util.copy(temp_iso, iso_image)
def copySdkBinImp(dst, sourcecode, sourcebinaries, toolkitX64, toolkitX86,
testfolder):
# copies ths binary SDK from 'src' folder to 'dst' one.
# 'bGenHelp' turns on/off generating of help file
# 'bCopyToolkit' turns on/off copying of toolkit packages
from pathlib import Path
import os
import util
import vs
# source files to be copied
source = {
"include": ["*.h"],
r"include\GL": ["*.h"],
"ikernel": ["*.h"],
"ikernel_imp": ["ii_kernel.h", "ii_std.h"],
"math_ex": ["mx_exam.h", "mx_std.h"],
"math_kit": ["mk_const.h"],
"kit": [
"kt_assert.h", "kt_exam.h", "kt_io.h", "kt_job.h", "kt_log.h",
"kt_out.h", "kt_std.h", "kt_str.h", "kt_timing.h", "kt_xin.h",
"kt_xml.h", "kt_xml_cfg.h", "kt_xml_elem.h", "kt_xml_out.h",
"kt_test.h", "kt_xml_dom.h", "kt_tools.h", "kt_set.h"
]
}
# binaries to be copied
bin = [
"sample.exe", "cmd_sw.dll", "quality.dll", "win.dll", "graphics.dll",
"converter.exe", "ikernel.dll", "ikernel_imp.dll", "kit.dll",
"math.dll", "math_ap.dll", "math_ex.dll", "math_kit.dll",
"modeler.dll", "modeler_ex.dll", "net_server_util.dll", "opt_ui.dll",
"registration.exe", "toolkit.dll", "toolkit_reg.dll", "tr_asc.dll",
"tr_config.dll", "tr_mesh.dll", "tr_vrml.dll", "tr_step.dll",
"tr_xml.dll", "translator.dll", "util_lm.dll", "win_kit.dll",
"win_kit_mfc.dll", "glew32.dll", "vi.exe", "xmlvalidate.exe",
"xsl_transform.exe"
]
# resources from 'Data' folder
resources = [
"3dv_string.csv", "3dv_string_repair.csv",
"3dv_strings_translator.csv", "basic_opt.xml",
"healing_env_compare.xml", "healing_env_default.xml", "sa_string.csv",
"toolkit.xml", "transformRead.xsl", "transformReadLog.xsl",
"transformRepairStat.xsl"
]
# configurations
configurations = ["x64", "win32"]
# make path to destination folder
dst = Path(dst)
if not dst.is_absolute(): dst = os.getcwd() / dst
# TODO: create destination folder by SVN latest revision number
#if dst.exists() : util.rmDir(dst)
if dst.exists(): return False
dst.mkdir()
# create folders for binary and libraries
for cfg in configurations:
Path(dst, "bin", cfg).mkdir(parents=True)
Path(dst, "lib", cfg).mkdir(parents=True)
# copy source code and corresponting .lib files
dirCodeDst = Path(dst, "include")
dirCodeDst.mkdir(parents=True)
for folder in source:
dirCodeSrc = Path(sourcecode, folder)
# copy source code files
for mask in source[folder]:
for file in list(dirCodeSrc.glob(mask)):
util.copy(file, dirCodeDst)
pass
# copy corresponding .lib file
for cfg in configurations:
lib = Path(sourcebinaries,
"bin/{0}_gold/{1}.lib".format(cfg, folder))
if lib.exists(): util.copy(lib, Path(dst, "lib", cfg))
pass
# copy binaries and resource data
for cfg in configurations:
dirBinDst = Path(dst, r"bin", cfg)
dirBinSrc = Path(sourcebinaries, "bin/{0}_gold".format(cfg))
for file in bin:
fSrc = Path(dirBinSrc, file)
if fSrc.exists(): util.copy(fSrc, dirBinDst)
pass
dirDataSrc = Path(sourcecode, "data")
for res in resources:
fSrc = Path(dirDataSrc, res)
if fSrc.exists(): util.copy(fSrc, dirBinDst)
pass
# copy sample project
vs.copySln(Path(sourcecode, "sample\sample_sdk.sln"), Path(dst, "sample"))
# copy test files for sample
util.copyTests(Path(testfolder, r"sample\_all_sdk.xml"),
Path(dst, r"sample\test"))
util.copy(Path(testfolder, r"sample\_all_sdk.xml"),
Path(dst, r"sample\test"))
util.copy(Path(testfolder, r"sample\options.xml"),
Path(dst, r"sample\test"))
util.copy(Path(testfolder, r"sample\options_auxgeom.xml"),
Path(dst, r"sample\test"))
# copy redist
util.copyDirs(Path(sourcecode, "install"), Path(dst, "install"),
["fonts", "dotNetFx", "vcredist_*"])
# copy readme
util.copy(Path(sourcecode, "install/doc/3DTransVidiaSDK_Installation.pdf"),
dst.joinpath("3DTransVidiaSDK_Installation.pdf"))
# copy help file
# TODO:if bGenHelp : util.genHelp(dst, Path(dst, "sdk_api_bin.chm"))
util.copy(Path(sourcecode, "doc/doxygen/sdk_api_bin.chm"), dst)
# copy qif schema
util.copyDirs(Path(sourcecode, "data/qif2"), Path(dst, "xsd"),
["QIFApplications", "QIFLibrary"])
# copy toolkit
dirDstToolkit = Path(dst, "toolkit")
dirDstToolkit.mkdir()
util.copy(toolkitX64, dirDstToolkit)
util.copy(toolkitX86, dirDstToolkit)
pass
def gen_iso(iso_image, authorized_key, mode=None):
with tempfile.TemporaryDirectory() as d:
inclusion = []
with open(os.path.join(d, "dns_bootstrap_lines"), "w") as outfile:
outfile.write(setup.dns_bootstrap_lines())
inclusion += ["dns_bootstrap_lines"]
util.copy(authorized_key, os.path.join(d, "authorized.pub"))
util.writefile(os.path.join(d, "keyservertls.pem"),
authority.get_pubkey_by_filename("./server.pem"))
resource.copy_to("postinstall.sh", os.path.join(d, "postinstall.sh"))
os.chmod(os.path.join(d, "postinstall.sh"), 0o755)
inclusion += ["authorized.pub", "keyservertls.pem", "postinstall.sh"]
for variant in configuration.KEYCLIENT_VARIANTS:
util.writefile(os.path.join(d, "keyclient-%s.yaml" % variant),
configuration.get_keyclient_yaml(variant).encode())
inclusion.append("keyclient-%s.yaml" % variant)
resource.copy_to("sshd_config", os.path.join(d, "sshd_config.new"))
preseeded = resource.get_resource("preseed.cfg.in")
generated_password = util.pwgen(20)
creation_time = datetime.datetime.now().isoformat()
git_hash = get_git_version().encode()
add_password_to_log(generated_password, creation_time)
print("generated password added to log")
preseeded = preseeded.replace(b"{{HASH}}",
util.mkpasswd(generated_password))
preseeded = preseeded.replace(b"{{BUILDDATE}}", creation_time.encode())
preseeded = preseeded.replace(b"{{GITHASH}}", git_hash)
mirror = configuration.get_config().mirror
if mirror.count("/") < 1 or mirror.count(".") < 1:
command.fail(
"invalid mirror specification '%s'; must be of the form HOST.NAME/PATH"
)
mirror_host, mirror_dir = mirror.split("/", 1)
preseeded = preseeded.replace(b"{{MIRROR-HOST}}", mirror_host.encode())
preseeded = preseeded.replace(b"{{MIRROR-DIR}}",
("/" + mirror_dir).encode())
realm = configuration.get_config().realm
preseeded = preseeded.replace(b"{{KERBEROS-REALM}}", realm.encode())
cidr_nodes, upstream_dns_servers = configuration.get_config(
).cidr_nodes, configuration.get_config().dns_upstreams
node_cidr_prefix = ".".join(str(cidr_nodes.ip).split(".")[:-1]) + "."
preseeded = preseeded.replace(b"{{IP-PREFIX}}",
node_cidr_prefix.encode())
node_cidr_gateway = cidr_nodes.gateway()
preseeded = preseeded.replace(b"{{GATEWAY}}",
str(node_cidr_gateway).encode())
node_cidr_netmask = cidr_nodes.netmask()
preseeded = preseeded.replace(b"{{NETMASK}}",
str(node_cidr_netmask).encode())
preseeded = preseeded.replace(
b"{{NAMESERVERS}}",
" ".join(str(server_ip)
for server_ip in upstream_dns_servers).encode())
util.writefile(os.path.join(d, "preseed.cfg"), preseeded)
inclusion += ["sshd_config.new", "preseed.cfg"]
for package_name, (short_filename,
package_bytes) in packages.verified_download_full(
PACKAGES).items():
assert "/" not in short_filename, "invalid package name: %s for %s" % (
short_filename, package_name)
assert short_filename.startswith(
package_name +
"_"), "invalid package name: %s for %s" % (short_filename,
package_name)
assert short_filename.endswith(
"_amd64.deb"), "invalid package name: %s for %s" % (
short_filename, package_name)
util.writefile(os.path.join(d, short_filename), package_bytes)
inclusion.append(short_filename)
cddir = os.path.join(d, "cd")
os.mkdir(cddir)
subprocess.check_call(
["bsdtar", "-C", cddir, "-xzf", "/usr/share/homeworld/debian.iso"])
subprocess.check_call(["chmod", "+w", "--recursive", cddir])
if mode is not None:
if mode not in MODES:
command.fail("no such ISO mode: %s" % mode)
MODES[mode](d, cddir, inclusion)
with gzip.open(os.path.join(cddir, "initrd.gz"), "ab") as f:
subprocess.run(["cpio", "--create", "--format=newc"],
check=True,
stdout=f,
input="".join("%s\n" % filename
for filename in inclusion).encode(),
cwd=d)
files_for_md5sum = subprocess.check_output(
["find", ".", "-follow", "-type", "f", "-print0"],
cwd=cddir).decode().split("\0")
assert files_for_md5sum.pop() == ""
md5s = subprocess.check_output(["md5sum", "--"] + files_for_md5sum,
cwd=cddir)
util.writefile(os.path.join(cddir, "md5sum.txt"), md5s)
temp_iso = os.path.join(d, "temp.iso")
subprocess.check_call([
"xorriso", "-as", "mkisofs", "-quiet", "-o", temp_iso, "-r", "-J",
"-c", "boot.cat", "-b", "isolinux.bin", "-no-emul-boot",
"-boot-load-size", "4", "-boot-info-table", cddir
])
subprocess.check_call(["isohybrid", "-h", "64", "-s", "32", temp_iso])
util.copy(temp_iso, iso_image)
given_path="tmp_file.msa"
dl_gdrive.download_file_from_google_drive( data_googleid, tmp_file )
# if remote was gzipped
was_gz = False
if is_gz_file( given_path ):
was_gz = True
print("Input was zipped. unzipping!")
suffix="." + given_path.split('.')[-1]
cmd = []
cmd.append("gunzip")
cmd.append("--keep")
cmd.append("--suffix")
cmd.append(suffix)
cmd.append( given_path )
#print((" ".join(cmd)))
subprocess.call(cmd)
given_path=given_path[:-len(suffix)]
if was_gz or not local:
print("Moving the data over:\n " + given_path + " -> " + p.raw_sequences)
util.move( given_path, p.raw_sequences )
else:
print("Making a copy of the data:\n " + given_path + " -> " + p.raw_sequences)
util.copy( given_path, p.raw_sequences )
print("")
print("Version string: " + p.version)
def gen_iso(iso_image, authorized_key, cdpack=None):
with tempfile.TemporaryDirectory() as d:
inclusion = []
util.copy(authorized_key, os.path.join(d, "authorized.pub"))
util.writefile(os.path.join(d, "keyservertls.pem"),
authority.get_pubkey_by_filename("./server.pem"))
resource.copy_to("postinstall.sh", os.path.join(d, "postinstall.sh"))
os.chmod(os.path.join(d, "postinstall.sh"), 0o755)
inclusion += ["authorized.pub", "keyservertls.pem", "postinstall.sh"]
for variant in configuration.KEYCLIENT_VARIANTS:
util.writefile(os.path.join(d, "keyclient-%s.yaml" % variant),
configuration.get_keyclient_yaml(variant).encode())
inclusion.append("keyclient-%s.yaml" % variant)
resource.copy_to("sshd_config", os.path.join(d, "sshd_config.new"))
preseeded = resource.get_resource("preseed.cfg.in")
generated_password = util.pwgen(20)
add_password_to_log(generated_password)
print("generated password added to log")
preseeded = preseeded.replace(b"{{HASH}}",
util.mkpasswd(generated_password))
util.writefile(os.path.join(d, "preseed.cfg"), preseeded)
inclusion += ["sshd_config.new", "preseed.cfg"]
for package_name, (short_filename,
package_bytes) in packages.verified_download_full(
PACKAGES).items():
assert "/" not in short_filename, "invalid package name: %s for %s" % (
short_filename, package_name)
assert short_filename.startswith(
package_name +
"_"), "invalid package name: %s for %s" % (short_filename,
package_name)
assert short_filename.endswith(
"_amd64.deb"), "invalid package name: %s for %s" % (
short_filename, package_name)
util.writefile(os.path.join(d, short_filename), package_bytes)
inclusion.append(short_filename)
if cdpack is not None:
subprocess.check_call(["tar", "-C", d, "-xzf", cdpack, "cd"])
else:
subprocess.check_output(
["tar", "-C", d, "-xz", "cd"],
input=resource.get_resource("debian-9.2.0-cdpack.tgz"))
subprocess.check_output([
"cpio", "--create", "--append", "--format=newc", "--file=cd/initrd"
],
input="".join(
"%s\n" % filename
for filename in inclusion).encode(),
cwd=d)
subprocess.check_call(["gzip", os.path.join(d, "cd/initrd")])
files_for_md5sum = subprocess.check_output(
["find", ".", "-follow", "-type", "f", "-print0"],
cwd=os.path.join(d, "cd")).decode().split("\0")
assert files_for_md5sum.pop() == ""
md5s = subprocess.check_output(["md5sum", "--"] + files_for_md5sum,
cwd=os.path.join(d, "cd"))
util.writefile(os.path.join(d, "cd", "md5sum.txt"), md5s)
subprocess.check_call([
"genisoimage", "-quiet", "-o", iso_image, "-r", "-J",
"-no-emul-boot", "-boot-load-size", "4", "-boot-info-table", "-b",
"isolinux.bin", "-c", "isolinux.cat",
os.path.join(d, "cd")
])
def copySdkReverseImp(dst, sourcecode, sourcebinaries, testfolder) :
from pathlib import Path
import util
# source files to be copied
source = {"include": ["in_assert.h", "in_base.h", "in_ref.h"],
"kit" : ["kt_assert.h", "kt_std.h"],
"3DTransVidiaReverse" : ["tvr_reverse.h", "tvr_std.h"],
"math_ex" : ["mx_reverse_def.h"]
}
# binaries to be copied
bin = ["3DTransVidiaKit.exe", "3DTransVidiaReverse.dll", "ikernel.dll", "ikernel_imp.dll", "kit.dll", "math.dll", "math_ap.dll", "math_ex.dll", "math_kit.dll", "modeler.dll", "modeler_ex.dll", "tr_config.dll", "tr_step.dll", "tr_xml.dll", "translator.dll", "util_lm.dll", "win_kit.dll", "win_kit_mfc.dll", "opt_ui.dll", "glew32.dll"]
# libraries (TODO: why for SDK reverse are copied ikernel.lib and ikernel_imp.lib but not kit.lib and math_ex.lib according to .h-files)
libs = ["3DTransVidiaReverse.lib", "ikernel.lib", "ikernel_imp.lib" ]
# resources from 'Data' folder
resources = ["3dv_string.csv", "3dv_string_repair.csv", "3dv_strings_translator.csv"]
# configurations
configurations = ["x64", "win32"]
# make path to destination folder
dst = Path(dst)
if not dst.is_absolute() : dst = os.getcwd() / dst
# TODO: create destination folder by SVN latest revision number
#if dst.exists() : util.rmDir(dst)
if dst.exists() : return False
dst.mkdir()
# create folders for binary and libraries
for cfg in configurations :
Path(dst, "bin", cfg).mkdir(parents = True)
Path(dst, "lib", cfg).mkdir(parents = True)
# copy source code and corresponting .lib files
dirCodeDst = Path(dst, "include")
dirCodeDst.mkdir(parents = True)
for folder in source :
dirCodeSrc = Path(sourcecode, folder)
# copy source code files
for mask in source[folder] :
for file in list(dirCodeSrc.glob(mask)) :
util.copy(file, dirCodeDst)
pass
# copy corresponding .lib file
'''
for cfg in configurations :
lib = Path(sourcebinaries, "bin/{0}_gold/{1}.lib".format(cfg, folder))
if lib.exists() : util.copy(lib, Path(dst, "lib", cfg))
pass
'''
for lib in libs:
for cfg in configurations :
file = Path(sourcebinaries, "bin/{0}_gold/{1}".format(cfg, lib))
if file.exists() : util.copy(file, Path(dst, "lib", cfg))
pass
# copy binaries and resource data
for cfg in configurations :
dirBinDst = Path(dst, r"bin", cfg)
dirBinSrc = Path(sourcebinaries, "bin/{0}_gold".format(cfg))
for file in bin :
fSrc = Path(dirBinSrc, file)
if fSrc.exists() : util.copy(fSrc, dirBinDst)
pass
dirDataSrc = Path(sourcecode, "data")
for res in resources :
fSrc = Path(dirDataSrc, res)
if fSrc.exists() : util.copy(fSrc, dirBinDst)
pass
# copy redist
util.copyDirs(Path(sourcecode, "install"), Path(dst, "prerequisites"), ["dotNetFx", "vcredist_*_2012"])
# copy sample tests
util.copyDir(Path(testfolder, r"reverse\sdk_reverse"), Path(dst, "sample"))
# copy readme
util.copy(Path(sourcecode, r"doc\sdk_reverse\3DTransVidiaReverseSDK_Installation.pdf"), dst.joinpath("3DTransVidiaReverseSDK_Installation.pdf"))
pass
def copySdkBinImp(dst, sourcecode, sourcebinaries, toolkitX64, toolkitX86, testfolder) :
# copies ths binary SDK from 'src' folder to 'dst' one.
# 'bGenHelp' turns on/off generating of help file
# 'bCopyToolkit' turns on/off copying of toolkit packages
from pathlib import Path
import os
import util
import vs
# source files to be copied
source = {"include": ["*.h"],
r"include\GL" : ["*.h"],
"ikernel" : [ "*.h"],
"ikernel_imp" : ["ii_kernel.h", "ii_std.h"],
"math_ex" : ["mx_exam.h", "mx_std.h"],
"math_kit": ["mk_const.h"],
"kit": ["kt_assert.h", "kt_exam.h", "kt_io.h", "kt_job.h", "kt_log.h", "kt_out.h", "kt_std.h", "kt_str.h", "kt_timing.h", "kt_xin.h", "kt_xml.h", "kt_xml_cfg.h", "kt_xml_elem.h", "kt_xml_out.h", "kt_test.h", "kt_xml_dom.h", "kt_tools.h", "kt_set.h"]
}
# binaries to be copied
bin = ["sample.exe", "cmd_sw.dll", "quality.dll", "win.dll", "graphics.dll", "converter.exe", "ikernel.dll", "ikernel_imp.dll", "kit.dll", "math.dll", "math_ap.dll", "math_ex.dll", "math_kit.dll", "modeler.dll", "modeler_ex.dll", "net_server_util.dll", "opt_ui.dll", "registration.exe", "toolkit.dll", "toolkit_reg.dll", "tr_asc.dll", "tr_config.dll", "tr_mesh.dll", "tr_vrml.dll", "tr_step.dll", "tr_xml.dll", "translator.dll", "util_lm.dll", "win_kit.dll", "win_kit_mfc.dll", "glew32.dll", "vi.exe", "xmlvalidate.exe", "xsl_transform.exe"]
# resources from 'Data' folder
resources = ["3dv_string.csv", "3dv_string_repair.csv", "3dv_strings_translator.csv", "basic_opt.xml", "healing_env_compare.xml", "healing_env_default.xml", "sa_string.csv", "toolkit.xml", "transformRead.xsl", "transformReadLog.xsl", "transformRepairStat.xsl"]
# configurations
configurations = ["x64", "win32"]
# make path to destination folder
dst = Path(dst)
if not dst.is_absolute() : dst = os.getcwd() / dst
# TODO: create destination folder by SVN latest revision number
#if dst.exists() : util.rmDir(dst)
if dst.exists() : return False
dst.mkdir()
# create folders for binary and libraries
for cfg in configurations :
Path(dst, "bin", cfg).mkdir(parents = True)
Path(dst, "lib", cfg).mkdir(parents = True)
# copy source code and corresponting .lib files
dirCodeDst = Path(dst, "include")
dirCodeDst.mkdir(parents = True)
for folder in source :
dirCodeSrc = Path(sourcecode, folder)
# copy source code files
for mask in source[folder] :
for file in list(dirCodeSrc.glob(mask)) :
util.copy(file, dirCodeDst)
pass
# copy corresponding .lib file
for cfg in configurations :
lib = Path(sourcebinaries, "bin/{0}_gold/{1}.lib".format(cfg, folder))
if lib.exists() : util.copy(lib, Path(dst, "lib", cfg))
pass
# copy binaries and resource data
for cfg in configurations :
dirBinDst = Path(dst, r"bin", cfg)
dirBinSrc = Path(sourcebinaries, "bin/{0}_gold".format(cfg))
for file in bin :
fSrc = Path(dirBinSrc, file)
if fSrc.exists() : util.copy(fSrc, dirBinDst)
pass
dirDataSrc = Path(sourcecode, "data")
for res in resources :
fSrc = Path(dirDataSrc, res)
if fSrc.exists() : util.copy(fSrc, dirBinDst)
pass
# copy sample project
vs.copySln(Path(sourcecode, "sample\sample_sdk.sln"), Path(dst, "sample"))
# copy test files for sample
util.copyTests(Path(testfolder, r"sample\_all_sdk.xml"), Path(dst, r"sample\test"))
util.copy(Path(testfolder, r"sample\_all_sdk.xml"), Path(dst, r"sample\test"))
util.copy(Path(testfolder, r"sample\options.xml"), Path(dst, r"sample\test"))
util.copy(Path(testfolder, r"sample\options_auxgeom.xml"), Path(dst, r"sample\test"))
# copy redist
util.copyDirs(Path(sourcecode, "install"), Path(dst, "install"), ["fonts", "dotNetFx", "vcredist_*"])
# copy readme
util.copy(Path(sourcecode, "install/doc/3DTransVidiaSDK_Installation.pdf"), dst.joinpath("3DTransVidiaSDK_Installation.pdf"))
# copy help file
# TODO:if bGenHelp : util.genHelp(dst, Path(dst, "sdk_api_bin.chm"))
util.copy(Path(sourcecode, "doc/doxygen/sdk_api_bin.chm"), dst)
# copy qif schema
util.copyDirs(Path(sourcecode, "data/qif2"), Path(dst, "xsd"), ["QIFApplications", "QIFLibrary"])
# copy toolkit
dirDstToolkit = Path(dst, "toolkit")
dirDstToolkit.mkdir()
util.copy(toolkitX64, dirDstToolkit)
util.copy(toolkitX86, dirDstToolkit)
pass
def revert(self, prompt=lambda x, y: True):
if prompt(self.reponame, self.destination):
util.copy(self.reponame, self.destination)