def main():
os.chdir(common.root + '/shared')
modulepath = common.deps()
sources = common.glob('java', '*.java')
common.javac(sources,
'target/classes',
classpath=modulepath,
modulepath=modulepath,
release='9')
return 0
def run():
m_info = configure.readConfig()
work_dir = m_info['work_dir']
exist_runs = common.glob(work_dir + '/[0-9]*_[0-9]*')
print 'Launch: '
print '%s: %s' %(0, 'Create new')
print '\n'.join('%s: %s' %(i + 1, e) for i, e in enumerate(exist_runs))
idx = int(raw_input('Select work:'))
if idx == 0:
s_timestamp = workdir.setup(m_info)
else:
s_timestamp = common.getBasename(exist_runs[idx - 1])
workdir.makeWorkDirs(m_info, s_timestamp)
runWork(m_info, s_timestamp)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--skija-version')
(args, _) = parser.parse_known_args()
modulepath = []
if args.skija_version:
modulepath += [
common.fetch_maven('org.jetbrains.skija',
'skija-shared',
args.skija_version,
repo=common.space_skija),
common.fetch_maven('org.jetbrains.skija',
'skija-' + common.classifier,
args.skija_version,
repo=common.space_skija)
]
else:
build.main()
modulepath += [
'../shared/target/classes', '../platform/target/classes'
]
os.chdir(common.root + '/tests')
sources = common.glob('java', '*.java')
common.javac(sources,
'target/classes',
modulepath=modulepath,
add_modules=[common.module])
common.check_call([
'java',
# '--class-path', common.classpath_separator.join(modulepath + ['target/classes']),
'--class-path',
'target/classes',
'--module-path',
common.classpath_separator.join(modulepath),
'--add-modules',
common.module
] + (['-XstartOnFirstThread'] if 'macos' == common.system else []) + [
'-Djava.awt.headless=true', '-enableassertions',
'-enablesystemassertions', '-Xcheck:jni', '-Dskija.logLevel=DEBUG',
'org.jetbrains.skija.test.TestSuite'
])
return 0
def main():
os.chdir(common.root + '/shared')
print('Generating', 'target/generated-sources/delombok/**.java')
subprocess.check_call([
"java", "-jar",
common.deps()[0], "delombok", "java", "--module-path",
common.classpath_separator.join(common.deps()), "-d",
"target/generated-sources/delombok/org/jetbrains/skija"
])
if os.path.exists('target/generated-sources/delombok/module-info.java'):
os.remove('target/generated-sources/delombok/module-info.java')
print('Generating', '../docs/apidocs/**')
sources = common.glob('target/generated-sources/delombok', '*.java')
common.check_call([
'javadoc', '--module-path',
common.classpath_separator.join(common.deps()), '-d',
'../docs/apidocs', '-quiet', '-Xdoclint:all,-missing'
] + sources)
return 0
def runWork(m_info, s_timestamp):
work_dir = m_info['work_dir']
script_dir = '%s/%s/scripts' %(work_dir, s_timestamp)
config_dir = '%s/%s/partition/config' %(work_dir, s_timestamp)
common.makeDirOrPass(script_dir)
worklist = common.glob('%s/config*' %config_dir)
#create scripts
for work in worklist:
launch_id = common.getBasename(work).lstrip('config')
config_file = work
exec_file = m_info['exec']
out_root = '%s/%s/out' %(work_dir, s_timestamp)
log_root = '%s/%s/log/launch%s' %(work_dir, s_timestamp, launch_id)
common.makeDirOrPass(log_root) #log_root need to be created
pbs_setting = pbs.PBSOption()
resources = 'nodes=%s:ppn=%s:gpus=%s,walltime=%s:00:00' %(
settings.nodes_per_launch, settings.gpus_per_node,
settings.gpus_per_node, settings.node_time_lim)
d_values = {
'out_file': '%s/terminal_out' %log_root,
'err_file': '%s/terminal_err' %log_root,
'res_list': resources,
}
pbs_setting.setOptVals(d_values)
s_script = pbs_setting.getScriptStr(
getCmdStr(exec_file, config_file, log_root, out_root))
f_script = open('%s/pbslaunch%s.sh' %(script_dir, launch_id), 'w')
f_script.write(s_script)
f_script.close()
#launch
for work in worklist:
launch_id = common.getBasename(work).lstrip('config')
logger.info('pbs submit work: %s/pbslaunch%s.sh' %(
script_dir, launch_id))
pbs.submitWork('%s/pbslaunch%s.sh' %(script_dir, launch_id))
def wsj_extend_dict(dir_13_32_1, data_root, src_dict_suffix, mincount=2):
src_dict_dir = os.path.join(data_root, "local", "dict" + src_dict_suffix)
dst_dict_dir = os.path.join(data_root, "local",
"dict" + src_dict_suffix + "_larger")
if os.path.isdir(dst_dict_dir):
rmtree(dst_dict_dir)
copytree(src_dict_dir, dst_dict_dir)
# lexicon1_raw_nosil.txt is an unsorted (?) version of dict.cmu
dict_cmu = os.path.join(dst_dict_dir, "dict.cmu")
pipe_to(
sort(set(cat(os.path.join(src_dict_dir, "lexicon1_raw_nosil.txt")))),
dict_cmu)
pipe_to(
sort(set(x.split()[0] for x in cat(dict_cmu))),
os.path.join(dst_dict_dir, "wordlist.cmu"),
)
cleaned_gz = os.path.join(dst_dict_dir, "cleaned.gz")
train_data_root = os.path.join(dir_13_32_1, "wsj1", "doc", "lng_modl",
"lm_train", "np_data")
assert os.path.isdir(train_data_root)
train_data_files = []
for subdir in ("87", "88", "89"):
train_data_files.extend(
glob(os.path.join(train_data_root, subdir), r"*.z"))
isword = set(x.strip()
for x in cat(os.path.join(dst_dict_dir, "wordlist.cmu")))
with gzip.open(cleaned_gz, "wt") as out:
for train_data_file in train_data_files:
with open(train_data_file, "rb") as in_:
compressed = in_.read()
decompressed = unlzw(compressed)
in_ = io.TextIOWrapper(io.BytesIO(decompressed))
for line in in_:
if line.startswith("<"):
continue
A = line.strip().upper().split(" ")
for n, a in enumerate(A):
if a not in isword and len(a) > 1 and a.endswith("."):
out.write(a[:-1])
if n < len(A) - 1:
out.write("\n")
else:
out.write(a + " ")
out.write("\n")
del in_, compressed, decompressed
counts = Counter()
with gzip.open(cleaned_gz, "rt") as cleaned:
for line in cleaned:
for token in line.strip().split():
counts[token] += 1
counts = sorted(((v, k) for (k, v) in counts.items()), reverse=True)
digits = set(str(x) for x in range(10))
oov_counts_path = os.path.join(dst_dict_dir, "oov.counts")
oovlist_path = os.path.join(dst_dict_dir, "oovlist")
with open(os.path.join(dst_dict_dir, "unigrams"),
"w") as unigrams, open(oov_counts_path,
"w") as oov_cnts, open(oovlist_path,
"w") as oov_lst:
for count, word in counts:
line = "{} {}\n".format(count, word)
unigrams.write(line)
if word not in isword:
oov_cnts.write(line)
if not (set(word) & digits) and count >= mincount:
oov_lst.write(word + "\n")
del counts
dict_acronyms_path = os.path.join(dst_dict_dir, "dict.acronyms")
pipe_to(
dict_get_acronym_prons(oovlist_path, dict_cmu),
dict_acronyms_path,
)
f_dir = os.path.join(dst_dict_dir, "f")
b_dir = os.path.join(dst_dict_dir, "b")
mkdir(f_dir, b_dir)
banned = set(",;")
pipe_to(
(x for x in cat(dict_cmu) if not (set(x.split()[0]) & banned)),
os.path.join(f_dir, "dict"),
)
pipe_to(
(x for x in cat(os.path.join(dst_dict_dir, "oovlist"))
if not (set(x.split()[0]) & banned)),
os.path.join(f_dir, "oovs"),
)
pipe_to(
(" ".join([w[::-1]] + p.split()[::-1])
for (w, p) in (x.split(" ", 1)
for x in cat(os.path.join(f_dir, "dict")))),
os.path.join(b_dir, "dict"),
)
pipe_to((x[::-1] for x in cat(os.path.join(f_dir, "oovs"))),
os.path.join(b_dir, "oovs"))
for dir_ in (f_dir, b_dir):
dict_path = os.path.join(dir_, "dict")
rules_path = os.path.join(dir_, "rules")
hierarchy_path = os.path.join(dir_, "hierarchy")
oovs_path = os.path.join(dir_, "oovs")
rule_counts_path = os.path.join(dir_, "rule.counts")
rules_with_scores_path = os.path.join(dir_, "rules.with_scores")
oov_candidates_path = os.path.join(dir_, "oovs.candidates")
pipe_to(dict_get_rules(cat(dict_path)), rules_path)
pipe_to(dict_get_rule_hierarchy(rules_path), hierarchy_path)
pipe_to(
dict_count_rules(
dict_score_prons(
dict_path,
dict_limit_candidate_prons(
hierarchy_path,
dict_get_candidate_prons(rules_path, dict_path,
dict_path),
),
)),
rule_counts_path,
)
pipe_to(
sorted(
dict_score_rules(rule_counts_path),
key=lambda x: (float(x.split(";")[2]), x),
reverse=True,
),
rules_with_scores_path,
)
pipe_to(
dict_limit_candidate_prons(
hierarchy_path,
dict_get_candidate_prons(rules_with_scores_path, dict_path,
oovs_path),
),
oov_candidates_path,
)
oov_candidates_path = os.path.join(dst_dict_dir, "oovs.candidates")
pipe_to(
sorted(
cat(
dict_reverse_candidates(os.path.join(b_dir,
"oovs.candidates")),
os.path.join(f_dir, "oovs.candidates"),
)),
oov_candidates_path,
)
dict_oovs_path = os.path.join(dst_dict_dir, "dict.oovs")
pipe_to(
("{0} {1}".format(*x.split(";"))
for x in dict_select_candidate_prons(oov_candidates_path)),
dict_oovs_path,
)
dict_oovs_merged_path = os.path.join(dst_dict_dir, "dict.oovs_merged")
pipe_to(
sorted(set(cat(dict_acronyms_path, dict_oovs_path))),
dict_oovs_merged_path,
)
pipe_to(
sorted(
set(
cat(
[
"!SIL SIL", "<SPOKEN_NOISE> SPN", "<UNK> SPN",
"<NOISE> NSN"
],
dict_cmu,
dict_oovs_merged_path,
))),
os.path.join(dst_dict_dir, "lexicon.txt"),
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--debug', action='store_true')
parser.add_argument('--arch', default=common.arch)
parser.add_argument('--skia-dir')
parser.add_argument('--skia-release', default='m93-87e8842e8c')
parser.add_argument('--skija-version')
(args, _) = parser.parse_known_args()
# Fetch Skia
build_type = 'Debug' if args.debug else 'Release'
if args.skia_dir:
skia_dir = os.path.abspath(args.skia_dir)
os.chdir(common.root + '/platform')
else:
os.chdir(common.root + '/platform')
skia_dir = "Skia-" + args.skia_release + "-" + common.system + "-" + build_type + '-' + common.arch
if not os.path.exists(skia_dir):
zip = skia_dir + '.zip'
common.fetch('https://github.com/JetBrains/skia-build/releases/download/' + args.skia_release + '/' + zip, zip)
with zipfile.ZipFile(zip, 'r') as f:
print("Extracting", zip)
f.extractall(skia_dir)
os.remove(zip)
skia_dir = os.path.abspath(skia_dir)
print("Using Skia from", skia_dir)
# CMake
os.makedirs("build", exist_ok = True)
common.check_call([
"cmake",
"-G", "Ninja",
"-DCMAKE_BUILD_TYPE=" + build_type,
"-DSKIA_DIR=" + skia_dir,
"-DSKIA_ARCH=" + common.arch]
+ (["-DCMAKE_OSX_ARCHITECTURES=" + {"x64": "x86_64", "arm64": "arm64"}[common.arch]] if common.system == "macos" else [])
+ [".."],
cwd=os.path.abspath('build'))
# Ninja
common.check_call(["ninja"], cwd=os.path.abspath('build'))
# Codesign
if common.system == "macos" and os.getenv("APPLE_CODESIGN_IDENTITY"):
subprocess.call(["codesign",
# "--force",
# "-vvvvvv",
"--deep",
"--sign",
os.getenv("APPLE_CODESIGN_IDENTITY"),
"build/libskija_" + common.arch + ".dylib"])
# javac
modulepath = []
if args.skija_version:
modulepath += [
common.fetch_maven('org.jetbrains.skija', 'skija-shared', args.skija_version, repo='https://packages.jetbrains.team/maven/p/skija/maven')
]
else:
build_shared.main()
modulepath += ['../shared/target/classes']
os.chdir(common.root + '/platform')
sources = common.glob('java-' + common.classifier, '*.java')
common.javac(sources, 'target/classes', modulepath = modulepath, release = '9')
# Copy files
target = 'target/classes/org/jetbrains/skija'
if common.classifier == 'macos-x64':
common.copy_newer('build/libskija_x64.dylib', target + '/macos/x64/libskija_x64.dylib')
elif common.classifier == 'macos-arm64':
common.copy_newer('build/libskija_arm64.dylib', target + '/macos/arm64/libskija_arm64.dylib')
elif common.classifier == 'linux':
common.copy_newer('build/libskija.so', target + '/linux/libskija.so')
elif common.classifier == 'windows':
common.copy_newer('build/skija.dll', target + '/windows/skija.dll')
common.copy_newer(skia_dir + '/out/' + build_type + '-' + common.arch + '/icudtl.dat',
target + '/windows/icudtl.dat')
return 0
