def _load_docs(self):
logging.info("[+] loading docs metadata")
title_parser = document.DocumentParser(
document.parse_document(),
document.pad_document(1, self.config.title_shape)
)
body_parser = document.DocumentParser(
document.parse_document(),
document.pad_document(1, self.config.body_shape)
)
doc_parser = document.DocumentParser(
document.parse_document(),
document.pad_document(1, self.config.title_shape + self.config.body_shape)
)
vert2idx = {}
vert_cnt = 1
subvert2idx = {}
subvert_cnt = 1
vert2idx_path = self.config.vertical2idx_input
with utils.open(vert2idx_path[0]) as file:
for line in file:
vert, idx = line.strip('\n').split('\t')
vert2idx[vert] = idx
vert_cnt = vert_cnt + 1
with utils.open(vert2idx_path[1]) as file:
for line in file:
vert, idx = line.strip('\n').split('\t')
subvert2idx[vert] = idx
subvert_cnt = subvert_cnt + 1
self.vert_cnt = vert_cnt
self.subvert_cnt = subvert_cnt
with utils.open(self.config.doc_meta_input) as file:
docs = [line.strip('\n').split('\t') for line in file]
self.docs = {
int(line[1]): self.News(
title_parser(line[4])[0],
body_parser(line[5])[0],
doc_parser(line[4] + ' ' + line[5])[0],
[int(vert2idx[line[2]])],
[int(subvert2idx[line[3]])]
) for line in docs}
self.doc_count = max(self.docs.keys()) + 1
doc_example = self.docs[self.doc_count - 1]
self.docs[0] = self.News(
np.zeros_like(doc_example.title),
np.zeros_like(doc_example.body),
np.zeros_like(doc_example.doc),
[0], [0])
logging.info("[-] loaded docs metadata")
def _load_users(self):
logging.info("[+] loading users metadata")
with utils.open(self.config.user_meta_input) as file:
users = [line.split('\t') for line in file]
self.users = {int(line[2]): line[0] for line in users}
self.user_count = max(self.users.keys()) + 1
logging.info("[-] loaded {} users".format(self.user_count))
def _load_data(self):
logging.info("[+] loading training data")
with utils.open(self.config.training_data_input) as file:
user_ch = [line.split('\t') for line in file]
self.user_ch = {
int(line[0]): np.array(line[1].split(' '), dtype=np.int32)
for line in user_ch
}
training_ch = [v for k, v in self.user_ch.items() if k % 10 != 0]
validation_ch = [v for k, v in self.user_ch.items() if k % 10 == 0]
self.training_pairs = np.vstack(
utils.rolling_window(line, self.config.window_size + 1)
for line in training_ch if len(line > self.config.window_size + 1))
self.validation_pairs = np.vstack(
utils.rolling_window(line, self.config.window_size + 1)
for line in validation_ch
if len(line > self.config.window_size + 1))
self.training_pairs_count = len(self.training_pairs)
self.validation_pairs_count = len(self.validation_pairs)
self.training_step = np.ceil(self.training_pairs_count /
self.config.batch_size)
self.validation_step = np.ceil(self.validation_pairs_count /
self.config.batch_size)
logging.info("[-] loaded {}+{} pairs".format(
self.training_pairs_count, self.validation_pairs_count))
def test_gen(self, doc2vec=None):
def _gen_doc(clicked, impression):
for p in impression.pos:
vec = doc2vec[p]
yield clicked, vec, 1
for n in impression.neg:
vec = doc2vec[n]
yield clicked, vec, 0
with utils.open(
self.config.training_data_input if self.is_training else self.config.testing_data_input) as file:
for line in file:
line = line.strip('\n').split('\t')
if line[2] and line[3]:
ih1 = self._extract_impressions(line[2])
ih2 = self._extract_impressions(line[3])
ch = self.Window(self.docs, self.config.test_window_size)
for impression in ih1:
for pos in impression.pos:
ch.push(pos)
for impression in ih2:
clicked = np.stack([doc2vec[i] for i in ch.click_history])
yield list(_gen_doc(clicked, impression))
for pos in impression.pos:
ch.push(pos)
def _load_docs(self):
logging.info("[+] loading docs metadata")
title_parser = document.DocumentParser(
document.parse_document(),
document.pad_document(1, self.config.title_shape)
)
body_parser = document.DocumentParser(
document.parse_document(),
document.pad_document(1, self.config.body_shape)
)
with utils.open(self.config.doc_meta_input) as file:
docs = [line.strip('\n').split('\t') for line in file]
self.docs = {
int(line[1]): self.News(
title_parser(line[4])[0],
body_parser(line[5])[0],
) for line in docs}
self.doc_count = max(self.docs.keys()) + 1
doc_example = self.docs[self.doc_count - 1]
self.docs[0] = self.News(
np.zeros_like(doc_example.title),
np.zeros_like(doc_example.body))
logging.info("[-] loaded docs metadata")
def load_patch_file(fn):
filepath = os.path.join('data', 'script', fn + ".txt")
try:
with open(filepath, "r") as f:
lines = f.readlines()
except IOError:
print(f"failed to open data/script/{fn}.txt")
return
for i, line in enumerate(lines):
s = line.split(':', 1)
try:
script_idx = int(s[0].strip())
except ValueError:
print(f"{fn}.txt: line {i} - {s[0]} is not a valid caption index")
continue
changes = s[1].rstrip('\n').split('|')
for c in changes:
if '->' not in c: continue
chgfrom, chgto = c.split('->', 1)
if '@' in chgfrom:
chgfrom, match_idx = chgfrom.split('@', 1)
try:
match_idx = int(match_idx)
except ValueError:
print(
f"{fn}.txt: line {i} - {match_idx} is not a valid match index"
)
match_idx = 0
else:
match_idx = None
if chgto == "*": chgto = None
patch_dialogue(script_idx, chgfrom, chgto, index=match_idx)
def _test_user_vec_gen(self):
users_path = self.config.pipeline_inputs[1]
with utils.open(users_path) as file:
for line in file:
line = line.strip('\n').split('\t')
user = line[0] + line[1]
clicks = line[2].split('#N#')
yield user, clicks
def _test_doc_vec_gen(self):
docs_path = self.config.pipeline_inputs[0]
with utils.open(docs_path) as file:
for line in file:
line = line.strip('\n').split('\t')
doc = line[0]
inputs = line[2]
yield doc, inputs
def __init__(self, id=0, port=None):
self.x = 126 # angle range is [51..201]
[self._device, idString] = utils.open(utils.getPort(id, port))
if idString == "joystick streaming angle, Ready!":
logging.debug("Device is a joystick ({}): {}".format(len(idString), idString))
else:
logging.error("Device is NOT a joystick ({}): {}".format(len(idString), idString))
def writeScreenCoordsToFile(self):
return
fh=utils.open("logs/screen.log","wb")
fh.write(str(self._display))
#+"\n$$$\n")
# for j in i._display:
# fh.write(str(j)+"\n")
fh.write("\n")
fh.close()
def writeScreenCoordsToFile(self):
return
fh = utils.open("logs/screen.log", "wb")
fh.write(str(self._display))
# +"\n$$$\n")
# for j in i._display:
# fh.write(str(j)+"\n")
fh.write("\n")
fh.close()
def _test_user_doc_gen(self):
pair_path = self.config.pipeline_inputs[2]
with utils.open(pair_path) as file:
for line in file:
line = line.strip('\n').split('\t')
user_id = line[0]
user_type = line[1]
doc = line[2]
yield user_id, user_type, doc
def write_allele_counts(output, vcf_data):
with open(output, 'wt') as outfile:
header = ['chrom', 'coord', 'ref', 'alt', 'n_ref_count', 'n_alt_count']
header = '\t'.join(header) + '\n'
outfile.write(header)
for key, key_data in vcf_data.items():
outstr = key + key_data[1]
outstr = '\t'.join(outstr) + '\n'
outfile.write(outstr)
def read_opera_mml(file):
try:
file = os.path.join('custom', 'opera', f'{file}.mml')
with open(file, "r") as f:
mml = f.read()
return mml
except IOError:
print(f"Failed to read {file}")
raise
def test_user_doc_score(self):
pair_cnt = 0
real_cnt = 0
cnt = 0
out_path = self.config.pipeline_output
batch_size = self.config.batch_size
users = []
docs = []
users_vec = []
docs_vec = []
user_vec = self.user_vec
doc_vec = self.doc_vec
score_encoder = self.get_score_encoder()
with utils.open(out_path, 'w') as ff:
for user_id, user_type, doc in self._test_user_doc_gen():
pair_cnt = pair_cnt + 1
user = user_id + user_type
if user in user_vec and doc in doc_vec:
users.append((user_id, user_type))
docs.append(doc)
users_vec.append(user_vec[user])
docs_vec.append(doc_vec[doc])
real_cnt = real_cnt + 1
cnt = cnt + 1
if cnt == batch_size:
outputs = score_encoder.predict(
[np.stack(users_vec),
np.stack(docs_vec)])
for idtp, do, out in zip(users, docs, outputs):
ff.write(idtp[0] + '\t' + idtp[1] + '\t' + do +
'\t' + str(out[0]) + '\n')
users = []
docs = []
users_vec = []
docs_vec = []
cnt = 0
if cnt != 0:
outputs = score_encoder.predict(
[np.stack(users_vec),
np.stack(docs_vec)])
for idtp, do, out in zip(users, docs, outputs):
ff.write(idtp[0] + '\t' + idtp[1] + '\t' + do + '\t' +
str(out[0]) + '\n')
users = []
docs = []
users_vec = []
docs_vec = []
cnt = 0
def score(config: settings.Config):
task_handler = task.get(config)
with utils.open(next(task_handler), 'w') as file:
model = next(task_handler)
for batch_info, batch_data in task_handler:
batch_pred = model.predict_on_batch(batch_data)
for (session, label, score), pred in zip(batch_info, batch_pred):
file.write('{}\t{}\t{}\t{}\n'.format(session, label, score,
pred[0]))
return 0
def __init__(self, id=0, port=None):
[self._device, idString] = utils.open(utils.getPort(id, port))
if not self._device:
logging.error("No BITSI extended buttonbox connected.")
elif idString == "BITSI_extend mode, Ready!":
logging.debug(
"Device is a BITSI extended buttonbox ({}): {}".format(
len(idString), idString))
else:
logging.error(
"Device is NOT a BITSI extended buttonbox ({}): {}".format(
len(idString), idString))
self.calibrated = False
def _load_data(self):
logging.info("[+] loading training data")
with utils.open(self.config.training_data_input) as file:
pairs = [line.split('\t') for line in file]
pairs = np.array([[int(line[0]), int(line[1])] for line in pairs])
self.partition = np.arange(len(pairs)) % 10 == 0
self.training_pairs = pairs[~self.partition]
self.validation_pairs = pairs[self.partition]
self.training_pairs_count = len(self.training_pairs)
self.validation_pairs_count = len(self.validation_pairs)
self.training_step = np.ceil(self.training_pairs_count / self.config.batch_size)
self.validation_step = np.ceil(self.validation_pairs_count / self.config.batch_size)
logging.info("[-] loaded {}+{} pairs".format(self.training_pairs_count, self.validation_pairs_count))
def _train_gen(self, once=False):
window_size = self.config.window_size
docs_window = self.DocsWindow(self.docs, window_size)
while True:
with utils.open(self.config.training_data_input) as file:
for line in file:
line = line.strip('\n').split('\t')
if line[2]:
ih = self._extract_impressions(line[2])
all_pos = []
all_ims = []
if self.config.max_impression != -1 and len(ih) > self.config.max_impression:
continue
for impression in ih:
if self.config.max_impression_pos != -1 and len(
impression.pos) > self.config.max_impression_pos:
continue
if self.config.max_impression_neg != -1 and len(
impression.neg) > self.config.max_impression_neg:
continue
for pos in impression.pos:
all_pos.append(pos)
all_ims.append(impression)
if len(all_pos) <= 1:
continue
i = -1
pos_len = len(all_pos) - 1
for pos in all_pos:
i = i + 1
seq = all_pos[:i] + all_pos[i + 1:]
if pos_len >= window_size:
seq = np.random.choice(seq, size=window_size, replace=False)
else:
seq = np.random.choice(seq, size=pos_len, replace=False)
for sam in self.train_sample(docs_window, seq, pos,
all_ims[i].negative_samples(self.config.negative_samples)):
yield sam
if once:
break
def get_vcf_header(vcf_files):
with open(vcf_files[0], 'rt') as invcf:
header = []
for line in invcf:
if '##INFO' in line or '##FORMAT' in line:
continue
if line.startswith('##'):
header.append(line.strip())
elif line.startswith('#'):
line = '\t'.join(line.strip().split()[:8])
header.append(line)
else:
break
return header
def write_vcf(output, vcf_files, data, tempdir, sample_id):
temp_output = os.path.join(tempdir, '{}.vcf'.format(sample_id))
with open(temp_output, 'wt') as outfile:
header = get_vcf_header(vcf_files)
for line in header:
outfile.write(line + '\n')
for key, key_data in data.items():
chrom, pos, ref, alt = key
outstr = (chrom, pos, '.', ref, alt) + key_data[0] + ('.', )
outstr = '\t'.join(outstr) + '\n'
outfile.write(outstr)
sort_vcf_file(temp_output, output)
def _load_docs(self):
logging.info("[+] loading docs metadata")
parser = document.DocumentParser(
document.parse_document(),
document.pad_document(1, self.config.title_shape))
with utils.open(self.config.doc_meta_input) as file:
docs = [line.split('\t') for line in file]
self.docs = {int(line[1]): parser(line[4])[0] for line in docs}
self.doc_count = max(self.docs.keys()) + 1
self.doc_freq = np.zeros(self.doc_count)
for line in docs:
self.doc_freq[int(line[1])] = int(line[2])
self.doc_freq = self.doc_freq**0.75
self.doc_freq = self.doc_freq / np.sum(self.doc_freq)
logging.info("[-] loaded {} docs".format(self.doc_count))
def get_doc_parser(self):
doc_encoder = self.model.get_layer('doc_encoder')
doc_input_shape = doc_encoder.layers[0].input_shape[1:]
split_tokens = []
with utils.open(self.config.doc_punc_index_input) as file:
for line in file:
line = line.strip('\n').split('\t')
if line[0] == '.' or line[0] == '?' or line[0] == '!':
split_tokens.append(int(line[1]))
body_parser = document.DocumentParser(
document.parse_document(), document.clause(split_tokens),
document.pad_docs(doc_input_shape[0], doc_input_shape[1]))
return body_parser
def __init__(self, id=0, port=None):
self._port = utils.getPort(id, port)
self._device = None
if not self._port:
raise Exception(
"No USB serial device connected, could not get port name.")
[self._device, idString] = utils.open(self._port)
if not self._device:
logging.critical(
"No BITSI buttonbox connected, could not connect to port {}".
format(self._port))
elif idString == "BITSI mode, Ready!" or idString == "BITSI event mode, Ready!":
logging.debug("Device is a BITSI buttonbox ({}): {}".format(
len(idString), idString))
else:
logging.error("Device is NOT a BITSI buttonbox ({}): {}".format(
len(idString), idString))
def post_processing(self):
names = self.cw.names
filename = os.path.join(self.dir, '[merged] {}.txt'.format(self.title))
try:
with utils.open(filename, 'wb') as f:
f.write(' {}\n\n 作者：{}\n\n\n'.format(
self.info['title'], self.artist).encode('utf8'))
if self.info['novel_ex']:
f.write(self.info['novel_ex'].encode('utf8'))
for i, file in enumerate(names):
self.cw.pbar.setFormat('[%v/%m] {} [{}/{}]'.format(
tr_('병합...'), i, len(names)))
with open(file, 'rb') as f_:
text = f_.read()
f.write(b'\n\n\n\n')
f.write(text)
finally:
self.cw.pbar.setFormat('[%v/%m]')
def __init__(self):
super(PartitionGadget, self).__init__()
self.schedule_phase = 'pre_vlog'
static_vkits = gvars.StaticVkits
vkits = [it for it in gvars.Vkits if it not in static_vkits]
cells = gvars.TB.PARTITION_CELLS
part_cfg_name = '.partition.cfg'
self.turds.append(part_cfg_name)
with utils.open(part_cfg_name, 'w') as pfile:
if static_vkits:
static_vkit_pkgs = ' '.join([it.get_pkg_name() for it in static_vkits])
print("partition package %s ;" % static_vkit_pkgs, file=pfile)
for cell in cells:
print("partition cell %s ;" % cell, file=pfile)
for vkit in vkits:
print("partition package %s ;" % vkit.get_pkg_name(), file=pfile)
def post_processing(self):
names = self.cw.names
filename = clean_title(u'[merged] [{}] {}'.format(
self.artist, self.info['title']),
n=-4) + '.txt'
filename = os.path.join(self.dir, filename)
try:
with utils.open(filename, 'wb') as f:
f.write(u' {}\n\n \u4f5c\u8005\uff1a{}\n\n\n'.format(
self.info['title'], self.artist).encode('utf8'))
f.write(self.info['description'].encode('utf8'))
for i, file in enumerate(names):
self.cw.pbar.setFormat('[%v/%m] {} [{}/{}]'.format(
tr_(u'\ubcd1\ud569...'), i, len(names)))
with open(file, 'rb') as f_:
text = f_.read()
f.write(b'\n\n\n\n')
f.write(text)
finally:
self.cw.pbar.setFormat('[%v/%m]')
def _valid_gen(self):
while True:
with utils.open(self.config.training_data_input) as file:
for line in file:
line = line.strip('\n').split('\t')
if line[2] and line[3]:
ih1 = self._extract_impressions(line[2])
ih2 = self._extract_impressions(line[3])
ch = self.Window(self.docs, self.config.window_size)
for impression in ih1:
for pos in impression.pos:
ch.push(pos)
for impression in ih2:
for pos in impression.pos:
if ch.count:
for sam in self.valid_sample(ch, pos, impression.negative_samples(
self.config.negative_samples)):
yield sam
ch.push(pos)
def load_patch_file(fn):
filepath = os.path.join('data', 'script', fn + ".txt")
# NEW
# this prepends the absolute file path of the parent/calling script
# to filepath created above - GreenKnight5
filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)),
filepath)
try:
with open(filepath, "r") as f:
lines = f.readlines()
except IOError:
print(f"failed to open data/script/{fn}.txt")
return
for i, line in enumerate(lines):
s = line.split(':', 1)
try:
script_idx = int(s[0].strip())
except ValueError:
print(f"{fn}.txt: line {i} - {s[0]} is not a valid caption index")
continue
changes = s[1].rstrip('\n').split('|')
for c in changes:
if '->' not in c:
continue
chgfrom, chgto = c.split('->', 1)
if '@' in chgfrom:
chgfrom, match_idx = chgfrom.split('@', 1)
try:
match_idx = int(match_idx)
except ValueError:
print(
f"{fn}.txt: line {i} - {match_idx} is not a valid match index"
)
match_idx = 0
else:
match_idx = None
if chgto == "*":
chgto = None
patch_dialogue(script_idx, chgfrom, chgto, index=match_idx)
def post_processing(self):
if self.single:
return
names = self.cw.names
filename = os.path.join(self.dir,
(u'[merged] {}.txt').format(self.title))
try:
with utils.open(filename, 'wb') as f:
f.write(u' {}\n\n \u4f5c\u8005\uff1a{}\n\n\n'.format(
self.__title, self.artist).encode('utf8'))
if self.novel_ex:
f.write(self.novel_ex.encode('utf8'))
for i, file in enumerate(names):
self.cw.pbar.setFormat(u"[%v/%m] {} [{}/{}]".format(
tr_(u'\ubcd1\ud569...'), i, len(names)))
with open(file, 'rb') as f_:
text = f_.read()
f.write(b'\n\n\n\n')
f.write(text)
finally:
self.cw.pbar.setFormat("[%v/%m]")
def manage_opera(fout, affect_music):
fout.seek(0)
data = fout.read()
SAMPLE_MAX_SIZE = 3746
global opera_log
#Determine opera cast
class OperaSinger:
def __init__(self, name, title, sprite, gender, file, sampletype,
sample, octave, volume, init):
self.name = name
self.title = title
self.sprite = sprite
self.gender = gender
self.file = file
self.sampletype = sampletype
self.sample = sample
self.octave = octave
self.volume = volume
self.init = init
self.sid = ("ext"
if sampletype == "brr" else sampletype) + ":" + sample
if self.sampletype not in ["brr", "ext"]:
try:
self.sample = int(self.sample, 16)
except ValueError:
print(
f"WARNING: in opera config: {name}: invalid sample id for {sampletype}: {sample}"
)
self.sample = 1
def get_sample_text(self, prg):
if self.sampletype in ["brr", "ext"]:
return f"\n#BRR 0x{prg:02X}; {self.sample}\n"
elif self.sampletype == "legacy":
return f"\n#BRR 0x{prg:02X}; {get_legacy_import(self.sample, subpath='music')}\n"
else:
return f"\n#WAVE 0x{prg:02X} 0x{self.sample:02X}\n"
#voice range notes
# Overture (Draco) ranges from B(o-1) to B(o)
# Aria (Maria) ranges from D(o) to D(o+1)
# Duel 1 (Draco) ranges from B(o-1) to E(o)
# Duel 2 (Maria) ranges from A(o) to F(o+1)
# Duel 3 (Ralse) ranges from E(o) to C(o+1)
# Duel 4 (Draco) ranges from B(o-1) to F(o)
# Duel 5 (Ralse) ranges from E(o) to B(o)
singer_options = []
opera_config = configparser.ConfigParser(interpolation=None)
opera_config.optionxform = lambda option: option
try:
with open(os.path.join('custom', 'opera.txt'), "r") as f:
opera_config.read_file(f)
except OSError:
print("WARNING: failed to load opera config")
return
if 'Singers' not in opera_config or 'Factions' not in opera_config:
print("WARNING: opera config is invalid, or failed to load")
return
for k, v in opera_config.items('Singers'):
line = [k] + v.split('|')
singer_options.append([l.strip() for l in line])
singer_options = [OperaSinger(*s) for s in singer_options]
#categorize by voice sample (unless not changing music)
if affect_music:
voices = {}
for s in singer_options:
if s.sid in voices:
voices[s.sid].append(s)
else:
voices[s.sid] = [s]
#find a set of voices that doesn't overflow SPC RAM
while True:
vchoices = random.sample(list(voices.values()), 3)
test_mml = read_opera_mml('duel')
for i, c in enumerate(vchoices):
test_mml += c[0].get_sample_text(i + 0x2A)
memusage = get_spc_memory_usage(test_mml, subpath="music")
if memusage <= SAMPLE_MAX_SIZE:
break
#select characters
charpool = []
char = {}
#by voice, for singers
for v in vchoices:
charpool.append(random.choice(v))
random.shuffle(charpool)
for c in ["Maria", "Draco", "Ralse"]:
char[c] = charpool.pop()
opera_log += str(str(c) + ": ").ljust(17) + string.capwords(
str(char[c].name)) + "\n"
#by sprite/name, for impresario
charpool = [c for c in singer_options if c not in char.values()]
char["Impresario"] = random.choice(charpool)
opera_log += str("Impresario: ").ljust(17) + string.capwords(
str(char["Impresario"].name)) + "\n"
else:
print(
"\nalasdraco -- note: opera voices will not be changed unless a" +
" random music code (johnnydmad/johnnyachaotic) is also used." +
" Sprites and text will still be affected.")
cchoices = random.sample(singer_options, 4)
char = {}
for c in ["Maria", "Draco", "Ralse", "Impresario"]:
char[c] = cchoices.pop()
opera_log += str(str(c) + ": ").ljust(17) + string.capwords(
str(char[c].name)) + "\n"
#reassign sprites in npc data
locations = get_locations()
#first, free up space for a unique Ralse
#choose which other NPCs get merged:
# 0. id for new scholar, 1. offset for new scholar, 2. id for merged sprite, 3. offset for merged sprite, 4. spritesheet filename, 5. extra pose type
merge_options = [
(32, 0x172C00, 33, 0x1731C0, "dancer.bin",
"flirt"), #fancy gau -> dancer
(32, 0x172C00, 35, 0x173D40, "gau-fancy.bin",
"sideeye"), #clyde -> fancy gau
(60, 0x17C4C0, 35, 0x173D40, "daryl.bin", "sideeye"), #clyde -> daryl
(42, 0x176580, 35, 0x173D40, "katarin.bin",
"sideeye"), #clyde -> katarin
(60, 0x17C4C0, 42, 0x176580, "daryl.bin",
"sideeye"), #katarin -> daryl
(60, 0x17C4C0, 42, 0x176580, "katarin.bin",
"sideeye"), #daryl -> katarin
(60, 0x17C4C0, 41, 0x175FC0, "daryl.bin",
"sleeping"), #rachel -> daryl
(60, 0x17C4C0, 41, 0x175FC0, "rachel.bin",
"sleeping"), #daryl -> rachel
(60, 0x17C4C0, 30, 0x172080, "returner.bin",
"prone"), #daryl -> returner
(53, 0x17A1C0, 59, 0x17BFC0, "figaroguard.bin",
None), #conductor -> figaro guard
(45, 0x1776C0, 48, 0x178800, "maduin.bin", "prone"), #yura -> maduin
]
merge = random.choice(merge_options)
#merge sacrifice into new slot
replace_npc(locations, (merge[0], None), merge[2])
#move scholar into sacrifice slot
for i in [0, 1, 3, 4, 5]:
replace_npc(locations, (27, i), (merge[0], i))
#randomize palettes of shuffled characters
for i in [0x1A, 0x1B, 0x1C, 0x2B]:
palette = random.randint(0, 5)
for l in locations:
for npc in l.npcs:
if npc.graphics == i:
#npc.palette = palette
pass
#debug info
# for l in locations:
# for npc in l.npcs:
# if npc.graphics in [118,138]:
# print()
# print(f"graphics {npc.graphics} found at ${npc.pointer:X}, in location 0x{l.locid:X}")
# print(f"palette {npc.palette}, facing byte {npc.facing:X}")
# print(f"facing {npc.facing & 3:X}, change {npc.facing>>2 & 1:X}")
# print(f"bglayer {npc.facing>>3 & 3:X}, unknown1 {npc.facing>>5 & 1:X}")
# print(f"mirror {npc.facing>>6 & 2:X}, unknown2 {npc.facing>>7 & 1:X}")
#randomize item thrown off balcony
balcony = get_location(0xEC)
for npc in balcony.npcs:
if npc.graphics == 88:
item = random.choice([
#(84, 0x24, "chest"), #treasure box (palette broken)
(87, 0x44, "statue"),
(88, 0x54, "flowers"), #bouquet
(89, 0x54, "letter"), #letter
(91, 0x54, "magicite"), #magicite
(92, 0x44, "book"), #book
##DO NOT THROW THE BABY
(96, 0x44, "crown"), #slave crown
(97, 0x54, "weight"), #4ton weight
(100, 0x54, "bandana"), #locke's bandana
##(124, 0x02, "helmet") #a shiny thing (didn't work)
])
npc.graphics = item[0]
npc.palette = random.choice(range(6))
npc.facing = item[1]
set_dialogue_var("OperaItem", item[2])
opera_log += str("Opera Flowers: ").ljust(17) + string.capwords(
item[2]) + "\n"
#print(f"opera item is {npc.graphics}, palette {npc.palette} ({item[2]})")
#print(f"at address {npc.pointer:X}")
#4 ton weight
for npc in get_location(0xEB).npcs:
if npc.graphics == 97:
item = random.choice([
(89, 0x00, "letter"), # letter
(92, 0x44, "book"), # book
(108, 0x00, "rat"), # rat
(87, 0x44, "mini statue"), # mini statue
(93, 0x54, "baby"
), # ultros is allowed to try to throw the baby (STOP HIM)
(97, 0x54, "4 ton weight"), # 4ton weight
(112, 0x43, "fire"), # fire
###(118, 0x10), #rock (didn't work)
###(138, 0x12) #leo's sword (didn't work)
])
npc.graphics = item[0]
npc.palette = random.choice(range(6))
npc.facing = item[1]
opera_log += str("Opera Rafters: ").ljust(17) + string.capwords(
item[2]) + "\n"
#print(f"ultros item is {npc.graphics}, palette {npc.palette}")
#print(f"at address {npc.pointer:X}")
#set up some spritesheet locations
pose = {
'singing':
[0x66, 0x67, 0x68, 0x69, 0x64, 0x65, 0x60, 0x61, 0x62, 0x63],
'ready': list(range(0x3E, 0x44)),
'prone': list(range(0x51, 0x57)),
'angry': list(range(0x76, 0x7C)),
'flirt': [0xA3, 0xA4, 0x9C, 0x99, 0x9A, 0x9B],
'sideeye': [0x92, 0x93, 0x94, 0x95, 0x08, 0x09],
'sleeping': [0x86, 0x87, 0x88, 0x89, 0x08, 0x09]
}
opath = os.path.join("custom", "opera")
#load scholar graphics
try:
with open(os.path.join(opath, "ralse.bin"), "rb") as f:
sprite = f.read()
except IOError:
print(f"failed to open custom/opera/ralse.bin")
sprite = None
if sprite:
new_sprite = create_sprite(sprite)
data = byte_insert(data, merge[1], new_sprite)
#load new graphics into merged slot
try:
with open(os.path.join(opath, f"{merge[4]}"), "rb") as f:
sprite = f.read()
except IOError:
try:
with open(os.path.join("custom", "sprites", f"{merge[4]}"),
"rb") as f:
sprite = f.read()
except:
print(
f"failed to open custom/opera/{merge[4]} or custom/sprites/{merge[4]}"
)
sprite = None
if sprite:
#print(f"merge {merge}, pose {pose}")
new_sprite = create_sprite(
sprite, pose[merge[5]] if merge[5] is not None else [])
data = byte_insert(data, merge[3], new_sprite)
#load new graphics into opera characters
char_offsets = {
"Maria": (0x1705C0, pose['ready'] + pose['singing']),
"Draco": (0x1713C0, pose['prone'] + pose['singing']),
"Ralse": (0x170CC0, pose['prone'] + pose['singing']),
"Impresario": (0x176B40, pose['angry'])
}
for cname, c in char.items():
#print(f"{cname} -> {c.name}")
try:
with open(os.path.join(opath, f"{c.sprite}.bin"), "rb") as f:
sprite = f.read()
except IOError:
try:
with open(os.path.join("custom", "sprites", f"{c.sprite}.bin"),
"rb") as f:
sprite = f.read()
except:
print(
f"failed to open custom/opera/{c.sprite}.bin or custom/sprites/{c.sprite}.bin"
)
continue
offset, extra_tiles = char_offsets[cname]
#tiles = list(range(0x28)) + extra_tiles
#new_sprite = bytearray()
#for t in tiles:
# loc = t*32
# new_sprite.extend(sprite[loc:loc+32])
#data = byte_insert(data, offset, new_sprite)
new_sprite = create_sprite(sprite, extra_tiles)
data = byte_insert(data, offset, new_sprite)
### adjust script
load_patch_file("opera")
factions = []
for f, _ in opera_config.items('Factions'):
factions.append([s.strip() for s in f.split('|')])
factions = random.choice(factions)
if random.choice([False, True]):
factions = (factions[1], factions[0])
opera_log += str("Opera Factions: ").ljust(17) + string.capwords(
factions[0]) + " VS " + string.capwords(factions[1]) + "\n"
set_dialogue_var("OperaEast", factions[0])
set_dialogue_var("OperaWest", factions[1])
set_dialogue_var("maria", char['Maria'].name)
set_dialogue_var("draco", char['Draco'].name)
set_dialogue_var("ralse", char['Ralse'].name)
set_dialogue_var("impresario", char['Impresario'].name)
set_dialogue_var("mariatitle", char['Maria'].title)
set_dialogue_var("dracotitle", char['Draco'].title)
set_dialogue_var("ralsetitle", char['Ralse'].title)
set_dialogue_var("impresariotitle", char['Impresario'].title)
char['Maria'].gender = set_pronoun('Maria', char['Maria'].gender)
char['Draco'].gender = set_pronoun('Draco', char['Draco'].gender)
char['Ralse'].gender = set_pronoun('Ralse', char['Ralse'].gender)
char['Impresario'].gender = set_pronoun('Impresario',
char['Impresario'].gender)
#due to the variance in power relations connoted by "make X my queen" and "make X my king", this line will be altered in all variations so that it means roughly the same thing no matter the Maria replacement's gender
if char['Maria'].gender == "female":
set_dialogue_var("MariaTheGirl", "the girl")
set_dialogue_var("MariaQueenBad", "mine")
set_dialogue_var("MariaQueen", "queen")
set_dialogue_var("MariaWife", "wife")
elif char['Maria'].gender == "male":
set_dialogue_var("MariaTheGirl", "the guy")
set_dialogue_var("MariaQueenBad", "mine")
set_dialogue_var("MariaQueen", "king")
set_dialogue_var("MariaWife", "husband")
elif char['Maria'].gender == "object":
set_dialogue_var("MariaTheGirl",
char['Maria'].title + " " + char['Maria'].name)
set_dialogue_var("MariaQueenBad", "mine")
set_dialogue_var("MariaQueen", "prize")
set_dialogue_var("MariaWife", "collection")
else:
set_dialogue_var("MariaTheGirl", "the girl")
set_dialogue_var("MariaQueenBad", "mine")
set_dialogue_var("MariaQueen", "consort")
set_dialogue_var("MariaWife", "partner")
if char['Impresario'].gender == "male":
set_dialogue_var("ImpresarioMan", "man") # from "music man"
elif char['Impresario'].gender == "female":
set_dialogue_var("ImpresarioMan", "madam")
elif char['Impresario'].gender == "object":
set_dialogue_var("ImpresarioMan", "machine")
else:
set_dialogue_var("ImpresarioMan", "maker")
### adjust music
opera = {}
maria_sample = char['Maria'].get_sample_text(0x2A)
draco_sample = char['Draco'].get_sample_text(0x2B)
ralse_sample = char['Ralse'].get_sample_text(0x2C)
try:
overture = read_opera_mml('overture')
overture += draco_sample
overture += f"\n#def draco= |B o{char['Draco'].octave[0]} v{char['Draco'].volume} {char['Draco'].init}\n"
seg = read_opera_mml(f"{char['Draco'].file}_overture")
overture += seg
aria = read_opera_mml('aria')
aria += maria_sample
aria += f"\n#def maria= |A o{char['Maria'].octave[1]} v{char['Maria'].volume} {char['Maria'].init}\n"
seg = read_opera_mml(f"{char['Maria'].file}_aria")
aria += seg
duel = read_opera_mml('duel')
duel += maria_sample
duel += f"\n#def maria= |A o{char['Maria'].octave[3]} v{char['Maria'].volume} {char['Maria'].init}\n"
duel += draco_sample
duel += f"\n#def draco= |B o{char['Draco'].octave[2]} v{char['Draco'].volume} {char['Draco'].init}\n"
duel += f"\n#def draco2= |B o{char['Draco'].octave[5]} v{char['Draco'].volume} {char['Draco'].init}\n"
duel += ralse_sample
duel += f"\n#def ralse= |C o{char['Ralse'].octave[4]} v{char['Ralse'].volume} {char['Ralse'].init}\n"
duel += f"\n#def ralse2= |C o{char['Ralse'].octave[6]} v{char['Ralse'].volume} {char['Ralse'].init}\n"
duelists = ["Draco", "Maria", "Ralse", "Draco", "Ralse"]
for i in range(5):
seg = read_opera_mml(f"{char[duelists[i]].file}_duel{i+1}")
duel += seg
#print(overture)
#print("########")
#print(duel)
#print("########")
#print(aria)
opera['opera_draco'] = overture
opera['wed_duel'] = duel
opera['aria'] = aria
except IOError:
print("opera music generation failed, reverting to default")
affect_music = False
fout.seek(0)
fout.write(data)
return opera if affect_music else None
# -*- coding: utf-8 -*-
import utils
import pandas as pd
import pickle
df = pd.read_csv('../data/Vocab.tsv',sep='\t',names=['word','index','vec'])
punc_idx = [
('.',df[df['word']=='.']['index'].values[0]),
('!',df[df['word']=='!']['index'].values[0]),
('?',df[df['word']=='?']['index'].values[0]),
(';',df[df['word']==';']['index'].values[0]),
(',',df[df['word']==',']['index'].values[0])
]
with utils.open('../data/PuncIndex.tsv','w') as file:
for punc,idx in punc_idx:
file.write(punc+'\t'+str(idx)+'\n')
vocab2idx = {word:index for word,index,_ in df.values}
pickle.dump(vocab2idx,open('../data/vocab2idx.pkl','wb'))
def extractGuideFastq(fastqFile,outputFolder='',gzip=False):
'''
takes a fastq and extracts candidate guide RNAs
'''
#get the full absolute path for the fastq File
fastqFile = os.path.abspath(fastqFile)
fastq = utils.open(fastqFile,'r')
#get the fastq name and root
if len(outputFolder) == 0:
outputFolder = utils.getParentFolder(fastqFile)
#makes sure the output folder exists
utils.formatFolder(outputFolder,True)
#grab the name info from the fastq
fastqName = fastqFile.split('/')[-1]
fastqRoot = string.replace(fastqName,'.fastq','')
fastqRoot = string.replace(fastqRoot,'.gz','')
#guideFastqFile output
guideFastqFile = '%s%s.gecko.fastq' % (outputFolder,fastqRoot)
guideFastq = utils.open(guideFastqFile,'w')
print('processing %s' % (fastqName))
print('million reads processed:')
ticker = 0
found = 0
while True:
if ticker%1000000 == 0:
print(ticker/1000000)
fastqLines = []
#now load the fastq lines
try:
for i in range(4):
fastqLines.append(fastq.next())
except StopIteration:
break
#see if you can find a cut site
seq = fastqLines[1].rstrip()
try:
cutPosition = seq.index(cutSeq)
found+=1
except ValueError:
ticker+=1
continue
guideStart = cutPosition + cutOffset
guideStop = guideStart + guideLength
#pulling out the guide seq in the fastqLines
fastqLines[1] = fastqLines[1][guideStart:guideStop] + '\n'
fastqLines[3] = fastqLines[3][guideStart:guideStop] + '\n'
for line in fastqLines:
guideFastq.write(line)
ticker+=1
# if ticker == 100000:
# print(ticker)
# print(found)
# print(float(found)/float(ticker))
# break
print('SUMMARY STATISTICS')
print(ticker)
print(found)
print(float(found)/float(ticker))
#close the fastq
guideFastq.close()
#gzip the fastq
if gzip:
os.system('gzip %s &' % (guideFastqFile))
guideFastqFile += '.gz'
return guideFastqFile
