def _load_docs(self): logging.info("[+] loading docs metadata") title_parser = document.DocumentParser( document.parse_document(), document.pad_document(1, self.config.title_shape) ) body_parser = document.DocumentParser( document.parse_document(), document.pad_document(1, self.config.body_shape) ) doc_parser = document.DocumentParser( document.parse_document(), document.pad_document(1, self.config.title_shape + self.config.body_shape) ) vert2idx = {} vert_cnt = 1 subvert2idx = {} subvert_cnt = 1 vert2idx_path = self.config.vertical2idx_input with utils.open(vert2idx_path[0]) as file: for line in file: vert, idx = line.strip('\n').split('\t') vert2idx[vert] = idx vert_cnt = vert_cnt + 1 with utils.open(vert2idx_path[1]) as file: for line in file: vert, idx = line.strip('\n').split('\t') subvert2idx[vert] = idx subvert_cnt = subvert_cnt + 1 self.vert_cnt = vert_cnt self.subvert_cnt = subvert_cnt with utils.open(self.config.doc_meta_input) as file: docs = [line.strip('\n').split('\t') for line in file] self.docs = { int(line[1]): self.News( title_parser(line[4])[0], body_parser(line[5])[0], doc_parser(line[4] + ' ' + line[5])[0], [int(vert2idx[line[2]])], [int(subvert2idx[line[3]])] ) for line in docs} self.doc_count = max(self.docs.keys()) + 1 doc_example = self.docs[self.doc_count - 1] self.docs[0] = self.News( np.zeros_like(doc_example.title), np.zeros_like(doc_example.body), np.zeros_like(doc_example.doc), [0], [0]) logging.info("[-] loaded docs metadata")
def _load_users(self): logging.info("[+] loading users metadata") with utils.open(self.config.user_meta_input) as file: users = [line.split('\t') for line in file] self.users = {int(line[2]): line[0] for line in users} self.user_count = max(self.users.keys()) + 1 logging.info("[-] loaded {} users".format(self.user_count))
def _load_data(self): logging.info("[+] loading training data") with utils.open(self.config.training_data_input) as file: user_ch = [line.split('\t') for line in file] self.user_ch = { int(line[0]): np.array(line[1].split(' '), dtype=np.int32) for line in user_ch } training_ch = [v for k, v in self.user_ch.items() if k % 10 != 0] validation_ch = [v for k, v in self.user_ch.items() if k % 10 == 0] self.training_pairs = np.vstack( utils.rolling_window(line, self.config.window_size + 1) for line in training_ch if len(line > self.config.window_size + 1)) self.validation_pairs = np.vstack( utils.rolling_window(line, self.config.window_size + 1) for line in validation_ch if len(line > self.config.window_size + 1)) self.training_pairs_count = len(self.training_pairs) self.validation_pairs_count = len(self.validation_pairs) self.training_step = np.ceil(self.training_pairs_count / self.config.batch_size) self.validation_step = np.ceil(self.validation_pairs_count / self.config.batch_size) logging.info("[-] loaded {}+{} pairs".format( self.training_pairs_count, self.validation_pairs_count))
def test_gen(self, doc2vec=None): def _gen_doc(clicked, impression): for p in impression.pos: vec = doc2vec[p] yield clicked, vec, 1 for n in impression.neg: vec = doc2vec[n] yield clicked, vec, 0 with utils.open( self.config.training_data_input if self.is_training else self.config.testing_data_input) as file: for line in file: line = line.strip('\n').split('\t') if line[2] and line[3]: ih1 = self._extract_impressions(line[2]) ih2 = self._extract_impressions(line[3]) ch = self.Window(self.docs, self.config.test_window_size) for impression in ih1: for pos in impression.pos: ch.push(pos) for impression in ih2: clicked = np.stack([doc2vec[i] for i in ch.click_history]) yield list(_gen_doc(clicked, impression)) for pos in impression.pos: ch.push(pos)
def _load_docs(self): logging.info("[+] loading docs metadata") title_parser = document.DocumentParser( document.parse_document(), document.pad_document(1, self.config.title_shape) ) body_parser = document.DocumentParser( document.parse_document(), document.pad_document(1, self.config.body_shape) ) with utils.open(self.config.doc_meta_input) as file: docs = [line.strip('\n').split('\t') for line in file] self.docs = { int(line[1]): self.News( title_parser(line[4])[0], body_parser(line[5])[0], ) for line in docs} self.doc_count = max(self.docs.keys()) + 1 doc_example = self.docs[self.doc_count - 1] self.docs[0] = self.News( np.zeros_like(doc_example.title), np.zeros_like(doc_example.body)) logging.info("[-] loaded docs metadata")
def load_patch_file(fn): filepath = os.path.join('data', 'script', fn + ".txt") try: with open(filepath, "r") as f: lines = f.readlines() except IOError: print(f"failed to open data/script/{fn}.txt") return for i, line in enumerate(lines): s = line.split(':', 1) try: script_idx = int(s[0].strip()) except ValueError: print(f"{fn}.txt: line {i} - {s[0]} is not a valid caption index") continue changes = s[1].rstrip('\n').split('|') for c in changes: if '->' not in c: continue chgfrom, chgto = c.split('->', 1) if '@' in chgfrom: chgfrom, match_idx = chgfrom.split('@', 1) try: match_idx = int(match_idx) except ValueError: print( f"{fn}.txt: line {i} - {match_idx} is not a valid match index" ) match_idx = 0 else: match_idx = None if chgto == "*": chgto = None patch_dialogue(script_idx, chgfrom, chgto, index=match_idx)
def _test_user_vec_gen(self): users_path = self.config.pipeline_inputs[1] with utils.open(users_path) as file: for line in file: line = line.strip('\n').split('\t') user = line[0] + line[1] clicks = line[2].split('#N#') yield user, clicks
def _test_doc_vec_gen(self): docs_path = self.config.pipeline_inputs[0] with utils.open(docs_path) as file: for line in file: line = line.strip('\n').split('\t') doc = line[0] inputs = line[2] yield doc, inputs
def __init__(self, id=0, port=None): self.x = 126 # angle range is [51..201] [self._device, idString] = utils.open(utils.getPort(id, port)) if idString == "joystick streaming angle, Ready!": logging.debug("Device is a joystick ({}): {}".format(len(idString), idString)) else: logging.error("Device is NOT a joystick ({}): {}".format(len(idString), idString))
def writeScreenCoordsToFile(self): return fh=utils.open("logs/screen.log","wb") fh.write(str(self._display)) #+"\n$$$\n") # for j in i._display: # fh.write(str(j)+"\n") fh.write("\n") fh.close()
def writeScreenCoordsToFile(self): return fh = utils.open("logs/screen.log", "wb") fh.write(str(self._display)) # +"\n$$$\n") # for j in i._display: # fh.write(str(j)+"\n") fh.write("\n") fh.close()
def _test_user_doc_gen(self): pair_path = self.config.pipeline_inputs[2] with utils.open(pair_path) as file: for line in file: line = line.strip('\n').split('\t') user_id = line[0] user_type = line[1] doc = line[2] yield user_id, user_type, doc
def write_allele_counts(output, vcf_data): with open(output, 'wt') as outfile: header = ['chrom', 'coord', 'ref', 'alt', 'n_ref_count', 'n_alt_count'] header = '\t'.join(header) + '\n' outfile.write(header) for key, key_data in vcf_data.items(): outstr = key + key_data[1] outstr = '\t'.join(outstr) + '\n' outfile.write(outstr)
def read_opera_mml(file): try: file = os.path.join('custom', 'opera', f'{file}.mml') with open(file, "r") as f: mml = f.read() return mml except IOError: print(f"Failed to read {file}") raise
def test_user_doc_score(self): pair_cnt = 0 real_cnt = 0 cnt = 0 out_path = self.config.pipeline_output batch_size = self.config.batch_size users = [] docs = [] users_vec = [] docs_vec = [] user_vec = self.user_vec doc_vec = self.doc_vec score_encoder = self.get_score_encoder() with utils.open(out_path, 'w') as ff: for user_id, user_type, doc in self._test_user_doc_gen(): pair_cnt = pair_cnt + 1 user = user_id + user_type if user in user_vec and doc in doc_vec: users.append((user_id, user_type)) docs.append(doc) users_vec.append(user_vec[user]) docs_vec.append(doc_vec[doc]) real_cnt = real_cnt + 1 cnt = cnt + 1 if cnt == batch_size: outputs = score_encoder.predict( [np.stack(users_vec), np.stack(docs_vec)]) for idtp, do, out in zip(users, docs, outputs): ff.write(idtp[0] + '\t' + idtp[1] + '\t' + do + '\t' + str(out[0]) + '\n') users = [] docs = [] users_vec = [] docs_vec = [] cnt = 0 if cnt != 0: outputs = score_encoder.predict( [np.stack(users_vec), np.stack(docs_vec)]) for idtp, do, out in zip(users, docs, outputs): ff.write(idtp[0] + '\t' + idtp[1] + '\t' + do + '\t' + str(out[0]) + '\n') users = [] docs = [] users_vec = [] docs_vec = [] cnt = 0
def score(config: settings.Config): task_handler = task.get(config) with utils.open(next(task_handler), 'w') as file: model = next(task_handler) for batch_info, batch_data in task_handler: batch_pred = model.predict_on_batch(batch_data) for (session, label, score), pred in zip(batch_info, batch_pred): file.write('{}\t{}\t{}\t{}\n'.format(session, label, score, pred[0])) return 0
def __init__(self, id=0, port=None): [self._device, idString] = utils.open(utils.getPort(id, port)) if not self._device: logging.error("No BITSI extended buttonbox connected.") elif idString == "BITSI_extend mode, Ready!": logging.debug( "Device is a BITSI extended buttonbox ({}): {}".format( len(idString), idString)) else: logging.error( "Device is NOT a BITSI extended buttonbox ({}): {}".format( len(idString), idString)) self.calibrated = False
def _load_data(self): logging.info("[+] loading training data") with utils.open(self.config.training_data_input) as file: pairs = [line.split('\t') for line in file] pairs = np.array([[int(line[0]), int(line[1])] for line in pairs]) self.partition = np.arange(len(pairs)) % 10 == 0 self.training_pairs = pairs[~self.partition] self.validation_pairs = pairs[self.partition] self.training_pairs_count = len(self.training_pairs) self.validation_pairs_count = len(self.validation_pairs) self.training_step = np.ceil(self.training_pairs_count / self.config.batch_size) self.validation_step = np.ceil(self.validation_pairs_count / self.config.batch_size) logging.info("[-] loaded {}+{} pairs".format(self.training_pairs_count, self.validation_pairs_count))
def _train_gen(self, once=False): window_size = self.config.window_size docs_window = self.DocsWindow(self.docs, window_size) while True: with utils.open(self.config.training_data_input) as file: for line in file: line = line.strip('\n').split('\t') if line[2]: ih = self._extract_impressions(line[2]) all_pos = [] all_ims = [] if self.config.max_impression != -1 and len(ih) > self.config.max_impression: continue for impression in ih: if self.config.max_impression_pos != -1 and len( impression.pos) > self.config.max_impression_pos: continue if self.config.max_impression_neg != -1 and len( impression.neg) > self.config.max_impression_neg: continue for pos in impression.pos: all_pos.append(pos) all_ims.append(impression) if len(all_pos) <= 1: continue i = -1 pos_len = len(all_pos) - 1 for pos in all_pos: i = i + 1 seq = all_pos[:i] + all_pos[i + 1:] if pos_len >= window_size: seq = np.random.choice(seq, size=window_size, replace=False) else: seq = np.random.choice(seq, size=pos_len, replace=False) for sam in self.train_sample(docs_window, seq, pos, all_ims[i].negative_samples(self.config.negative_samples)): yield sam if once: break
def get_vcf_header(vcf_files): with open(vcf_files[0], 'rt') as invcf: header = [] for line in invcf: if '##INFO' in line or '##FORMAT' in line: continue if line.startswith('##'): header.append(line.strip()) elif line.startswith('#'): line = '\t'.join(line.strip().split()[:8]) header.append(line) else: break return header
def write_vcf(output, vcf_files, data, tempdir, sample_id): temp_output = os.path.join(tempdir, '{}.vcf'.format(sample_id)) with open(temp_output, 'wt') as outfile: header = get_vcf_header(vcf_files) for line in header: outfile.write(line + '\n') for key, key_data in data.items(): chrom, pos, ref, alt = key outstr = (chrom, pos, '.', ref, alt) + key_data[0] + ('.', ) outstr = '\t'.join(outstr) + '\n' outfile.write(outstr) sort_vcf_file(temp_output, output)
def _load_docs(self): logging.info("[+] loading docs metadata") parser = document.DocumentParser( document.parse_document(), document.pad_document(1, self.config.title_shape)) with utils.open(self.config.doc_meta_input) as file: docs = [line.split('\t') for line in file] self.docs = {int(line[1]): parser(line[4])[0] for line in docs} self.doc_count = max(self.docs.keys()) + 1 self.doc_freq = np.zeros(self.doc_count) for line in docs: self.doc_freq[int(line[1])] = int(line[2]) self.doc_freq = self.doc_freq**0.75 self.doc_freq = self.doc_freq / np.sum(self.doc_freq) logging.info("[-] loaded {} docs".format(self.doc_count))
def get_doc_parser(self): doc_encoder = self.model.get_layer('doc_encoder') doc_input_shape = doc_encoder.layers[0].input_shape[1:] split_tokens = [] with utils.open(self.config.doc_punc_index_input) as file: for line in file: line = line.strip('\n').split('\t') if line[0] == '.' or line[0] == '?' or line[0] == '!': split_tokens.append(int(line[1])) body_parser = document.DocumentParser( document.parse_document(), document.clause(split_tokens), document.pad_docs(doc_input_shape[0], doc_input_shape[1])) return body_parser
def __init__(self, id=0, port=None): self._port = utils.getPort(id, port) self._device = None if not self._port: raise Exception( "No USB serial device connected, could not get port name.") [self._device, idString] = utils.open(self._port) if not self._device: logging.critical( "No BITSI buttonbox connected, could not connect to port {}". format(self._port)) elif idString == "BITSI mode, Ready!" or idString == "BITSI event mode, Ready!": logging.debug("Device is a BITSI buttonbox ({}): {}".format( len(idString), idString)) else: logging.error("Device is NOT a BITSI buttonbox ({}): {}".format( len(idString), idString))
def post_processing(self): names = self.cw.names filename = os.path.join(self.dir, '[merged] {}.txt'.format(self.title)) try: with utils.open(filename, 'wb') as f: f.write(' {}\n\n 作者:{}\n\n\n'.format( self.info['title'], self.artist).encode('utf8')) if self.info['novel_ex']: f.write(self.info['novel_ex'].encode('utf8')) for i, file in enumerate(names): self.cw.pbar.setFormat('[%v/%m] {} [{}/{}]'.format( tr_('병합...'), i, len(names))) with open(file, 'rb') as f_: text = f_.read() f.write(b'\n\n\n\n') f.write(text) finally: self.cw.pbar.setFormat('[%v/%m]')
def __init__(self): super(PartitionGadget, self).__init__() self.schedule_phase = 'pre_vlog' static_vkits = gvars.StaticVkits vkits = [it for it in gvars.Vkits if it not in static_vkits] cells = gvars.TB.PARTITION_CELLS part_cfg_name = '.partition.cfg' self.turds.append(part_cfg_name) with utils.open(part_cfg_name, 'w') as pfile: if static_vkits: static_vkit_pkgs = ' '.join([it.get_pkg_name() for it in static_vkits]) print("partition package %s ;" % static_vkit_pkgs, file=pfile) for cell in cells: print("partition cell %s ;" % cell, file=pfile) for vkit in vkits: print("partition package %s ;" % vkit.get_pkg_name(), file=pfile)
def post_processing(self): names = self.cw.names filename = clean_title(u'[merged] [{}] {}'.format( self.artist, self.info['title']), n=-4) + '.txt' filename = os.path.join(self.dir, filename) try: with utils.open(filename, 'wb') as f: f.write(u' {}\n\n \u4f5c\u8005\uff1a{}\n\n\n'.format( self.info['title'], self.artist).encode('utf8')) f.write(self.info['description'].encode('utf8')) for i, file in enumerate(names): self.cw.pbar.setFormat('[%v/%m] {} [{}/{}]'.format( tr_(u'\ubcd1\ud569...'), i, len(names))) with open(file, 'rb') as f_: text = f_.read() f.write(b'\n\n\n\n') f.write(text) finally: self.cw.pbar.setFormat('[%v/%m]')
def _valid_gen(self): while True: with utils.open(self.config.training_data_input) as file: for line in file: line = line.strip('\n').split('\t') if line[2] and line[3]: ih1 = self._extract_impressions(line[2]) ih2 = self._extract_impressions(line[3]) ch = self.Window(self.docs, self.config.window_size) for impression in ih1: for pos in impression.pos: ch.push(pos) for impression in ih2: for pos in impression.pos: if ch.count: for sam in self.valid_sample(ch, pos, impression.negative_samples( self.config.negative_samples)): yield sam ch.push(pos)
def load_patch_file(fn): filepath = os.path.join('data', 'script', fn + ".txt") # NEW # this prepends the absolute file path of the parent/calling script # to filepath created above - GreenKnight5 filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), filepath) try: with open(filepath, "r") as f: lines = f.readlines() except IOError: print(f"failed to open data/script/{fn}.txt") return for i, line in enumerate(lines): s = line.split(':', 1) try: script_idx = int(s[0].strip()) except ValueError: print(f"{fn}.txt: line {i} - {s[0]} is not a valid caption index") continue changes = s[1].rstrip('\n').split('|') for c in changes: if '->' not in c: continue chgfrom, chgto = c.split('->', 1) if '@' in chgfrom: chgfrom, match_idx = chgfrom.split('@', 1) try: match_idx = int(match_idx) except ValueError: print( f"{fn}.txt: line {i} - {match_idx} is not a valid match index" ) match_idx = 0 else: match_idx = None if chgto == "*": chgto = None patch_dialogue(script_idx, chgfrom, chgto, index=match_idx)
def post_processing(self): if self.single: return names = self.cw.names filename = os.path.join(self.dir, (u'[merged] {}.txt').format(self.title)) try: with utils.open(filename, 'wb') as f: f.write(u' {}\n\n \u4f5c\u8005\uff1a{}\n\n\n'.format( self.__title, self.artist).encode('utf8')) if self.novel_ex: f.write(self.novel_ex.encode('utf8')) for i, file in enumerate(names): self.cw.pbar.setFormat(u"[%v/%m] {} [{}/{}]".format( tr_(u'\ubcd1\ud569...'), i, len(names))) with open(file, 'rb') as f_: text = f_.read() f.write(b'\n\n\n\n') f.write(text) finally: self.cw.pbar.setFormat("[%v/%m]")
def manage_opera(fout, affect_music): fout.seek(0) data = fout.read() SAMPLE_MAX_SIZE = 3746 global opera_log #Determine opera cast class OperaSinger: def __init__(self, name, title, sprite, gender, file, sampletype, sample, octave, volume, init): self.name = name self.title = title self.sprite = sprite self.gender = gender self.file = file self.sampletype = sampletype self.sample = sample self.octave = octave self.volume = volume self.init = init self.sid = ("ext" if sampletype == "brr" else sampletype) + ":" + sample if self.sampletype not in ["brr", "ext"]: try: self.sample = int(self.sample, 16) except ValueError: print( f"WARNING: in opera config: {name}: invalid sample id for {sampletype}: {sample}" ) self.sample = 1 def get_sample_text(self, prg): if self.sampletype in ["brr", "ext"]: return f"\n#BRR 0x{prg:02X}; {self.sample}\n" elif self.sampletype == "legacy": return f"\n#BRR 0x{prg:02X}; {get_legacy_import(self.sample, subpath='music')}\n" else: return f"\n#WAVE 0x{prg:02X} 0x{self.sample:02X}\n" #voice range notes # Overture (Draco) ranges from B(o-1) to B(o) # Aria (Maria) ranges from D(o) to D(o+1) # Duel 1 (Draco) ranges from B(o-1) to E(o) # Duel 2 (Maria) ranges from A(o) to F(o+1) # Duel 3 (Ralse) ranges from E(o) to C(o+1) # Duel 4 (Draco) ranges from B(o-1) to F(o) # Duel 5 (Ralse) ranges from E(o) to B(o) singer_options = [] opera_config = configparser.ConfigParser(interpolation=None) opera_config.optionxform = lambda option: option try: with open(os.path.join('custom', 'opera.txt'), "r") as f: opera_config.read_file(f) except OSError: print("WARNING: failed to load opera config") return if 'Singers' not in opera_config or 'Factions' not in opera_config: print("WARNING: opera config is invalid, or failed to load") return for k, v in opera_config.items('Singers'): line = [k] + v.split('|') singer_options.append([l.strip() for l in line]) singer_options = [OperaSinger(*s) for s in singer_options] #categorize by voice sample (unless not changing music) if affect_music: voices = {} for s in singer_options: if s.sid in voices: voices[s.sid].append(s) else: voices[s.sid] = [s] #find a set of voices that doesn't overflow SPC RAM while True: vchoices = random.sample(list(voices.values()), 3) test_mml = read_opera_mml('duel') for i, c in enumerate(vchoices): test_mml += c[0].get_sample_text(i + 0x2A) memusage = get_spc_memory_usage(test_mml, subpath="music") if memusage <= SAMPLE_MAX_SIZE: break #select characters charpool = [] char = {} #by voice, for singers for v in vchoices: charpool.append(random.choice(v)) random.shuffle(charpool) for c in ["Maria", "Draco", "Ralse"]: char[c] = charpool.pop() opera_log += str(str(c) + ": ").ljust(17) + string.capwords( str(char[c].name)) + "\n" #by sprite/name, for impresario charpool = [c for c in singer_options if c not in char.values()] char["Impresario"] = random.choice(charpool) opera_log += str("Impresario: ").ljust(17) + string.capwords( str(char["Impresario"].name)) + "\n" else: print( "\nalasdraco -- note: opera voices will not be changed unless a" + " random music code (johnnydmad/johnnyachaotic) is also used." + " Sprites and text will still be affected.") cchoices = random.sample(singer_options, 4) char = {} for c in ["Maria", "Draco", "Ralse", "Impresario"]: char[c] = cchoices.pop() opera_log += str(str(c) + ": ").ljust(17) + string.capwords( str(char[c].name)) + "\n" #reassign sprites in npc data locations = get_locations() #first, free up space for a unique Ralse #choose which other NPCs get merged: # 0. id for new scholar, 1. offset for new scholar, 2. id for merged sprite, 3. offset for merged sprite, 4. spritesheet filename, 5. extra pose type merge_options = [ (32, 0x172C00, 33, 0x1731C0, "dancer.bin", "flirt"), #fancy gau -> dancer (32, 0x172C00, 35, 0x173D40, "gau-fancy.bin", "sideeye"), #clyde -> fancy gau (60, 0x17C4C0, 35, 0x173D40, "daryl.bin", "sideeye"), #clyde -> daryl (42, 0x176580, 35, 0x173D40, "katarin.bin", "sideeye"), #clyde -> katarin (60, 0x17C4C0, 42, 0x176580, "daryl.bin", "sideeye"), #katarin -> daryl (60, 0x17C4C0, 42, 0x176580, "katarin.bin", "sideeye"), #daryl -> katarin (60, 0x17C4C0, 41, 0x175FC0, "daryl.bin", "sleeping"), #rachel -> daryl (60, 0x17C4C0, 41, 0x175FC0, "rachel.bin", "sleeping"), #daryl -> rachel (60, 0x17C4C0, 30, 0x172080, "returner.bin", "prone"), #daryl -> returner (53, 0x17A1C0, 59, 0x17BFC0, "figaroguard.bin", None), #conductor -> figaro guard (45, 0x1776C0, 48, 0x178800, "maduin.bin", "prone"), #yura -> maduin ] merge = random.choice(merge_options) #merge sacrifice into new slot replace_npc(locations, (merge[0], None), merge[2]) #move scholar into sacrifice slot for i in [0, 1, 3, 4, 5]: replace_npc(locations, (27, i), (merge[0], i)) #randomize palettes of shuffled characters for i in [0x1A, 0x1B, 0x1C, 0x2B]: palette = random.randint(0, 5) for l in locations: for npc in l.npcs: if npc.graphics == i: #npc.palette = palette pass #debug info # for l in locations: # for npc in l.npcs: # if npc.graphics in [118,138]: # print() # print(f"graphics {npc.graphics} found at ${npc.pointer:X}, in location 0x{l.locid:X}") # print(f"palette {npc.palette}, facing byte {npc.facing:X}") # print(f"facing {npc.facing & 3:X}, change {npc.facing>>2 & 1:X}") # print(f"bglayer {npc.facing>>3 & 3:X}, unknown1 {npc.facing>>5 & 1:X}") # print(f"mirror {npc.facing>>6 & 2:X}, unknown2 {npc.facing>>7 & 1:X}") #randomize item thrown off balcony balcony = get_location(0xEC) for npc in balcony.npcs: if npc.graphics == 88: item = random.choice([ #(84, 0x24, "chest"), #treasure box (palette broken) (87, 0x44, "statue"), (88, 0x54, "flowers"), #bouquet (89, 0x54, "letter"), #letter (91, 0x54, "magicite"), #magicite (92, 0x44, "book"), #book ##DO NOT THROW THE BABY (96, 0x44, "crown"), #slave crown (97, 0x54, "weight"), #4ton weight (100, 0x54, "bandana"), #locke's bandana ##(124, 0x02, "helmet") #a shiny thing (didn't work) ]) npc.graphics = item[0] npc.palette = random.choice(range(6)) npc.facing = item[1] set_dialogue_var("OperaItem", item[2]) opera_log += str("Opera Flowers: ").ljust(17) + string.capwords( item[2]) + "\n" #print(f"opera item is {npc.graphics}, palette {npc.palette} ({item[2]})") #print(f"at address {npc.pointer:X}") #4 ton weight for npc in get_location(0xEB).npcs: if npc.graphics == 97: item = random.choice([ (89, 0x00, "letter"), # letter (92, 0x44, "book"), # book (108, 0x00, "rat"), # rat (87, 0x44, "mini statue"), # mini statue (93, 0x54, "baby" ), # ultros is allowed to try to throw the baby (STOP HIM) (97, 0x54, "4 ton weight"), # 4ton weight (112, 0x43, "fire"), # fire ###(118, 0x10), #rock (didn't work) ###(138, 0x12) #leo's sword (didn't work) ]) npc.graphics = item[0] npc.palette = random.choice(range(6)) npc.facing = item[1] opera_log += str("Opera Rafters: ").ljust(17) + string.capwords( item[2]) + "\n" #print(f"ultros item is {npc.graphics}, palette {npc.palette}") #print(f"at address {npc.pointer:X}") #set up some spritesheet locations pose = { 'singing': [0x66, 0x67, 0x68, 0x69, 0x64, 0x65, 0x60, 0x61, 0x62, 0x63], 'ready': list(range(0x3E, 0x44)), 'prone': list(range(0x51, 0x57)), 'angry': list(range(0x76, 0x7C)), 'flirt': [0xA3, 0xA4, 0x9C, 0x99, 0x9A, 0x9B], 'sideeye': [0x92, 0x93, 0x94, 0x95, 0x08, 0x09], 'sleeping': [0x86, 0x87, 0x88, 0x89, 0x08, 0x09] } opath = os.path.join("custom", "opera") #load scholar graphics try: with open(os.path.join(opath, "ralse.bin"), "rb") as f: sprite = f.read() except IOError: print(f"failed to open custom/opera/ralse.bin") sprite = None if sprite: new_sprite = create_sprite(sprite) data = byte_insert(data, merge[1], new_sprite) #load new graphics into merged slot try: with open(os.path.join(opath, f"{merge[4]}"), "rb") as f: sprite = f.read() except IOError: try: with open(os.path.join("custom", "sprites", f"{merge[4]}"), "rb") as f: sprite = f.read() except: print( f"failed to open custom/opera/{merge[4]} or custom/sprites/{merge[4]}" ) sprite = None if sprite: #print(f"merge {merge}, pose {pose}") new_sprite = create_sprite( sprite, pose[merge[5]] if merge[5] is not None else []) data = byte_insert(data, merge[3], new_sprite) #load new graphics into opera characters char_offsets = { "Maria": (0x1705C0, pose['ready'] + pose['singing']), "Draco": (0x1713C0, pose['prone'] + pose['singing']), "Ralse": (0x170CC0, pose['prone'] + pose['singing']), "Impresario": (0x176B40, pose['angry']) } for cname, c in char.items(): #print(f"{cname} -> {c.name}") try: with open(os.path.join(opath, f"{c.sprite}.bin"), "rb") as f: sprite = f.read() except IOError: try: with open(os.path.join("custom", "sprites", f"{c.sprite}.bin"), "rb") as f: sprite = f.read() except: print( f"failed to open custom/opera/{c.sprite}.bin or custom/sprites/{c.sprite}.bin" ) continue offset, extra_tiles = char_offsets[cname] #tiles = list(range(0x28)) + extra_tiles #new_sprite = bytearray() #for t in tiles: # loc = t*32 # new_sprite.extend(sprite[loc:loc+32]) #data = byte_insert(data, offset, new_sprite) new_sprite = create_sprite(sprite, extra_tiles) data = byte_insert(data, offset, new_sprite) ### adjust script load_patch_file("opera") factions = [] for f, _ in opera_config.items('Factions'): factions.append([s.strip() for s in f.split('|')]) factions = random.choice(factions) if random.choice([False, True]): factions = (factions[1], factions[0]) opera_log += str("Opera Factions: ").ljust(17) + string.capwords( factions[0]) + " VS " + string.capwords(factions[1]) + "\n" set_dialogue_var("OperaEast", factions[0]) set_dialogue_var("OperaWest", factions[1]) set_dialogue_var("maria", char['Maria'].name) set_dialogue_var("draco", char['Draco'].name) set_dialogue_var("ralse", char['Ralse'].name) set_dialogue_var("impresario", char['Impresario'].name) set_dialogue_var("mariatitle", char['Maria'].title) set_dialogue_var("dracotitle", char['Draco'].title) set_dialogue_var("ralsetitle", char['Ralse'].title) set_dialogue_var("impresariotitle", char['Impresario'].title) char['Maria'].gender = set_pronoun('Maria', char['Maria'].gender) char['Draco'].gender = set_pronoun('Draco', char['Draco'].gender) char['Ralse'].gender = set_pronoun('Ralse', char['Ralse'].gender) char['Impresario'].gender = set_pronoun('Impresario', char['Impresario'].gender) #due to the variance in power relations connoted by "make X my queen" and "make X my king", this line will be altered in all variations so that it means roughly the same thing no matter the Maria replacement's gender if char['Maria'].gender == "female": set_dialogue_var("MariaTheGirl", "the girl") set_dialogue_var("MariaQueenBad", "mine") set_dialogue_var("MariaQueen", "queen") set_dialogue_var("MariaWife", "wife") elif char['Maria'].gender == "male": set_dialogue_var("MariaTheGirl", "the guy") set_dialogue_var("MariaQueenBad", "mine") set_dialogue_var("MariaQueen", "king") set_dialogue_var("MariaWife", "husband") elif char['Maria'].gender == "object": set_dialogue_var("MariaTheGirl", char['Maria'].title + " " + char['Maria'].name) set_dialogue_var("MariaQueenBad", "mine") set_dialogue_var("MariaQueen", "prize") set_dialogue_var("MariaWife", "collection") else: set_dialogue_var("MariaTheGirl", "the girl") set_dialogue_var("MariaQueenBad", "mine") set_dialogue_var("MariaQueen", "consort") set_dialogue_var("MariaWife", "partner") if char['Impresario'].gender == "male": set_dialogue_var("ImpresarioMan", "man") # from "music man" elif char['Impresario'].gender == "female": set_dialogue_var("ImpresarioMan", "madam") elif char['Impresario'].gender == "object": set_dialogue_var("ImpresarioMan", "machine") else: set_dialogue_var("ImpresarioMan", "maker") ### adjust music opera = {} maria_sample = char['Maria'].get_sample_text(0x2A) draco_sample = char['Draco'].get_sample_text(0x2B) ralse_sample = char['Ralse'].get_sample_text(0x2C) try: overture = read_opera_mml('overture') overture += draco_sample overture += f"\n#def draco= |B o{char['Draco'].octave[0]} v{char['Draco'].volume} {char['Draco'].init}\n" seg = read_opera_mml(f"{char['Draco'].file}_overture") overture += seg aria = read_opera_mml('aria') aria += maria_sample aria += f"\n#def maria= |A o{char['Maria'].octave[1]} v{char['Maria'].volume} {char['Maria'].init}\n" seg = read_opera_mml(f"{char['Maria'].file}_aria") aria += seg duel = read_opera_mml('duel') duel += maria_sample duel += f"\n#def maria= |A o{char['Maria'].octave[3]} v{char['Maria'].volume} {char['Maria'].init}\n" duel += draco_sample duel += f"\n#def draco= |B o{char['Draco'].octave[2]} v{char['Draco'].volume} {char['Draco'].init}\n" duel += f"\n#def draco2= |B o{char['Draco'].octave[5]} v{char['Draco'].volume} {char['Draco'].init}\n" duel += ralse_sample duel += f"\n#def ralse= |C o{char['Ralse'].octave[4]} v{char['Ralse'].volume} {char['Ralse'].init}\n" duel += f"\n#def ralse2= |C o{char['Ralse'].octave[6]} v{char['Ralse'].volume} {char['Ralse'].init}\n" duelists = ["Draco", "Maria", "Ralse", "Draco", "Ralse"] for i in range(5): seg = read_opera_mml(f"{char[duelists[i]].file}_duel{i+1}") duel += seg #print(overture) #print("########") #print(duel) #print("########") #print(aria) opera['opera_draco'] = overture opera['wed_duel'] = duel opera['aria'] = aria except IOError: print("opera music generation failed, reverting to default") affect_music = False fout.seek(0) fout.write(data) return opera if affect_music else None
# -*- coding: utf-8 -*- import utils import pandas as pd import pickle df = pd.read_csv('../data/Vocab.tsv',sep='\t',names=['word','index','vec']) punc_idx = [ ('.',df[df['word']=='.']['index'].values[0]), ('!',df[df['word']=='!']['index'].values[0]), ('?',df[df['word']=='?']['index'].values[0]), (';',df[df['word']==';']['index'].values[0]), (',',df[df['word']==',']['index'].values[0]) ] with utils.open('../data/PuncIndex.tsv','w') as file: for punc,idx in punc_idx: file.write(punc+'\t'+str(idx)+'\n') vocab2idx = {word:index for word,index,_ in df.values} pickle.dump(vocab2idx,open('../data/vocab2idx.pkl','wb'))
def extractGuideFastq(fastqFile,outputFolder='',gzip=False): ''' takes a fastq and extracts candidate guide RNAs ''' #get the full absolute path for the fastq File fastqFile = os.path.abspath(fastqFile) fastq = utils.open(fastqFile,'r') #get the fastq name and root if len(outputFolder) == 0: outputFolder = utils.getParentFolder(fastqFile) #makes sure the output folder exists utils.formatFolder(outputFolder,True) #grab the name info from the fastq fastqName = fastqFile.split('/')[-1] fastqRoot = string.replace(fastqName,'.fastq','') fastqRoot = string.replace(fastqRoot,'.gz','') #guideFastqFile output guideFastqFile = '%s%s.gecko.fastq' % (outputFolder,fastqRoot) guideFastq = utils.open(guideFastqFile,'w') print('processing %s' % (fastqName)) print('million reads processed:') ticker = 0 found = 0 while True: if ticker%1000000 == 0: print(ticker/1000000) fastqLines = [] #now load the fastq lines try: for i in range(4): fastqLines.append(fastq.next()) except StopIteration: break #see if you can find a cut site seq = fastqLines[1].rstrip() try: cutPosition = seq.index(cutSeq) found+=1 except ValueError: ticker+=1 continue guideStart = cutPosition + cutOffset guideStop = guideStart + guideLength #pulling out the guide seq in the fastqLines fastqLines[1] = fastqLines[1][guideStart:guideStop] + '\n' fastqLines[3] = fastqLines[3][guideStart:guideStop] + '\n' for line in fastqLines: guideFastq.write(line) ticker+=1 # if ticker == 100000: # print(ticker) # print(found) # print(float(found)/float(ticker)) # break print('SUMMARY STATISTICS') print(ticker) print(found) print(float(found)/float(ticker)) #close the fastq guideFastq.close() #gzip the fastq if gzip: os.system('gzip %s &' % (guideFastqFile)) guideFastqFile += '.gz' return guideFastqFile