def preprocess(self): # Initialize variables files = [] self.wrongs_idx = [[]] i = 0 converter = cv.Converter() # Get all files in the directory path = os.path.join(utils.getRoot(), self.directory) for file in os.listdir(path): files.append(utils.getPath(self.directory, file)) # Convert all files retrieved for file in files: base = file.split('/')[-1].split('.')[0] ext = file.split('/')[-1].split('.')[-1] outname = "".join([base, ".txt"]) if ext == "xml": converter.reset() self.datas.append([]) (score_info, _, notes) = utils.xml_parser(file) converter.setKappa(score_info.divisions) for note in notes: matrix = converter.convert_note(note) for elem in matrix: self.datas[i].append(elem) with open(utils.getPath(self.directory, outname), 'r') as file: for line in file: self.wrongs_idx[i].append(int(line.rstrip('\n'))) self.wrongs_idx.append([]) i += 1
def preprocess(self): # Initialize variables files = [] num_file = 0 converter = cv.Converter() self.wrongs_idx = [[]] # Get all files in the directory path = os.path.join(utils.getRoot(), self.directory) for file in os.listdir(path): files.append(utils.getPath(self.directory, file)) # Convert all files retrieved for data in files: converter.reset() self.datas.append([]) (score_info, _, notes) = utils.xml_parser(data) converter.setKappa(score_info.divisions) num_note = 0 for note in notes: matrix = converter.convert_note(note) for elem in matrix: new_note, has_changed = self.random_note(elem) self.datas[num_file].append(new_note) if has_changed: self.wrongs_idx[num_file].append(num_note) num_note += 1 self.wrongs_idx.append([]) num_file += 1
def __init__(self): """Constructor. Raises: Exception: ROS node does not exist """ argvs = sys.argv argc = len(argvs) # Initialize if not (argc != 5 or argc != 7): raise Exception('Not enough arguments') self.output_path = argvs[1] self.log_dir_path = argvs[2] self.main_func_path = argvs[3] self.fop = file_operator.FileOperator(self.output_path) self.convert_file_path, self.data_file_path = self.fop._getWorkFilesPath( ) self.logger = cfs_logger.CfsLogger(self.log_dir_path) self.searcher = searcher.Searcher() self.topic_manager = topic_manager.TopicManager( self.convert_file_path, self.data_file_path) self.conv = converter.Converter(self.output_path, self.convert_file_path, self.data_file_path, self.fop, self.logger, self.topic_manager) self.nh_name = None
def main(): parser = argparse.ArgumentParser(description="Converts a dependency structure to a phrase structure") parser.add_argument("data", type=str, help="Dependency graph in Malt-TAB format") parser.add_argument("projections", type=argparse.FileType(mode="r"), help="Projections in JSON format") parser.add_argument("arguments", type=argparse.FileType(mode="r"), help="Argument table in JSON format") parser.add_argument("modifiers", type=argparse.FileType(mode="r"), help="Modification table in JSON format") parser.add_argument("-d", "--draw", action='store_true', help="Draw the graph (windowmanager needed)") parser.add_argument("-q", "--qtree", action='store_true', help="Print latex qTree to the console") parser.add_argument("-o", "--outfile", dest="outfile", type=argparse.FileType(mode="w"), help="Write tree to file") args = parser.parse_args() deps = parse.DependencyGraph.load(args.data) proj = json.load(args.projections) arg = json.load(args.arguments) mod = json.load(args.modifiers) for dep in deps: c = converter.Converter(dep, proj, arg, mod) ret = c.convert() print(ret.pprint()) if args.qtree: print(ret.pprint_latex_qtree()) if args.outfile: args.outfile.write(ret.pprint()) args.outfile.write(os.linesep) args.outfile.flush() if args.draw: ret.draw()
def __init__(self, config, cons): assert config['bit_depth'] >= 1 assert config['pixels_per_cycle'] >= 1 self.bd = config['bit_depth'] self.ps = config['pixels_per_cycle'] enc_out_bits = min(31, 16 + self.bd) * self.ps #pixels in self.pixels_in = Array( Signal(self.bd, name="pixel_in") for _ in range(self.ps)) self.enc_out = Signal(enc_out_bits) self.enc_out_ctr = Signal(max=enc_out_bits + 1) self.latch_output = Signal(1) #valid in & out self.nready = Signal(1, reset=1) self.valid_in = Signal(1) self.valid_out = Signal(1) self.out_end = Signal(1) self.integration_1 = integration_1.Integration1(config, cons) self.lj92_pipeline_fifo = lj92_pipeline_fifo.LJ92PipelineFifo( config, cons) self.converter = converter.Converter(config, cons) self.converter_fifo = converter_fifo.ConverterFifo(config, cons) self.ios = \ [pixel_in for pixel_in in self.pixels_in] + \ [self.enc_out, self.enc_out_ctr] + \ [self.latch_output, self.nready] + \ [self.valid_in, self.valid_out] + \ [self.integration_1.fend_out]
def __init__(self, *args, **kwargs): wx.Frame.__init__(self, *args, **kwargs) self.tbIcon = tray.TaskBarIcon(self) self.hotKeyId = 100 self.RegisterHotKey(self.hotKeyId, wx.MOD_CONTROL, wx.WXK_F11) self.Bind(wx.EVT_HOTKEY, self.handleConvert, id=self.hotKeyId) self.convert = converter.Converter()
def setUpClass(cls): """ Run one-time before any testing is performed in this class """ cls.filename = "test_recursion.json" cls.nodes = conv.Converter() cls.nodes.load(cls.filename) cls.nodes.to_call_list() cls.nodes.save(cls.filename)
def test_convert_method_yard_to_inch_human_readable(self): """ given a converter with the 3 unit converters registered and 3 yards set as value when I convert the string '79.4yd' into 'in' with a result as string then I get 2858.4007 """ converter_instance = converter.Converter() result = converter_instance.convert('79.4yd', 'in', as_string=True) self.assertEquals(result, '2858.5 in')
def __init__(self, output, jsonFile): self.bidsDir = os.path.abspath(output) self.masterDicomsDir = os.path.dirname(os.path.abspath(jsonFile)) self.json = utils.loadJsonFile(jsonFile) self.participant = participant.Participant(self.json["participant"]) self.session = session.Session(self.json) self.session.setDir(self.bidsDir, self.participant.getName()) self.acquisitions = self.__setAcquisitions() self.converter = converter.Converter()
def test_convert_method_yard_to_inch(self): """ given a converter with the 3 unit converters registered and 3 yards set as value when I convert the string '79.4yd' into 'in' then I get 2858.4007 """ converter_instance = converter.Converter() result = converter_instance.convert('79.4yd', 'in') self.assertAlmostEqual(result, 2858.4007, places=4)
def test_check_registered_short_unit_unknow_raise(self): """ given a converter with the 3 unit converters registered when I search the unit converter for short name ft then I get an UnitConvertNotFound exception """ converter_instance = converter.Converter() with self.assertRaises(converter.UnitConvertNotFound): converter_instance._get_unit_converter('ft')
def __init__(self): # initialize our window object root = tkinter.Tk() root.title('Temperature Converter v1.0') root.geometry('515x140') root.resizable(False, False) self.model = converter.Converter() self.view = windows.MyFrame(self) self.view.mainloop()
def test_value_convertion_yard_to_inche(self): """ given a converter with the 4 unit converters registered and 3 yards set as value when I get converted value in inches then I get 144 inches """ converter_instance = converter.Converter() converter_instance.set_value(4, 'yd') result = converter_instance.convert_to_unit('in') self.assertAlmostEqual(result, 144, places=4)
def __init__(self, args, ext, remove_invalid=True): self.args = args self.remove_invalid = remove_invalid if self.remove_invalid: self.conv = converter.Converter(tables=getattr( args, 'tables', 'data/spider/tables'), db=getattr(args, 'db', 'data/database')) self.sql_vocab = ext['sql_voc'] self.evaluator = evaluation.Evaluator()
def try_converter(): """ 输入: bin/test.md, src/temp.yaml 输出: /outputs/test_md_folder/sample_item.zh.md """ c = converter.Converter(TEST_MD, "test_md_folder") print(c) io = io_handler.IOHandler(c) # print(c) io.write_yaml()
def test_value_convertion_yard_to_feet(self): """ given a converter with the 3 unit converters registered and 3 yards set as value when I get converted value in feets then I get an UnitConvertNotFound exception """ converter_instance = converter.Converter() converter_instance.set_value(23, 'yd') with self.assertRaises(converter.UnitConvertNotFound): converter_instance.convert_to_unit('ft')
def test_value_convertion_yard_to_inches_as_string(self): """ given a converter with the 3 unit converters registered and 3 yards set as value when I get converted value in meters with the as_string flag then I get litteraly "112.98 m" """ converter_instance = converter.Converter() converter_instance.set_value(123.56, 'yd') result = converter_instance.convert_to_unit('m', as_string=True) self.assertEquals(result, "113 m")
def test_check_registered_short_unit(self): """ given a converter with the 3 unit converters registered when I search the unit converter for each short unit name then I get unit_convert for each name """ converter_instance = converter.Converter() for short_unit_name in ('m', 'yd', 'in'): unit_converter = converter_instance._get_unit_converter(short_unit_name) self.assertEquals(short_unit_name, unit_converter.short_unit)
def run_Convert(): valid = c.validateForConversion() if valid: converter = c.Converter(inputBase=int(e.input_BaseInput.get()), outputBase=int(e.input_BaseOutput.get()), inputValue=str(e.input_Value.get())) converted = converter.convertToOutput() if len(converter.warnings) > 0: for warning in converter.warnings: converted = converted + " : " + warning e.box_Output.update(converted)
def __init__(self): """ This starts the Tk framework up, instantiates the Model (a Counter object), instantiates the View (a MyFrame object), and starts the event loop that waits for the user to press a Button on the View. """ root = tkinter.Tk() self.model = converter.Converter() self.view = myFrame.MyFrame(self) self.view.mainloop() root.destroy()
def try_dump(): c = converter.Converter(TEST_MD, "test_md_folder") yh = yaml_handler.YAMLHandler(c) yh.set_title("测试标题") yh.set_date(datetime.today()) yh.set_summary(True) yh.set_taxonomy(['学习'], ['tag1', 'tag2', '中文tag']) yaml_dict = yh.get_yaml_dict() print(yaml_dict, "<- yaml dict") yh.dump_yaml()
def __init__(self, args, ext): super().__init__(args, ext) self.conv = converter.Converter(tables=getattr(args, 'tables', 'data/spider/tables'), db=getattr(args, 'db', 'data/database')) self.bert_tokenizer = DistilBertTokenizer.from_pretrained( args.dcache + '/vocab.txt', cache_dir=args.dcache) self.bert_embedder = DistilBertModel.from_pretrained( args.dcache, cache_dir=args.dcache) self.value_bert_embedder = DistilBertModel.from_pretrained( args.dcache, cache_dir=args.dcache) self.denc = 768 self.demb = args.demb self.sql_vocab = ext['sql_voc'] self.sql_emb = nn.Embedding.from_pretrained(ext['sql_emb'], freeze=False) self.pad_id = self.sql_vocab.word2index('PAD') self.dropout = nn.Dropout(args.dropout) self.bert_dropout = nn.Dropout(args.bert_dropout) self.table_sa_scorer = nn.Linear(self.denc, 1) self.col_sa_scorer = nn.Linear(self.denc, 1) self.col_trans = nn.LSTM(self.denc, self.demb // 2, bidirectional=True, batch_first=True) self.table_trans = nn.LSTM(self.denc, args.drnn, bidirectional=True, batch_first=True) self.pointer_decoder = decoder.PointerDecoder( demb=self.demb, denc=2 * args.drnn, ddec=args.drnn, dropout=args.dec_dropout, num_layers=args.num_layers) self.utt_trans = nn.LSTM(self.denc, self.demb // 2, bidirectional=True, batch_first=True) self.value_decoder = decoder.PointerDecoder(demb=self.demb, denc=self.denc, ddec=args.drnn, dropout=args.dec_dropout, num_layers=args.num_layers) self.evaluator = evaluation.Evaluator() if 'reranker' in ext: self.reranker = ext['reranker'] else: self.reranker = rank_max.Module(args, ext, remove_invalid=True)
def test_set_value_register_meter_value(self): """ given a converter with the 3 unit converters registered when I set a value with set_value then the intial_length is register as meter """ yard_value = 2 meter_value = 1.8288 converter_instance = converter.Converter() converter_instance.set_value(2, 'yd') self.assertAlmostEqual(converter_instance.intial_length, meter_value, places=4) self.assertEquals(converter_instance.intial_unit, 'yd')
def __init__(self): ''' starts TK framework instantiates model (converter) instantiates view (MyFrame) starts event loop ''' root = tkinter.Tk() self.model = converter.Converter() self.view = myFrame.MyFrame(self) self.view.mainloop() root.destroy()
def from_xml_to_n2c(file): """ Convert a .xml file to .n2c extension file. Note are encoding with 16 bits (or 4 hex) Parameters ---------- file: XML file File to be compressed. Returns ------- _ : int 0 if success, else 1. Raise ----- AssertionError """ # Initialize Variables compressed_notes = [] basename, ext = os.path.splitext(file) if ext == ".n2c" or ext == ".txt": return 1 # Convert all files retrieved converter = conv.Converter() converter.reset() (score_info, _, notes) = utils.xml_parser(file) converter.setKappa(score_info.divisions) for note in notes: try: matrix = converter.convert_note(note) for elem in matrix: temp = from_array_to_hex(elem) compressed_notes.append(temp) temp_note, _ = extract.from_hex_to_array(temp) np.testing.assert_array_equal(elem, temp_note) except AssertionError: print file return 1 # Assertions evaluation for x in compressed_notes: assert len(x) == 4, "{}, {}".format(file, x) assert len("".join(compressed_notes)) % 4 == 0, compressed_notes with open("".join([basename, ".n2c"]), 'wb') as compressed_file: compressed_file.write("".join(compressed_notes)) return 0
def from_txt_to_n2c(xml, txt): """ Convert a .xml file with a .txt to .n2c extension file. Note are encoding with 16 bits. Txt file consists of wrong notes indexes. Xml and txt files must have the same basename (only extension is different). Parameters ---------- xml: XML file File to be compressed. txt: TXT file File with wrongs notes indexes. Numbers are comma-separated. """ # Initialize variables with open(txt, 'rb') as text_file: wrongs = [int(line) for line in text_file.read().split(',')] compressed_notes = [] basename, ext = os.path.splitext(xml) if ext == ".n2c" or ext == ".txt": return 1 # Convert all files retrieved converter = conv.Converter() converter.reset() (score_info, _, notes) = utils.xml_parser(xml) converter.setKappa(score_info.divisions) for idx, note in enumerate(notes): try: matrix = converter.convert_note(note) for elem in matrix: is_wrong = idx in wrongs temp = from_array_to_hex(elem, is_wrong) compressed_notes.append(temp) temp_note, _ = extract.from_hex_to_array(temp) np.testing.assert_array_equal(elem, temp_note) except AssertionError: print file return 1 # Assertions evaluation for x in compressed_notes: assert len(x) == 4, "Error in file {file}: Hexa converted equals to {hexa}".format(file=file, hexa=x) joined_compressed_notes = "".join(compressed_notes) assert len(joined_compressed_notes) % 4 == 0, "Compressed file length is not a multiple of 4 : {length}".format(length=len(joined_compressed_notes)) with open("".join([basename, '.n2c']), 'wb') as compressed_file: compressed_file.write(joined_compressed_notes) return 0
def __convert(self): self.__outfilename = os.path.join(self.__outdir, str(self.outputFileLineName.text())) if not os.path.exists(self.__infile) : raise FileNotExistException(self) self.checkFile(__OUTFILE) if self.__infile == self.__outfilename: raise SameFileException(self) if os.path.exists(self.__outfilename): if not self.overwriteBox.isChecked() : raise OverWriteException(self) overwrite = self.overwriteBox.isChecked() self.starter = converter.Converter(self.__infile, self.__outfilename, overwrite, self) self.starter.convert()
def __init__(self, **kwargs): """ Initialize a dataset from datas (already converted) Parameters ---------- See dyci2 module (run function) """ self._past_index = 0 self._future_index = 0 self._right_index = 1 self.directory = kwargs["directory"] self.datas = [[]] self.converter = cv.Converter() self.config(**kwargs) self.time_size = self.past + self.future + 1
def compile(filename, format, **options): conv = converter.Converter(filename, **options) modules = conv.parse() text = '\n'.join(modules[fname] for fname in sorted(modules.keys())) lib = os.path.join(options.get('lib_dir', '.'), 'pjslib.js') data = {'file': os.path.abspath(filename), 'text': text, 'lib': lib} data['path'] = sys.path if options.get('rhino', False): template = rhino_out elif options.get('html', False): template = html_out else: template = js_out return template % data
def from_file(cls, root, dspider, dcache, debug=False): train_database, dev_database = editsql_preprocess.read_db_split(dspider) conv = converter.Converter() kmaps = evaluation.build_foreign_key_map_from_json(os.path.join(dspider, 'tables.json')) splits = {} for k in ['train', 'dev']: with open(os.path.join(root, '{}.json'.format(k)), 'rb') as f: splits[k] = [] for ex in json.load(f): splits[k].append(ex) if debug and len(splits[k]) > 100: break tokenizer = DistilBertTokenizer.from_pretrained(BERT_MODEL, cache_dir=dcache) sql_voc = Vocab(['PAD', 'EOS', 'GO', 'SEP', '`', "'", '1', '%', 'yes', '2', '.', '5', 'f', 'm', 'name', 'song', 't', 'l']) # make contexts and populate vocab for s, data in splits.items(): proc = [] for i, ex in enumerate(tqdm.tqdm(data, desc='preprocess {}'.format(s))): for turn_i, turn in enumerate(ex['interaction']): turn['id'] = '{}/{}:{}'.format(ex['database_id'], i, turn_i) turn['db_id'] = ex['database_id'] turn['prev'] = ex['interaction'][turn_i-1] if turn_i > 0 else None new = cls.make_example(turn, tokenizer, sql_voc, kmaps, conv, train=s=='train') if new is not None and (s != 'train' or not new['invalid']): proc.append(new) splits[s] = proc # make candidate list using vocab for s, data in splits.items(): for ex in data: ex['cands_query'], ex['cands_value'] = cls.make_cands(ex, sql_voc) splits[s] = data # make pointers for training data for ex in splits['train']: ex['pointer_query'], ex['pointer_value'] = cls.make_query_pointer(ex['sup_query'], ex['cands_query'], ex['cands_value'], sql_voc) # look up pretrained word embeddings emb = E.ConcatEmbedding([E.GloveEmbedding(), E.KazumaCharEmbedding()], default='zero') sql_emb = torch.tensor([emb.emb(w) for w in sql_voc._index2word]) ext = dict(sql_voc=sql_voc, sql_emb=sql_emb) return splits, ext