def preprocess(self):
# Initialize variables
files = []
self.wrongs_idx = [[]]
i = 0
converter = cv.Converter()
# Get all files in the directory
path = os.path.join(utils.getRoot(), self.directory)
for file in os.listdir(path):
files.append(utils.getPath(self.directory, file))
# Convert all files retrieved
for file in files:
base = file.split('/')[-1].split('.')[0]
ext = file.split('/')[-1].split('.')[-1]
outname = "".join([base, ".txt"])
if ext == "xml":
converter.reset()
self.datas.append([])
(score_info, _, notes) = utils.xml_parser(file)
converter.setKappa(score_info.divisions)
for note in notes:
matrix = converter.convert_note(note)
for elem in matrix:
self.datas[i].append(elem)
with open(utils.getPath(self.directory, outname), 'r') as file:
for line in file:
self.wrongs_idx[i].append(int(line.rstrip('\n')))
self.wrongs_idx.append([])
i += 1
def preprocess(self):
# Initialize variables
files = []
num_file = 0
converter = cv.Converter()
self.wrongs_idx = [[]]
# Get all files in the directory
path = os.path.join(utils.getRoot(), self.directory)
for file in os.listdir(path):
files.append(utils.getPath(self.directory, file))
# Convert all files retrieved
for data in files:
converter.reset()
self.datas.append([])
(score_info, _, notes) = utils.xml_parser(data)
converter.setKappa(score_info.divisions)
num_note = 0
for note in notes:
matrix = converter.convert_note(note)
for elem in matrix:
new_note, has_changed = self.random_note(elem)
self.datas[num_file].append(new_note)
if has_changed:
self.wrongs_idx[num_file].append(num_note)
num_note += 1
self.wrongs_idx.append([])
num_file += 1
def __init__(self):
"""Constructor.
Raises:
Exception: ROS node does not exist
"""
argvs = sys.argv
argc = len(argvs)
# Initialize
if not (argc != 5 or argc != 7):
raise Exception('Not enough arguments')
self.output_path = argvs[1]
self.log_dir_path = argvs[2]
self.main_func_path = argvs[3]
self.fop = file_operator.FileOperator(self.output_path)
self.convert_file_path, self.data_file_path = self.fop._getWorkFilesPath(
)
self.logger = cfs_logger.CfsLogger(self.log_dir_path)
self.searcher = searcher.Searcher()
self.topic_manager = topic_manager.TopicManager(
self.convert_file_path, self.data_file_path)
self.conv = converter.Converter(self.output_path,
self.convert_file_path,
self.data_file_path, self.fop,
self.logger, self.topic_manager)
self.nh_name = None
def main():
parser = argparse.ArgumentParser(description="Converts a dependency structure to a phrase structure")
parser.add_argument("data", type=str, help="Dependency graph in Malt-TAB format")
parser.add_argument("projections", type=argparse.FileType(mode="r"), help="Projections in JSON format")
parser.add_argument("arguments", type=argparse.FileType(mode="r"), help="Argument table in JSON format")
parser.add_argument("modifiers", type=argparse.FileType(mode="r"), help="Modification table in JSON format")
parser.add_argument("-d", "--draw", action='store_true', help="Draw the graph (windowmanager needed)")
parser.add_argument("-q", "--qtree", action='store_true', help="Print latex qTree to the console")
parser.add_argument("-o", "--outfile", dest="outfile", type=argparse.FileType(mode="w"), help="Write tree to file")
args = parser.parse_args()
deps = parse.DependencyGraph.load(args.data)
proj = json.load(args.projections)
arg = json.load(args.arguments)
mod = json.load(args.modifiers)
for dep in deps:
c = converter.Converter(dep, proj, arg, mod)
ret = c.convert()
print(ret.pprint())
if args.qtree:
print(ret.pprint_latex_qtree())
if args.outfile:
args.outfile.write(ret.pprint())
args.outfile.write(os.linesep)
args.outfile.flush()
if args.draw:
ret.draw()
def __init__(self, config, cons):
assert config['bit_depth'] >= 1
assert config['pixels_per_cycle'] >= 1
self.bd = config['bit_depth']
self.ps = config['pixels_per_cycle']
enc_out_bits = min(31, 16 + self.bd) * self.ps
#pixels in
self.pixels_in = Array(
Signal(self.bd, name="pixel_in") for _ in range(self.ps))
self.enc_out = Signal(enc_out_bits)
self.enc_out_ctr = Signal(max=enc_out_bits + 1)
self.latch_output = Signal(1)
#valid in & out
self.nready = Signal(1, reset=1)
self.valid_in = Signal(1)
self.valid_out = Signal(1)
self.out_end = Signal(1)
self.integration_1 = integration_1.Integration1(config, cons)
self.lj92_pipeline_fifo = lj92_pipeline_fifo.LJ92PipelineFifo(
config, cons)
self.converter = converter.Converter(config, cons)
self.converter_fifo = converter_fifo.ConverterFifo(config, cons)
self.ios = \
[pixel_in for pixel_in in self.pixels_in] + \
[self.enc_out, self.enc_out_ctr] + \
[self.latch_output, self.nready] + \
[self.valid_in, self.valid_out] + \
[self.integration_1.fend_out]
def __init__(self, *args, **kwargs):
wx.Frame.__init__(self, *args, **kwargs)
self.tbIcon = tray.TaskBarIcon(self)
self.hotKeyId = 100
self.RegisterHotKey(self.hotKeyId, wx.MOD_CONTROL, wx.WXK_F11)
self.Bind(wx.EVT_HOTKEY, self.handleConvert, id=self.hotKeyId)
self.convert = converter.Converter()
def setUpClass(cls):
""" Run one-time before any testing is performed in this class
"""
cls.filename = "test_recursion.json"
cls.nodes = conv.Converter()
cls.nodes.load(cls.filename)
cls.nodes.to_call_list()
cls.nodes.save(cls.filename)
def test_convert_method_yard_to_inch_human_readable(self):
"""
given a converter with the 3 unit converters registered and 3 yards set as value
when I convert the string '79.4yd' into 'in' with a result as string
then I get 2858.4007
"""
converter_instance = converter.Converter()
result = converter_instance.convert('79.4yd', 'in', as_string=True)
self.assertEquals(result, '2858.5 in')
def __init__(self, output, jsonFile):
self.bidsDir = os.path.abspath(output)
self.masterDicomsDir = os.path.dirname(os.path.abspath(jsonFile))
self.json = utils.loadJsonFile(jsonFile)
self.participant = participant.Participant(self.json["participant"])
self.session = session.Session(self.json)
self.session.setDir(self.bidsDir, self.participant.getName())
self.acquisitions = self.__setAcquisitions()
self.converter = converter.Converter()
def test_convert_method_yard_to_inch(self):
"""
given a converter with the 3 unit converters registered and 3 yards set as value
when I convert the string '79.4yd' into 'in'
then I get 2858.4007
"""
converter_instance = converter.Converter()
result = converter_instance.convert('79.4yd', 'in')
self.assertAlmostEqual(result, 2858.4007, places=4)
def test_check_registered_short_unit_unknow_raise(self):
"""
given a converter with the 3 unit converters registered
when I search the unit converter for short name ft
then I get an UnitConvertNotFound exception
"""
converter_instance = converter.Converter()
with self.assertRaises(converter.UnitConvertNotFound):
converter_instance._get_unit_converter('ft')
def __init__(self):
# initialize our window object
root = tkinter.Tk()
root.title('Temperature Converter v1.0')
root.geometry('515x140')
root.resizable(False, False)
self.model = converter.Converter()
self.view = windows.MyFrame(self)
self.view.mainloop()
def test_value_convertion_yard_to_inche(self):
"""
given a converter with the 4 unit converters registered and 3 yards set as value
when I get converted value in inches
then I get 144 inches
"""
converter_instance = converter.Converter()
converter_instance.set_value(4, 'yd')
result = converter_instance.convert_to_unit('in')
self.assertAlmostEqual(result, 144, places=4)
def __init__(self, args, ext, remove_invalid=True):
self.args = args
self.remove_invalid = remove_invalid
if self.remove_invalid:
self.conv = converter.Converter(tables=getattr(
args, 'tables', 'data/spider/tables'),
db=getattr(args, 'db',
'data/database'))
self.sql_vocab = ext['sql_voc']
self.evaluator = evaluation.Evaluator()
def try_converter():
"""
输入: bin/test.md, src/temp.yaml
输出: /outputs/test_md_folder/sample_item.zh.md
"""
c = converter.Converter(TEST_MD, "test_md_folder")
print(c)
io = io_handler.IOHandler(c)
# print(c)
io.write_yaml()
def test_value_convertion_yard_to_feet(self):
"""
given a converter with the 3 unit converters registered and 3 yards set as value
when I get converted value in feets
then I get an UnitConvertNotFound exception
"""
converter_instance = converter.Converter()
converter_instance.set_value(23, 'yd')
with self.assertRaises(converter.UnitConvertNotFound):
converter_instance.convert_to_unit('ft')
def test_value_convertion_yard_to_inches_as_string(self):
"""
given a converter with the 3 unit converters registered and 3 yards set as value
when I get converted value in meters with the as_string flag
then I get litteraly "112.98 m"
"""
converter_instance = converter.Converter()
converter_instance.set_value(123.56, 'yd')
result = converter_instance.convert_to_unit('m', as_string=True)
self.assertEquals(result, "113 m")
def test_check_registered_short_unit(self):
"""
given a converter with the 3 unit converters registered
when I search the unit converter for each short unit name
then I get unit_convert for each name
"""
converter_instance = converter.Converter()
for short_unit_name in ('m', 'yd', 'in'):
unit_converter = converter_instance._get_unit_converter(short_unit_name)
self.assertEquals(short_unit_name, unit_converter.short_unit)
def run_Convert():
valid = c.validateForConversion()
if valid:
converter = c.Converter(inputBase=int(e.input_BaseInput.get()),
outputBase=int(e.input_BaseOutput.get()),
inputValue=str(e.input_Value.get()))
converted = converter.convertToOutput()
if len(converter.warnings) > 0:
for warning in converter.warnings:
converted = converted + " : " + warning
e.box_Output.update(converted)
def __init__(self):
"""
This starts the Tk framework up, instantiates the Model (a Counter object),
instantiates the View (a MyFrame object), and starts the event loop that waits
for the user to press a Button on the View.
"""
root = tkinter.Tk()
self.model = converter.Converter()
self.view = myFrame.MyFrame(self)
self.view.mainloop()
root.destroy()
def try_dump():
c = converter.Converter(TEST_MD, "test_md_folder")
yh = yaml_handler.YAMLHandler(c)
yh.set_title("测试标题")
yh.set_date(datetime.today())
yh.set_summary(True)
yh.set_taxonomy(['学习'], ['tag1', 'tag2', '中文tag'])
yaml_dict = yh.get_yaml_dict()
print(yaml_dict, "<- yaml dict")
yh.dump_yaml()
def __init__(self, args, ext):
super().__init__(args, ext)
self.conv = converter.Converter(tables=getattr(args, 'tables',
'data/spider/tables'),
db=getattr(args, 'db',
'data/database'))
self.bert_tokenizer = DistilBertTokenizer.from_pretrained(
args.dcache + '/vocab.txt', cache_dir=args.dcache)
self.bert_embedder = DistilBertModel.from_pretrained(
args.dcache, cache_dir=args.dcache)
self.value_bert_embedder = DistilBertModel.from_pretrained(
args.dcache, cache_dir=args.dcache)
self.denc = 768
self.demb = args.demb
self.sql_vocab = ext['sql_voc']
self.sql_emb = nn.Embedding.from_pretrained(ext['sql_emb'],
freeze=False)
self.pad_id = self.sql_vocab.word2index('PAD')
self.dropout = nn.Dropout(args.dropout)
self.bert_dropout = nn.Dropout(args.bert_dropout)
self.table_sa_scorer = nn.Linear(self.denc, 1)
self.col_sa_scorer = nn.Linear(self.denc, 1)
self.col_trans = nn.LSTM(self.denc,
self.demb // 2,
bidirectional=True,
batch_first=True)
self.table_trans = nn.LSTM(self.denc,
args.drnn,
bidirectional=True,
batch_first=True)
self.pointer_decoder = decoder.PointerDecoder(
demb=self.demb,
denc=2 * args.drnn,
ddec=args.drnn,
dropout=args.dec_dropout,
num_layers=args.num_layers)
self.utt_trans = nn.LSTM(self.denc,
self.demb // 2,
bidirectional=True,
batch_first=True)
self.value_decoder = decoder.PointerDecoder(demb=self.demb,
denc=self.denc,
ddec=args.drnn,
dropout=args.dec_dropout,
num_layers=args.num_layers)
self.evaluator = evaluation.Evaluator()
if 'reranker' in ext:
self.reranker = ext['reranker']
else:
self.reranker = rank_max.Module(args, ext, remove_invalid=True)
def test_set_value_register_meter_value(self):
"""
given a converter with the 3 unit converters registered
when I set a value with set_value
then the intial_length is register as meter
"""
yard_value = 2
meter_value = 1.8288
converter_instance = converter.Converter()
converter_instance.set_value(2, 'yd')
self.assertAlmostEqual(converter_instance.intial_length, meter_value, places=4)
self.assertEquals(converter_instance.intial_unit, 'yd')
def __init__(self):
'''
starts TK framework
instantiates model (converter)
instantiates view (MyFrame)
starts event loop
'''
root = tkinter.Tk()
self.model = converter.Converter()
self.view = myFrame.MyFrame(self)
self.view.mainloop()
root.destroy()
def from_xml_to_n2c(file):
"""
Convert a .xml file to .n2c extension file.
Note are encoding with 16 bits (or 4 hex)
Parameters
----------
file: XML file
File to be compressed.
Returns
-------
_ : int
0 if success, else 1.
Raise
-----
AssertionError
"""
# Initialize Variables
compressed_notes = []
basename, ext = os.path.splitext(file)
if ext == ".n2c" or ext == ".txt":
return 1
# Convert all files retrieved
converter = conv.Converter()
converter.reset()
(score_info, _, notes) = utils.xml_parser(file)
converter.setKappa(score_info.divisions)
for note in notes:
try:
matrix = converter.convert_note(note)
for elem in matrix:
temp = from_array_to_hex(elem)
compressed_notes.append(temp)
temp_note, _ = extract.from_hex_to_array(temp)
np.testing.assert_array_equal(elem, temp_note)
except AssertionError:
print file
return 1
# Assertions evaluation
for x in compressed_notes:
assert len(x) == 4, "{}, {}".format(file, x)
assert len("".join(compressed_notes)) % 4 == 0, compressed_notes
with open("".join([basename, ".n2c"]), 'wb') as compressed_file:
compressed_file.write("".join(compressed_notes))
return 0
def from_txt_to_n2c(xml, txt):
"""
Convert a .xml file with a .txt to .n2c extension file.
Note are encoding with 16 bits. Txt file consists of wrong notes indexes.
Xml and txt files must have the same basename (only extension is different).
Parameters
----------
xml: XML file
File to be compressed.
txt: TXT file
File with wrongs notes indexes. Numbers are comma-separated.
"""
# Initialize variables
with open(txt, 'rb') as text_file:
wrongs = [int(line) for line in text_file.read().split(',')]
compressed_notes = []
basename, ext = os.path.splitext(xml)
if ext == ".n2c" or ext == ".txt":
return 1
# Convert all files retrieved
converter = conv.Converter()
converter.reset()
(score_info, _, notes) = utils.xml_parser(xml)
converter.setKappa(score_info.divisions)
for idx, note in enumerate(notes):
try:
matrix = converter.convert_note(note)
for elem in matrix:
is_wrong = idx in wrongs
temp = from_array_to_hex(elem, is_wrong)
compressed_notes.append(temp)
temp_note, _ = extract.from_hex_to_array(temp)
np.testing.assert_array_equal(elem, temp_note)
except AssertionError:
print file
return 1
# Assertions evaluation
for x in compressed_notes:
assert len(x) == 4, "Error in file {file}: Hexa converted equals to {hexa}".format(file=file, hexa=x)
joined_compressed_notes = "".join(compressed_notes)
assert len(joined_compressed_notes) % 4 == 0, "Compressed file length is not a multiple of 4 : {length}".format(length=len(joined_compressed_notes))
with open("".join([basename, '.n2c']), 'wb') as compressed_file:
compressed_file.write(joined_compressed_notes)
return 0
def __convert(self): self.__outfilename = os.path.join(self.__outdir, str(self.outputFileLineName.text())) if not os.path.exists(self.__infile) : raise FileNotExistException(self) self.checkFile(__OUTFILE) if self.__infile == self.__outfilename: raise SameFileException(self) if os.path.exists(self.__outfilename): if not self.overwriteBox.isChecked() : raise OverWriteException(self) overwrite = self.overwriteBox.isChecked() self.starter = converter.Converter(self.__infile, self.__outfilename, overwrite, self) self.starter.convert()
def __init__(self, **kwargs):
"""
Initialize a dataset from datas (already converted)
Parameters
----------
See dyci2 module (run function)
"""
self._past_index = 0
self._future_index = 0
self._right_index = 1
self.directory = kwargs["directory"]
self.datas = [[]]
self.converter = cv.Converter()
self.config(**kwargs)
self.time_size = self.past + self.future + 1
def compile(filename, format, **options):
conv = converter.Converter(filename, **options)
modules = conv.parse()
text = '\n'.join(modules[fname] for fname in sorted(modules.keys()))
lib = os.path.join(options.get('lib_dir', '.'), 'pjslib.js')
data = {'file': os.path.abspath(filename), 'text': text, 'lib': lib}
data['path'] = sys.path
if options.get('rhino', False):
template = rhino_out
elif options.get('html', False):
template = html_out
else:
template = js_out
return template % data
def from_file(cls, root, dspider, dcache, debug=False):
train_database, dev_database = editsql_preprocess.read_db_split(dspider)
conv = converter.Converter()
kmaps = evaluation.build_foreign_key_map_from_json(os.path.join(dspider, 'tables.json'))
splits = {}
for k in ['train', 'dev']:
with open(os.path.join(root, '{}.json'.format(k)), 'rb') as f:
splits[k] = []
for ex in json.load(f):
splits[k].append(ex)
if debug and len(splits[k]) > 100:
break
tokenizer = DistilBertTokenizer.from_pretrained(BERT_MODEL, cache_dir=dcache)
sql_voc = Vocab(['PAD', 'EOS', 'GO', 'SEP', '`', "'", '1', '%', 'yes', '2', '.', '5', 'f', 'm', 'name', 'song', 't', 'l'])
# make contexts and populate vocab
for s, data in splits.items():
proc = []
for i, ex in enumerate(tqdm.tqdm(data, desc='preprocess {}'.format(s))):
for turn_i, turn in enumerate(ex['interaction']):
turn['id'] = '{}/{}:{}'.format(ex['database_id'], i, turn_i)
turn['db_id'] = ex['database_id']
turn['prev'] = ex['interaction'][turn_i-1] if turn_i > 0 else None
new = cls.make_example(turn, tokenizer, sql_voc, kmaps, conv, train=s=='train')
if new is not None and (s != 'train' or not new['invalid']):
proc.append(new)
splits[s] = proc
# make candidate list using vocab
for s, data in splits.items():
for ex in data:
ex['cands_query'], ex['cands_value'] = cls.make_cands(ex, sql_voc)
splits[s] = data
# make pointers for training data
for ex in splits['train']:
ex['pointer_query'], ex['pointer_value'] = cls.make_query_pointer(ex['sup_query'], ex['cands_query'], ex['cands_value'], sql_voc)
# look up pretrained word embeddings
emb = E.ConcatEmbedding([E.GloveEmbedding(), E.KazumaCharEmbedding()], default='zero')
sql_emb = torch.tensor([emb.emb(w) for w in sql_voc._index2word])
ext = dict(sql_voc=sql_voc, sql_emb=sql_emb)
return splits, ext
