def produce_function():
window.focus_set()
if file_input.get() != "請輸入檔案名字" and isDigit(seed_input.get()):
check_button.config(state="disabled")
cancel_button.config(state="disabled")
file_input.config(state="disabled")
seed_input.config(state="disabled")
state_label_2.config(text="開始轉換")
start = time.time()
message, succeed = data.convert_data(file_input.get())
if succeed is False:
state_label_2.config(text=message)
check_button.config(state="normal")
cancel_button.config(state="normal")
file_input.config(state="normal")
seed_input.config(state="normal")
end = time.time()
print("執行時間 : " + str(end - start))
return
message, succeed = produce_feature_table_function.produce(
message, seed_input.get())
state_label_2.config(text=message)
check_button.config(state="normal")
cancel_button.config(state="normal")
file_input.config(state="normal")
seed_input.config(state="normal")
end = time.time()
print("執行時間 : " + str(end - start))
return
else:
messagebox.showinfo(title='Warning', message='請填完所有空格或格式錯誤')
def combine_data(self):
"""Combine users near-edge data with extended spectrum data."""
self.Full_E = None
self.Imaginary_Spectrum = None
if self.raw_file is not None:
logger.info("Convert to scattering factors")
self.NearEdgeData = data.convert_data(self.raw_file,self.DataTypeCombo.GetValue(),'ASF')
# if self.InvertDataCheckBox.GetValue():
# self.NearEdgeData[:,1] = numpy.abs(self.NearEdgeData[:,1] - 2*numpy.mean(self.NearEdgeData[:,1]))
logger.info("Combine Data")
# Get splice points
splice_eV = numpy.array([10.0, 30000.0]) # Henke limits
if self.SpliceText1.GetValue() == "Start":
if self.raw_file is not None:
splice_eV[0] = self.NearEdgeData[0, 0]
else:
splice_eV[0] = float(self.SpliceText1.GetValue())
if self.SpliceText2.GetValue() == "End":
if self.raw_file is not None:
splice_eV[1] = self.NearEdgeData[-1, 0]
else:
splice_eV[1] = float(self.SpliceText2.GetValue())
if self.raw_file is not None and self.ASF_Data is None:
self.Full_E, self.Imaginary_Spectrum, self.NearEdgeData, self.splice_ind = data.merge_spectra(self.NearEdgeData, self.ASF_E, self.ASF_Data, merge_points=splice_eV, add_background=self.AddBackgroundCheckBox.GetValue(), plotting_extras=True)
elif self.raw_file is None and self.ASF_Data is not None:
self.Full_E = self.ASF_E
self.Imaginary_Spectrum = self.ASF_Data
elif self.raw_file is not None and self.ASF_Data is not None:
self.Full_E, self.Imaginary_Spectrum, self.NearEdgeData, self.splice_ind = data.merge_spectra(self.NearEdgeData, self.ASF_E, self.ASF_Data, merge_points=splice_eV, add_background=self.AddBackgroundCheckBox.GetValue(), fix_distortions=self.FixDistortionsCheckBox.GetValue(), plotting_extras=True)
def kk_calculate_real(NearEdgeDataFile,
ChemicalFormula,
load_options=None,
input_data_type=None,
merge_points=None,
add_background=False,
fix_distortions=False,
curve_tolerance=None,
curve_recursion=50):
"""Do all data loading and processing and then calculate the kramers-Kronig transform.
Parameters
----------
NearEdgeDataFile : string
Path to file containg near-edge data
ChemicalFormula : string
A standard chemical formula string consisting of element symbols, numbers and parentheses.
merge_points : list or tuple pair of `float` values, or None
The photon energy values (low, high) at which the near-edge and scattering factor data values
are set equal so as to ensure continuity of the merged data set.
Returns
-------
This function returns a numpy array with columns consisting of the photon energy, the real and the imaginary parts of the scattering factors.
"""
Stoichiometry = data.ParseChemicalFormula(ChemicalFormula)
Relativistic_Correction = calc_relativistic_correction(Stoichiometry)
Full_E, Imaginary_Spectrum = data.calculate_asf(Stoichiometry)
if NearEdgeDataFile is not None:
NearEdge_Data = data.convert_data(data.load_data(
NearEdgeDataFile, load_options),
FromType=input_data_type,
ToType='asf')
Full_E, Imaginary_Spectrum = data.merge_spectra(
NearEdge_Data,
Full_E,
Imaginary_Spectrum,
merge_points=merge_points,
add_background=add_background,
fix_distortions=fix_distortions)
Real_Spectrum = KK_PP(Full_E, Full_E, Imaginary_Spectrum,
Relativistic_Correction)
if curve_tolerance is not None:
output_data = improve_accuracy(Full_E, Real_Spectrum,
Imaginary_Spectrum,
Relativistic_Correction,
curve_tolerance, curve_recursion)
else:
Imaginary_Spectrum_Values = data.coeffs_to_ASF(
Full_E, numpy.vstack((Imaginary_Spectrum, Imaginary_Spectrum[-1])))
output_data = numpy.vstack(
(Full_E, Real_Spectrum, Imaginary_Spectrum_Values)).T
return output_data
import sys
import glob
import data
if len(sys.argv) != 2:
sys.exit("Usage: python create_data <train/test>")
data_name = sys.argv[1]
assert data_name in ["train", "test"]
paths_csv = "./data/csv/" + data_name + "/*"
print("you csv path is: %s" % paths_csv)
print("Converting data ...")
data.convert_data(paths_csv)
def main():
ap = argparse.ArgumentParser()
ap.add_argument('-b', dest='batch_size', default=10, type=int)
ap.add_argument('--clip', default=10., type=float)
ap.add_argument('--gpu', default=-1, type=int)
ap.add_argument('--max_word_length', default=50, type=int)
ap.add_argument('--max_epoch', default=50, type=int)
ap.add_argument('--num_steps', default=20, type=int)
ap.add_argument('--input_file')
ap.add_argument('--test_file')
ap.add_argument('--lr', default=0.2, type=float)
ap.add_argument('--vocab_file')
args = ap.parse_args()
vocab = Vocabulary()
vocab.load(args.vocab_file)
char_vocab = UnicodeCharVocabulary()
lm = LanguageModel(vocab)
device = 'cpu'
if args.gpu > -1 and torch.cuda.is_available():
device = 'cuda:%d' % args.gpu
lm = lm.to(device)
optimizer = optim.Adagrad(lm.parameters(), lr=args.lr)
print('#loading training data')
train_sentences = data.convert_data(args.input_file, vocab, char_vocab,
True)
print('#loading test data')
test_sentences = data.convert_data(args.test_file, vocab, char_vocab,
False)
def test(model, test_data):
model.eval()
test_loss = 0.
seen_batches = 0
for batch in data.get_batch(test_data, args.batch_size, args.num_steps,
args.max_word_length, device):
optimizer.zero_grad()
loss = lm.forward_loss(batch['token_characters'],
batch['next_token_ids'])
seen_batches += 1
test_loss += loss.item() / batch['token_characters'].size(0)
model.train()
return test_loss / seen_batches
print('#training data', len(train_sentences))
for e in range(args.max_epoch):
loss_epoch = 0.
start_at = time.time()
seen_batches = 0
for i, batch in enumerate(
data.get_batch(train_sentences,
args.batch_size,
args.num_steps,
args.max_word_length,
device,
shuffle=True)):
optimizer.zero_grad()
loss = lm.forward_loss(batch['token_characters'],
batch['next_token_ids'])
loss_epoch += loss.item() / batch['token_characters'].size(0)
seen_batches += 1
loss.backward()
torch.nn.utils.clip_grad_norm_(lm.parameters(), args.clip)
optimizer.step()
if (i + 1) % 1000 == 0:
duration = time.time() - start_at
print('epoch %d\titer %d\t%.2f\t%.3f' %
(e, i + 1, duration, loss_epoch / seen_batches))
duration = time.time() - start_at
test_loss = test(lm, test_sentences)
print('------------------------')
print('epoch:%d\ttrain loss:%.3f\ttest loss:%.3f\telapsed::%.2f' %
(e, loss_epoch / seen_batches, test_loss, duration))
print('------------------------')
torch.save(lm.elmo.state_dict(), 'elmo-checkpoint.pt')
"{:s}/{:s}_{:s}.npy".format(directory, task, var)
for var in ["desired", "model", "state"]
]
n_gate = 1
size = 11
print(task)
if not np.all([os.path.exists(f) for f in files]):
# Random generator initialization
np.random.seed(1)
# Training data
n = 25000
values = np.random.randint(0, 10, n)
ticks = np.random.uniform(0, 1, (n, n_gate)) < 0.1
train_data_ = generate_data(values, ticks)
train_data = convert_data(train_data_, size, noise=0.)
# Testing data
n = 50
values = np.random.randint(0, 10, n)
ticks = np.random.uniform(0, 1, (n, n_gate)) < 0.1
test_data_ = generate_data(values, ticks)
test_data = convert_data(test_data_, size, noise=0.)
# Model
model = generate_model(shape=(train_data["input"].shape[1], 1000,
n_gate),
sparsity=0.5,
radius=0.1,
scaling=(1.0, 1.0),
leak=1.0,
def onReturn(*event):
convert_data(entry_1.get(), entry_2.get(), entry_3.get(), entry_4.get(),
entry_5.get(), entry_6.get(), entry_7.get(), entry_8.get(),
entry_9.get(), output_key, output_value, output_unit)
entry_1.grid(row=1, column=1)
entry_2.grid(row=2, column=1)
entry_3.grid(row=3, column=1)
entry_4.grid(row=4, column=1)
entry_5.grid(row=5, column=1)
entry_6.grid(row=6, column=1)
entry_7.grid(row=7, column=1)
entry_8.grid(row=9, column=1)
entry_9.grid(row=10, column=1)
button_1 = \
tk.Button(left, text='OK', width=20, bd=5,
command=lambda: convert_data(entry_1.get(), entry_2.get(),
entry_3.get(), entry_4.get(),
entry_5.get(), entry_6.get(),
entry_7.get(), entry_8.get(),
entry_9.get(), output_key,
output_value, output_unit))
button_1.focus_set()
def onReturn(*event):
convert_data(entry_1.get(), entry_2.get(), entry_3.get(), entry_4.get(),
entry_5.get(), entry_6.get(), entry_7.get(), entry_8.get(),
entry_9.get(), output_key, output_value, output_unit)
root.bind('<Return>', onReturn)
button_1.grid(row=11, column=1)