def rws_test():
size = 10000
selection = 1000
random_state = RandomState()
probs = random_state.uniform(size=size)
probs /= sum(probs)
random_state.seed(5)
def standard_method():
t.tic()
result = []
cum_probs = np.cumsum(probs)
for _ in range(selection):
r = random_state.random()
for i in range(size):
if r <= cum_probs[i]:
result.append(i)
break
return result
def numpy_method():
return random_state.choice(size, size=selection, replace=True, p=probs)
t = TicToc()
t.tic()
result_standard_method = standard_method()
elp_std = t.tocvalue(restart=True)
result_numpy_method = numpy_method()
elp_np = t.tocvalue()
print('standard: {}'.format(elp_std))
print('numpy: {}'.format(elp_np))
print(result_numpy_method)
print(result_standard_method)
def run_dt_solver(
traces,
subsetSize=config.DT_SUBSET_SIZE,
txtFile="treeRepresentation.txt",
strategy=config.DT_SAMPLING_STRATEGY,
decreaseRate=config.DT_DECREASE_RATE,
repetitionsInsideSampling=config.DT_REPETITIONS_INSIDE_SAMPLING,
restartsOfSampling=config.DT_RESTARTS_OF_SAMPLING,
q=None,
encoder=DagSATEncoding,
misclassification=0,
timeout=float("inf"),
record_result=dict(), # output
):
#try:
config.encoder = encoder
separate_process = q is not None
ab = AtomBuilder()
ab.getExamplesFromTraces(traces)
#samplingStrategy = config.DT_SAMPLING_STRATEGY
samplingStrategy = strategy
#decreaseRate = config.DT_DECREASE_RATE
decreaseRate = decreaseRate
t = TicToc()
t.tic()
(atoms, atomTraceEvaluation) = ab.buildAtoms(
sizeOfPSubset=subsetSize,
strategy=samplingStrategy,
sizeOfNSubset=subsetSize,
probabilityDecreaseRate=decreaseRate,
numRepetitionsInsideSampling=repetitionsInsideSampling,
numRestartsOfSampling=restartsOfSampling,
timeout=timeout - t.tocvalue(),
)
fb = DTFormulaBuilder(
features=ab.atoms,
data=ab.getMatrixRepresentation(),
labels=ab.getLabels(),
stoppingVal=misclassification,
# timeout=timeout-t.tocvalue(), #TODO
)
fb.createASeparatingFormula()
timePassed = t.tocvalue()
atomsFile = "atoms.txt"
treeTxtFile = txtFile
ab.writeAtomsIntoFile(atomsFile)
numberOfUsedPrimitives = fb.numberOfNodes()
fb.tree_to_text_file(treeTxtFile)
fb.tree_to_dot_file("atoms.dot")
record_result['formulaTree'] = fb.tree_to_DecisionTreeFormula()
# return (timePassed, len(atoms), numberOfUsedPrimitives)
if separate_process:
q.put([timePassed, len(atoms), numberOfUsedPrimitives])
else:
return [timePassed, len(atoms), numberOfUsedPrimitives]
def get_finite_witness(f,
trace_length=5,
operators=[
encodingConstants.G, encodingConstants.F,
encodingConstants.LAND, encodingConstants.LOR,
encodingConstants.ENDS, encodingConstants.LNOT,
encodingConstants.BEFORE,
encodingConstants.STRICTLY_BEFORE,
encodingConstants.UNTIL
],
wall_locations=[],
water_locations=None,
robot_position=None,
items_locations=None,
testing=False):
t = TicToc()
solvingTic = TicToc()
t.tic()
all_variables = [str(v) for v in f.getAllVariables()]
fg = SATOfLTLEncoding(f,
trace_length,
0,
operators=None,
literals=all_variables,
wall_positions=wall_locations,
water_locations=water_locations,
robot_position=robot_position,
items_locations=items_locations,
testing=testing)
fg.encodeFormula()
stats_log.debug("creation time was {}".format(t.tocvalue()))
solvingTic.tic()
solverRes = fg.solver.check()
stats_log.debug("solving time was {}".format(solvingTic.tocvalue()))
if solverRes == sat:
solverModel = fg.solver.model()
(cex_trace, init_world, path) = fg.reconstructWitnessTrace(solverModel)
return (cex_trace, init_world, path)
elif solverRes == unknown:
return constants.UNKNOWN_SOLVER_RES
else:
# logging.debug(solverRes)
# pdb.set_trace()
if constants.DEBUG_UNSAT_CORE is True:
filename = "debug_files/unsatCore"
os.makedirs(os.path.dirname(filename), exist_ok=True)
with open(filename, "w") as unsat_core_file:
unsat_core_file.write(str(fg.solver.unsat_core()))
return "unsat"
def poll_for_valid_message(consumer,
expected_file_identifier=b"f142",
timeout=15.0):
"""
Polls the subscribed topics by the consumer and checks the buffer is not empty or malformed.
Skips connection status messages.
:param consumer: The consumer object
:param expected_file_identifier: The schema id we expect to find in the message
:param timeout: give up if we haven't found a message with expected_file_identifier after this length of time
:return: Tuple of the message payload and the key
"""
timer = TicToc()
timer.tic()
while timer.tocvalue() < timeout:
msg = consumer.poll(timeout=1.0)
assert msg is not None
if msg.error():
raise MsgErrorException("Consumer error when polling: {}".format(
msg.error()))
if expected_file_identifier is None:
return msg.value(), msg.key()
elif expected_file_identifier is not None:
message_file_id = msg.value()[4:8]
assert (
expected_file_identifier == message_file_id
or message_file_id == b"ep00"
), f"Expected message to have schema id of {expected_file_identifier}, but it has {message_file_id}"
if message_file_id == b"f142":
return LogData.LogData.GetRootAsLogData(msg.value(),
0), msg.key()
def _throw(self):
dice = None
if self.method == 1: # insert from keyboard
flag = True
while flag:
padding = " " * (len(str(self.max_ind)) - len(str(self.ind)))
str_input = "throw " + padding + str(self.ind) + "/" + str(
self.max_ind) + ", result: "
inp = input(str_input)
if all(c in "0123456789" for c in inp) and inp != "":
dice = int(inp) - 1
if dice in range(self.base):
flag = False
else:
print("invalid result, insert again")
else:
print("invalid result, insert again")
if self.method == 2: # dice result always 2
dice = 1
padding = " " * (len(str(self.max_ind)) - len(str(self.ind)))
print("throw " + padding + str(self.ind) + "/" +
str(self.max_ind) + ", result: " + str(dice + 1))
if self.method == 3: # throw pressing keyboard
t = TicToc()
t.tic()
input("Press enter to throw")
dice = int(t.tocvalue() * 10**8) % self.base
padding = " " * (len(str(self.max_ind)) - len(str(self.ind)))
print("throw " + padding + str(self.ind) + "/" +
str(self.max_ind) + ", result: " + str(dice + 1))
return dice
def __merge_knnr_data(self, knnsr_data):
if self.verbosity == 2:
print("Combining KNNSR data")
self.views = {}
ptids = self.data.get_ptids()
tictoc = TicToc()
tictoc.tic()
self.valid_ptids = []
for m in self.data.get_modalities():
self.views[m] = {}
for t in knnsr_data[m].keys():
xy_measurement = self.data.getXY(
ptids, m, target=t, split=False, impute_ptids=self.train_ptids
)
modality_ts = knnsr_data[m][t]
if not isinstance(modality_ts, self.data.backend.DataFrame):
modality_ts = self.data.backend.DataFrame(modality_ts)
view = self.merge_view(xy_measurement, modality_ts)
self.views[m][t] = view
self.valid_ptids += view[TADPOLEData.PTID].values.tolist()
if self.verbosity == 2:
print("Mergin KNNSR data took %f" % tictoc.tocvalue())
self.valid_ptids = list(set(self.valid_ptids))
if self.verbosity == 2:
print("%i patients in set" % len(self.valid_ptids))
def run_single():
tictoc = TicToc()
tictoc.tic()
result = [eval(rs) for rs in range(CYCLES)]
ellapsed = tictoc.tocvalue()
print("Single: {}".format(ellapsed))
return result
def _throw(self):
dice = None
if self.method == 1: # insert from keyboard
flag = True
while flag:
padding = " " * (len(str(self.max_ind)) - len(str(self.ind)))
str_input = "throw " + padding + str(self.ind) + "/" + str(self.max_ind) + ", result: "
inp = input(str_input)
if all(c in "0123456789" for c in inp) and inp != "":
dice = int(inp) - 1
if dice in range(self.base):
flag = False
else:
print("invalid result, insert again")
else:
print("invalid result, insert again")
if self.method == 2: # dice result always 2
dice = 1
padding = " " * (len(str(self.max_ind)) - len(str(self.ind)))
print("throw " + padding + str(self.ind) + "/" + str(self.max_ind) + ", result: " + str(dice + 1))
if self.method == 3: # throw pressing keyboard
t = TicToc()
t.tic()
input("Press enter to throw")
dice = int(t.tocvalue() * 10**8) % self.base
padding = " " * (len(str(self.max_ind)) - len(str(self.ind)))
print("throw " + padding + str(self.ind) + "/" + str(self.max_ind) + ", result: " + str(dice + 1))
return dice
def run_dt_solver(traces, subsetSize=config.DT_SUBSET_SIZE, txtFile="treeRepresentation.txt", strategy=config.DT_SAMPLING_STRATEGY, decreaseRate=config.DT_DECREASE_RATE,\
repetitionsInsideSampling=config.DT_REPETITIONS_INSIDE_SAMPLING, restartsOfSampling=config.DT_RESTARTS_OF_SAMPLING, q = None, encoder=DagSATEncoding,):
#try:
config.encoder = encoder
if q != None:
separateProcess = True
else:
separateProcess = False
ab = AtomBuilder()
ab.getExamplesFromTraces(traces)
#samplingStrategy = config.DT_SAMPLING_STRATEGY
samplingStrategy = strategy
#decreaseRate = config.DT_DECREASE_RATE
decreaseRate = decreaseRate
t = TicToc()
t.tic()
(atoms, atomTraceEvaluation) = ab.buildAtoms(sizeOfPSubset=subsetSize, strategy = samplingStrategy, sizeOfNSubset=subsetSize, probabilityDecreaseRate=decreaseRate,\
numRepetitionsInsideSampling=repetitionsInsideSampling, numRestartsOfSampling = restartsOfSampling)
fb = DTFormulaBuilder(features=ab.atoms,
data=ab.getMatrixRepresentation(),
labels=ab.getLabels())
fb.createASeparatingFormula()
timePassed = t.tocvalue()
atomsFile = "atoms.txt"
treeTxtFile = txtFile
ab.writeAtomsIntoFile(atomsFile)
numberOfUsedPrimitives = fb.numberOfNodes()
fb.tree_to_text_file(treeTxtFile)
# return (timePassed, len(atoms), numberOfUsedPrimitives)
if separateProcess == True:
q.put([timePassed, len(atoms), numberOfUsedPrimitives])
else:
return [timePassed, len(atoms), numberOfUsedPrimitives]
def run_parallel():
tictoc = TicToc()
tictoc.tic()
result = Parallel(n_jobs=JOBS,
prefer=None)(delayed(eval)(rs) for rs in range(CYCLES))
ellapsed = tictoc.tocvalue()
print("Parallel: {}".format(ellapsed))
return result
class ElapsedTime(object):
"""Measure the elapsed time between Tic and Toc"""
def __init__(self):
self.t = TicToc()
self.t.tic()
def elapsed(self):
_elapsed = self.t.tocvalue()
d = timedelta(seconds=_elapsed)
logger.debug('< {} >'.format(d))
def perform_cv(args):
splitter = KFold(args.folds, random_state=0, shuffle=True)
data = TADPOLEData(
data=args.modality_path + args.data_file,
modality_path=args.modality_path,
modality_k=args.modality_k,
)
# Split the ptids into n folds
ptids = data.get_ptids(min_time_points=2, target=args.target)
print("Total patients in CV: %i" % len(ptids))
t = TicToc()
print("CV mode %s" % args.mode)
sys.stdout.flush()
predictions = []
modality_ranks = []
for fold, (train_index, test_index) in enumerate(splitter.split(ptids)):
print("Fold %i/%i" % (fold, args.folds - 1))
sys.stdout.flush()
train_ptids = [ptids[i] for i in train_index]
test_ptids = [ptids[i] for i in test_index]
aug_data = AugmentedTADPOLEData(
data, args.precomputed_path + "merged_%i.p" % fold, train_ptids)
model = SMNSR(aug_data,
n_jobs=args.cpus,
forecast=False,
mode=args.mode)
print("Fitting model")
t.tic()
model.fit(train_ptids)
print("Trainig took %s seconds" % t.tocvalue())
print("Performing forecasting")
sys.stdout.flush()
# Fetch known target values for the patients
y = aug_data.getY(test_ptids, target=args.target)
prediction_definition = y[[TADPOLEData.PTID, TADPOLEData.C_MONTH]]
print("Patients with more than one measurement in fold %i: %i" %
(fold, y[TADPOLEData.PTID].unique().shape[0]))
y_hat = model.predict(prediction_definition, target=args.target)
prediction = y.merge(
y_hat,
left_on=[TADPOLEData.PTID, TADPOLEData.C_MONTH],
right_on=[TADPOLEData.PTID, TO],
)
predictions.append((prediction))
modality_ranks.append(model.ranked_modalities)
fold += 1
predictions = pd.concat(predictions, ignore_index=True)
with open(args.output_path + args.result_file_name, "wb") as file:
pickle.dump(prediction, file)
evaluate_predictions(predictions, data)
return prediction
def upload_dcm_files(workers, accession_number=None):
global R, instance_manifest, compress
pool = Pool(processes=workers)
initial_size = orthanc.size()
instance_manifest = orthanc.do_get('instances')
initial_count = len(instance_manifest)
logging.info("-----------------------------------")
logging.info("DCM Pre-Index File Uploader")
logging.info("-----------------------------------")
logging.info(" Workers: {}".format(workers))
if compress:
logging.info(" J2K Compress: ON")
t = TicToc()
t.tic()
if accession_number:
# Upload single accession
fps = Q.sget(accession_number)
logging.info(" Upload: Accession {}".format(accession_number))
else:
# Upload _all_
fps = R.keys()
logging.info(" Upload: All data")
pool.map(upload_dcm_file, fps, 20)
toc_ = float(t.tocvalue())
final_size = orthanc.size()
instance_manifest = orthanc.do_get('instances')
final_count = len(instance_manifest)
count = final_count - initial_count
upload_mb = final_size - initial_size
n_per_sec = float(count) / toc_
time40m = 40000000.0 / (n_per_sec or 1) / 60 / 60
mb_per_sec = float(upload_mb) / toc_
time15t = 15000000.0 / (mb_per_sec or 1) / 60 / 60
logging.info(" Time: {} sec".format(toc_))
logging.info(" Num uploaded: {}".format(count))
logging.info(" Num/sec: {}".format(n_per_sec))
logging.info(" Hrs for 40M: {}".format(time40m))
logging.info(" MB uploaded: {}".format(upload_mb))
logging.info(" MB/sec: {}".format(mb_per_sec))
logging.info(" Hrs for 15TB: {}".format(time15t))
logging.info("-----------------------------------")
def components(
nodes
): #randomize graph and calculate connected components given number of nodes
G = Graph.Erdos_Renyi(nodes, p=0.3)
t = TicToc()
#create an adjacency matrix.
#An = (sparse(1:nodes,assig,np.ones(1,nodes),nodes,nodes))
tables = G.components(mode=WEAK)
largest = max(tables)
t.tic()
adj = G.get_adjacency()
t.toc()
return (tables, largest, adj, t.tocvalue())
def run_solver(
*,
q=None,
encoder=DagSATEncoding,
**solver_args,
):
separate_process = q is not None
t = TicToc()
t.tic()
results = get_models(encoder=encoder, **solver_args)
time_passed = t.tocvalue()
if separate_process == True:
q.put([results, time_passed])
else:
return [results, time_passed]
def components(
nodes
): #randomize graph and calculate connected components given number of nodes
G = nx.erdos_renyi_graph(nodes, 0.3)
t = TicToc()
#t.tic()
#create an adjacency matrix.
#An = (sparse(1:nodes,assig,np.ones(1,nodes),nodes,nodes))
tables = nx.connected_components(G)
largest = max(tables)
t.tic()
adj = G.adjacency()
t.toc()
return (tables, largest, adj, t.tocvalue())
def test_fuzzy_partition(D: np.array, K, m, T, err):
"""
Calculate the fuzzy partition for a given input, measures the time and
saves the results
"""
t = TicToc()
mvf = MVFuzzy()
print("---------------------------------------------")
print("Calculating fuzzy partition...", end="", flush=True)
t.tic()
mvf.run(D, K, m, T, err)
elapsed = t.tocvalue()
print("done!")
print_formatted("Last Iteration:", mvf.lastIteration, suffix=False)
print_formatted("Last J_t:", mvf.lastAdequacy, suffix=False)
print_formatted("Elapsed time:", elapsed)
np.save("G_medoids", mvf.bestMedoidVectors)
np.save("W_weights", mvf.bestWeightVectors)
np.save("U_membership", mvf.bestMembershipVectors)
def run_rec_dt(
*,
traces,
q=None,
encoder=DagSATEncoding,
**solver_args,
):
separate_process = q is not None
t = TicToc()
t.tic()
result = get_rec_dt(traces=traces, encoder=encoder, **solver_args)
time_passed = t.tocvalue()
result.writeDotFile("recdt.dot", traces=traces)
if separate_process == True:
q.put([result, time_passed])
else:
return [result, time_passed]
def poll_for_connection_status_message(consumer, timeout=15.0):
"""
Polls the subscribed topics by the consumer and checks the buffer is not empty or malformed.
Skips connection status messages.
:param consumer: The consumer object
:param timeout: give up if we haven't found a connection status message after this length of time
:return: The LogData flatbuffer from the message payload
"""
timer = TicToc()
timer.tic()
while timer.tocvalue() < timeout:
msg = consumer.poll(timeout=1.0)
assert msg is not None
if msg.error():
raise MsgErrorException("Consumer error when polling: {}".format(
msg.error()))
message_file_id = msg.value()[4:8]
if message_file_id == b"ep00":
return EpicsConnectionInfo.EpicsConnectionInfo.GetRootAsEpicsConnectionInfo(
msg.value(), 0)
def process_file(audio_fname, speakers_number, config_dict, is_remote_file,
gt_fname):
prepare_outdir(RESULTS_PATH)
remote_path = ""
if is_remote_file:
print("Remote file selected")
remote_path = config_dict['bucket_url'] + "/" + audio_fname
else:
# local file, upload to bucket and set "remote_path" variable
print("Local file selected, will upload before processing...")
print("----------------------------------------")
file_path = Path(audio_fname)
remote_path = config_dict['bucket_url'] + "/" + config_dict[
'bucket_new_dir'] + "/" + file_path.name
upload_to_bucket(file_path, remote_path)
print("----------------------------------------\n")
print("Waiting for transcription...")
t = TicToc()
t.tic()
transcribed_str = request_transcription(remote_path, speakers_number,
config_dict)
elapsed_time = t.tocvalue()
print("Time to transcribe: {:.2f} minutes ({:.2f} seconds)".format(
elapsed_time / 60, elapsed_time))
out_transcribed_fname = Path(audio_fname).stem + "_transcribed.txt"
str_to_file(RESULTS_PATH / out_transcribed_fname, transcribed_str)
print("Transcription completed, saved to '{}'".format(
RESULTS_PATH / out_transcribed_fname))
if (gt_fname):
print(
"\nGT file provided, will evaluate results with several metrics..."
)
eval_result_str = evaluate_transcription(transcribed_str, gt_fname)
print(eval_result_str)
out_eval_fname = Path(audio_fname).stem + "_eval.txt"
str_to_file(RESULTS_PATH / out_eval_fname, eval_result_str)
print("Results saved to '{}'".format(RESULTS_PATH / out_eval_fname))
def ReadRegister(self, nReadings, sampleInteveralMs, address):
self.client = ModbusTcpClient(
self.host) # use ModbusTcpClient from pymodbus
#Method to read nReadings of a single modbus regeister at a sample interval sampleInteveralMs
#Uses a pause of sampleInterval rather than re-sampling
#Make an array for the the NReadings of values read from the register
allValues = np.ones(nReadings)
allValues = allValues * np.NAN
t = TicToc()
t.tic() #Start timer
registerValue = np.NAN
#bug fix - read a registor once, to set the correct registor address
request = self.client.read_holding_registers(address, 1)
registerValue = request.registers
registerValue = np.NAN
for k in range(0, nReadings):
try:
#address = int(self.config['READ_MODBUS_REG'])
request = self.client.read_holding_registers(address, 1)
registerValue = request.registers
except:
registerValue = np.NAN # return Nan if the register can't be read
#print(registerValue)
#print(type(registerValue))
#print(allValues)
#exit(0)
allValues[k] = registerValue[0]
time.sleep(
sampleInteveralMs / 1e3
) #pause before next reading. NB we are consitently sampling the same phase of the chopper
#Return the array of nReadings of a single modbus register and the time taken to acquire.
elapsedTime = t.tocvalue()
time.sleep(1) #Dwell to allow return of reponse
self.client.close()
return allValues, elapsedTime
def run_solver(finalDepth,
traces,
maxNumOfFormulas=1,
startValue=1,
step=1,
q=None,
encoder=DagSATEncoding):
if q is not None:
separate_process = True
else:
separate_process = False
t = TicToc()
t.tic()
results = get_models(finalDepth, traces, startValue, step, encoder,
maxNumOfFormulas)
time_passed = t.tocvalue()
if separate_process == True:
q.put([results, time_passed])
else:
return [results, time_passed]
def poll_for_valid_message(
consumer: Consumer,
expected_file_identifier: Optional[bytes] = b"f142",
timeout: float = 15.0,
) -> Tuple[bytes, bytes]:
"""
Polls the subscribed topics by the consumer and checks the buffer is not empty or malformed.
Skips connection status messages.
:param consumer: The consumer object
:param expected_file_identifier: The schema id we expect to find in the message
:param timeout: give up if we haven't found a message with expected_file_identifier after this length of time
:return: Tuple of the message payload and the key
"""
timer = TicToc()
timer.tic()
while timer.tocvalue() < timeout:
msg = consumer.poll(timeout=1.0)
if msg is None:
continue
if msg.error():
raise MsgErrorException("Consumer error when polling: {}".format(
msg.error()))
if expected_file_identifier is None:
return msg.value(), msg.key()
elif expected_file_identifier is not None:
message_file_id = msg.value()[4:8]
# Skip ep00 messages if we are looking for something else
if expected_file_identifier != b"ep00" and message_file_id == b"ep00":
continue
assert (
message_file_id == expected_file_identifier
), f"Expected message to have schema id of {expected_file_identifier}, but it has {message_file_id}"
return msg.value(), msg.key()
def index_dcm_dirs(dirs, workers):
pool = Pool(processes=workers)
# 1 workers = 64 seconds, 8 workers = 18 seconds (quiet)/21 sec (logged)
logging.info("-----------------------------------")
logging.info("DCM Directory Pre-Indexer")
logging.info("-----------------------------------")
logging.info(" Workers: {}".format(workers))
logging.info(" Dir: {}".format(s.base_dir))
t = TicToc()
t.tic()
pool.map(index_dcm_dir, dirs)
toc_ = float(t.tocvalue())
n_per_sec = float(len(R)) / toc_
time40m = 40000000.0 / (n_per_sec or 1) / 60 / 60
logging.info(" Time: {} sec".format(toc_))
logging.info(" Num indexed: {}".format(len(R)))
logging.info(" Num/sec: {}".format(n_per_sec))
logging.info(" Hrs for 40M: {}".format(time40m))
def run_alg(loggers, loaders, model, optimizer, scheduler):
global comms
from pytictoc import TicToc
t = TicToc()
# currently have only one batch containing all data
mod = None
pred = None
for batch in loaders[0]:
# compute modularity with baseline algorithm
# consider bipartite / hyper later (use bipartite projection then, nx supports
# that)
nxG = batch.G[0]
import networkx.algorithms.community as nx_comm
t.tic()
comms = nx_comm.greedy_modularity_communities(nxG)
# NOTE this is crisp modularity!
# todo for crisp assignments, should be the same?
mod = nx_comm.modularity(nxG, comms)
toctime = t.tocvalue()
# ... thus also only one logger is relevant
logger = loggers[0]
logger.update_stats(true=None,
pred=comms,
loss=mod * -1,
lr=0,
time_used=toctime,
params=1,
loss_main=mod * -1,
loss_reg=0)
logger.write_epoch(0, loader=loaders[0])
for logger in loggers:
logger.close()
def run_multi(lam_array,num_params=1024,maxiter=50,trials=5):
tictoc = TicToc()
tictoc.tic()
for i in range(trials):
out_name = 'DIF_SF_' + str(i)
print('Run:' + out_name)
re.run_evo(data_init=False,sparse=False,out_name=out_name,num_params=num_params,maxiter=maxiter)
out_name = 'DIT_SF_' + str(i)
print('Run:' + out_name)
re.run_evo(data_init=True,sparse=False,out_name=out_name,num_params=num_params,maxiter=maxiter)
for lam in lam_array:
norm = 0
# out_name = 'DIF_ST_' + str(lam) + '_' + str(i)
# print('Run:' + out_name)
# re.run_evo(data_init=False,sparse=True,lam=lam,out_name=out_name,num_params=num_params,maxiter=maxiter)
out_name = 'DIT_ST_' + str(lam) + '_' + str(i)
print('Run:' + out_name)
re.run_evo(data_init=True,sparse=True,lam=lam,out_name=out_name,num_params=num_params,maxiter=maxiter,norm=norm)
elapsed = timedelta(seconds = tictoc.tocvalue())
print('Total Execution Time: {}'.format(elapsed))
class DataLoader(object):
def __init__(self,
random_seed=0,
reduction_dimension=128,
reduction_iterations=30):
"""
Constructor of DataLoader
:param random_seed: random seed
:param reduction_dimension: input feature dimension (SVD)
:param reduction_iterations: number of iterations required by SVD
"""
self.random_seed = random_seed
np.random.seed(self.random_seed)
self.reduction_dimension = reduction_dimension
self.reduction_iterations = reduction_iterations
self.timer = TicToc()
def load(self, data_path, heldout_ratio=0.2):
"""
Load data and split the data into training and test.
:param data_path: path for dataset
:param heldout_ratio: heldout ratio between training and test
:return: loaded data
"""
logger.info('Start loading the signed network...')
X = np.loadtxt(data_path, dtype='int', delimiter='\t')
y = X[:, 2]
num_nodes = np.amax(X[:, 0:2]) + 1
num_edges = X.shape[0]
train_X, test_X, train_y, test_y = train_test_split(X, y,
test_size=heldout_ratio,
random_state=self.random_seed)
logger.info('Start creating input features with random_seed: {}...'.format(self.random_seed))
self.timer.tic()
H = self.generate_input_features(train_X, num_nodes)
gen_time = self.timer.tocvalue()
logger.info('Generation input features completed in {:.4} sec'.format(gen_time))
data = DotMap()
data.train.X = train_X
data.train.y = train_y
data.test.X = test_X
data.test.y = test_y
data.H = H # input feature matrix
data.num_nodes = num_nodes
neg_idx = train_X[:, 2] < 0
neg_ratio = train_X[neg_idx, :].shape[0] / float(train_X.shape[0])
data.neg_ratio = neg_ratio
data.class_weights = np.asarray([1.0, 1.0])
return data
def generate_input_features(self, train_edges, num_nodes):
"""
Create spectral features based on SVD
:return: SVD input features
"""
src = train_edges[:, 0]
dst = train_edges[:, 1]
sign = train_edges[:, 2]
shaping = (num_nodes, num_nodes)
signed_A = sparse.csr_matrix((sign, (src, dst)),
shape=shaping,
dtype=np.float32)
svd = TruncatedSVD(n_components=self.reduction_dimension,
n_iter=self.reduction_iterations,
random_state=self.random_seed)
X = svd.fit_transform(signed_A) # equivalent to U * Sigma
return X
return pdata, mdata
Timer = 0
period = 150
vel = 0
pos = 0
prevPos = 0
kp = 5
kd = 0
offset = 0
PWM = 0
t.tic()
t.tocvalue()
while True: #Loop to send 5000 to arduino and read it when arduino sends it bac
if (t.tocvalue() * 1000 - Timer > period):
Timer = t.tocvalue() * 1000
print("Timer")
print(Timer)
pos = getPEncoderPos()
print(pos)
vel = pos - prevPos
print(vel)
PWM = kp * pos + offset + kd * vel
print(PWM)
prevPos = pos
if (PWM > 0):
writePWM(PWM, "L")
print("Start training with {} epochs".format(args.num_epochs))
t = TicToc()
for epoch in tqdm(range(int(args.num_epochs))):
sketch_loss_weight = 1 if epoch < args.loss_epoch_threshold else args.sketch_loss_weight
t.tic()
global_step = train(global_step,
tb_writer,
train_loader,
table_data,
model,
optimizer,
args.clip_grad,
sketch_loss_weight=sketch_loss_weight)
train_time = t.tocvalue()
tqdm.write(
"Training of epoch {0} finished after {1:.2f} seconds. Evaluate now on the dev-set"
.format(epoch, train_time))
with torch.no_grad():
sketch_acc, acc, _, predictions = evaluate(model, dev_loader,
table_data,
args.beam_size)
with open(os.path.join(output_path, 'predictions_sem_ql.json'),
'w',
encoding='utf-8') as f:
json.dump(predictions, f, indent=2)
eval_results_string = "Epoch: {} Sketch-Accuracy: {} Accuracy: {}".format(
def test_matrix_iterative(D: np.array, K, m, T, err):
"""
Compares the performance of the iterative vs matrix implementations of the
several equations to compute the fuzzy partition.
"""
n_elems = D.shape[1]
p_views = D.shape[0]
t = TicToc()
mvf = MVFuzzy()
# Initial medoids selection
np.random.seed(RANDOM_SEED)
G_medoids = np.random.randint(n_elems, size=(K, p_views))
# Initial weight vector
W_weights = np.ones(shape=[K, p_views], dtype=float)
# ---------------------------------------------------------------------------
# Membership degree vector calculation
# iterative
print("---------------------------------------------")
print("Membership Vector (U | Eq. 6)")
t.tic()
U_membDegree_iterative = mvf_iterative.calc_membership_degree(
D, G_medoids, W_weights, K, m)
elapsed = t.tocvalue()
print_formatted("Iterative: ", elapsed)
# matrix (optimized)
t.tic()
U_membDegree_matrix = mvf._calc_membership_degree(D, G_medoids, W_weights,
K, m)
elapsed = t.tocvalue()
print_formatted("Matrix: ", elapsed)
# check if matrices are identical
areEqual_U = U_membDegree_iterative == U_membDegree_matrix
areEqual_U = areEqual_U.all()
print_formatted("U | Iterative == Matrix:", areEqual_U)
print_formatted(
"Diff: ",
str(np.sum(abs(U_membDegree_iterative - U_membDegree_matrix))))
# ---------------------------------------------------------------------------
# Adequacy calculation
#
# iterative
print("---------------------------------------------")
print("Adequacy (J | Eq. 1)")
t.tic()
J_adequacy_iterative = mvf_iterative.calc_adequacy(D, G_medoids, W_weights,
U_membDegree_iterative,
K, m)
elapsed = t.tocvalue()
print_formatted("Iterative: ", elapsed)
# matrix (optimized)
t.tic()
J_adequacy_matrix = mvf._calc_adequacy_np(D, G_medoids, W_weights,
U_membDegree_matrix, K, m)
elapsed = t.tocvalue()
print_formatted("Matrix: ", elapsed)
# check if results are identical
areEqual_J = J_adequacy_iterative == J_adequacy_matrix
print_formatted("J | Iterative == Matrix:", areEqual_J)
print_formatted("Diff: ",
str(abs(J_adequacy_iterative - J_adequacy_matrix)))
# ---------------------------------------------------------------------------
# Best medoid vector calculation
#
# iterative
print("---------------------------------------------")
print("Best Medoids Vector (G | Eq. 4)")
t.tic()
G_bestMedoids_iterative = mvf_iterative.calc_best_medoids(
D, U_membDegree_matrix, K, m)
elapsed = t.tocvalue()
print_formatted("Iterative:", elapsed)
# matrix (optimized)
t.tic()
G_bestMedoids_matrix = mvf._calc_best_medoids(D, U_membDegree_matrix, K, m)
elapsed = t.tocvalue()
print_formatted("Matrix:", elapsed)
# check if results are identical
areEqual_G = G_bestMedoids_iterative == G_bestMedoids_matrix
areEqual_G = areEqual_G.all()
print_formatted("G | Iterative == Matrix:", areEqual_G)
# ---------------------------------------------------------------------------
# Best weights vector calculation
#
# iterative
print("---------------------------------------------")
print("Best Weights Vector (W | Eq. 5)")
t.tic()
W_bestWeights_iterative = mvf_iterative.calc_best_weights(
D, U_membDegree_matrix, G_bestMedoids_matrix, K, m)
elapsed = t.tocvalue()
print_formatted("Iterative:", elapsed)
# matrix (optimized)
t.tic()
W_bestWeights_matrix = mvf._calc_best_weights(D, U_membDegree_matrix,
G_bestMedoids_matrix, K, m)
elapsed = t.tocvalue()
print_formatted("Matrix:", elapsed)
# check if results are identical
areEqual_W = W_bestWeights_iterative == W_bestWeights_matrix
areEqual_W = areEqual_W.all()
print_formatted("W | Iterative == Matrix:", areEqual_W)
print_formatted(
"Diff: ",
str(np.sum(abs(W_bestWeights_iterative - W_bestWeights_matrix))))
# ---------------------------------------------------------------------------
# Membership with weights
# iterative
print("---------------------------------------------")
print("WEIGHTED Membership Vector (U | Eq. 6)")
t.tic()
U_membDegree_iterative = mvf_iterative.calc_membership_degree(
D, G_bestMedoids_matrix, W_bestWeights_iterative, K, m)
elapsed = t.tocvalue()
print_formatted("Iterative: ", elapsed)
# matrix (optimized)
t.tic()
U_membDegree_matrix = mvf._calc_membership_degree(D, G_bestMedoids_matrix,
W_bestWeights_matrix, K,
m)
elapsed = t.tocvalue()
print_formatted("Matrix: ", elapsed)
# check if matrices are identical
areEqual_U = U_membDegree_iterative == U_membDegree_matrix
areEqual_U = areEqual_U.all()
print_formatted("Uw | Iterative == Matrix:", areEqual_U)
print_formatted(
"Diff: ",
str(np.sum(abs(U_membDegree_iterative - U_membDegree_matrix))))
signalFDataSquare = signalFDataN * signalFDataN
meanSignalFDataSquare = np.mean(signalFDataSquare)
rmsFSignal = np.sqrt(meanSignalFDataSquare)
signalFAmp = np.sqrt(2) * rmsFSignal * 1e3
print(signalFourierAmplitude)
print(signalAmp)
numpyAllSignal = numpy.append(numpyAllSignal, signalAmp)
numpyAllFourierSignal = numpy.append(
numpyAllFourierSignal, signalFAmp)
numpyAllthermistorSignal = numpy.append(
numpyAllthermistorSignal, thermistorsignal)
numpyAllDCSignal = numpy.append(numpyAllDCSignal, dCData)
elapsedTime = t.tocvalue()
elapsedTime = float(elapsedTime)
elapsedTimeHours = elapsedTime / 3600
print("Writing to csv file " + str(fileName))
arrayOut = str(elapsedTime) + "," + str(
furnaceTempString) + "," + str(
ambientSetTemp) + "," + str(
thermistorsignal) + "," + str(
dCData) + "," + str(signalFAmp) + '\n'
hFile.write(arrayOut)
peakToPeakSignal = 2 * np.mean(numpyAllFourierSignal)
peakToPeakNoise = 2 * np.std(numpyAllFourierSignal)
signalToNoise = peakToPeakSignal / peakToPeakNoise
