def queries(training_set, test_set, train_vectors, test_vectors, path):
threshold_start = 1
threshold_end = 10
thresholds = []
metrics_obj_list = []
for i in range(threshold_start, threshold_end + 1):
thresholds.append(i)
metrics_obj_list.append(Metrics())
fw = FileWriter(path)
eval = Evaluator(training_set)
for i in range(len(test_vectors)):
scores = query(train_vectors, test_vectors[i], threshold_end)
query_doc = test_set[i]
for j in range(len(thresholds)):
threshold = thresholds[j]
eval.query([training_set[x] for (x, y) in scores[0:threshold]],
query_doc)
eval.calculate()
metrics_obj_list[j].updateConfusionMatrix(eval)
metrics_obj_list[j].updateMacroAverages(eval)
for obj in metrics_obj_list:
obj.calculate(len(test_set))
fw.writeToFiles(metrics_obj_list, thresholds)
class SerialFileWriter(Observer):
"""
This class intercepts logs and writes these to its own file writer.
"""
def __init__(self, log_file_path, callback):
self.logger = logging.getLogger(self.__class__.__name__)
super(SerialFileWriter, self).__init__(self.__class__.__name__)
self._file_writer = FileWriter(log_file_path, callback)
def __repr__(self):
return '{}({!r})'.format(self.__class__.__name__, self.name)
def start(self):
self._file_writer.start()
def stop(self):
self._file_writer.stop()
def is_alive(self):
return self._file_writer.is_alive()
def update(self, data):
log_line = data[0] # data is a tuple
if self._file_writer.is_alive():
self.logger.debug('writing: {}'.format(log_line))
self._file_writer.put(
log_line) # log lines are written to file writer's queue
def __init__(self, initial_vaccinated, initial_infected, initial_healthy, virus, resultsfilename):
'''Set up the initial simulation values'''
self.virus = virus
self.initial_infected = initial_infected
self.initial_healthy = initial_healthy
self.initial_vaccinated = initial_vaccinated
self.population = []
self.population_size = initial_infected + initial_healthy + initial_vaccinated
self.total_dead = 0
self.total_vaccinated = initial_vaccinated
self.file_writer = FileWriter(resultsfilename)
def write_file(self, name, obj, dest):
if hasattr(obj, 'toData'):
data = obj.toData()
path, name = os.path.split(name)
name = self.get_file_name(obj, name)
log.info("Extracting %s to %s...", name, dest + '/' + name)
if not self.readOnly:
with FileWriter(dest + '/' + name) as output:
output.write(data)
else:
for obj in obj.objects:
self.write_file(name, obj, dest)
def mkfile(self, path:Path, mode:fopenMode='wb') -> BinOutput:
"""Create a file within the output directory.
Allows multiple nested paths. Creates the directories as needed.
Returns the file object.
"""
log.debug("%s.mkfile(%s) dest=%s", type(self).__name__,
path, self.destPath)
_, name = os.path.split(self.input.name)
path += '/' + name + '.' + self.defaultFileExt
#path = self.destPath + '/' + path
if self.dry: return DummyFileWriter(path, mode)
return FileWriter(path, mode)
def extract_recursive(self, path, dest):
with self.makeFileReader(path) as file:
decoder = codec.getDecoderForFile(file)
decoder = decoder(file, None)
log.debug("get_files(%s, %s)", decoder, path)
items = self.get_files(decoder, path)
for item in items:
log.info("Extracting %s/%s...", dest, item['name'])
if not self.readOnly:
with FileWriter(dest + '/' + item['name']) as output:
if 'file' in item:
item['file'].seek(0)
output.write(item['file'].read())
else:
output.write(item['obj'].toData())
def unpack(self):
"""Unpack this file to `self.destPath`."""
objs = list(self._iter_objects())
for obj in objs:
name = obj.name
if name == '' or name.startswith('.'):
log.warning("Object named '%s' output to '%s'",
name, 'data'+name)
name = 'data'+name
if hasattr(obj, 'defaultFileExt'):
name += '.' + obj.defaultFileExt
if self.destPath != '':
name = self.destPath+'/'+name
log.info('Extract "%s" to "%s"', obj.name, name)
with FileWriter(name) as file:
file.write(obj.toData())
from filewriter import FileWriter
if len(sys.argv) > 1:
log = sys.argv[1] # name of log file
else:
log = 'webmet.log' # default log file
if len(sys.argv) > 2:
n = int(sys.argv[2]) # number of observations
else:
n = 10 # default single observation
if len(sys.argv) > 3:
delay = int(sys.argv[3]) # delay between observations (sec)
else:
delay = 30 # default delay 30 sec
if len(sys.argv) > 4:
elevation = float(sys.argv[4]) # elevation of start point
else:
elevation = 100 # default elevation for start point
mu = WebMetMeasureUnit(
msg="q=budapest&appid=13152b0308b85a39cc9a161e241ec2cf")
wi = WebIface("demo", "http://api.openweathermap.org/data/2.5/weather",
"json")
fw = FileWriter(fname=log,
filt=['pressure', 'temp', 'humidity', 'datetime'])
web = WebMet('WebMet', mu, wi)
for i in range(n):
data = web.GetPressure()
data['temp'] = web.GetTemp()['temp']
fw.WriteData(data)
print(data)
time.sleep(delay)
def false_positive(self, partitions: ClusterPoints,
clusters: ClusterPoints):
return self.sum_of_pairs(clusters) - self.true_positive(
partitions, clusters)
@staticmethod
def sum_of_pairs(cluster_points: ClusterPoints):
combinations = 0
for cluster_id in cluster_points.cluster_ids():
cluster_count = cluster_points.points_count(cluster_id)
combinations += cluster_count * (cluster_count - 1)
return combinations / 2
nmi = ClusterEvaluator(NormalizedMutualInformation(), FileReader("", " "))
jcs = ClusterEvaluator(JaccardSimilarity(), FileReader("", " "))
results = list()
for iii in range(1, 6):
nmi_result = nmi.evaluate("data/partitions.txt",
"data/clustering_" + str(iii) + ".txt")
jcs_result = jcs.evaluate("data/partitions.txt",
"data/clustering_" + str(iii) + ".txt")
print("////// " + str(iii) + " ///////")
print(nmi_result)
print(jcs_result)
print()
results.append([nmi_result, jcs_result])
writer = FileWriter("data/scores.txt", " ")
writer.write_list_of_rows_to_file(results)
from trimble5500 import Trimble5500
mu = Trimble5500()
else:
print "unsupported instrument type"
#sys.exit(1)
# set port
port = 'COM5'
if len(sys.argv) > 4:
port = sys.argv[4]
iface = SerialIface("test", port)
# set output file name
fn = 'measmtrx.txt'
if len(sys.argv) > 5:
fn = sys.argv[5]
# write out measurements
wrt = FileWriter(angle='DEG', dist='.3f', fname=fn)
if wrt.GetState() != wrt.WR_OK:
sys.exit(-1) # open error
ts = TotalStation(stationtype, mu, iface, wrt)
ts.SetATR(0) # turn ATR off
ts.SetEDMMode('RLSTANDARD') # reflectorless distance measurement
ts.SetRedLaser(1) # turn red laser on
w = raw_input("Target on lower left corner and press Enter")
w1 = ts.GetAngles()
w = raw_input("Target on upper right corner and press Enter")
w2 = ts.GetAngles()
dh = (w2['hz'].GetAngle() - w1['hz'].GetAngle()) / dh_nr
dv = (w2['v'].GetAngle() - w1['v'].GetAngle()) / dv_nr
# measurement loops
if len(sys.argv) > 1:
cm = sys.argv[1]
else:
cm = '/dev/ttyUSB0'
if re.search('COM[0-9]$', cm) or '/dev/ttyUSB0' == cm:
from serialiface import SerialIface
iface = SerialIface('test', cm)
else:
from bluetoothiface import BluetoothIface
iface = BluetoothIface('test', cm, 1)
#from localiface import LocalIface
#iface = LocalIface('test', 'output.nmea')
from nmeagnssunit import NmeaGnssUnit
from filewriter import FileWriter
from gnss import Gnss
#Making measurement unit
mu = NmeaGnssUnit()
#Writer unit creating
if len(sys.argv) > 2:
fn = sys.argv[2]
else:
fn = 'stdout'
wrt = FileWriter('', 'DEG', '.3f', '%Y-%m-%d %H:%M:%S', None, fn)
#Get NMEA data
g = Gnss('', mu, iface, wrt)
while g.measureIface.state == g.measureIface.IF_OK:
g.Measure()
session = requests.Session()
start_time = getTime()
training_set_json = json.load(open(training_set_path))["documents"]
if test_set_limit != -1:
test_set = json.load(open(test_set_path))["documents"][0:test_set_limit]
else:
test_set = json.load(open(test_set_path))["documents"]
test_set_pmids = [doc["pmid"] for doc in test_set]
tlog("Test set read.")
fw = FileWriter()
for i in range(threshold_start, threshold_end + 1):
thresholds.append(i)
metrics_obj_list.append(Metrics())
eval = Evaluator()
doc_results = {}
results_file = open("results.json", "r")
pmid_results = json.load(results_file)
result_documents_file = open("result_documents.json", "r")
result_documents = json.load(result_documents_file)
def __init__(self, log_file_path, callback):
self.logger = logging.getLogger(self.__class__.__name__)
super(SerialFileWriter, self).__init__(self.__class__.__name__)
self._file_writer = FileWriter(log_file_path, callback)
class Simulation:
def __init__(self, initial_vaccinated, initial_infected, initial_healthy, virus, resultsfilename):
'''Set up the initial simulation values'''
self.virus = virus
self.initial_infected = initial_infected
self.initial_healthy = initial_healthy
self.initial_vaccinated = initial_vaccinated
self.population = []
self.population_size = initial_infected + initial_healthy + initial_vaccinated
self.total_dead = 0
self.total_vaccinated = initial_vaccinated
self.file_writer = FileWriter(resultsfilename)
def create_population(self):
'''Creates the population (a list of Person objects) consisting of initial infected people, initial healthy non-vaccinated people, and
initial healthy vaccinated people. Adds them to the population list'''
for i in range(self.initial_infected):
person = Person(False, virus)
self.population.append(person)
for i in range(self.initial_healthy):
person = Person(False, None)
self.population.append(person)
for i in range(self.initial_vaccinated):
person = Person(True, None)
self.population.append(person)
def print_population(self):
'''Prints out every person in the population and their current attributes'''
#TODO: finish this method
for person in self.population:
print(person.is_alive)
print(person.is_vaccinated)
print("Population Size: {}".format(self.population_size))
print("Initial Infected: {}".format(self.initial_infected))
print("Initial Healthy: {}".format(self.initial_healthy))
print("Initial Vaccinated: {}".format(self.initial_vaccinated))
print("Total Dead: {}".format(self.total_dead))
print("Total Vaccinated: {}".format(self.total_vaccinated))
def get_infected(self):
'''Gets all the infected people from the population and returns them as a list'''
#TODO: finish this method
infected_list = []
for person in self.population:
if person.infection is not None:
infected_list.append(person)
return infected_list
def simulation_should_continue(self):
'''Determines whether the simulation should continue.
If everyone in the population is dead then return False, the simulation should not continue
If everyone in the population is vaccinated return False
If there are no more infected people left and everyone is either vaccinated or dead return False
In all other cases return True'''
#TODO: finish this method
for i in self.population:
if self.total_dead >= self.population_size:
return False
elif self.total_vaccinated >= self.population_size:
return False
elif len(self.get_infected()) == 0:
return False
else:
return True
def run(self):
''' This method should run the simulation until all requirements for ending
the simulation are met.
'''
self.create_population()
random.shuffle(self.population)
self.print_population()
time_step_counter = 0
should_continue = True
self.file_writer.init_file(self.virus, self.population_size, self.initial_vaccinated, self.initial_healthy, self.initial_infected)
#keep looping until the simulation ends
while self.simulation_should_continue():
#save the current infected
old_infected = self.get_infected()
self.time_step(old_infected)
#time step will create newly infected people, just determine the survivial of the previous infected people
self.determine_survival(old_infected)
time_step_counter += 1
print(f'The simulation has ended after {time_step_counter} turns.')
self.file_writer.write_results(time_step_counter, self.total_dead, self.total_vaccinated)
def determine_survival(self, infected):
'''Check if the current infected people survive their infection
Call the did_survive_infection() method
if it returns false then the person is no longer alive, does not have an infection and one is added to total dead
if it returns true then the person no longer has an infection and is vaccinated, one is added to total vaccinated'''
#TODO: finish this method
for self.person in infected:
if self.person.did_survive_infection() == False:
self.total_dead += 1
elif self.person.did_survive_infection() == True:
self.total_vaccinated += 1
else:
print("Error")
def time_step(self, infected):
''' For every infected person interact with a random person from the population 10 times'''
# self.newly_infected = []
# self.newly_dead = 0
# for person in self.population:
for infected_person in infected:
interaction_count = 0
for i in range(10):
#TODO: get a random index for the population list
random_index = random.randint(0, len(self.population)-1)
#TODO: using the random index get a random person from the population
random_person = self.population[random_index]
#TODO: call interaction() with the current infected person and the random person
self.interaction(infected_person, random_person)
def interaction(self, infected, random_person):
'''If the infected person is the same object as the random_person return and do nothing
if the random person is not alive return and do nothing
if the random person is vaccinated return and do nothing
if the random person is not vaccinated:
generate a random float between 0 and 1
if the random float is less then the infected person's virus reproduction number then the random person is infected
othersie the random person is vaccinated and one is added to the total vaccinated'''
#TODO: finish this method
if infected == random_person:
return
elif random_person.is_vaccinated == True:
return
elif random_person.is_alive == False:
return
elif random_person.is_vaccinated == False and random_person.is_alive == True:
random_result = random.uniform(0,1)
if random_result < self.virus.reproduction_num:
random_person.infection = self.virus
return random_person
else:
random_person.is_vaccinated = True
return random_person
from math import sqrt
word_file_path = "./types.txt"
vector_file_path = "./vectors.txt"
test_set_path = "./testSet"
training_set_path = "./trainingSet"
test_set_limit = 200
threshold_start = 1
threshold_end = 10
thresholds = []
metrics_obj_list = []
fw = FileWriter()
for i in range(threshold_start, threshold_end + 1):
thresholds.append(i)
metrics_obj_list.append(Metrics())
def getTime():
return str(datetime.datetime.time(datetime.datetime.now()))
def tlog(msg):
print("[" + getTime() + "] " + msg)
def cossim(v1, v2):
log = sys.argv[1] # name of log file
else:
log = 'bmp180.log' # default log file
if len(sys.argv) > 2:
n = int(sys.argv[2]) # number of observations
else:
n = 10 # default single observation
if len(sys.argv) > 3:
delay = int(sys.argv[3]) # delay between observations (sec)
else:
delay = 30 # default delay 30 sec
if len(sys.argv) > 4:
elevation = float(sys.argv[4]) # elevation of start point
else:
elevation = 100 # default elevation for start point
mu = BMP180MeasureUnit()
i2c = I2CIface(None, 0x77)
fw = FileWriter(fname='bmp180.log',
filt=['elev', 'pressure', 'temp', 'datetime'])
bmp = BMP180('BMP180', mu, i2c)
bmp.LoadCalibration()
bmp.SetSealevel(elevation)
#bmp.GetTemp()
for i in range(n):
data = bmp.GetPressure()
data['elev'] = bmp.GetAltitude()
data['temp'] = bmp.GetTemp()['temp']
fw.WriteData(data)
print(data)
time.sleep(delay)
