def test_csvwriter_append():
fileobj = StringIO()
writer1 = CsvWriter(fileobj)
writer1 << [{
"f1": 2,
"f2": 5,
"f3": "xyz"
}, {
"f1": 1,
"f2": "abc",
"f3": "f"
}]
writer2 = CsvWriter(fileobj)
writer2 << {"f1": 2, "f2": 5, "f3": "xyz"}
reader = csv.DictReader(StringIO(fileobj.getvalue()))
assert list(reader) == [{
"f1": '2',
"f2": '5',
"f3": "xyz"
}, {
"f1": '1',
"f2": "abc",
"f3": "f"
}, {
"f1": '2',
"f2": '5',
"f3": "xyz"
}]
def save_dataset_to_navigator(self, data_name=None):
'''
Code was taken from here
https://github.com/bigartm/bigartm-book/blob/master/BigartmNavigatorExample.ipynb
'''
if data_name is None:
if not hasattr(self, "data_name"):
warnings.warn("Dataset name isn't specified. Skip saving.")
return
data_name = self.data_name
def in_dataset_folder(filename):
return os.path.join(data_name, filename)
id = 1
with CsvWriter(open(in_dataset_folder('modalities.csv'), 'w')) as out:
out << [dict(id=id, name='words')]
with open(in_dataset_folder('docword.{}.txt'.format(data_name))) as f:
D = int(f.readline())
W = int(f.readline())
n = int(f.readline())
ndw_s = [map(int, line.split()) for line in f.readlines()]
ndw_s = [(d - 1, w - 1, cnt) for d, w, cnt in ndw_s]
with CsvWriter(open(in_dataset_folder('documents.csv'), 'w')) as out:
out << (
dict(id=d,
title='Document #{}'.format(d),
slug='document-{}'.format(d),
file_name='.../{}'.format(d))
for d in range(D)
)
with open(in_dataset_folder('vocab.{}.txt'.format(data_name))) as f, \
CsvWriter(open(in_dataset_folder('terms.csv'), 'w')) as out:
out << (
dict(id=i,
modality_id=id,
text=line.strip())
for i, line in enumerate(f)
)
with CsvWriter(open(in_dataset_folder('document_terms.csv'), 'w')) as out:
out << (
dict(document_id=d,
modality_id=id,
term_id=w,
count=cnt)
for d, w, cnt in ndw_s
)
print('Files saved.')
output = Experiment.run_navigator('add_dataset')
self.dataset_id = re.search('Added Dataset #(\d+)', output).group(1)
Experiment.run_navigator('load_dataset', '--dataset-id', self.dataset_id,
'--title', data_name, '-dir', os.path.abspath(data_name))
def test_csvwriter_write():
fileobj = StringIO()
writer = CsvWriter(fileobj)
writer.writerow({"f1": 2, "f2": 5, "f3": "xyz"})
reader = csv.DictReader(StringIO(fileobj.getvalue()))
assert list(reader) == [{"f1": '2', "f2": '5', "f3": "xyz"}]
def main(algorithm, board_number):
""" Runs Rush Hour game with the algorithm """
# Creates board
board = Board(f"data/Rushhour{sys.argv[2]}.csv")
x = algorithm
if memory_clearer:
algorithm = algorithm + "_memory_clearer"
# Initializes the first node
first_node_name = (0, )
first_node = Node(board, first_node_name)
# Sets and runs the algorithm
x_first_algorithm = X_first(first_node, memory_clearer, x)
if x == "depth_first" or x == "breadth_first":
solution, time_elapsed, nodes_dict, board = x_first_algorithm.run()
elif x == "randomize":
solution, time_elapsed = randomize(board)
elif x == "improved_random":
solution, time_elapsed = improved_random(board)
# Prints the time that the algorithm runned and prints the amount of moves
board.end_game(solution, time_elapsed)
# Writes the solution to a CSV file
writer = CsvWriter(algorithm, board.name)
writer.write_to_csv(time_elapsed, board.name, algorithm, solution)
:returns: list of available wifis
"""
wlist = wifi.Cell.all(self.measureUnit.wlan)
res = []
for w in wlist:
r = {
'ssid': w.ssid,
'address': w.address,
'quality': eval(w.quality + '.0'),
'signal': w.signal
}
if len(r):
if self.writerUnit is not None:
self.writerUnit.WriteData(r)
res.append(r)
return res
if __name__ == '__main__':
from wifiunit import WifiUnit
from csvwriter import CsvWriter
wu = WifiUnit()
wr = CsvWriter(fname='wifitest.csv',
mode='w',
filt=['ssid', 'address', 'quality', 'signal', 'datetime'])
wc = WifiCollector('wc', wu, None, wr)
while 1:
wc.GetWifis()
from trimble5500 import Trimble5500
mu = Trimble5500()
else:
print("unsupported instrument type")
exit(1)
if len(sys.argv) > 3:
port = sys.argv[3]
else:
port = 'COM1'
iface = SerialIface("rs-232", port)
if otype == 'geo':
geo_wrt = GeoWriter(dist='.4f', angle='RAD', fname=ofname+'.geo', mode='w')
coo_wrt = GeoWriter(dist='.4f', angle='RAD', fname=ofname + '.coo', mode='w')
else:
geo_wrt = CsvWriter(dist='.4f', fname=ofname+'.dmp', \
header=True, filt=['station', 'id', 'hz', 'v', 'faces'])
coo_wrt = CsvWriter(dist='.4f', fname=ofname+'.csv', \
header=True, filt=['id', 'east', 'north', 'elev'])
ts = TotalStation(stationtype, mu, iface)
# get station data
coo = {}
coo['id'] = raw_input("Station id: ")
coo['east'] = float(raw_input("Station east: "))
coo['north'] = float(raw_input("Station north: "))
coo['elev'] = float(raw_input("Station elev: "))
coo_wrt.WriteData(coo)
ih = float(raw_input("Instrument height: "))
geo = {}
if otype == 'geo':
geo['station'] = coo['id']
port = 'COM7'
if len(sys.argv) > 4:
maxa = float(sys.argv[4]) / 180.0 * math.pi
else:
maxa = PI2
tol = 0.01
if len(sys.argv) > 5:
tol = float(sys.argv[5])
maxiter = 10 # number of iterations to find point on horizontal plan
if len(sys.argv) > 6:
maxiter = int(sys.argv[6])
iface = SerialIface("rs-232", port)
wrt = CsvWriter(
angle='DMS',
dist='.3f',
filt=['id', 'hz', 'v', 'distance', 'east', 'north', 'elev'],
fname='stdout',
mode='a',
sep=';')
ts = TotalStation(stationtype, mu, iface)
ts.SetEDMMode('RLSTANDARD') # reflectorless distance measurement
ts.Measure() # initial measurement for startpoint
startp = ts.GetMeasure()
if ts.measureIface.state != ts.measureIface.IF_OK or 'errorCode' in startp:
print 'FATAL Cannot measure startpoint'
exit(1)
act = Angle(0) # actual angle from startpoint
# height of startpoint above the horizontal axis
height0 = math.cos(startp['v'].GetAngle()) * startp['distance']
w = True
return self._process(msg)
def Temperature(self):
""" Get temperature
:returns: internal temperature
"""
msg = self.measureUnit.TemperatureMsg()
return self._process(msg)
if __name__ == "__main__":
from leicadnaunit import LeicaDnaUnit
from serialiface import SerialIface
from csvwriter import CsvWriter
mu = LeicaDnaUnit()
iface = SerialIface('x', '/dev/ttyUSB0')
wrt = CsvWriter(angle='DMS',
dist='.5f',
filt=['id', 'distance', 'staff', 'datetime'],
fname='stdout',
mode='a',
sep=';')
dna = DigitalLevel('DNA03', mu, iface, wrt)
dna.SetAutoOff(0)
#print (dna.GetAutoOff())
print(dna.Temperature())
for i in range(10):
dna.Measure()
# Serial port
com = '/dev/ttyUSB0'
if len(sys.argv) > 4:
com = sys.argv[4]
if re.search('^COM[0-9]+', com) or re.search('^/dev/.*tty', com):
iface = SerialIface("rs-232", com)
else:
iface = LocalIface("testIface",
com) # Local iface for testing the module
# Writer
if len(sys.argv) > 5:
wrt = CsvWriter(
angle='GON',
dist='.3f',
dt='%Y-%m-%d %H:%M:%S',
filt=['id', 'hz', 'v', 'distance', 'east', 'north', 'elev'],
fname=sys.argv[5],
mode='a',
sep=';')
else:
wrt = EchoWriter()
ts = TotalStation("Leica", mu, iface)
slopeDist = 0
if edm not in mu.edmModes:
edm = 'FAST'
ts.SetEDMMode(edm)
# initialize instrument and variables
if mode > 0:
ts.SetLock(0)
ts.SetATR(1)
else:
g = CsvReader(fname=ifname[:-4] + '.dmp')
f = CsvReader(fname=ifname[:-4] + '.csv', filt=coo_filt)
directions = g.Load()
coordinates = f.Load()
# writers
if ofname[-4:] == '.dmp' or ofname[-4:] == '.csv' or ofname == 'stdout':
# dmp/csv file or console output
if ofname[-4:] == '.dmp' or ofname[-4:] == '.csv':
ofname1 = ofname[:-4] + '.dmp'
ofname2 = ofname[:-4] + '.csv'
else:
ofname1 = ofname2 = ofname
dmp_wrt = CsvWriter(angle = 'DMS', dist = '.4f', \
filt = ['station', 'id','hz','v','distance', 'datetime'], \
fname = ofname1, mode = 'a', sep = ';')
coo_wrt = CsvWriter(dist = '.4f', \
filt = ['id', 'east', 'north', 'elev', 'datetime'], \
fname = ofname2, mode = 'a', sep = ';')
elif ofname[-4:] == '.geo' or ofname[-4:] == '.coo':
# geo/coo file output
if ofname[-4:] == '.geo' or ofname[-4:] == '.coo':
ofname1 = ofname[:-4] + '.geo'
ofname2 = ofname[:-4] + '.coo'
dmp_wrt = GeoWriter(angle = 'RAD', dist = '.4f', \
filt = ['station', 'ih', 'id', 'th', 'hz', 'v', 'distance', \
'datetime'], fname = ofname1, mode = 'a')
coo_wrt = GeoWriter(dist = '.4f', \
filt = ['id', 'east', 'north', 'elev', 'datetime'], \
fname = ofname2, mode = 'a')
else:
numberOfPoints = 2
tol = 0.01
if len(sys.argv) > 5:
tol = float(sys.argv[5])
# Number of iterations to find point on the chosen plane
maxiter = 10
if len(sys.argv) > 6:
maxiter = int(sys.argv[6])
arange = PI2
if len(sys.argv) > 7:
arange = float(sys.argv[7]) / 360.0 * PI2
iface = SerialIface("rs-232", port) ## eomRead='\n'
wrt = CsvWriter(angle='DMS',
dist='.3f',
filt=['id', 'east', 'north', 'elev'],
fname='section.txt',
mode='w',
sep=';')
ts = TotalStation(stationtype, mu, iface)
if isinstance(mu, Trimble5500):
print("Set Trimble 550x to direct reflex (MNU 722)")
raw_input()
else:
ts.SetATR(0) # ATR mode off
ts.SetLock(0) # Lock mode off
ts.SetEDMMode("RLSTANDARD") # Reflectorless distance measurement
ts.SetRedLaser(1)
# Direction of the points to defining the plane
points = []
for i in range(numberOfPoints):
if len(sys.argv) > 4:
station_ih = float(sys.argv[4])
# load input data set
if ifname[-4:] == '.coo':
g = GeoReader(fname=ifname, filt=['id', 'east', 'north', 'elev'])
else:
g = CsvReader(fname=ifname, filt=['id', 'east', 'north', 'elev'])
data = g.Load()
if ofname[-4:] == '.geo':
geo_wrt = GeoWriter(dist = '.4f', angle = 'RAD', fname = ofname, \
filt = ['station', 'id', 'hz', 'v', 'distance', 'faces', 'ih', \
'code'], mode = 'w')
else:
geo_wrt = CsvWriter(dist = '.4f', angle = 'RAD', fname = ofname, \
header = True, mode = 'w', \
filt = ['station', 'id', 'hz', 'v', 'distance', 'faces', 'ih', \
'code'])
og = ObsGen(data, station_id, station_ih)
if og.station_east is None or og.station_north is None or og.station_elev is None:
print "station coordinates not found: ", station_id
exit(-1)
observations = og.run()
for obs in observations:
# heck for dmp output
if ofname[-4:] != '.geo':
if 'station' in obs:
station = obs['station']
continue
else:
obs['station'] = station
def test_csvwriter_with_closes():
fileobj = StringIO()
with CsvWriter(fileobj) as writer:
writer << {"f1": 2, "f2": 5, "f3": "xyz"}
assert fileobj.closed
def test_csvwriter_doesnt_close():
fileobj = StringIO()
writer = CsvWriter(fileobj)
writer << {"f1": 2, "f2": 5, "f3": "xyz"}
assert not fileobj.closed
print('invalid interface given')
sys.exit(1)
# output
if args.output is None:
wrt = EchoWriter(
'', 'DEG', '.3f', '%Y-%m-%d %H:%M:%S',
['id', 'latitude', 'longitude', 'altitude', 'datetime'])
elif re.search('^https?://', args.output):
wrt = HttpWriter(
angle='DEG',
url=args.output,
filt=['longitude', 'latitude', 'altitude', 'datetime'])
elif re.search('\.csv$', args.output):
wrt = CsvWriter(
'', 'DEG', '.3f', '%Y-%m-%d %H:%M:%S',
['id', 'latitude', 'longitude', 'altitude', 'datetime'],
args.output)
else:
print('invalid output given')
sys.exit(2)
# nmea data OK?
if li.state != li.IF_OK:
print("input file/device?")
exit(1)
# nmea processing unit
mu = NmeaGnssUnit()
# instrument
g = Gnss('test', mu, li, wrt)
while g.measureIface.state == g.measureIface.IF_OK:
def save_next_topics_batch_to_navigator(self, topic_batch_size):
'''
Code was taken from here
https://github.com/bigartm/bigartm-book/blob/master/BigartmNavigatorExample.ipynb
'''
topics = self.topics_pool.get_next_topics(topic_batch_size)
topics_ids = [int(topic[5:]) for topic in topics] # topic123 -> 123
pwt = self.topics_pool.get_basic_phi()[topics].as_matrix()
ptd = self.topics_pool.get_basic_theta().loc[topics].as_matrix()
pd = 1.0 / ptd.shape[1]
pt = (ptd * pd).sum(axis=1)
pw = (pwt * pt).sum(axis=1)
ptw = pwt * pt / pw[:, np.newaxis]
pdt = ptd * pd / pt[:, np.newaxis]
def in_dataset_folder(filename):
return os.path.join(self.data_name, filename)
with CsvWriter(open(in_dataset_folder('topics.csv'), 'w')) as out:
out << [
dict(id=0,
level=0,
id_in_level=0,
is_background=False,
probability=1)
] # the single zero-level topic with id=0 is required
out << (
dict(
id=1 + topics_ids[t], # any unique ids
level=
1, # for a flat non-hierarchical model just leave 1 here
id_in_level=topics_ids[t],
is_background=
False, # if you have background topics, they should have True here
probability=p) for t, p in enumerate(pt))
with CsvWriter(open(in_dataset_folder('topic_terms.csv'), 'w')) as out:
out << (
dict(
topic_id=1 + topics_ids[t], # same ids as above
modality_id=1,
term_id=w,
prob_wt=pwt[w, t],
prob_tw=ptw[w, t]) for w, t in zip(*np.nonzero(pwt)))
with CsvWriter(open(in_dataset_folder('document_topics.csv'),
'w')) as out:
out << (
dict(
topic_id=1 + topics_ids[t], # same ids as above
document_id=d,
prob_td=ptd[t, d],
prob_dt=pdt[t, d]) for t, d in zip(*np.nonzero(ptd)))
with CsvWriter(open(in_dataset_folder('topic_edges.csv'), 'w')) as out:
out << (dict(
parent_id=0, child_id=1 + topics_ids[t], probability=p)
for t, p in enumerate(pt))
if self.dataset_id is None:
warnings.warn("Dataset wasn't loaded to navigator.")
else:
output = Experiment.run_navigator('add_topicmodel', '--dataset-id',
self.dataset_id)
self.topic_model_id = re.search(
'Added Topic Model #(\d+) for Dataset #(\d+)', output).group(1)
Experiment.run_navigator('load_topicmodel', '--topicmodel-id',
self.topic_model_id, '--title',
self.data_name, '-dir',
os.path.abspath(self.data_name))
else:
logging.warning("meteorological data not available")
if 'met_wr' in cr.json:
if re.search('^http[s]?://', cr.json['met_wr']):
wrtm = HttpWriter(
name='met', url=cr.json['met_wr'], mode='POST',
filt=['id', 'temp', 'pressure', 'humidity', 'wettemp',
'datetime'])
elif re.search('^sqlite:', cr.json['met_wr']):
wrtm = SqLiteWriter(db=cr.json['met_wr'][7:],
table='monitoring_met',
filt=['id', 'pressure', 'temp', 'humidity',
'wettemp', 'datetime'])
else:
wrtm = CsvWriter(name='met', fname=cr.json['met_wr'],
filt=['id', 'temp', 'pressure', 'humidity',
'wettemp', 'datetime'], mode='a')
data = {'id': cr.json['station_id'], 'temp': temp,
'pressure': pres, 'humidity': humi, 'wettemp': wet}
if wrtm.WriteData(data) == -1:
logging.error('Met data write failed')
# get station coordinates
print("Loading station coords...")
if re.search('^http[s]?://', cr.json['coo_rd']):
rd_st = HttpReader(url=cr.json['coo_rd'], ptys=['STA'], \
filt=['id', 'east', 'north', 'elev'])
else:
rd_st = GeoReader(fname=cr.json['coo_rd'], \
filt=['id', 'east', 'north', 'elev'])
w = rd_st.Load()
st_coord = [x for x in w if x['id'] == cr.json['station_id']]
res = super(SenseHat, self).get_accelerometer_raw()
res2 = {}
for key, val in res.items():
res2['accel_' + key] = val
res2['time'] = time.time()
self.write(res2)
return res2
@property
def accel(self):
return self.get_accelerometer()
if __name__ == '__main__':
from csvwriter import CsvWriter
wrt = CsvWriter(fname='/home/bence/proba.csv',
dist='.17f',
filt=[
'hum', 'ori_pitch', 'ori_roll', 'ori_yaw',
'gyro_raw_x', 'gyro_raw_y', 'gyro_raw_z', 'accel_x',
'accel_y', 'accel_z'
])
a = SenseHat('test', wrt)
while 1:
a.accel
time.sleep(0.5)