def testParseDataWithMissingOffSet(getFixtures):
try:
parser = DataParser()
parsedData = parser.parseData(getFixtures.id, None, getFixtures.data)
fail("parser.parseData should have thrown an error on null offset")
except Exception as ex:
assert (type(ex) is ValueError)
def testParseDataWithNullData(getFixtures):
try:
parser = DataParser()
parsedData = parser.parseData(getFixtures.id, 1, None)
fail("parser.parseData should have thrown an error on null data")
except Exception as ex:
assert (type(ex) is ValueError)
def fillTableWithDataFromFile(self, fileName):
# Parse data into booking class
dataParser = DataParser(fileName)
bookings = dataParser.GetAllBookings()
# Show data in UI table
self.dataTable.setRowCount(0)
for row, booking in enumerate(bookings):
self.dataTable.insertRow(row)
dateItem = QtWidgets.QTableWidgetItem(booking.date)
bookoingTypeItem = QtWidgets.QTableWidgetItem(booking.bookingType)
nameItem = QtWidgets.QTableWidgetItem(booking.name)
purposeItem = QtWidgets.QTableWidgetItem(booking.purpose)
valueItem = QtWidgets.QTableWidgetItem(booking.value)
valueItem.setTextAlignment(Qt.AlignRight)
self.dataTable.setItem(row, 0, dateItem)
self.dataTable.setItem(row, 1, bookoingTypeItem)
self.dataTable.setItem(row, 2, nameItem)
self.dataTable.setItem(row, 3, purposeItem)
self.dataTable.setItem(row, 4, valueItem)
self.dataTable.resizeColumnsToContents()
def __init__(self, parameters, sc):
parameters = json.loads(parameters)
schema = parameters.get('schema', None)
header = parameters.get('header', False)
self._parser = DataParser(schema, header)
self._sc = sc
def testParseDataWithNonJsonData(getFixtures):
try:
parser = DataParser()
data = "this is not JSON"
parsedData = parser.parseData(getFixtures.id, 1, data)
fail("parser.parseData should have thrown an error on non JSON data")
except Exception as ex:
assert (type(ex) is ValueError)
def test_04_parser_parse_raw_data(self):
input = "empid=D011\ngender=M\nage=29"
parser = DataParser()
parser.parse_raw_data(input)
expected = [{'empid': 'D011', 'gender': 'M', 'age': '29'}]
actual = parser.get_data()
self.assertEqual(expected, actual)
def testOffsetTimeGenerationWithInvalidArgs(getFixtures):
parser = DataParser()
try:
strTimeNow = "2016-10-09T15:48:54"
dtOffset = parser.getOffsetTime(strTimeNow, getFixtures.offset)
fail("should have thrown exception b/c invalid timestamp")
except Exception as ex:
assert (type(ex) is ValueError)
def testParseDataWithInvalidJsonData(getFixtures):
try:
parser = DataParser()
data = {'foobar': 'goo'}
parsedData = parser.parseData(getFixtures.id, 1, data)
fail(
"parser.parseData should have thrown an error on incorrect JSON data"
)
except Exception as ex:
assert (type(ex) is KeyError)
def setUp(self):
self.parser = DataParser()
self.cmd_view = CmdView()
self.file_view = FileView()
self.validator = Validator()
self.db = DatabaseView("test.db")
self.vis = Visualiser()
# self.val = Validator()
self.controller = Controller(self.cmd_view, self.file_view,
self.parser, self.validator, self.db,
self.vis)
def setUp(self):
self.parser = DataParser()
self.cmd_view = CmdView()
self.file_reader = FileReader()
self.validator = Validator()
self.db = Database("test.db")
self.vis = Visualiser()
self.val = Validator()
self.serial = Serializer()
self.controller = Controller(self.cmd_view, self.file_reader,
self.parser, self.validator, self.db,
self.vis, self.serial)
self.init()
def test_10_controller_validate_fail(self):
self.controller = Controller(self.cmd_view, self.file_reader,
DataParser(), self.validator, self.db,
self.vis, self.serial)
captured_output = io.StringIO()
sys.stdout = captured_output
self.controller.validate()
expected = "* No data has been read.\n-- Type 'help get' for more details.\n"
actual = captured_output.getvalue()
sys.stdout = sys.__stdout__
self.assertEqual(expected, actual)
def parse(self):
struct = self.dictFromListBox(self.fields_list)
struct["separator"] = self.tokenSeparator.get()
parser = DataParser(struct)
with open(self.logFilename.get(), 'r') as logfile:
testline = logfile.readline()
print(testline, struct)
data = parser.parse_line(testline)
if data == None:
print(
"Could not match line structure to the log's [first] line")
return None
all_data = parser.parse_file(self.logFilename.get())
self.generateStatistics(all_data)
def analyze(self):
if self.fname is None:
print "File is not setted."
return
datagetter = DataParser(self.fname[0])
if self.regexTextField.toPlainText() is None:
print "Regexp is not setted."
return
datagetter.init_regex(self.regexTextField.toPlainText())
datagetter.filled_data()
if self.allowStatistics.isChecked():
datagetter.print_statistics()
if self.csvReport.isChecked():
datagetter.save_to_csv()
if self.allowGraphs.isChecked():
datagetter.print_graphs(self.separateGraphs.isChecked())
def testParsedDataWithValidArgs(getFixtures):
try:
parser = DataParser()
parsedData = parser.parseData(getFixtures.id, getFixtures.offset,
getFixtures.data)
assert (parsedData != None)
#logging.debug(parsedData)
assert (len(parsedData) > 0)
for data in parsedData:
assert data['id'] != None
assert data['time'] != None
assert data['heartRate'] != None
assert data['coordinates'] != None
except Exception as ex:
logging.debug(str(ex))
fail("should not have an exception when parsedData has valid input"
) # cause a failure
def testOffsetTimeGeneration(getFixtures):
try:
parser = DataParser()
strTimeNow = "2016-10-09T15:48:54Z"
dtOffset = parser.getOffsetTime(strTimeNow, getFixtures.offset)
assert (dtOffset.second == 59)
strTimeNow = "2016-10-09T15:48:55Z"
dtOffset = parser.getOffsetTime(strTimeNow, getFixtures.offset)
assert (dtOffset.second == 0)
assert (dtOffset.minute == 49)
strTimeNow = "2016-10-09T15:48:56Z"
dtOffset = parser.getOffsetTime(strTimeNow, getFixtures.offset)
assert (dtOffset.second == 1)
assert (dtOffset.minute == 49)
except Exception as ex:
logging.debug(str(ex))
fail("should not have an exception when parsing valid time and offset"
) # cause a failure
# Instiate the models
model1 = BCNN()
model2 = TCNN()
# Load models from last saved state
model1.load_state_dict(torch.load("./bcnn_model.pt"))
model2.load_state_dict(torch.load("./tcnn_model.pt"))
model1.cuda()
model2.cuda()
# Set optimizer as Adam
optimizer = torch.optim.Adam(
(list(model1.parameters()) + list(model2.parameters())), lr=1e-4)
# Load Training data
trainset = DataParser('01')
trainloader = torch.utils.data.DataLoader(trainset, batch_size=8, shuffle=True)
# Load testing data
testset = DataParser('04')
testloader = torch.utils.data.DataLoader(testset, batch_size=1, shuffle=True)
# criterion is MSELoss
criterion = nn.MSELoss().cuda()
epochs = 15
i = 0
for e in range(epochs):
model1.train()
model2.train()
__author__ = 'Radim Spigel'
__version__ = '1.0'
import sys
from dataparser import print_help, DataParser
from qtgui import qt_main
if __name__ == "__main__":
if len(sys.argv) > 1:
print sys.argv
if '-h' in sys.argv:
print_help()
sys.exit()
datagetter = DataParser(sys.argv[1])
datagetter.from_command_line(sys.argv[1:])
elif len(sys.argv) > 6:
print_help()
else:
qt_main()
if __name__ == '__main__':
parser = OptionParser()
parser.add_option("-d", "--dataset", dest="db_type", default="berlin")
parser.add_option("-p", "--dataset_path", dest="path", default="")
(options, args) = parser.parse_args(sys.argv)
db_type = options.db_type
path = options.path
print("Loading data from " + db_type + " dataset...")
if db_type not in ('berlin'):
sys.exit("Dataset not registered. Please create a method to read it")
db = DataParser(path, db_type)
# k_folds = len(db.test_sets)
# splits = zip(db.train_sets, db.test_sets)
callback_list = [
EarlyStopping(
monitor='acc',
patience=1,
verbose=1
),
ModelCheckpoint(
filepath='cnnlstm_model.h5',
monitor='val_loss',
save_best_only='True'
)
# Instantiate models
model1 = BCNN()
model2 = TCNN()
# Load models from files
model1.load_state_dict(torch.load("./bcnn_model.pt"))
model2.load_state_dict(torch.load("./tcnn_model.pt"))
model1 = model1.cuda()
model2 = model2.cuda()
# Set to eval mode
model1.eval()
model2.eval()
# Load Data from testing set
testset = DataParser('04')
testloader = torch.utils.data.DataLoader(testset, batch_size=1, shuffle=True)
total = 0
err_x = 0
err_z = 0
err_t = 0
# Run for all testing data
for counter, d in enumerate(testloader, 0):
dtype = torch.cuda.FloatTensor
x1 = d["img_l1"].type(dtype)
x2 = d["img_l2"].type(dtype)
yx = d["dx"].type(dtype)
yz = d["dz"].type(dtype)
yt = d["dth"].type(dtype)
def get_logs():
dp = DataParser(constants.log_file_path)
mentions_by_ticker = dp.mentions_by_ticker()
import time
from pprint import pprint
import matplotlib.pyplot as plt
import requests
import smopy
from google.transit import gtfs_realtime_pb2
from protobuf_to_dict import protobuf_to_dict
from dataparser import DataParser
FEED_URL = "https://gtfsrt.api.translink.com.au/Feed/SEQ"
MAP_FILENAME = "bne.bin"
UPDATE_EVERY = 30
BBOX = (152.7, 153.3, -27.7, -27.2)
data = DataParser()
def get_entity_type(routes_dict, route_id):
# Get TransLink URL and extract portion of path with vehicle type
# Edit this for different feeds
return routes_dict[route_id]['route_url'].split('/')[5]
def get_feed(url):
while True:
response = requests.get(url)
if response.status_code == 200:
break
print(f"Got status code {response.status_code}, retrying...")
time.sleep(0.5)
#!/usr/bin/python3
from dataparser import DataParser
import statistics
structure = {
"my_date": "date",
"my_int": "integer",
"my_float": "float",
"my_str": "str",
"separator": ";"
}
parser = DataParser(structure)
## LINE PARSING
line = "1-12-2021;123;-123.456;xa"
x = parser.parse_line(line)
print("Single Line:\n" + str(x))
## FILE PARSING
all_data = parser.parse_file('my_log.log')
print("\nFull File: \n" + str(all_data))
##GENERATE STATISTICS
g = statistics.generate_data_x_data(all_data['my_int'], all_data['my_float'],
"my_int", "my_float")
print(g)
def test_02_parser_to_dict(self):
expected = {'empid': 'D011', 'gender': 'M', 'age': '29'}
actual = DataParser()._to_dict(['empid=D011', 'gender=M', 'age=29'])
self.assertEqual(expected, actual)
def test_01_parser_to_list(self):
expected = ['empid=D011', 'gender=M', 'age=29']
actual = DataParser()._to_list("empid=D011\ngender=M\nage=29")
self.assertEqual(expected, actual)
from controller import Controller
from cmdview import CmdView
from file_reader import FileReader
from dataparser import DataParser
from validator import Validator
from database import Database
from visualiser import Visualiser
from serializer import Serializer
if __name__ == "__main__":
parser = DataParser()
cmd_view = CmdView()
file_reader = FileReader()
validator = Validator()
db = Database("test.db")
vis = Visualiser()
serial = Serializer()
con = Controller(cmd_view, file_reader, parser, validator, db, vis, serial)
cmd_view.set_controller(con)
# run program
cmd_view.cmdloop()
def test_03_parser_scrub_db_list(self):
expected = [14, 25]
actual = DataParser().scrub_db_list([(14, ), (25, )])
self.assertEqual(expected, actual)
def getDataParser(self):
return DataParser()
