def main(full=False):
paths = return_files_paths()
print(paths, len(paths))
# check the full flag
if not full:
l = randint(0, len(paths) - 1)
# try the first thousand rows of one random file
dataset = [return_dataset(paths[l])]
luke = (create_generator_from_dataset(d, 1000) for d in dataset)
else:
# dump all the files
dataset = []
dataset += [return_dataset(p) for p in paths]
luke = (create_generator_from_dataset(d) for d in dataset)
#print(luke, )
# Luke is a >> generator of generators <<
# Feel the Force
print('DUMPING...')
from src.formatdata import bulk_dump
i = 0
while True:
try:
_, n = bulk_dump(next(luke))
i += n
except StopIteration:
print('>>> {} Xco2 data dumped <<<'.format(i))
break
except KeyboardInterrupt:
break
sys.exit(0)
def util_populate_table(dataset, lentest):
"""
Populate the t_co2 table with n rows
:param numpyArray dataset:
:param int lentest:
:param Session session:
:return list: tuples containing primary keys of points and areas
"""
# create a generator from the first lentest records in the dataset
luke = create_generator_from_dataset(dataset, lentest)
samples = []
for d in luke:
new = Xco2(
xco2=d.xco2,
timestamp=d.timestamp,
latitude=d.latitude,
longitude=d.longitude
)
samples.append(
new.store_xco2()
)
return samples
def test_create_generator_from_dataset(self):
"""Test crate_generator_from_dataset()"""
gen = list(create_generator_from_dataset(self.dataset, 10))
[
self.assertAlmostEqual(l.xco2, self.dataset['xco2'][i], delta=0.001)
for i, l in enumerate(gen)
]
def test_create_generator_from_dataset(self):
"""Test crate_generator_from_dataset()"""
gen = list(create_generator_from_dataset(self.dataset, 10))
[
self.assertAlmostEqual(l.xco2,
self.dataset['xco2'][i],
delta=0.001) for i, l in enumerate(gen)
]
def test_bulk_dump(self):
"""Test Xco2.bulk_dump()"""
from src.formatdata import bulk_dump
print('#### TEST5 ####')
session2 = dbProxy.create_session(db='test', engine=self.engine)
util_truncate_table(session2, [Xco2, Areas])
bulk_dump(create_generator_from_dataset(self.dataset, 8))
rows = self.session.query(Xco2).count()
try:
self.assertEqual(rows, 8)
print('PASSED')
except AssertionError:
print('FAILED')
def test_bulk_dump(self):
"""Test Xco2.bulk_dump()"""
from src.formatdata import bulk_dump
print('#### TEST5 ####')
session2 = dbProxy.create_session(db='test', engine=self.engine)
util_truncate_table(session2, [Xco2, Areas])
bulk_dump(
create_generator_from_dataset(self.dataset, 8)
)
rows = self.session.query(Xco2).count()
try:
self.assertEqual(rows, 8)
print('PASSED')
except AssertionError:
print('FAILED')
