def main(full=False):
paths = return_files_paths()
print(paths, len(paths))
# check the full flag
if not full:
l = randint(0, len(paths) - 1)
# try the first thousand rows of one random file
dataset = [return_dataset(paths[l])]
luke = (create_generator_from_dataset(d, 1000) for d in dataset)
else:
# dump all the files
dataset = []
dataset += [return_dataset(p) for p in paths]
luke = (create_generator_from_dataset(d) for d in dataset)
#print(luke, )
# Luke is a >> generator of generators <<
# Feel the Force
print('DUMPING...')
from src.formatdata import bulk_dump
i = 0
while True:
try:
_, n = bulk_dump(next(luke))
i += n
except StopIteration:
print('>>> {} Xco2 data dumped <<<'.format(i))
break
except KeyboardInterrupt:
break
sys.exit(0)
def setUpClass(cls):
print(cls.__doc__)
_, cls.engine = start_postgre_engine('test', False)
cls.conn = cls.engine.connect()
cls.paths = return_files_paths()
cls.dataset = return_dataset(cls.paths[0])
cls.session = dbProxy.create_session(cls.engine)
def setUpClass(cls):
print(cls.__doc__)
_, cls.engine = start_postgre_engine('test', False)
cls.conn = cls.engine.connect()
cls.paths = return_files_paths()
cls.dataset = return_dataset(cls.paths[0])
cls.session = dbProxy.create_session(cls.engine)
def setUpClass(cls):
print(cls.__doc__)
_, cls.engine = start_postgre_engine('test', False)
cls.conn = cls.engine.connect()
cls.paths = return_files_paths()
cls.dataset = return_dataset(cls.paths[0])
# create an OCOpoint from the first record in the first file
cls.luke = createOCOpoint(
**{
'latitude': cls.dataset['latitude'][0],
'longitude': cls.dataset['longitude'][0],
'xco2': cls.dataset['xco2'][0],
'date': cls.dataset['date'][0],
})
def setUpClass(cls):
print(cls.__doc__)
_, cls.engine = start_postgre_engine('test', False)
cls.conn = cls.engine.connect()
cls.paths = return_files_paths()
cls.dataset = return_dataset(cls.paths[0])
# create an OCOpoint from the first record in the first file
cls.luke = createOCOpoint(**{
'latitude': cls.dataset['latitude'][0],
'longitude': cls.dataset['longitude'][0],
'xco2': cls.dataset['xco2'][0],
'date': cls.dataset['date'][0],
}
)
def test_get_files_paths(self):
paths = return_files_paths()
self.assertTrue(isinstance(paths, list))
self.assertTrue(len(paths) >= 2)
del paths
def setUp(self):
self.paths = return_files_paths()
def setUpClass(cls):
print(cls.__doc__)
cls.paths = return_files_paths()
cls.dataset = return_dataset(cls.paths[0])
cls.length = len(cls.dataset['latitude'])
def setUpClass(cls):
print(cls.__doc__)
cls.paths = return_files_paths()
cls.dataset = return_dataset(cls.paths[0])
cls.length = len(cls.dataset['latitude'])
