def main(full=False): paths = return_files_paths() print(paths, len(paths)) # check the full flag if not full: l = randint(0, len(paths) - 1) # try the first thousand rows of one random file dataset = [return_dataset(paths[l])] luke = (create_generator_from_dataset(d, 1000) for d in dataset) else: # dump all the files dataset = [] dataset += [return_dataset(p) for p in paths] luke = (create_generator_from_dataset(d) for d in dataset) #print(luke, ) # Luke is a >> generator of generators << # Feel the Force print('DUMPING...') from src.formatdata import bulk_dump i = 0 while True: try: _, n = bulk_dump(next(luke)) i += n except StopIteration: print('>>> {} Xco2 data dumped <<<'.format(i)) break except KeyboardInterrupt: break sys.exit(0)
def setUpClass(cls): print(cls.__doc__) _, cls.engine = start_postgre_engine('test', False) cls.conn = cls.engine.connect() cls.paths = return_files_paths() cls.dataset = return_dataset(cls.paths[0]) cls.session = dbProxy.create_session(cls.engine)
def setUpClass(cls): print(cls.__doc__) _, cls.engine = start_postgre_engine('test', False) cls.conn = cls.engine.connect() cls.paths = return_files_paths() cls.dataset = return_dataset(cls.paths[0]) # create an OCOpoint from the first record in the first file cls.luke = createOCOpoint( **{ 'latitude': cls.dataset['latitude'][0], 'longitude': cls.dataset['longitude'][0], 'xco2': cls.dataset['xco2'][0], 'date': cls.dataset['date'][0], })
def setUpClass(cls): print(cls.__doc__) _, cls.engine = start_postgre_engine('test', False) cls.conn = cls.engine.connect() cls.paths = return_files_paths() cls.dataset = return_dataset(cls.paths[0]) # create an OCOpoint from the first record in the first file cls.luke = createOCOpoint(**{ 'latitude': cls.dataset['latitude'][0], 'longitude': cls.dataset['longitude'][0], 'xco2': cls.dataset['xco2'][0], 'date': cls.dataset['date'][0], } )
def test_get_files_paths(self): paths = return_files_paths() self.assertTrue(isinstance(paths, list)) self.assertTrue(len(paths) >= 2) del paths
def setUp(self): self.paths = return_files_paths()
def setUpClass(cls): print(cls.__doc__) cls.paths = return_files_paths() cls.dataset = return_dataset(cls.paths[0]) cls.length = len(cls.dataset['latitude'])