df=df_transformed, method='infer', create='infer') connection.close_connection() def chunks(l, n): """Yield successive n-sized chunks from l.""" for i in range(0, len(l), n): yield l[i:i + n] if __name__ == "__main__": config_path = locate_config(sys.argv) # Configuration file reader config = ConfigParser() config.read(config_path) print("Configuration file read.") traffic_meta_path = config.get('Paths', 'meta_path') csv_files = config.get('Paths', 'data_path') print(csv_files) file_paths = utils.get_file_names(csv_files, extension=file_ext) print("Number of traffic files found: ", len(file_paths)) if DEBUG: end = batch_limit
import sys #sys.path.append('./src') import logging import boto3 from configparser import ConfigParser import pandas as pd import threading import time from threading import Thread import process_utils as utils #from omnisci_connector.omni_connect import OmnisciConnect from MetaData import MetaData from TrafficData import TrafficData from utils import locate_config from botocore.exceptions import ClientError import os #############################INPUTS############################# table_name = "test_2" # initial parameters for reading in traffic data threshold = 0.01 interest_col = 'speed' grouper = 'station' batch_limit = 1 file_ext = '.txt' thread_num = 1 DEBUG = False ################################################################ """ def upload_file(file_name, bucket, object_name=None): Upload a file to an S3 bucket
import sys sys.path.append('./src') from configparser import ConfigParser import pandas as pd #import threading import time from threading import Thread import process_utils as utils #from omnisci_connector.omni_connect import OmnisciConnect from MetaData import MetaData from TrafficData import TrafficData from utils import locate_config #############################INPUTS############################# table_name = "test_2" # initial parameters for reading in traffic data threshold = 0.01 interest_col = 'speed' grouper = 'station' batch_limit = 1 file_ext = '.txt' thread_num = 1 DEBUG = False ################################################################ def transform_and_load(config_path, paths, meta_path, output_path): # Send data to omnisci: #print("connect to omnisci with thread: ", thread) #connection = OmnisciConnect(config_path) #connection.start_connection() df_batch = []
import sys #sys.path.append('./src') from pems_extractor import PemsExtractor from utils import locate_config if __name__ == "__main__": config_path = locate_config(sys.argv) pems = PemsExtractor(config_path) links = pems.extract_links() print("extracted Links, found: ", len(links)) pems.get_files(links)