df=df_transformed,
method='infer',
create='infer')
connection.close_connection()
def chunks(l, n):
"""Yield successive n-sized chunks from l."""
for i in range(0, len(l), n):
yield l[i:i + n]
if __name__ == "__main__":
config_path = locate_config(sys.argv)
# Configuration file reader
config = ConfigParser()
config.read(config_path)
print("Configuration file read.")
traffic_meta_path = config.get('Paths', 'meta_path')
csv_files = config.get('Paths', 'data_path')
print(csv_files)
file_paths = utils.get_file_names(csv_files, extension=file_ext)
print("Number of traffic files found: ", len(file_paths))
if DEBUG:
end = batch_limit
import sys
#sys.path.append('./src')
import logging
import boto3
from configparser import ConfigParser
import pandas as pd
import threading
import time
from threading import Thread
import process_utils as utils
#from omnisci_connector.omni_connect import OmnisciConnect
from MetaData import MetaData
from TrafficData import TrafficData
from utils import locate_config
from botocore.exceptions import ClientError
import os
#############################INPUTS#############################
table_name = "test_2"
# initial parameters for reading in traffic data
threshold = 0.01
interest_col = 'speed'
grouper = 'station'
batch_limit = 1
file_ext = '.txt'
thread_num = 1
DEBUG = False
################################################################
"""
def upload_file(file_name, bucket, object_name=None):
Upload a file to an S3 bucket
import sys
sys.path.append('./src')
from configparser import ConfigParser
import pandas as pd
#import threading
import time
from threading import Thread
import process_utils as utils
#from omnisci_connector.omni_connect import OmnisciConnect
from MetaData import MetaData
from TrafficData import TrafficData
from utils import locate_config
#############################INPUTS#############################
table_name = "test_2"
# initial parameters for reading in traffic data
threshold = 0.01
interest_col = 'speed'
grouper = 'station'
batch_limit = 1
file_ext = '.txt'
thread_num = 1
DEBUG = False
################################################################
def transform_and_load(config_path, paths, meta_path, output_path):
# Send data to omnisci:
#print("connect to omnisci with thread: ", thread)
#connection = OmnisciConnect(config_path)
#connection.start_connection()
df_batch = []
import sys
#sys.path.append('./src')
from pems_extractor import PemsExtractor
from utils import locate_config
if __name__ == "__main__":
config_path = locate_config(sys.argv)
pems = PemsExtractor(config_path)
links = pems.extract_links()
print("extracted Links, found: ", len(links))
pems.get_files(links)