def getSwarmCache(intersection, overwrite=False): """ :param intersection: :return: the csv file where cache data is stored for this intersection to be used in swarming """ cache_file = os.path.join(getEngineDir(), SWARM_DATA_CACHE, intersection + '.csv') if os.path.exists(cache_file) and not overwrite: print "Using existing data cache file:", cache_file else: if overwrite: print "Overwriting cache file for", intersection, " in ", cache_file else: print "No cache file exists for", intersection, ".... creating new one in", cache_file with open(cache_file, 'wb') as csv_out, pymongo.MongoClient(mongo_uri) as client: db = client[mongo_database] collection = db[mongo_collection] readings = collection.find({'site_no': intersection}) sensors = getSensors(intersection) fieldnames = ['timestamp'] + sensors writer = csv.DictWriter(csv_out, fieldnames=fieldnames) writer.writeheader() headerRow2 = {f: 'int' for f in sensors} headerRow2['timestamp'] = 'datetime' headerRow3 = {'timestamp': 'T'} writer.writerow(headerRow2) writer.writerow(headerRow3) if readings.count() == 0: raise Exception( "No such intersection with site_no '%s' exists" % intersection) else: for i in readings: # nupic model data format expects this format row = { 'timestamp': i['datetime'].strftime('%Y-%m-%d %H:%M') } for s, c in i['readings'].items(): if c > 2040: row[s] = None else: row[s] = c writer.writerow(row) return "file://" + cache_file
def getSwarmCache(intersection, overwrite=False): """ :param intersection: :return: the csv file where cache data is stored for this intersection to be used in swarming """ cache_file = os.path.join(getEngineDir(), SWARM_DATA_CACHE, intersection+'.csv') if os.path.exists(cache_file) and not overwrite: print "Using existing data cache file:", cache_file else: if overwrite: print "Overwriting cache file for", intersection, " in ", cache_file else: print "No cache file exists for", intersection, ".... creating new one in", cache_file with open(cache_file, 'wb') as csv_out, pymongo.MongoClient(mongo_uri) as client: db = client[mongo_database] collection = db[mongo_collection] readings = collection.find({'site_no': intersection}) sensors = getSensors(intersection) fieldnames = ['timestamp'] + sensors writer = csv.DictWriter(csv_out, fieldnames=fieldnames) writer.writeheader() headerRow2 = {f: 'int' for f in sensors} headerRow2['timestamp'] = 'datetime' headerRow3 = {'timestamp': 'T'} writer.writerow(headerRow2) writer.writerow(headerRow3) if readings.count() == 0: raise Exception("No such intersection with site_no '%s' exists" % intersection) else: for i in readings: # nupic model data format expects this format row = {'timestamp': i['datetime'].strftime('%Y-%m-%d %H:%M')} for s, c in i['readings'].items(): if c > 2040: row[s] = None else: row[s] = c writer.writerow(row) return "file://"+cache_file
import os import sys import argparse import pymongo import pluck import csv import yaml from index import SWARM_CONFIGS_DIR, getEngineDir import pprint import operator from collections import Counter __author__ = 'Jonathan Mackenzie' try: path = os.path.join(os.path.dirname(getEngineDir()), 'connection.yaml') with open(path, 'r') as f: conf = yaml.load(f) mongo_uri = conf['mongo_uri'] mongo_collection = conf['mongo_collection'] mongo_database = conf['mongo_database'] max_vehicles = conf['max_vehicles'] except: raise Exception( 'No connection.yaml with mongo_uri defined! please make one with a mongo_uri variable' ) SWARM_DATA_CACHE = 'swarm_data_cache' DESCRIPTION = "Create a swarm config for a given intersection if it doesn't already exist" parser = argparse.ArgumentParser(description=DESCRIPTION)
import os import sys import argparse import pymongo import pluck import csv import yaml from index import SWARM_CONFIGS_DIR, getEngineDir import pprint import operator from collections import Counter __author__ = 'Jonathan Mackenzie' try: path = os.path.join(os.path.dirname(getEngineDir()), 'connection.yaml') with open(path, 'r') as f: conf = yaml.load(f) mongo_uri = conf['mongo_uri'] mongo_collection = conf['mongo_collection'] mongo_database = conf['mongo_database'] max_vehicles = conf['max_vehicles'] except: raise Exception('No connection.yaml with mongo_uri defined! please make one with a mongo_uri variable') SWARM_DATA_CACHE = 'swarm_data_cache' DESCRIPTION = "Create a swarm config for a given intersection if it doesn't already exist" parser = argparse.ArgumentParser(description=DESCRIPTION) parser.add_argument('--max', dest='max', type=int, help="Max number of cars to use in the input fields",