#!/usr/bin/env python
import json
import os
import gdata.spreadsheet.service as service
import config
import database
import database_util
from model.preload import ParameterType, ValueEncoding, CodeSet, Unit, FillValue, FunctionType, ParameterFunction, \
Parameter, Stream, StreamDependency
database.initialize_connection(database.PreloadDatabaseMode.EMPTY_FILE)
database.open_connection()
key = config.SPREADSHEET_KEY
use_cache = config.USE_CACHED_SPREADSHEET
cachedir = '.cache'
IGNORE_SCENARIOS = ['VOID']
def sheet_generator(name):
cache_path = os.path.join(cachedir, name)
rows = []
if use_cache and os.path.exists(cache_path):
try:
rows.extend(json.load(open(cache_path)))
print 'used cache'
for row in rows:
yield row
return
#!/usr/bin/env python
import codecs
import os
import shutil
import logging
import jinja2
import database
from model.preload import Stream
database.initialize_connection(database.PreloadDatabaseMode.POPULATED_MEMORY)
database.open_connection()
DROP_KEYSPACE = 'drop keyspace ooi;\n\n'
CREATE_KEYSPACE = "create keyspace ooi with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };\n\n"
CREATE_METADATA = '''create table ooi.stream_metadata
( subsite text, node text, sensor text, method text, stream text, count bigint, first double, last double,
primary key ((subsite, node, sensor), method, stream));
'''
CREATE_PROVENANCE = '''
CREATE TABLE ooi.dataset_l0_provenance (
subsite text,
node text,
sensor text,
method text,
deployment int,
id uuid,
fileName text,
#!/usr/bin/env python
import codecs
import os
import shutil
import logging
import jinja2
import database
from model.preload import Stream
database.initialize_connection(database.PreloadDatabaseMode.POPULATED_MEMORY)
database.open_connection()
DROP_KEYSPACE = 'drop keyspace ooi;\n\n'
CREATE_KEYSPACE = "create keyspace ooi with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };\n\n"
CREATE_PROVENANCE = '''
CREATE TABLE ooi.dataset_l0_provenance (
subsite text,
node text,
sensor text,
method text,
deployment int,
id uuid,
fileName text,
parserName text,
parserVersion text,
PRIMARY KEY((subsite, node, sensor), method, deployment, id)
);
'''
def setUpClass(cls):
database.initialize_connection(PreloadDatabaseMode.POPULATED_MEMORY)
database.open_connection()