Example #1
0
 def update_schema(self, s):
     name = s['name']
     if name in self._schemas:
         s = self.client.schemas.update(name, s)
         self._schemas[name].update_schema(s)
     else:
         s = self.client.schemas.create(s)
         self._schemas[name] = schema.create_schema(self, s)
Example #2
0
def read_zillow(sqlc, basepath, database, table, **kwargs):
    '''
    read in zillow table as spark dataframes. Assumes that the
    `basepath` directory contains the Layout.xlsx file, as well
    as the databases (ZTrans and ZAsmt directories).
    '''

    layout = os.path.join(basepath, 'Layout.xlsx')
    filepath = os.path.join(basepath, database, table + '.txt')

    schema = create_schema(layout, database=database, table=table)

    df = sqlc.read.csv(filepath,
                       sep='|',
                       ignoreLeadingWhiteSpace=True,
                       ignoreTrailingWhiteSpace=True,
                       schema=schema,
                       encoding='windows-1252')

    return df
Example #3
0
 def _add_schema(self, s):
     name = s['name']
     if name not in self._schemas:
         self._schemas[name] = schema.create_schema(self, s)
     else:
         self._schemas[name].update_schema(s)
Example #4
0
import csv
import sqlite3
import schema

schema.create_schema()
conn = sqlite3.connect('tsla.db')
c = conn.cursor()
filename = "tsla.csv"
with open(filename, 'r') as csvfile:
    reader = csv.reader(csvfile)
    # emp_dict = csv.DictReader(csvfile)
    for row in reader:
        c.execute(
            "INSERT INTO stock (Date, Open, High, Low, Close, Adj_Close, Volume) VALUES (?, ?, ?, ?, ?, ?, ?)",
            row)
    for row in c.execute("SELECT * FROM stock"):
        print(row)
Example #5
0
from aiohttp import web
from routes import setup_routes
import aiosqlite as sql

import logging
logger = logging.getLogger(__name__)

import schema
import db
import config

logger.info('Configuring Schema')
conf = config.init()
sch = schema.create_schema(conf.schema)


async def initialize_app():
    logger.info('Initializing Graph')
    await db.make_graph(conf.database)

    logger.info('Initializing Attributes')
    tables = schema.schema_to_tables(conf.schema)
    await db.attributes(conf.database, tables)

    logger.info('Initializing Webapp')
    app = web.Application()
    app['conf'] = {'database': conf.database, 'types': tables.keys()}
    app['schema'] = sch
    setup_routes(app)

    return app
Example #6
0
#!/usr/bin/python

import sys
import re
import csv
import traceback
import schema

# Main caller
program_name = sys.argv[0]

if len(sys.argv) < 3:
    print "usage: " + program_name + " <debug_level : 1-4> <schema.csv> ... "
    sys.exit(1)

debug_level = int(sys.argv[1])
schema_filename = sys.argv[2]
#db_filename = sys.argv[3]

if debug_level > 1:
    print 'args :', len(sys.argv)

schema = schema.Schema()

schema.set_debug_level(debug_level)

schema.create_schema(schema_filename)
# schema.create_schema(db_filename, schema_filename)

# schema.print_phase1(out_filename)