def update_schema(self, s):
name = s['name']
if name in self._schemas:
s = self.client.schemas.update(name, s)
self._schemas[name].update_schema(s)
else:
s = self.client.schemas.create(s)
self._schemas[name] = schema.create_schema(self, s)
def read_zillow(sqlc, basepath, database, table, **kwargs):
'''
read in zillow table as spark dataframes. Assumes that the
`basepath` directory contains the Layout.xlsx file, as well
as the databases (ZTrans and ZAsmt directories).
'''
layout = os.path.join(basepath, 'Layout.xlsx')
filepath = os.path.join(basepath, database, table + '.txt')
schema = create_schema(layout, database=database, table=table)
df = sqlc.read.csv(filepath,
sep='|',
ignoreLeadingWhiteSpace=True,
ignoreTrailingWhiteSpace=True,
schema=schema,
encoding='windows-1252')
return df
def _add_schema(self, s):
name = s['name']
if name not in self._schemas:
self._schemas[name] = schema.create_schema(self, s)
else:
self._schemas[name].update_schema(s)
import csv
import sqlite3
import schema
schema.create_schema()
conn = sqlite3.connect('tsla.db')
c = conn.cursor()
filename = "tsla.csv"
with open(filename, 'r') as csvfile:
reader = csv.reader(csvfile)
# emp_dict = csv.DictReader(csvfile)
for row in reader:
c.execute(
"INSERT INTO stock (Date, Open, High, Low, Close, Adj_Close, Volume) VALUES (?, ?, ?, ?, ?, ?, ?)",
row)
for row in c.execute("SELECT * FROM stock"):
print(row)
from aiohttp import web
from routes import setup_routes
import aiosqlite as sql
import logging
logger = logging.getLogger(__name__)
import schema
import db
import config
logger.info('Configuring Schema')
conf = config.init()
sch = schema.create_schema(conf.schema)
async def initialize_app():
logger.info('Initializing Graph')
await db.make_graph(conf.database)
logger.info('Initializing Attributes')
tables = schema.schema_to_tables(conf.schema)
await db.attributes(conf.database, tables)
logger.info('Initializing Webapp')
app = web.Application()
app['conf'] = {'database': conf.database, 'types': tables.keys()}
app['schema'] = sch
setup_routes(app)
return app
#!/usr/bin/python
import sys
import re
import csv
import traceback
import schema
# Main caller
program_name = sys.argv[0]
if len(sys.argv) < 3:
print "usage: " + program_name + " <debug_level : 1-4> <schema.csv> ... "
sys.exit(1)
debug_level = int(sys.argv[1])
schema_filename = sys.argv[2]
#db_filename = sys.argv[3]
if debug_level > 1:
print 'args :', len(sys.argv)
schema = schema.Schema()
schema.set_debug_level(debug_level)
schema.create_schema(schema_filename)
# schema.create_schema(db_filename, schema_filename)
# schema.print_phase1(out_filename)