def process_all_metadata():
counter = 0
model = 'BMW' # Audi
category_dir = paths.N_SCOUT_META_DIR
if input('Do you want to re-create database? yes - no') == 'yes':
connection.drop_and_create()
load_basic_tables()
session = connection.open_session()
print('Lets get started..')
for category in os.listdir(category_dir):
if model is not None and category != model:
continue
model_dir = category_dir + category
for model in os.listdir(model_dir):
make_dir = model_dir + '/' + model
for car in os.listdir(make_dir):
file_name = make_dir + '/' + car
# if counter >= 2000:
# break
try:
read_file(session, file_name)
counter += 1
except Exception as e:
print("Car not processed %s" % e)
session.rollback()
session.close()
print("Session closed")
session = connection.open_session()
session.close()
def load_basic_tables():
from dao.base import connection
from dao.models.base_tables import Make, Model
session = connection.open_session()
model_data_filename = paths.ROOT_DIR + '/model_backup.csv'
make_data_filename = paths.ROOT_DIR + '/make_backup.csv'
with open(make_data_filename) as f:
reader = csv.DictReader(f)
for row in reader:
m = Make(row['make'])
m.set_make(row['make_id'], row['make'])
session.add(m)
session.commit()
print("Make table data loaded")
with open(model_data_filename) as f:
reader = csv.DictReader(f)
for row in reader:
m = Model(row['model'])
m.set_model(row['model_id'], row['model'], row['make_id'])
session.add(m)
session.commit()
print("Model table data loaded")
session.close()
def query_for_property_value(property_name, property_value):
s = connection.open_session()
results = s.query(ScoutCar).filter(property_name + "= '" +
str(property_value) +
"'").all() # this can be very big
s.close()
return results
def makemodel_lookup_for_id(make_v, model_v):
s = connection.open_session()
model_id, make_id = s.query(Make.make_id, Model.model_id)\
.filter(Make.make_id == Model.make_id)\
.filter(Make.name == make_v)\
.filter(Model.name == model_v).one()
s.close()
return make_id, model_id
def query_for_properties_values(property_dict):
s = connection.open_session()
q = s.query(ScoutCar)
cond = and_(*[
p['name'] + " " + p['operation'] + " {}".format(p['value'])
for p in property_dict
])
q = q.filter(cond)
results = q.all()
s.close()
return results
def execute_sql(sql):
sql = text(sql)
s = connection.open_session()
result = s.execute(sql).fetchall()
s.close()
return result
def lookup_property_distinct_values(property_name):
s = connection.open_session()
o = getattr(ScoutCar, property_name)
values = s.query(o).distinct().all()
s.close()
return values
def lookup_for_id(id):
s = connection.open_session()
r = s.query(ScoutCar).get(id)
s.close()
return r