def consumer_queue(proc_id, queue):
loader = Loader(conf)
while True:
try:
consumer_data = queue.get(proc_id, 1)
if consumer_data == 'STOP':
queue.put('STOP')
break
for data in consumer_data:
v = Validation()
v.create(data)
loader.load(v)
except Empty:
pass
loader.close()
import json
import glob
import os
from mpp.loaders import Loader
from mpp.models import Response, Identity
from sqlalchemy.exc import IntegrityError
files = glob.glob(
'/home/ubuntu/semantics_pipeline/pipeline_tests/identified/*.json')
with open('local_rds.conf', 'r') as f:
conf = json.loads(f.read())
loader = Loader(conf)
for i, f in enumerate(files[100:]):
with open(f, 'r') as g:
data = json.loads(g.read())
fname = f.split('/')[-1].split('_')[0]
fmt = data.get('response_datatype', 'unknown')
identity = data.get('identity', [])
try:
r = Response()
r.create(data)
r_id = loader.load(r)
except Exception as ex:
print fname, ex
continue
