def consumer_queue(proc_id, queue):
loader = Loader(conf)
while True:
try:
consumer_data = queue.get(proc_id, 1)
if consumer_data == 'STOP':
queue.put('STOP')
break
for data in consumer_data:
v = Validation()
v.create(data)
loader.load(v)
except Empty:
pass
loader.close()
conf = json.loads(f.read())
reader = ResponseReader(conf)
loader = Loader(conf)
# get the set, validate, store outputs
# but need to paginate because of ram issues
for i in xrange(0, 668110, 25):
print 'QUERYING {0}:{1}'.format(i, 25)
for response in reader.read('', limit=25, offset=i):
print response.source_url
xml = response.cleaned_content
stderr = validate_in_memory(xml)
data = {
"response_id": response.id,
"valid": 'Error at' not in stderr,
"validated_on": datetime.now()
}
if stderr:
data.update({"errors": [s.strip() for s in stderr.split('\n\n')]})
print '\t{0}'.format(stderr[:100])
try:
v = Validation()
v.create(data)
loader.load(v)
except Exception as ex:
print ex
