def processCollection():
collection = helper.mongoConnect()
pages = bhl_process_tasks.getPagesForProcessing(collection)
while (pages is not None):
bhl_process_tasks.processPages(pages, collection)
pages.close()
pages = bhl_process_tasks.getPagesForProcessing(collection)
def processCollection():
collection = helper.mongoConnect()
pages = getPagesForProcessing(collection)
while (pages is not None):
processPages(pages, collection)
pages.close()
pages = getPagesForProcessing(collection)
def processCollection():
collection = helper.mongoConnect()
pages = getPagesForProcessing(collection)
while (pages is not None):
processPages(pages, collection)
pages.close()
#pages = None
if os.path.exists("./stop.txt"):
print("Stop file found. Exiting.\n")
break
pages = getPagesForProcessing(collection)
def processCollection():
collection = helper.mongoConnect()
pages = getPagesForProcessing(collection)
while (pages is not None):
processPages(pages, collection)
pages.close()
#pages = None
if os.path.exists("./stop.txt"):
print("Stop file found. Exiting.\n")
break
pages = getPagesForProcessing(collection)
import subprocess
import time
from helper import log
from helper import mongoConnect
from helper import base_path
from ftplib import FTP
server_name = 'IMA'
ftp_host = ''
ftp_username = ''
ftp_password = ''
ftp_path = '/incoming/artoflife'
mongo_host = 'localhost:27017'
if __name__ == '__main__':
collection = mongoConnect()
export_date = time.strftime('%Y%m%d')
log.debug("updating records that need exported with today's date")
collection.update(
{'abbyy_complete':True, 'contrast_complete':True, 'processing_error':False, 'processing_lock_end':{'$gt': 0}, 'exported':{'$exists':False}},
{'$set': {'exported': True, 'export_date': export_date}},
multi=True
)
file_base_path = '%s/export' % (base_path)
if not os.path.exists(file_base_path):
os.mkdir(file_base_path)
output_filename = 'export_%s_%s.json' % (server_name, export_date)
import helper
from pprint import pprint
if __name__ == '__main__':
collection = helper.mongoConnect()
pages = collection.find({});
abbyyTotalPages = 0
abbyyTotalTime = 0
contrastTotalPages = 0
contrastTotalTime = 0
compressionTotalPages = 0
compressionTotalTime = 0
for page in pages:
accumulatedTime = 0
if "abbyy_processing_duration" in page:
abbyyTotalTime = abbyyTotalTime + page['abbyy_processing_duration']
accumulatedTime = page['abbyy_processing_duration']
abbyyTotalPages = abbyyTotalPages + 1
if "compression_processing_duration" in page:
accumulatedTime = page['compression_processing_duration']
compressionTotalTime = compressionTotalTime + page['compression_processing_duration']
compressionTotalPages = compressionTotalPages + 1
if "contrast_processing_duration" in page:
contrastTotalTime = contrastTotalTime + page['contrast_processing_duration'] - accumulatedTime
contrastTotalPages = contrastTotalPages + 1
abbyyAverage = abbyyTotalTime / abbyyTotalPages
