def processCollection(): collection = helper.mongoConnect() pages = bhl_process_tasks.getPagesForProcessing(collection) while (pages is not None): bhl_process_tasks.processPages(pages, collection) pages.close() pages = bhl_process_tasks.getPagesForProcessing(collection)
def processCollection(): collection = helper.mongoConnect() pages = getPagesForProcessing(collection) while (pages is not None): processPages(pages, collection) pages.close() pages = getPagesForProcessing(collection)
def processCollection(): collection = helper.mongoConnect() pages = getPagesForProcessing(collection) while (pages is not None): processPages(pages, collection) pages.close() #pages = None if os.path.exists("./stop.txt"): print("Stop file found. Exiting.\n") break pages = getPagesForProcessing(collection)
import subprocess import time from helper import log from helper import mongoConnect from helper import base_path from ftplib import FTP server_name = 'IMA' ftp_host = '' ftp_username = '' ftp_password = '' ftp_path = '/incoming/artoflife' mongo_host = 'localhost:27017' if __name__ == '__main__': collection = mongoConnect() export_date = time.strftime('%Y%m%d') log.debug("updating records that need exported with today's date") collection.update( {'abbyy_complete':True, 'contrast_complete':True, 'processing_error':False, 'processing_lock_end':{'$gt': 0}, 'exported':{'$exists':False}}, {'$set': {'exported': True, 'export_date': export_date}}, multi=True ) file_base_path = '%s/export' % (base_path) if not os.path.exists(file_base_path): os.mkdir(file_base_path) output_filename = 'export_%s_%s.json' % (server_name, export_date)
import helper from pprint import pprint if __name__ == '__main__': collection = helper.mongoConnect() pages = collection.find({}); abbyyTotalPages = 0 abbyyTotalTime = 0 contrastTotalPages = 0 contrastTotalTime = 0 compressionTotalPages = 0 compressionTotalTime = 0 for page in pages: accumulatedTime = 0 if "abbyy_processing_duration" in page: abbyyTotalTime = abbyyTotalTime + page['abbyy_processing_duration'] accumulatedTime = page['abbyy_processing_duration'] abbyyTotalPages = abbyyTotalPages + 1 if "compression_processing_duration" in page: accumulatedTime = page['compression_processing_duration'] compressionTotalTime = compressionTotalTime + page['compression_processing_duration'] compressionTotalPages = compressionTotalPages + 1 if "contrast_processing_duration" in page: contrastTotalTime = contrastTotalTime + page['contrast_processing_duration'] - accumulatedTime contrastTotalPages = contrastTotalPages + 1 abbyyAverage = abbyyTotalTime / abbyyTotalPages