def process(self, pkgname, method, url, id, branch): wp = WebParse(URL,PORT) errorMsg='' if method=='httpls': upstream=HTTPLS(url, pkgname, branch) (ver,loc,error) = upstream.process() print pkgname, ver, loc if method=='subdirhttpls': upstream=SubdirHTTPLS(url, pkgname, branch) (ver,loc,error) = upstream.process() print pkgname, ver, loc if method=='dualhttpls': upstream=DualHTTPLS(url, pkgname, branch) (ver,loc,error) = upstream.process() print pkgname, ver, loc if method=='lp': upstream=Launchpad(url, pkgname, branch) (ver,loc,error) = upstream.process() print pkgname, ver, loc if method=='svnls': upstream=SVNLS(url, pkgname, branch) (ver,loc,error) = upstream.process() print pkgname, ver, loc if method=='google': upstream=Google(url, pkgname, branch) (ver,loc,error) = upstream.process() print pkgname, ver, loc if method=='ftpls': upstream=FTPLS(url, pkgname, branch) (ver,loc,error) = upstream.process() print pkgname, ver, loc if method=='trac': upstream=Trac(url, pkgname, branch) (ver,loc,error) = upstream.process() print pkgname, ver, loc if method=='sf': upstream=SF(url, pkgname, branch) (ver,loc,error) = upstream.process() print pkgname, ver, loc if method=='custom': custom=Custom(url, pkgname, branch) (ver,loc,error) = custom.process() print pkgname, ver, loc if error==None: error=False errorMsg='None' else: errorMsg=error error=True wp.updateRecord('error', str(error).lower(), id) wp.updateRecord('errorMessage', errorMsg, id) wp.updateRecord('processed', 'true', id) wp.updateRecord('latest_ver', ver, id) wp.updateRecord('loc', loc, id)
THREAD_LIMIT = 2 QUEUE_LIMIT = 50 URL = 'http://localhost' PORT = '3000' THREAD_WAIT = 5 HOURS_LIMIT=1 # On an average, no of records processed is : 1 for every THREAD_WAIT seconds jobs = Queue.Queue(QUEUE_LIMIT) singlelock = threading.Lock() inputlist_ori = [] wp = WebParse(URL, PORT) records=wp.getRecords() if records==None: print 'No records found!' sys.exit(1) for record in records: pkgname=record['pkgname'] method=record['method'] url=record['url'] id=record['id'] processed=record['processed'] branch=record['branch'] updated=record['updated-at']