conf=XmlRpcConfig('taringa','test','D:/DEV_PY/PSPLoader/config') conf.load() #p=PostProcessor(conf) #p.process('D:/DEV_PY/PSPLoader/processed.csv') #user=UserProcessor(conf) #user.process(conf.usersPerBatch) #print user.usersPool #print user.getUser() #conf.queuePath='D:/DEV_PY/PSPLoader/' #conf.csvDelimiter=';' #def myCallback(file): # print 'i received a file named %s' % file daemon=PostsProcessorDaemon('192.168.1.44',8000,conf) daemon.start() print 'daemon started' print 'thread ID:%s' % daemon.getName() DirectoryMonitor('D:/DEV_PY/PSPLoader',myCallback).monitor()
#create the file queue fqueue=Queue.Queue(100) def pool_builder(files): for file in files: fqueue.put(file, True) #load the configuration print 'Loading configuration...' conf=XmlRpcConfig('taringa','posts','D:/DEV_PY/PSPCrawler/config') conf.load() #create the processors threads print 'Starting processor threads...' for x in xrange(2): PostProcessor(conf, fqueue).start() #starts the xmlrpc server print 'Starting daemon...' daemon=PostsProcessorDaemon('192.168.1.44',8000,conf) daemon.start() #start the directory monitor print 'Starting monitor...' DirectoryMonitor(os.path.dirname(conf.queuePath),pool_builder).monitor()