def runcommand(options, parser): logger = get_logger('RunCommand') if not options.command or not options.node: return parser.print_help() #ns = Pyro.naming.NameServerLocator().getNS() #uri = ns.resolve('%s.%s.jobservice' % (domain, options.node)) #js = Pyro.core.getProxyForURI(uri) js = Pyro.core.getProxyForURI("PYROLOC://localhost:7766/jobservice") if options.command == 'create': if not options.profile: return logger.error( 'crate should be called with profile name: -n nodename -c create -p profile' ) sess = Session() try: pf = sess.query(Profile).filter( profile.c.name == options.profile).first() status, workername = js.create(pf.configuration) except Exception, e: logger.error('No profile found with name: %s' % options.profile) logger.error("".join(getPyroTraceback(e))) return -1 if status: j = Job(workername, 'paused') pf.jobs.append(j) #sess.save(j) sess.add(j) sess.commit() sess.close() logger.info("worker with name %s created" % workername) return 0 logger.error('no worker created: %s' % workername) return -1
def runcommand(options, parser): logger = get_logger('RunCommand') if not options.command or not options.node: return parser.print_help() #ns = Pyro.naming.NameServerLocator().getNS() #uri = ns.resolve('%s.%s.jobservice' % (domain, options.node)) #js = Pyro.core.getProxyForURI(uri) js = Pyro.core.getProxyForURI("PYROLOC://localhost:7766/jobservice") if options.command == 'create': if not options.profile: return logger.error('crate should be called with profile name: -n nodename -c create -p profile') sess = Session() try: pf = sess.query(Profile).filter(profile.c.name==options.profile).first() status, workername = js.create(pf.configuration) except Exception, e: logger.error( 'No profile found with name: %s' % options.profile) logger.error("".join(getPyroTraceback(e))) return -1 if status: j = Job(workername, 'paused') pf.jobs.append(j) #sess.save(j) sess.add(j) sess.commit() sess.close() logger.info ("worker with name %s created" % workername) return 0 logger.error('no worker created: %s' % workername) return -1
def indexer(filename, max_count=100): logger = get_logger('Indexer') count = offset = 0 sess = Session() d = DArcReader('%s' % filename) logger.debug('Archive file %s opened with pointer %s' % (filename, d)) timestamp = filename.split('_')[1] # always profilename_timestamp_node.arc for chunk in d.chunks(): # url : offset count += 1 index = Index(filename, timestamp, chunk[0], offset) #sess.save(index) sess.add(index) if count % max_count == 0: sess.commit() sess = Session() offset = d.tell() sess.commit() d.close()
def indexer(filename, max_count=100): logger = get_logger('Indexer') count = offset = 0 sess = Session() d = DArcReader('%s' % filename) logger.debug('Archive file %s opened with pointer %s' % (filename, d)) timestamp = filename.split('_')[1] # always profilename_timestamp_node.arc for chunk in d.chunks(): # url : offset count += 1 index = Index(filename, timestamp, chunk[0], offset) #sess.save(index) sess.add(index) if count % max_count == 0: sess.commit() sess = Session() offset = d.tell() sess.commit() d.close()
try: status, retval = getattr(js, options.command)(options.worker) except Exception, e: logger.error("".join(getPyroTraceback(e))) d = {'kill':'killed', 'pause':'paused', 'resume':'running'} if options.command in d: s = Session() j = s.query(Job).filter(job.c.name==options.worker).one() j.status = d.get(options.command) if options.command == 'kill': j.end_date = datetime.now() s.commit() s.close() else: logger.info(retval) if __name__ == '__main__': parser = OptionParser() parser.add_option('--nlist', action='callback', callback=listnodes, help="List active nodes") parser.add_option('--plist', action='callback', callback=listprofiles, help="List profiles") parser.add_option('-c', '--command', choices=('create', 'pause', 'resume', 'report', 'status', 'list', 'kill', 'listjobs', 'nodeinfo'), help="Crawler command. \n\ create: create a remote worker \n\ pause: pause a running worker. \n\ resume: resume a paused worker. \n\ report: get report from the specified worker. \n\
return -1 try: status, retval = getattr(js, options.command)(options.worker) except Exception, e: logger.error("".join(getPyroTraceback(e))) d = {'kill': 'killed', 'pause': 'paused', 'resume': 'running'} if options.command in d: s = Session() j = s.query(Job).filter(job.c.name == options.worker).one() j.status = d.get(options.command) if options.command == 'kill': j.end_date = datetime.now() s.commit() s.close() else: logger.info(retval) if __name__ == '__main__': parser = OptionParser() parser.add_option('--nlist', action='callback', callback=listnodes, help="List active nodes") parser.add_option('--plist', action='callback', callback=listprofiles, help="List profiles")