def start(self): global __g_userSpace global __g_memDB __g_memDB = MemDB() __g_userSpace = UserSpace(__g_memDB) application = tornado.web.Application([ (RequestMapping.register_node, RegisterHandler), (RequestMapping.fetch_dashboard, DashBoardHandler), ]) application.listen( ApplicationProperties.configure( p_key='application.controller.register.server.port')) print('Application listeming...[Port:%s]' % (ApplicationProperties.configure( p_key='application.controller.register.server.port'))) workers = gen.multi([worker(idx) for idx in range(1)]) __g_userSpace.init() ioLoop = tornado.ioloop.IOLoop.current() #print(ioLoop) ioLoop.start()
async def post(self): if "action" not in self.args or self.args["action"] != "fetchip": for key, values in global_ip_dic.items(): print("%s - %s" % (key, values)) self.write({"msg": "unknown action"}) return while True: ips = list(global_ip_dic.keys()) if len(ips) < 1: self.write({"msg": "no ip"}) break #print("index %d arrived"%(int(self.args["index"]))) getint = random.randint(1, len(ips)) ip = ips[getint - 1] maxtimes = int( ApplicationProperties.configure( "application.shared.ippool.maxUseTimes")) expireOffset = int( ApplicationProperties.configure( "application.shared.ippool.expireOffset")) now = datetime.now() expire = global_ip_dic[ip][1] times = global_ip_dic[ip][2] if ((expire - now).seconds / 60 < expireOffset) or (times > maxtimes): print("ip[%s] is out of work - %s" % (ip, datetime.now())) print(global_ip_dic[ip]) global_ip_dic.pop(ip) else: global_ip_dic[ip][2] = global_ip_dic[ip][2] + 1 self.write({"ip": ip, "port": global_ip_dic[ip][0]}) break
def __init__(self): self._workdir = ApplicationProperties.configure( 'application.storage.filesystem.download.tempdir') self._contentdir = ApplicationProperties.configure( 'application.storage.filesystem.download.tempdir') + "/html" self._staticdir = ApplicationProperties.configure( 'application.storage.filesystem.download.staticdir')
def start(self): print('Application listeming...[Port:%s]' % (ApplicationProperties.configure( p_key='application.bluewhale.server.port'))) application = Application([ (RequestMapping.whale_collect_start, WhaleHandler), (RequestMapping.whale_collect_write, WhaleWriter), ]) application.listen( ApplicationProperties.configure( p_key='application.bluewhale.server.port')) IOLoop.current().start()
async def refreship(): maxtimes = int( ApplicationProperties.configure( "application.shared.ippool.maxUseTimes")) expireOffset = int( ApplicationProperties.configure( "application.shared.ippool.expireOffset")) now = datetime.now() for idx, rec in enumerate(global_ip_pool): expire = rec[2] times = rec[3] if ((expire - now) / 60 < expireOffset) or (times > maxtimes): del (global_ip_pool[idx])
async def async_fetch_https(req): url = req["url"] mid = req["materialId"] print('[%s] async_fetch_https | url[%s]' % (datetime.now().strftime('%Y-%m-%d %H:%M:%S'), url)) header = { "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) VAR/1.0.0.1" } datadir = ApplicationProperties.configure( 'application.storage.filesystem.download.tempdir') http_client = httpclient.AsyncHTTPClient() try: response = await http_client.fetch( url, method='GET', headers=header, connect_timeout=ApplicationProperties.configure( 'application.exchange.server.download.connectTimeOut'), request_timeout=ApplicationProperties.configure( 'application.exchange.server.download.requestTimeOut'), validate_cert=False) except Exception as e: print("async_fetch_https Error: %s" % e) else: selector = Selector(text=response.body) #print(response.body) metaChr = selector.xpath('//meta/@charset') charset = 'utf-8' if metaChr: charset = metaChr.get() else: metaChr = selector.xpath( "//meta[contains('http-equiv','Content-Type')]/@content") if metaChr: charsetM = re.search(r'(?<=charset=)\w+', metaChr.get().lower()) if charsetM: charset = charsetM.group(0) print('Page encoding charset is %s' % (charset)) seqno = str(mid) + datetime.now().strftime('%Y%m%d%H%M%S') req["writeTime"] = datetime.now().strftime('%Y-%m-%d %H:%M:%S') req["seqno"] = seqno req["charset"] = charset ini = open(datadir + "/" + seqno + ".ini", 'w+') ini.write(json.dumps(req) + "\n") fp = open(datadir + "/html/" + seqno, 'wb+') fp.write(response.body)
async def sjob(pid,scannerId,startSeq,endSeq): conn,cur = PgConnectionPool.getConn() try: print("job fired...%s,%s,%d,%d"%(pid,scannerId,startSeq,endSeq)) limit = int(ApplicationProperties.configure("application.storage.access.page.limit")) offset = 0 flag = limit while flag == limit: flag = 1 cur.execute('SELECT * FROM t_uc_material__%s where status = 2 and materialId >= %d and materialId < %d limit %d offset %d'%(pid,startSeq,endSeq,limit,offset)) results=cur.fetchall() print(results) for idx, r in enumerate(results): #print(r[2]) #definition = json.loads(r[2]) #print("task definition:%s"%(definition)) type = r[2]["type"] addr = r[2]["addr"] secondDomain = re.search(r"[A-Za-z0-9\-]+\.com|[A-Za-z0-9\-]+\.edu.cn|[A-Za-z0-9\-]+\.cn|[A-Za-z0-9\-]+\.com.cn|[A-Za-z0-9\-]+\.org|[A-Za-z0-9\-]+\.net|[A-Za-z0-9\-]+\.tv|[A-Za-z0-9\-]+\.vip|[A-Za-z0-9\-]+\.cc|[A-Za-z0-9\-]+\.gov.cn|[A-Za-z0-9\-]+\.gov|[A-Za-z0-9\-]+\.edu|[A-Za-z0-9\-]+\.biz|[A-Za-z0-9\-]+\.net.cn|[A-Za-z0-9\-]+\.org.cn",addr.lower()) #if type == 1 or type == 9: #todo: send to massCollect #{materialId, seqno, definition} #None #else: await inq.put({"pid":pid,"materialId":r[0],"uuid":r[1],"type":type,"url":addr,"domain":secondDomain.group(),"definition":r[2]}); flag=flag+idx offset=offset+limit except Exception as e: print(e) finally: PgConnectionPool.release(conn, cur)
async def post(self): fp = open(ApplicationProperties.configure( "application.storage.filesystem.uc.rootdir") + "/data", 'w+', encoding="utf-8") fp.write(json.dumps(WorkEnv.current_result, ensure_ascii=False)) fp.flush() fp.close() print("Write done!")
def initialize_env(): phases = WorkEnv.Environment.keys() #scheduler = TornadoScheduler() for idx, p in enumerate(phases): defi = WorkEnv.Environment[p] ini = {} ini["phase"] = idx + 1 fp = open(ApplicationProperties.configure( "application.storage.filesystem.uc.rootdir") + "/temp_data/phase" + str(ini["phase"]), 'w+', encoding="utf-8") if "pagination" in defi: bridgeq = queues.Queue() ini["pagination"] = defi["pagination"] ini["selectors"] = defi["selectors"] interim = ini["pagination"]["interim"] print("gen pworkers") pworkers = gen.multi([ pagination( ini["pagination"], defi["srcq"], bridgeq, "phase[" + str(ini["phase"]) + "]-pagination[" + str(_) + "]") for _ in range(p_concurrency) ]) #print(pworkers) print(defi["srcq"]) print(bridgeq) #scheduler.add_job(pagination,'interval', seconds=interim, args=(ini, defi["srcq"], bridgeq)) cworkers = gen.multi([ worker( Crawler(ini), bridgeq, defi["distq"], "phase[" + str(ini["phase"]) + "]-crawler[" + str(_) + "]", fp) for _ in range(c_concurrency) ]) print("gen cworkers") print(cworkers) print(bridgeq) print(defi["distq"]) else: ini["selectors"] = defi["selectors"] dq = None if "distq" in defi: dq = defi["distq"] print(defi["distq"]) cworkers = gen.multi([ worker( Crawler(ini), defi["srcq"], dq, "phase[" + str(ini["phase"]) + "]-crawler[" + str(_) + "]", fp) for _ in range(c_concurrency) ]) print("gen cworkers") print(cworkers) print(defi["srcq"])
def start(self): self._listenPort = ApplicationProperties.configure( "application.exchange.server.port") self._mode = ApplicationProperties.configure( "application.exchange.server.needLimitRequest") if self._mode == "False": self._server = Server() print("Nolimited Server starting...") elif self._mode == "True": self._timelineSche = TornadoScheduler() _scheduler.add_job(timeline, 'interval', seconds=5) self._timelineSche.start() self._server = LimitServer() print("Limited Server starting...") self._server.listen(self._listenPort) #if q: # q.put_nowait(self) IOLoop.current().start()
def start(self): self._scheduler = TornadoScheduler() self._scheduler.add_job( fetchips, 'interval', seconds=int( ApplicationProperties.configure( "application.shared.ippool.fetchInterval"))) self._scheduler.start() print('Application listeming...[Port:%s]' % (ApplicationProperties.configure( p_key='application.shared.ippool.server.port'))) application = Application([ (RequestMapping.dynamic_ip_assign, IPAssign), ]) application.listen( ApplicationProperties.configure( p_key='application.shared.ippool.server.port')) #IOLoop.current().start() IOLoop.current().run_sync(fetchips) IOLoop.current().start()
async def fetchips(): flag = int( ApplicationProperties.configure( "application.shared.ippool.refreshFlagNum")) if len(global_ip_dic) < flag: print("job fired, current ip num %d, need min num is %d" % (len(global_ip_dic), flag)) #for key,values in global_ip_dic.items(): # print ("%s - %s"%(key,values)) response = await fetch( 'http://api.wandoudl.com/api/ip?app_key=8fbe2e6d52ee4657e89580276d698abc&pack=210570&num=10&xy=1&type=2&mr=1&' ) print(response) if response: results = json.loads(response) if "data" in results: iplist = results["data"] for ip in iplist: ts = datetime.strptime(ip["expire_time"], '%Y-%m-%d %H:%M:%S') #global_ip_pool.append([ip["ip"], ip["port"], ts, 0, ]) global_ip_dic[ip["ip"]] = [ip["port"], ts, 0] print(global_ip_dic)
def load(self): #jobstores = { # 'mongo': {'type': 'mongodb'}, # 'default': SQLAlchemyJobStore(url='sqlite:///jobs.sqlite') #} executors = {'default': {'type': 'threadpool', 'max_workers': 5}} job_defaults = {'coalesce': False, 'max_instances': 3} self._massServerHost = ApplicationProperties.configure( "application.mass.server.host") self._massServerPort = ApplicationProperties.configure( "application.mass.server.port") print('Application listeming...[Port:%s]' % (ApplicationProperties.configure( p_key='application.collector.server.port'))) application = tornado.web.Application([ (RequestMapping.adjust_collect_job, JobHandler), (RequestMapping.fetch_collect_dashboard, DashBoardHandler), ]) application.listen( ApplicationProperties.configure( p_key='application.collector.server.port')) self._scheduler = TornadoScheduler() self._scheduler.add_job( asyncjob, 'interval', seconds=int( ApplicationProperties.configure( "application.collector.schedule.intervalSeconds")), args=(ApplicationProperties.configure( 'application.storage.filesystem.download.tempdir'), ApplicationProperties.configure( 'application.storage.filesystem.download.tempdir') + "/html", Collector())) #self._scheduler.configure(executors=executors, job_defaults=job_defaults) self._scheduler.start() tornado.ioloop.IOLoop.current().run_sync(self.send)
def start(self): global __g_scheduler __g_scheduler = TornadoScheduler() #ApplicationProperties.populate(p_command=self._command) scannerId = ApplicationProperties.configure("scannerId") print("Ready initialize scanner(id=%s)"%(scannerId)) conn,cur = PgConnectionPool.getConn() try: sql = 'SELECT p.pid,p.maxSeq,p.minSeq,p.splitNum,s.intervalSecs FROM t_uc_material_partition_scan s inner join t_uc_material_partition p on s.pid=p.pid where s.scannerId = %s' data = (scannerId) cur.execute(sql, data) results=cur.fetchall() print(results) record = results[0] splitNum = record[3] maxSeq = record[1] minSeq = record[2] intervalSecs = int(record[4]) pid = record[0] fragment = math.ceil((maxSeq-minSeq+1)/splitNum) sp = minSeq for i in range(splitNum): fromI = sp sp = sp + fragment print("add scanner, from %d to %d, interval %d seconds, partition id %s."%(fromI,sp,intervalSecs,pid)) #memS.insert(startSeq=fromI,endSeq=sp,intervalSecs=__g_properties.getProperty("application.scanner.schedule.intervalSeconds"),scannerId=scannerId,pid=unassiP["pid"]) __g_scheduler.add_job(sjob,'interval', seconds=intervalSecs, args=(pid,scannerId,fromI,sp)) __g_scheduler.start() #except Exception as e: # print(e) finally: PgConnectionPool.release(conn, cur) self._exchangeServerHost = ApplicationProperties.configure("application.exchange.server.host") self._exchangeServerPort = ApplicationProperties.configure("application.exchange.server.port") ''' jobstores = { 'mongo': {'type': 'mongodb'}, 'default': SQLAlchemyJobStore(url='sqlite:///jobs.sqlite') } executors = { 'default': {'type': 'threadpool', 'max_workers': 5} } job_defaults = { 'coalesce': False, 'max_instances': 1 } ''' ''' for idx, sche in enumerate(p_schedule): sid = sche["scannerId"]+"#"+str(idx) _scheduler.add_job(sjob,'interval', seconds=_properties.getProperty("application.scanner.schedule.intervalSeconds"), args=(sche["pid"],sche["tableName"],sid,sche["startSeq"],sche["endSeq"])) ''' #if q: # q.put_nowait(self) #newLoop = tornado.ioloop.IOLoop.current() #print(newLoop) tornado.ioloop.IOLoop.current().run_sync(self.send)
self._mode = None self._timelineSche = None def start(self): self._listenPort = ApplicationProperties.configure( "application.exchange.server.port") self._mode = ApplicationProperties.configure( "application.exchange.server.needLimitRequest") if self._mode == "False": self._server = Server() print("Nolimited Server starting...") elif self._mode == "True": self._timelineSche = TornadoScheduler() _scheduler.add_job(timeline, 'interval', seconds=5) self._timelineSche.start() self._server = LimitServer() print("Limited Server starting...") self._server.listen(self._listenPort) #if q: # q.put_nowait(self) IOLoop.current().start() if __name__ == '__main__': ApplicationProperties.populate( {"config": "/Users/apple/Documents/java/workspace/var-daemon"}) Exchange().start()
def run(self): urllib3.contrib.pyopenssl.inject_into_urllib3() start = time.time() fetching, fetched, dead = set(), set(), set() server = Server() server.listen( int(ApplicationProperties.configure( "application.mass.server.port"))) print( "Server listening: %s" % (ApplicationProperties.configure("application.mass.server.port"))) ''' properties = ApplicationProperties() application = web.Application([ (RequestMapping.submit_fetch_url_task, MainHandler), (RequestMapping.get_current_fetch_worker_queue_num, dashboard), ]) application.listen(properties.getProperty(p_key='server.port')) ''' async def fetch_url(task): definition = task["definition"] current_url = definition["addr"] if current_url in fetching: return print("fetching %s" % current_url) fetching.add(current_url) try: contents, urls = await get_links_from_url(definition) except Exception as e: print("Error: %s" % e) else: print("after fetching %s" % current_url) fetching.remove(current_url) print(contents, urls) if definition["level"] == 1: if len(urls) > 0: dataq[task["seqno"]]["queue"] = queues.Queue(len(urls)) dataq[task["seqno"]]["contents"] = contents dataq[task["seqno"]]["childrennum"] = len(urls) print("level 1 had done") print(dataq) else: if "queue" in dataq[task["seqno"]]: print("child content had done") print(contents) q = dataq[task["seqno"]]["queue"] q.put(contents) if q.qsize() == dataq[task["seqno"]]["childrennum"]: write(task, dataq[task["seqno"]]) if "child" in definition: for url in urls: #newurls = [urljoin(url, remove_fragment(new_url)) for new_url in get_links(task["extractLinkSelector"], s)] #for nu in newurls: #subtask = copy.deepcopy(task) subtask = {} subtask["pid"] = task["pid"] subtask["uuid"] = task["uuid"] subtask["type"] = task["type"] subtask["writeTime"] = task["writeTime"] subtask["charset"] = task["charset"] subtask["materialId"] = task["materialId"] subtask["seqno"] = task["seqno"] subtask["definition"] = {} subtask["definition"]["addr"] = url #["addr"] #subtask["definition"]["parent"] = url["parent"] subtask["definition"]["selectors"] = definition[ "child"]["selectors"] subtask["definition"]["level"] = definition["child"][ "level"] subtask["definition"]["wrap"] = definition["child"][ "wrap"] if "child" in definition["child"]: subtask["definition"]["child"] = definition[ "child"]["child"] await inboundq.put(subtask) fetched.add(current_url) #for new_url in urls: # Only follow links beneath the base URL #if new_url.startswith(base_url): # await q.put(new_url) # await inboundq .put(new_url) async def worker(): async for task in inboundq: if task is None: return try: if task["type"] == 1: definition = task["definition"] if definition["level"] == 1: seqno = task["seqno"] dataq[seqno] = {"queue": None} await fetch_url(task) except Exception as e: print("Exception: %s %s" % (e, url)) dead.add(url) finally: inboundq.task_done() #await q.put(base_url) # Start workers, then wait for the work queue to be empty. workers = gen.multi([worker() for _ in range(concurrency)]) ioloop.IOLoop.current().start()
except Exception as e: print("Exception: %s %s" % (e, url)) dead.add(url) finally: inboundq.task_done() #await q.put(base_url) # Start workers, then wait for the work queue to be empty. workers = gen.multi([worker() for _ in range(concurrency)]) ioloop.IOLoop.current().start() #io_loop = ioloop.IOLoop.current() #io_loop.run_sync(main) if __name__ == '__main__': ApplicationProperties.populate( {"config": "/Users/apple/Documents/var/workspace/var-daemon"}) PgConnectionPool.ini( host=ApplicationProperties.configure( "application.storage.postgres.connection.host"), port=ApplicationProperties.configure( "application.storage.postgres.connection.port"), user=ApplicationProperties.configure( "application.storage.postgres.connection.user"), password=ApplicationProperties.configure( "application.storage.postgres.connection.password"), database=ApplicationProperties.configure( "application.storage.postgres.connection.database")) LoadCollectors().run()
if __name__ == '__main__': props = {} if len(sys.argv) > 1: for idx in range(1, len(sys.argv)): arg = sys.argv[idx] if arg.startswith("-"): prop = arg[1:] pair = prop.split("=") props[pair[0]] = pair[1] print(props) #Main( p_command=props ) module = None if "module" in props: ApplicationProperties.populate(props) PgConnectionPool.ini( host=ApplicationProperties.configure( "application.storage.postgres.connection.host"), port=ApplicationProperties.configure( "application.storage.postgres.connection.port"), user=ApplicationProperties.configure( "application.storage.postgres.connection.user"), password=ApplicationProperties.configure( "application.storage.postgres.connection.password"), database=ApplicationProperties.configure( "application.storage.postgres.connection.database")) module = props["module"] print("Run application's module [%s]" % (module)) if module == "centricControl":