def run(task_drv, worker_pool): try: if not os.path.exists( sockets.get_socket_path( sockets.format_socket_uri(sockets.uatrains_bot_task_worker, drv=task_drv))): ctx = zmq.Context() uatrains_bot_task_worker_socket = ctx.socket(zmq.PUSH) uatrains_bot_task_worker_socket.bind( sockets.format_socket_uri(sockets.uatrains_bot_task_worker, drv=task_drv)) conn = orm.null_engine.connect() ses = orm.sescls(bind=conn) if task_drv == task_drvs.southwest: for tid in range(0, 5000): bot_task = orm.uatrains.BotTask() bot_task.data = str(tid) bot_task.drv = task_drvs.southwest ses.add(bot_task) elif task_drv == task_drvs.passengers: for tid in range(20000, 70000): bot_task = orm.uatrains.BotTask() bot_task.data = str(tid) bot_task.drv = task_drvs.passengers ses.add(bot_task) ses.commit() tasks = ses.query(orm.uatrains.BotTask).\ filter(orm.and_(orm.uatrains.BotTask.status == None, orm.uatrains.BotTask.drv == task_drv)).\ order_by(orm.desc(orm.cast(orm.uatrains.BotTask.data, orm.BigInteger))).all() manager = threading.Thread(target=sink.run, args=(len(tasks), task_drv)) manager.start() for wrk_num in range(worker_pool): thr = threading.Thread(target=worker.run, args=(task_drv, )) thr.start() for t in tasks: uatrains_bot_task_worker_socket.send_unicode(t.id) uatrains_bot_task_finish_socket = ctx.socket(zmq.SUB) uatrains_bot_task_finish_socket.connect( sockets.format_socket_uri(sockets.uatrains_bot_task_finish, drv=task_drv)) uatrains_bot_task_finish_socket.setsockopt_string( zmq.SUBSCRIBE, '') uatrains_bot_task_finish_socket.recv_unicode() ses.close() conn.close() else: bot.logger.error('uatrains bot info. Bot already started') except: bot.logger.error('uatrains bot - task ventilator error\r\n' + traceback.format_exc()) raise Exception('Task ventilator error. See log.')
def getStationSitemap(lng): now = datetime.datetime.now().strftime("%Y-%m-%d") conn = orm.q_engine.connect() ses = orm.sescls(bind=conn) ss = None try: ss = ses.query(orm.uatrains.E).filter( orm.uatrains.E.etype == etype.station).all() except: nlog.info('Uatrains error', 'Can\'t create sitemap\n' + traceback.format_exc()) sitemap = '<?xml version="1.0" encoding="UTF-8"?>' +\ '<urlset xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" ' +\ 'xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 ' +\ 'http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd" ' +\ 'xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">' if ss is not None: for s in ss: sitemap += '<url>' +\ '<loc>' + cherrypy.request.base + '/' + str(s.id) + '</loc>' +\ '<lastmod>' + now + '</lastmod>' +\ '<changefreq>daily</changefreq>' +\ '<priority>1.0</priority>' +\ '</url>' sitemap += '</urlset>' return sitemap
def getAll(): conn = orm.q_engine.connect() ses = orm.sescls(bind=conn) rawAliases = ses.query(orm.ukrainianside.Article).order_by( orm.desc(orm.ukrainianside.Article.order)).all() ses.close() conn.close() return rawAliases
def index(self): l = 'No data' try: conn = orm.q_engine.connect() ses = orm.sescls(bind=conn) user_agents = ses.query(orm.UserAgent).all() l = layout.getHome(user_agents) ses.close() conn.close() except: nlog.info('ugently - error', traceback.format_exc()) return l
def get_train_data(drv_module, tid, ua_dom_tree): ses = None conn = None try: conn = orm.null_engine.connect() ses = orm.sescls(bind=conn) e = drv_module.from_remote(ua_dom_tree, tid) if e is not None: if common.is_not_empty(e): t = common.get_t(e, ses) if t is None: if common.e_has_all_data(e): ses.add(e) ses.commit() t = e triggers.e.add_history(ses, t, orm.uatrains.htype.insert) else: bot.logger.error('Train has no all data\r\ntid: ' + str(tid) + '\r\n' +\ 'ua_title: ' + str(e.ua_title) + '\r\n' +\ 'value: ' + str(e.value) + '\r\n' +\ 'oid: ' + str(e.oid) + '\r\n') raise Exception( drv_module.name + ' driver train entity has empty fields') else: if (e.ua_title is not None and t.ua_title != e.ua_title) or \ (e.ua_period is not None and t.ua_period != e.ua_period) or \ (e.from_date is not None and t.from_date != e.from_date) or \ (e.to_date is not None and t.to_date != e.to_date): if e.ua_title is not None and t.ua_title != e.ua_title: t.ua_title = e.ua_title if e.ua_period is not None and t.ua_period != e.ua_period: t.ua_period = e.ua_period if e.from_date is not None and t.from_date != e.from_date: t.from_date = e.from_date if e.to_date is not None and t.to_date != e.to_date: t.to_date = e.to_date ses.commit() triggers.e.add_history(ses, t, orm.uatrains.htype.update) drv_module.link_to_station(ua_dom_tree, t, ses) ses.commit() ses.close() conn.close() except Exception as e: if ses is not None: ses.close() if conn is not None: conn.close() raise e
def index(self, eid=None): lng = get_lng() conn = orm.q_engine.connect() ses = orm.sescls(bind=conn) ts = [] pc = 5 try: q = None if lng == lngs.UA: q = ses.query(orm.uatrains.E).\ filter(orm.and_(orm.uatrains.E.etype == etype.train, orm.uatrains.E.ref_id == None)).\ order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.ua_title) if lng == lngs.RU: q = ses.query(orm.uatrains.E).\ filter(orm.and_(orm.uatrains.E.etype == etype.train, orm.uatrains.E.ref_id == None)).\ order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.ru_title) if lng == lngs.EN: q = ses.query(orm.uatrains.E).\ filter(orm.and_(orm.uatrains.E.etype == etype.train, orm.uatrains.E.ref_id == None)).\ order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.en_title) ts = q.limit(pc).all() except Exception: nlog.info('Uatrains error', 'Can\'t find trains\n' + traceback.format_exc()) ss = [] try: q = None if lng == lngs.UA: q = ses.query(orm.uatrains.E).filter(orm.uatrains.E.etype == etype.station).\ order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.ua_title) if lng == lngs.RU: q = ses.query(orm.uatrains.E).filter(orm.uatrains.E.etype == etype.station).\ order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.ru_title) if lng == lngs.EN: q = ses.query(orm.uatrains.E).filter(orm.uatrains.E.etype == etype.station).\ order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.en_title) ss = q.limit(pc).all() except Exception: nlog.info('Uatrains error', 'Can\'t find stations\n' + traceback.format_exc()) news = [] try: news = ses.query(orm.uatrains.New).filter(orm.uatrains.New.lng == lng).\ order_by(orm.uatrains.New.date.desc()).limit(3).all() except: nlog.info('Uatrains error', 'Can\'t find news\n' + traceback.format_exc()) ses.close() conn.close() return layout.getHome(ts, ss, news, lng)
def news(self): lng = get_lng() conn = orm.q_engine.connect() ses = orm.sescls(bind=conn) news = [] try: news = ses.query(orm.uatrains.New).filter(orm.uatrains.New.lng == lng).\ order_by(orm.uatrains.New.date.desc()).all() except: nlog.info('Uatrains error', 'Can\'t find news\n' + traceback.format_exc()) ses.close() conn.close() return layout.getNews(news, lng)
def es(self, srcht=0, ph='', fs='', ts='', pn=0, pc=9): lng = get_lng() conn = orm.q_engine.connect() ses = orm.sescls(bind=conn) pn = int(pn) pc = int(pc) es = [] has_next_p = False if int(srcht) == 0: es, has_next_p = search.from_to(ses, fs, ts, pc, pn) elif int(srcht) == 1: prepared_ph = ph.replace(' ', '%').replace('-', '%') es, has_next_p = search.full(ses, ph, pc, pn) ses.close() conn.close() return layout.getEs(es, srcht, ph, fs, ts, pn, has_next_p, lng)
def base_rnd(): if red.exists(red_keys.froxly_base_check_free_proxy) and \ red.scard(red_keys.froxly_base_check_free_proxy) > 0: return red.srandmember( red_keys.froxly_base_check_free_proxy) else: conn = orm.null_engine.connect() ses = orm.sescls(bind=conn) free_proxies = ses.query(orm.FreeProxy).filter( orm.and_(orm.FreeProxy.protocol == 'http', orm.FreeProxy.http_status == 200)).all() for free_proxy in free_proxies: sproxy = data_server_common.dbproxy2sproxy(free_proxy) red.sadd(red_keys.froxly_base_check_free_proxy, sproxy) ses.close() conn.close() return red.srandmember( red_keys.froxly_base_check_free_proxy)
def ts(self, ph='', pn=0): lng = get_lng() conn = orm.q_engine.connect() ses = orm.sescls(bind=conn) pn = int(pn) pc = 9 ts = [] has_next_p = False try: q = None if lng == lngs.UA: q = ses.query(orm.uatrains.E).\ filter(orm.and_(orm.or_(orm.uatrains.E.etype == etype.train, orm.uatrains.E.etype == etype.etrain, orm.uatrains.E.etype == etype.ptrain), orm.uatrains.E.ua_graph.ilike('%' + ph.lower().replace(' ', '%') + '%'), orm.uatrains.E.ua_title.ilike('%' + ph.lower() + '%'), orm.uatrains.E.ref_id == None)).\ order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.ua_title) if lng == lngs.RU: q = ses.query(orm.uatrains.E).\ filter(orm.and_(orm.or_(orm.uatrains.E.etype == etype.train, orm.uatrains.E.etype == etype.etrain, orm.uatrains.E.etype == etype.ptrain), orm.uatrains.E.ru_graph.ilike('%' + ph.lower().replace(' ', '%') + '%'), orm.uatrains.E.ru_title.ilike('%' + ph.lower() + '%'), orm.uatrains.E.ref_id == None)).\ order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.ru_title) if lng == lngs.EN: q = ses.query(orm.uatrains.E).\ filter(orm.and_(orm.or_(orm.uatrains.E.etype == etype.train, orm.uatrains.E.etype == etype.etrain, orm.uatrains.E.etype == etype.ptrain), orm.uatrains.E.en_graph.ilike('%' + ph.lower().replace(' ', '%') + '%'), orm.uatrains.E.en_title.ilike('%' + ph.lower() + '%'), orm.uatrains.E.ref_id == None)).\ order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.en_title) ts = q.limit(pc).offset(pn * pc).all() next_p_ts = q.limit(pc).offset((pn + 1) * pc).all() if len(next_p_ts) > 0: has_next_p = True except Exception: nlog.info('Uatrains error', 'Can\'t find trains\n' + traceback.format_exc()) ses.close() conn.close() return layout.getTrains(ts, ph, pn, has_next_p, lng)
def ss(self, ph='', pn=0): lng = get_lng() conn = orm.q_engine.connect() ses = orm.sescls(bind=conn) pn = int(pn) pc = 9 ss = [] has_next_p = False try: q = ses.query(orm.uatrains.E).filter(orm.uatrains.E.etype == etype.station).\ filter(orm.or_(orm.uatrains.E.ua_title.ilike('%' + ph.lower() + '%'), orm.uatrains.E.ru_title.ilike('%' + ph.lower() + '%'), orm.uatrains.E.en_title.ilike('%' + ph.lower() + '%'))).\ order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.ua_title) ss = q.limit(pc).offset(pn * pc).all() next_p_ss = q.limit(pc).offset((pn + 1) * pc).all() if len(next_p_ss) > 0: has_next_p = True except Exception: nlog.info('Uatrains error', 'Can\'t find stations\n' + traceback.format_exc()) ses.close() conn.close() return layout.getStations(ss, ph, pn, has_next_p, lng)
def t(self, tid): conn = orm.q_engine.connect() ses = orm.sescls(bind=conn) id = None try: ts = ses.query(orm.uatrains.E).filter( orm.and_(orm.uatrains.E.oid == int(tid), orm.uatrains.E.etype == etype.train)).all() t = ts[0] for train in ts: if t.id < train.id and t.ref_id is None: t = train id = t.id except: nlog.info('Uatrains error', 'Can\'t find train by tid = ' + str(tid) + '\n' +\ traceback.format_exc()) ses.close() conn.close() cherrypy.response.status = 301 if id is not None: cherrypy.response.headers['Location'] = '/' + str(id) else: cherrypy.response.headers['Location'] = '/' return ''
def s(self, sid): conn = orm.q_engine.connect() ses = orm.sescls(bind=conn) id = None try: ss = ses.query(orm.uatrains.E).filter( orm.and_(orm.uatrains.E.oid == int(sid), orm.uatrains.E.etype == etype.station)).all() s = ss[0] for station in ss: if s.id < station.id: s = station id = s.id except: nlog.info('Uatrains error', 'Can\'t find station by sid = ' + str(sid) + '\n' +\ traceback.format_exc()) ses.close() conn.close() cherrypy.response.status = 301 if id is not None: cherrypy.response.headers['Location'] = '/' + str(id) else: cherrypy.response.headers['Location'] = '/' return ''
import traceback import time import datetime from werp import orm, nlog, exec_log conn = None ses = None try: start_dt = datetime.datetime.now() start_time = time.time() conn = orm.null_engine.connect() ses = orm.sescls(bind=conn) trains = ses.query(orm.uatrains.E).\ filter(orm.or_(orm.uatrains.E.etype == 1, orm.uatrains.E.etype == 4, orm.uatrains.E.etype == 5)).all() for train in trains: # Clear halts of train for the same s_id halt_ids = ses.query(orm.distinct(orm.uatrains.TrainStation.s_id)).\ filter(orm.uatrains.TrainStation.t_id == train.id).all() for halt_id in halt_ids: halts = ses.query(orm.uatrains.TrainStation).\ filter(orm.and_(orm.uatrains.TrainStation.t_id == train.id, orm.uatrains.TrainStation.s_id == halt_id[0])).all() if len(halts) > 1: newest_halt = halts[0] for halt in halts: if halt.c_date > newest_halt.c_date: newest_halt = halt for halt in halts: if halt.id != newest_halt.id:
def run(task_drv): conn = None ses = None try: ctx = zmq.Context() froxly_data_server_socket = ctx.socket(zmq.REQ) froxly_data_server_socket.connect( sockets.format_socket_uri(sockets.froxly_data_server, drv=task_drv)) uatrains_bot_task_sink_socket = ctx.socket(zmq.PUSH) uatrains_bot_task_sink_socket.connect( sockets.format_socket_uri(sockets.uatrains_bot_task_sink, drv=task_drv)) uatrains_bot_task_worker_socket = ctx.socket(zmq.PULL) uatrains_bot_task_worker_socket.connect( sockets.format_socket_uri(sockets.uatrains_bot_task_worker, drv=task_drv)) uatrains_bot_task_finish_socket = ctx.socket(zmq.SUB) uatrains_bot_task_finish_socket.connect( sockets.format_socket_uri(sockets.uatrains_bot_task_finish, drv=task_drv)) uatrains_bot_task_finish_socket.setsockopt_string(zmq.SUBSCRIBE, '') poller = zmq.Poller() poller.register(uatrains_bot_task_worker_socket, zmq.POLLIN) poller.register(uatrains_bot_task_finish_socket, zmq.POLLIN) conn = orm.null_engine.connect() while True: socks = dict(poller.poll()) if uatrains_bot_task_worker_socket in socks and socks[ uatrains_bot_task_worker_socket] == zmq.POLLIN: ses = orm.sescls(bind=conn) task_id = uatrains_bot_task_worker_socket.recv_unicode() task = None try: task = ses.query(orm.uatrains.BotTask).filter( orm.uatrains.BotTask.id == task_id).one() except: bot.logger.error('uatrains bot - task worker error\r\n' + traceback.format_exc()) if task is not None: task.status = task_status.running ses.commit() task.http_status = 0 try_count = 0 ua_dom_tree = None current_drv = None parser = etree.HTMLParser() if task.drv == task_drvs.southwest: current_drv = drv.southwest elif task.drv == task_drvs.passengers: current_drv = drv.passengers while task.http_status <= 0 and try_count < TRY_COUNT: ua_res = None if ua_dom_tree is None: ua_url = current_drv.ua_url.replace( '(tid)', str(task.data)) ua_req = { 'method': 'request', 'params': { 'url': ua_url, 'charset': current_drv.charset, 'timeout': REQ_TIMEOUT } } froxly_data_server_socket.send_unicode( json.dumps(ua_req)) ua_res = json.loads( froxly_data_server_socket.recv_unicode()) if 'http_status' in ua_res['result'] and 'data' in ua_res['result'] and \ ua_res['result']['http_status'] == 200: try: ua_dom_tree = etree.parse( io.StringIO(ua_res['result']['data']), parser) drv.check_dom_tree(current_drv, ua_dom_tree) except Exception as e: ua_dom_tree = None ua_res['result']['http_status'] = -3 ua_res['result'][ 'http_status_reason'] = str(e) if ua_res is not None and 'http_status' in ua_res[ 'result'] and ua_res['result'][ 'http_status'] < 0: task.http_status = ua_res['result']['http_status'] task.http_status_reason = ua_res['result'][ 'http_status_reason'] try: if ua_dom_tree is not None: drv.get_train_data(current_drv, int(task.data), ua_dom_tree) task.http_status = 200 task.http_status_reason = None except Exception as e: task.http_status = -2 task.http_status_reason = str(e) bot.logger.error('uatrains bot - task worker error\r\n' + 'Task data: ' + \ str(task.data) + '\n\n' + traceback.format_exc()) try_count += 1 task.status = task_status.completed ses.commit() ses.close() uatrains_bot_task_sink_socket.send_unicode(str(1)) if uatrains_bot_task_finish_socket in socks and socks[ uatrains_bot_task_finish_socket] == zmq.POLLIN: break conn.close() except: bot.logger.error('uatrains bot - task worker error\r\n' + traceback.format_exc()) if ses is not None: ses.close() if conn is not None: conn.close()
def default(self, eid=None, *a, **kw): lng = get_lng() l = '' ref_id = None if eid is not None: prepared_eid = None try: prepared_eid = int(float(eid)) except: nlog.info('Uatrains error', 'Can\'t parse eid = ' + str(prepared_eid) + '\n' +\ traceback.format_exc()) if prepared_eid is not None: conn = orm.q_engine.connect() ses = orm.sescls(bind=conn) e = None try: e = ses.query(orm.uatrains.E).filter( orm.uatrains.E.id == prepared_eid).one() if e.ref_id is not None: ref_id = e.ref_id else: e.vc = e.vc + 1 ses.commit() except: nlog.info('Uatrains error', 'Can\'t find entity by eid = ' + str(prepared_eid) + '\n' +\ traceback.format_exc()) if e is not None and ref_id is None: if e.etype == etype.train: try: t = ses.query(orm.uatrains.E).\ options(orm.joinedload_all(orm.uatrains.E.t_ss, orm.uatrains.TrainStation.s)).\ filter(orm.uatrains.E.id == prepared_eid).one() l = layout.getTrain(t, t.t_ss, lng) except: nlog.info('Uatrains error', 'Can\'t find train by id = ' + str(prepared_eid) + '\n' +\ traceback.format_exc()) elif e.etype == etype.ptrain: try: t = ses.query(orm.uatrains.E).\ options(orm.joinedload_all(orm.uatrains.E.t_ss, orm.uatrains.TrainStation.s)).\ filter(orm.uatrains.E.id == prepared_eid).one() l = layout.getPTrain(t, t.t_ss, lng) except: nlog.info('Uatrains error', 'Can\'t find ptrain by id = ' + str(prepared_eid) + '\n' +\ traceback.format_exc()) elif e.etype == etype.station: try: s = ses.query(orm.uatrains.E).\ options(orm.joinedload_all(orm.uatrains.E.s_ts, orm.uatrains.TrainStation.t)).\ filter(orm.uatrains.E.id == prepared_eid).one() l = layout.getStation(s, s.s_ts, lng) except: nlog.info('Uatrains error', 'Can\'t find station by id = ' + str(prepared_eid) + '\n' +\ traceback.format_exc()) ses.close() conn.close() else: l = self.index(eid) if l == '': cherrypy.response.status = 301 if ref_id is not None: cherrypy.response.headers['Location'] = '/' + str(ref_id) else: cherrypy.response.headers['Location'] = '/' return l