def run(task_drv, worker_pool):
try:
if not os.path.exists(
sockets.get_socket_path(
sockets.format_socket_uri(sockets.uatrains_bot_task_worker,
drv=task_drv))):
ctx = zmq.Context()
uatrains_bot_task_worker_socket = ctx.socket(zmq.PUSH)
uatrains_bot_task_worker_socket.bind(
sockets.format_socket_uri(sockets.uatrains_bot_task_worker,
drv=task_drv))
conn = orm.null_engine.connect()
ses = orm.sescls(bind=conn)
if task_drv == task_drvs.southwest:
for tid in range(0, 5000):
bot_task = orm.uatrains.BotTask()
bot_task.data = str(tid)
bot_task.drv = task_drvs.southwest
ses.add(bot_task)
elif task_drv == task_drvs.passengers:
for tid in range(20000, 70000):
bot_task = orm.uatrains.BotTask()
bot_task.data = str(tid)
bot_task.drv = task_drvs.passengers
ses.add(bot_task)
ses.commit()
tasks = ses.query(orm.uatrains.BotTask).\
filter(orm.and_(orm.uatrains.BotTask.status == None, orm.uatrains.BotTask.drv == task_drv)).\
order_by(orm.desc(orm.cast(orm.uatrains.BotTask.data, orm.BigInteger))).all()
manager = threading.Thread(target=sink.run,
args=(len(tasks), task_drv))
manager.start()
for wrk_num in range(worker_pool):
thr = threading.Thread(target=worker.run, args=(task_drv, ))
thr.start()
for t in tasks:
uatrains_bot_task_worker_socket.send_unicode(t.id)
uatrains_bot_task_finish_socket = ctx.socket(zmq.SUB)
uatrains_bot_task_finish_socket.connect(
sockets.format_socket_uri(sockets.uatrains_bot_task_finish,
drv=task_drv))
uatrains_bot_task_finish_socket.setsockopt_string(
zmq.SUBSCRIBE, '')
uatrains_bot_task_finish_socket.recv_unicode()
ses.close()
conn.close()
else:
bot.logger.error('uatrains bot info. Bot already started')
except:
bot.logger.error('uatrains bot - task ventilator error\r\n' +
traceback.format_exc())
raise Exception('Task ventilator error. See log.')
def getStationSitemap(lng):
now = datetime.datetime.now().strftime("%Y-%m-%d")
conn = orm.q_engine.connect()
ses = orm.sescls(bind=conn)
ss = None
try:
ss = ses.query(orm.uatrains.E).filter(
orm.uatrains.E.etype == etype.station).all()
except:
nlog.info('Uatrains error',
'Can\'t create sitemap\n' + traceback.format_exc())
sitemap = '<?xml version="1.0" encoding="UTF-8"?>' +\
'<urlset xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" ' +\
'xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 ' +\
'http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd" ' +\
'xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
if ss is not None:
for s in ss:
sitemap += '<url>' +\
'<loc>' + cherrypy.request.base + '/' + str(s.id) + '</loc>' +\
'<lastmod>' + now + '</lastmod>' +\
'<changefreq>daily</changefreq>' +\
'<priority>1.0</priority>' +\
'</url>'
sitemap += '</urlset>'
return sitemap
def getAll():
conn = orm.q_engine.connect()
ses = orm.sescls(bind=conn)
rawAliases = ses.query(orm.ukrainianside.Article).order_by(
orm.desc(orm.ukrainianside.Article.order)).all()
ses.close()
conn.close()
return rawAliases
def index(self):
l = 'No data'
try:
conn = orm.q_engine.connect()
ses = orm.sescls(bind=conn)
user_agents = ses.query(orm.UserAgent).all()
l = layout.getHome(user_agents)
ses.close()
conn.close()
except:
nlog.info('ugently - error', traceback.format_exc())
return l
def get_train_data(drv_module, tid, ua_dom_tree):
ses = None
conn = None
try:
conn = orm.null_engine.connect()
ses = orm.sescls(bind=conn)
e = drv_module.from_remote(ua_dom_tree, tid)
if e is not None:
if common.is_not_empty(e):
t = common.get_t(e, ses)
if t is None:
if common.e_has_all_data(e):
ses.add(e)
ses.commit()
t = e
triggers.e.add_history(ses, t,
orm.uatrains.htype.insert)
else:
bot.logger.error('Train has no all data\r\ntid: ' + str(tid) + '\r\n' +\
'ua_title: ' + str(e.ua_title) + '\r\n' +\
'value: ' + str(e.value) + '\r\n' +\
'oid: ' + str(e.oid) + '\r\n')
raise Exception(
drv_module.name +
' driver train entity has empty fields')
else:
if (e.ua_title is not None and t.ua_title != e.ua_title) or \
(e.ua_period is not None and t.ua_period != e.ua_period) or \
(e.from_date is not None and t.from_date != e.from_date) or \
(e.to_date is not None and t.to_date != e.to_date):
if e.ua_title is not None and t.ua_title != e.ua_title:
t.ua_title = e.ua_title
if e.ua_period is not None and t.ua_period != e.ua_period:
t.ua_period = e.ua_period
if e.from_date is not None and t.from_date != e.from_date:
t.from_date = e.from_date
if e.to_date is not None and t.to_date != e.to_date:
t.to_date = e.to_date
ses.commit()
triggers.e.add_history(ses, t,
orm.uatrains.htype.update)
drv_module.link_to_station(ua_dom_tree, t, ses)
ses.commit()
ses.close()
conn.close()
except Exception as e:
if ses is not None:
ses.close()
if conn is not None:
conn.close()
raise e
def index(self, eid=None):
lng = get_lng()
conn = orm.q_engine.connect()
ses = orm.sescls(bind=conn)
ts = []
pc = 5
try:
q = None
if lng == lngs.UA:
q = ses.query(orm.uatrains.E).\
filter(orm.and_(orm.uatrains.E.etype == etype.train, orm.uatrains.E.ref_id == None)).\
order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.ua_title)
if lng == lngs.RU:
q = ses.query(orm.uatrains.E).\
filter(orm.and_(orm.uatrains.E.etype == etype.train, orm.uatrains.E.ref_id == None)).\
order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.ru_title)
if lng == lngs.EN:
q = ses.query(orm.uatrains.E).\
filter(orm.and_(orm.uatrains.E.etype == etype.train, orm.uatrains.E.ref_id == None)).\
order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.en_title)
ts = q.limit(pc).all()
except Exception:
nlog.info('Uatrains error',
'Can\'t find trains\n' + traceback.format_exc())
ss = []
try:
q = None
if lng == lngs.UA:
q = ses.query(orm.uatrains.E).filter(orm.uatrains.E.etype == etype.station).\
order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.ua_title)
if lng == lngs.RU:
q = ses.query(orm.uatrains.E).filter(orm.uatrains.E.etype == etype.station).\
order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.ru_title)
if lng == lngs.EN:
q = ses.query(orm.uatrains.E).filter(orm.uatrains.E.etype == etype.station).\
order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.en_title)
ss = q.limit(pc).all()
except Exception:
nlog.info('Uatrains error',
'Can\'t find stations\n' + traceback.format_exc())
news = []
try:
news = ses.query(orm.uatrains.New).filter(orm.uatrains.New.lng == lng).\
order_by(orm.uatrains.New.date.desc()).limit(3).all()
except:
nlog.info('Uatrains error',
'Can\'t find news\n' + traceback.format_exc())
ses.close()
conn.close()
return layout.getHome(ts, ss, news, lng)
def news(self):
lng = get_lng()
conn = orm.q_engine.connect()
ses = orm.sescls(bind=conn)
news = []
try:
news = ses.query(orm.uatrains.New).filter(orm.uatrains.New.lng == lng).\
order_by(orm.uatrains.New.date.desc()).all()
except:
nlog.info('Uatrains error',
'Can\'t find news\n' + traceback.format_exc())
ses.close()
conn.close()
return layout.getNews(news, lng)
def es(self, srcht=0, ph='', fs='', ts='', pn=0, pc=9):
lng = get_lng()
conn = orm.q_engine.connect()
ses = orm.sescls(bind=conn)
pn = int(pn)
pc = int(pc)
es = []
has_next_p = False
if int(srcht) == 0:
es, has_next_p = search.from_to(ses, fs, ts, pc, pn)
elif int(srcht) == 1:
prepared_ph = ph.replace(' ', '%').replace('-', '%')
es, has_next_p = search.full(ses, ph, pc, pn)
ses.close()
conn.close()
return layout.getEs(es, srcht, ph, fs, ts, pn, has_next_p, lng)
def base_rnd():
if red.exists(red_keys.froxly_base_check_free_proxy) and \
red.scard(red_keys.froxly_base_check_free_proxy) > 0:
return red.srandmember(
red_keys.froxly_base_check_free_proxy)
else:
conn = orm.null_engine.connect()
ses = orm.sescls(bind=conn)
free_proxies = ses.query(orm.FreeProxy).filter(
orm.and_(orm.FreeProxy.protocol == 'http',
orm.FreeProxy.http_status == 200)).all()
for free_proxy in free_proxies:
sproxy = data_server_common.dbproxy2sproxy(free_proxy)
red.sadd(red_keys.froxly_base_check_free_proxy, sproxy)
ses.close()
conn.close()
return red.srandmember(
red_keys.froxly_base_check_free_proxy)
def ts(self, ph='', pn=0):
lng = get_lng()
conn = orm.q_engine.connect()
ses = orm.sescls(bind=conn)
pn = int(pn)
pc = 9
ts = []
has_next_p = False
try:
q = None
if lng == lngs.UA:
q = ses.query(orm.uatrains.E).\
filter(orm.and_(orm.or_(orm.uatrains.E.etype == etype.train, orm.uatrains.E.etype == etype.etrain,
orm.uatrains.E.etype == etype.ptrain),
orm.uatrains.E.ua_graph.ilike('%' + ph.lower().replace(' ', '%') + '%'),
orm.uatrains.E.ua_title.ilike('%' + ph.lower() + '%'), orm.uatrains.E.ref_id == None)).\
order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.ua_title)
if lng == lngs.RU:
q = ses.query(orm.uatrains.E).\
filter(orm.and_(orm.or_(orm.uatrains.E.etype == etype.train, orm.uatrains.E.etype == etype.etrain,
orm.uatrains.E.etype == etype.ptrain),
orm.uatrains.E.ru_graph.ilike('%' + ph.lower().replace(' ', '%') + '%'),
orm.uatrains.E.ru_title.ilike('%' + ph.lower() + '%'), orm.uatrains.E.ref_id == None)).\
order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.ru_title)
if lng == lngs.EN:
q = ses.query(orm.uatrains.E).\
filter(orm.and_(orm.or_(orm.uatrains.E.etype == etype.train, orm.uatrains.E.etype == etype.etrain,
orm.uatrains.E.etype == etype.ptrain),
orm.uatrains.E.en_graph.ilike('%' + ph.lower().replace(' ', '%') + '%'),
orm.uatrains.E.en_title.ilike('%' + ph.lower() + '%'), orm.uatrains.E.ref_id == None)).\
order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.en_title)
ts = q.limit(pc).offset(pn * pc).all()
next_p_ts = q.limit(pc).offset((pn + 1) * pc).all()
if len(next_p_ts) > 0:
has_next_p = True
except Exception:
nlog.info('Uatrains error',
'Can\'t find trains\n' + traceback.format_exc())
ses.close()
conn.close()
return layout.getTrains(ts, ph, pn, has_next_p, lng)
def ss(self, ph='', pn=0):
lng = get_lng()
conn = orm.q_engine.connect()
ses = orm.sescls(bind=conn)
pn = int(pn)
pc = 9
ss = []
has_next_p = False
try:
q = ses.query(orm.uatrains.E).filter(orm.uatrains.E.etype == etype.station).\
filter(orm.or_(orm.uatrains.E.ua_title.ilike('%' + ph.lower() + '%'),
orm.uatrains.E.ru_title.ilike('%' + ph.lower() + '%'),
orm.uatrains.E.en_title.ilike('%' + ph.lower() + '%'))).\
order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.ua_title)
ss = q.limit(pc).offset(pn * pc).all()
next_p_ss = q.limit(pc).offset((pn + 1) * pc).all()
if len(next_p_ss) > 0:
has_next_p = True
except Exception:
nlog.info('Uatrains error',
'Can\'t find stations\n' + traceback.format_exc())
ses.close()
conn.close()
return layout.getStations(ss, ph, pn, has_next_p, lng)
def t(self, tid):
conn = orm.q_engine.connect()
ses = orm.sescls(bind=conn)
id = None
try:
ts = ses.query(orm.uatrains.E).filter(
orm.and_(orm.uatrains.E.oid == int(tid),
orm.uatrains.E.etype == etype.train)).all()
t = ts[0]
for train in ts:
if t.id < train.id and t.ref_id is None:
t = train
id = t.id
except:
nlog.info('Uatrains error', 'Can\'t find train by tid = ' + str(tid) + '\n' +\
traceback.format_exc())
ses.close()
conn.close()
cherrypy.response.status = 301
if id is not None:
cherrypy.response.headers['Location'] = '/' + str(id)
else:
cherrypy.response.headers['Location'] = '/'
return ''
def s(self, sid):
conn = orm.q_engine.connect()
ses = orm.sescls(bind=conn)
id = None
try:
ss = ses.query(orm.uatrains.E).filter(
orm.and_(orm.uatrains.E.oid == int(sid),
orm.uatrains.E.etype == etype.station)).all()
s = ss[0]
for station in ss:
if s.id < station.id:
s = station
id = s.id
except:
nlog.info('Uatrains error', 'Can\'t find station by sid = ' + str(sid) + '\n' +\
traceback.format_exc())
ses.close()
conn.close()
cherrypy.response.status = 301
if id is not None:
cherrypy.response.headers['Location'] = '/' + str(id)
else:
cherrypy.response.headers['Location'] = '/'
return ''
import traceback
import time
import datetime
from werp import orm, nlog, exec_log
conn = None
ses = None
try:
start_dt = datetime.datetime.now()
start_time = time.time()
conn = orm.null_engine.connect()
ses = orm.sescls(bind=conn)
trains = ses.query(orm.uatrains.E).\
filter(orm.or_(orm.uatrains.E.etype == 1, orm.uatrains.E.etype == 4,
orm.uatrains.E.etype == 5)).all()
for train in trains:
# Clear halts of train for the same s_id
halt_ids = ses.query(orm.distinct(orm.uatrains.TrainStation.s_id)).\
filter(orm.uatrains.TrainStation.t_id == train.id).all()
for halt_id in halt_ids:
halts = ses.query(orm.uatrains.TrainStation).\
filter(orm.and_(orm.uatrains.TrainStation.t_id == train.id,
orm.uatrains.TrainStation.s_id == halt_id[0])).all()
if len(halts) > 1:
newest_halt = halts[0]
for halt in halts:
if halt.c_date > newest_halt.c_date:
newest_halt = halt
for halt in halts:
if halt.id != newest_halt.id:
def run(task_drv):
conn = None
ses = None
try:
ctx = zmq.Context()
froxly_data_server_socket = ctx.socket(zmq.REQ)
froxly_data_server_socket.connect(
sockets.format_socket_uri(sockets.froxly_data_server,
drv=task_drv))
uatrains_bot_task_sink_socket = ctx.socket(zmq.PUSH)
uatrains_bot_task_sink_socket.connect(
sockets.format_socket_uri(sockets.uatrains_bot_task_sink,
drv=task_drv))
uatrains_bot_task_worker_socket = ctx.socket(zmq.PULL)
uatrains_bot_task_worker_socket.connect(
sockets.format_socket_uri(sockets.uatrains_bot_task_worker,
drv=task_drv))
uatrains_bot_task_finish_socket = ctx.socket(zmq.SUB)
uatrains_bot_task_finish_socket.connect(
sockets.format_socket_uri(sockets.uatrains_bot_task_finish,
drv=task_drv))
uatrains_bot_task_finish_socket.setsockopt_string(zmq.SUBSCRIBE, '')
poller = zmq.Poller()
poller.register(uatrains_bot_task_worker_socket, zmq.POLLIN)
poller.register(uatrains_bot_task_finish_socket, zmq.POLLIN)
conn = orm.null_engine.connect()
while True:
socks = dict(poller.poll())
if uatrains_bot_task_worker_socket in socks and socks[
uatrains_bot_task_worker_socket] == zmq.POLLIN:
ses = orm.sescls(bind=conn)
task_id = uatrains_bot_task_worker_socket.recv_unicode()
task = None
try:
task = ses.query(orm.uatrains.BotTask).filter(
orm.uatrains.BotTask.id == task_id).one()
except:
bot.logger.error('uatrains bot - task worker error\r\n' +
traceback.format_exc())
if task is not None:
task.status = task_status.running
ses.commit()
task.http_status = 0
try_count = 0
ua_dom_tree = None
current_drv = None
parser = etree.HTMLParser()
if task.drv == task_drvs.southwest:
current_drv = drv.southwest
elif task.drv == task_drvs.passengers:
current_drv = drv.passengers
while task.http_status <= 0 and try_count < TRY_COUNT:
ua_res = None
if ua_dom_tree is None:
ua_url = current_drv.ua_url.replace(
'(tid)', str(task.data))
ua_req = {
'method': 'request',
'params': {
'url': ua_url,
'charset': current_drv.charset,
'timeout': REQ_TIMEOUT
}
}
froxly_data_server_socket.send_unicode(
json.dumps(ua_req))
ua_res = json.loads(
froxly_data_server_socket.recv_unicode())
if 'http_status' in ua_res['result'] and 'data' in ua_res['result'] and \
ua_res['result']['http_status'] == 200:
try:
ua_dom_tree = etree.parse(
io.StringIO(ua_res['result']['data']),
parser)
drv.check_dom_tree(current_drv,
ua_dom_tree)
except Exception as e:
ua_dom_tree = None
ua_res['result']['http_status'] = -3
ua_res['result'][
'http_status_reason'] = str(e)
if ua_res is not None and 'http_status' in ua_res[
'result'] and ua_res['result'][
'http_status'] < 0:
task.http_status = ua_res['result']['http_status']
task.http_status_reason = ua_res['result'][
'http_status_reason']
try:
if ua_dom_tree is not None:
drv.get_train_data(current_drv, int(task.data),
ua_dom_tree)
task.http_status = 200
task.http_status_reason = None
except Exception as e:
task.http_status = -2
task.http_status_reason = str(e)
bot.logger.error('uatrains bot - task worker error\r\n' + 'Task data: ' + \
str(task.data) + '\n\n' + traceback.format_exc())
try_count += 1
task.status = task_status.completed
ses.commit()
ses.close()
uatrains_bot_task_sink_socket.send_unicode(str(1))
if uatrains_bot_task_finish_socket in socks and socks[
uatrains_bot_task_finish_socket] == zmq.POLLIN:
break
conn.close()
except:
bot.logger.error('uatrains bot - task worker error\r\n' +
traceback.format_exc())
if ses is not None:
ses.close()
if conn is not None:
conn.close()
def default(self, eid=None, *a, **kw):
lng = get_lng()
l = ''
ref_id = None
if eid is not None:
prepared_eid = None
try:
prepared_eid = int(float(eid))
except:
nlog.info('Uatrains error', 'Can\'t parse eid = ' + str(prepared_eid) + '\n' +\
traceback.format_exc())
if prepared_eid is not None:
conn = orm.q_engine.connect()
ses = orm.sescls(bind=conn)
e = None
try:
e = ses.query(orm.uatrains.E).filter(
orm.uatrains.E.id == prepared_eid).one()
if e.ref_id is not None:
ref_id = e.ref_id
else:
e.vc = e.vc + 1
ses.commit()
except:
nlog.info('Uatrains error', 'Can\'t find entity by eid = ' + str(prepared_eid) + '\n' +\
traceback.format_exc())
if e is not None and ref_id is None:
if e.etype == etype.train:
try:
t = ses.query(orm.uatrains.E).\
options(orm.joinedload_all(orm.uatrains.E.t_ss, orm.uatrains.TrainStation.s)).\
filter(orm.uatrains.E.id == prepared_eid).one()
l = layout.getTrain(t, t.t_ss, lng)
except:
nlog.info('Uatrains error', 'Can\'t find train by id = ' + str(prepared_eid) + '\n' +\
traceback.format_exc())
elif e.etype == etype.ptrain:
try:
t = ses.query(orm.uatrains.E).\
options(orm.joinedload_all(orm.uatrains.E.t_ss, orm.uatrains.TrainStation.s)).\
filter(orm.uatrains.E.id == prepared_eid).one()
l = layout.getPTrain(t, t.t_ss, lng)
except:
nlog.info('Uatrains error', 'Can\'t find ptrain by id = ' + str(prepared_eid) + '\n' +\
traceback.format_exc())
elif e.etype == etype.station:
try:
s = ses.query(orm.uatrains.E).\
options(orm.joinedload_all(orm.uatrains.E.s_ts, orm.uatrains.TrainStation.t)).\
filter(orm.uatrains.E.id == prepared_eid).one()
l = layout.getStation(s, s.s_ts, lng)
except:
nlog.info('Uatrains error', 'Can\'t find station by id = ' + str(prepared_eid) + '\n' +\
traceback.format_exc())
ses.close()
conn.close()
else:
l = self.index(eid)
if l == '':
cherrypy.response.status = 301
if ref_id is not None:
cherrypy.response.headers['Location'] = '/' + str(ref_id)
else:
cherrypy.response.headers['Location'] = '/'
return l
