def get_s(e, ses):
s = None
try:
prepared_ua_title = e.ua_title.replace(' ', '%').replace(
'-', '%') if e.ua_title is not None else ''
s = ses.query(orm.uatrains.E).\
filter(orm.and_(orm.uatrains.E.etype == e.etype,
orm.uatrains.E.oid == e.oid, orm.uatrains.E.ua_title.ilike(prepared_ua_title))).\
order_by(orm.asc(orm.uatrains.E.c_date)).first()
except orm.NoResultFound:
try:
if e.oid >= 20000:
s = ses.query(orm.uatrains.E).\
filter(orm.and_(orm.uatrains.E.etype == e.etype,
orm.uatrains.E.oid < 20000, orm.uatrains.E.ua_title.ilike(prepared_ua_title))).\
order_by(orm.asc(orm.uatrains.E.c_date)).first()
elif e.oid < 20000:
s = ses.query(orm.uatrains.E).\
filter(orm.and_(orm.uatrains.E.etype == e.etype,
orm.uatrains.E.oid >= 20000, orm.uatrains.E.ua_title.ilike(prepared_ua_title))).\
order_by(orm.asc(orm.uatrains.E.c_date)).first()
except orm.NoResultFound:
pass
except:
s = None
return s
def index(self, eid=None):
lng = get_lng()
conn = orm.q_engine.connect()
ses = orm.sescls(bind=conn)
ts = []
pc = 5
try:
q = None
if lng == lngs.UA:
q = ses.query(orm.uatrains.E).\
filter(orm.and_(orm.uatrains.E.etype == etype.train, orm.uatrains.E.ref_id == None)).\
order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.ua_title)
if lng == lngs.RU:
q = ses.query(orm.uatrains.E).\
filter(orm.and_(orm.uatrains.E.etype == etype.train, orm.uatrains.E.ref_id == None)).\
order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.ru_title)
if lng == lngs.EN:
q = ses.query(orm.uatrains.E).\
filter(orm.and_(orm.uatrains.E.etype == etype.train, orm.uatrains.E.ref_id == None)).\
order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.en_title)
ts = q.limit(pc).all()
except Exception:
nlog.info('Uatrains error',
'Can\'t find trains\n' + traceback.format_exc())
ss = []
try:
q = None
if lng == lngs.UA:
q = ses.query(orm.uatrains.E).filter(orm.uatrains.E.etype == etype.station).\
order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.ua_title)
if lng == lngs.RU:
q = ses.query(orm.uatrains.E).filter(orm.uatrains.E.etype == etype.station).\
order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.ru_title)
if lng == lngs.EN:
q = ses.query(orm.uatrains.E).filter(orm.uatrains.E.etype == etype.station).\
order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.en_title)
ss = q.limit(pc).all()
except Exception:
nlog.info('Uatrains error',
'Can\'t find stations\n' + traceback.format_exc())
news = []
try:
news = ses.query(orm.uatrains.New).filter(orm.uatrains.New.lng == lng).\
order_by(orm.uatrains.New.date.desc()).limit(3).all()
except:
nlog.info('Uatrains error',
'Can\'t find news\n' + traceback.format_exc())
ses.close()
conn.close()
return layout.getHome(ts, ss, news, lng)
def from_to(ses, fs, ts, pc, pn):
es = []
has_next_p = False
if fs != '' and ts == '':
es, has_next_p = full(ses,
fs.replace(' ', '%').replace('-', '%'), pc, pn)
elif fs == '' and ts != '':
es, has_next_p = full(ses,
ts.replace(' ', '%').replace('-', '%'), pc, pn)
else:
prepared_fs = fs.replace(' ', '%').replace('-', '%')
prepared_ts = ts.replace(' ', '%').replace('-', '%')
prepared_ph = prepared_fs + '%' + prepared_ts
try:
q = ses.query(orm.uatrains.E).\
filter(orm.and_(
orm.or_(orm.uatrains.E.ua_graph.ilike('%' + prepared_ph.lower() + '%'),
orm.uatrains.E.ru_graph.ilike('%' + prepared_ph.lower() + '%'),
orm.uatrains.E.en_graph.ilike('%' + prepared_ph.lower() + '%')),
orm.uatrains.E.ref_id == None)).\
order_by(orm.uatrains.E.etype.desc(), orm.uatrains.E.vc.desc(), orm.uatrains.E.ua_title)
es = q.limit(pc).offset(pn * pc).all()
next_p_es = q.limit(pc).offset((pn + 1) * pc).all()
if len(next_p_es) > 0:
has_next_p = True
except Exception:
nlog.info(
'Uatrains error',
'Can\'t find entities by fs and ts\n' + traceback.format_exc())
return es, has_next_p
def run(task_drv, worker_pool):
try:
if not os.path.exists(
sockets.get_socket_path(
sockets.format_socket_uri(sockets.uatrains_bot_task_worker,
drv=task_drv))):
ctx = zmq.Context()
uatrains_bot_task_worker_socket = ctx.socket(zmq.PUSH)
uatrains_bot_task_worker_socket.bind(
sockets.format_socket_uri(sockets.uatrains_bot_task_worker,
drv=task_drv))
conn = orm.null_engine.connect()
ses = orm.sescls(bind=conn)
if task_drv == task_drvs.southwest:
for tid in range(0, 5000):
bot_task = orm.uatrains.BotTask()
bot_task.data = str(tid)
bot_task.drv = task_drvs.southwest
ses.add(bot_task)
elif task_drv == task_drvs.passengers:
for tid in range(20000, 70000):
bot_task = orm.uatrains.BotTask()
bot_task.data = str(tid)
bot_task.drv = task_drvs.passengers
ses.add(bot_task)
ses.commit()
tasks = ses.query(orm.uatrains.BotTask).\
filter(orm.and_(orm.uatrains.BotTask.status == None, orm.uatrains.BotTask.drv == task_drv)).\
order_by(orm.desc(orm.cast(orm.uatrains.BotTask.data, orm.BigInteger))).all()
manager = threading.Thread(target=sink.run,
args=(len(tasks), task_drv))
manager.start()
for wrk_num in range(worker_pool):
thr = threading.Thread(target=worker.run, args=(task_drv, ))
thr.start()
for t in tasks:
uatrains_bot_task_worker_socket.send_unicode(t.id)
uatrains_bot_task_finish_socket = ctx.socket(zmq.SUB)
uatrains_bot_task_finish_socket.connect(
sockets.format_socket_uri(sockets.uatrains_bot_task_finish,
drv=task_drv))
uatrains_bot_task_finish_socket.setsockopt_string(
zmq.SUBSCRIBE, '')
uatrains_bot_task_finish_socket.recv_unicode()
ses.close()
conn.close()
else:
bot.logger.error('uatrains bot info. Bot already started')
except:
bot.logger.error('uatrains bot - task ventilator error\r\n' +
traceback.format_exc())
raise Exception('Task ventilator error. See log.')
def ts(self, ph='', pn=0):
lng = get_lng()
conn = orm.q_engine.connect()
ses = orm.sescls(bind=conn)
pn = int(pn)
pc = 9
ts = []
has_next_p = False
try:
q = None
if lng == lngs.UA:
q = ses.query(orm.uatrains.E).\
filter(orm.and_(orm.or_(orm.uatrains.E.etype == etype.train, orm.uatrains.E.etype == etype.etrain,
orm.uatrains.E.etype == etype.ptrain),
orm.uatrains.E.ua_graph.ilike('%' + ph.lower().replace(' ', '%') + '%'),
orm.uatrains.E.ua_title.ilike('%' + ph.lower() + '%'), orm.uatrains.E.ref_id == None)).\
order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.ua_title)
if lng == lngs.RU:
q = ses.query(orm.uatrains.E).\
filter(orm.and_(orm.or_(orm.uatrains.E.etype == etype.train, orm.uatrains.E.etype == etype.etrain,
orm.uatrains.E.etype == etype.ptrain),
orm.uatrains.E.ru_graph.ilike('%' + ph.lower().replace(' ', '%') + '%'),
orm.uatrains.E.ru_title.ilike('%' + ph.lower() + '%'), orm.uatrains.E.ref_id == None)).\
order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.ru_title)
if lng == lngs.EN:
q = ses.query(orm.uatrains.E).\
filter(orm.and_(orm.or_(orm.uatrains.E.etype == etype.train, orm.uatrains.E.etype == etype.etrain,
orm.uatrains.E.etype == etype.ptrain),
orm.uatrains.E.en_graph.ilike('%' + ph.lower().replace(' ', '%') + '%'),
orm.uatrains.E.en_title.ilike('%' + ph.lower() + '%'), orm.uatrains.E.ref_id == None)).\
order_by(orm.uatrains.E.vc.desc(), orm.uatrains.E.en_title)
ts = q.limit(pc).offset(pn * pc).all()
next_p_ts = q.limit(pc).offset((pn + 1) * pc).all()
if len(next_p_ts) > 0:
has_next_p = True
except Exception:
nlog.info('Uatrains error',
'Can\'t find trains\n' + traceback.format_exc())
ses.close()
conn.close()
return layout.getTrains(ts, ph, pn, has_next_p, lng)
def get_ts(ts, ses):
db_ts = None
try:
db_ts = ses.query(orm.uatrains.TrainStation).\
filter(orm.and_(orm.uatrains.TrainStation.t_id == ts.t_id, orm.uatrains.TrainStation.s_id == ts.s_id,
orm.uatrains.TrainStation.order == ts.order)).\
one()
except orm.NoResultFound:
pass
except:
db_ts = None
return db_ts
def get_t(e, ses):
t = None
try:
prepared_ua_title = e.ua_title if e.ua_title is not None else ''
t = ses.query(orm.uatrains.E).\
filter(orm.and_(orm.uatrains.E.etype == e.etype, orm.uatrains.E.oid == e.oid,
orm.uatrains.E.value == e.value, orm.uatrains.E.ua_title == prepared_ua_title)).\
order_by(orm.asc(orm.uatrains.E.c_date)).first()
except orm.NoResultFound:
pass
except:
t = None
return t
def base_rnd():
if red.exists(red_keys.froxly_base_check_free_proxy) and \
red.scard(red_keys.froxly_base_check_free_proxy) > 0:
return red.srandmember(
red_keys.froxly_base_check_free_proxy)
else:
conn = orm.null_engine.connect()
ses = orm.sescls(bind=conn)
free_proxies = ses.query(orm.FreeProxy).filter(
orm.and_(orm.FreeProxy.protocol == 'http',
orm.FreeProxy.http_status == 200)).all()
for free_proxy in free_proxies:
sproxy = data_server_common.dbproxy2sproxy(free_proxy)
red.sadd(red_keys.froxly_base_check_free_proxy, sproxy)
ses.close()
conn.close()
return red.srandmember(
red_keys.froxly_base_check_free_proxy)
def full(ses, ph, pc, pn):
es = []
has_next_p = False
prepared_ph = ph.replace(' ', '%').replace('-', '%')
try:
q = ses.query(orm.uatrains.E).\
filter(orm.and_(orm.or_(orm.uatrains.E.ua_title.ilike('%' + prepared_ph.lower() + '%'),
orm.uatrains.E.ru_title.ilike('%' + prepared_ph.lower() + '%'),
orm.uatrains.E.en_title.ilike('%' + prepared_ph.lower() + '%'),
orm.uatrains.E.ua_graph.ilike('%' + prepared_ph.lower() + '%'),
orm.uatrains.E.ru_graph.ilike('%' + prepared_ph.lower() + '%'),
orm.uatrains.E.en_graph.ilike('%' + prepared_ph.lower() + '%'),
orm.uatrains.E.value.op('similar to')('([0-9А-Яа-я]*/)?' + prepared_ph.lower() + \
'([А-Яа-я]*)?(/[0-9А-Яа-я]*)?(/[0-9А-Яа-я]*)?')),
orm.uatrains.E.ref_id == None)).\
order_by(orm.uatrains.E.etype.desc(), orm.uatrains.E.vc.desc(), orm.uatrains.E.ua_title)
es = q.limit(pc).offset(pn * pc).all()
next_p_es = q.limit(pc).offset((pn + 1) * pc).all()
if len(next_p_es) > 0:
has_next_p = True
except Exception:
nlog.info('Uatrains error',
'Can\'t find entities\n' + traceback.format_exc())
return es, has_next_p
def t(self, tid):
conn = orm.q_engine.connect()
ses = orm.sescls(bind=conn)
id = None
try:
ts = ses.query(orm.uatrains.E).filter(
orm.and_(orm.uatrains.E.oid == int(tid),
orm.uatrains.E.etype == etype.train)).all()
t = ts[0]
for train in ts:
if t.id < train.id and t.ref_id is None:
t = train
id = t.id
except:
nlog.info('Uatrains error', 'Can\'t find train by tid = ' + str(tid) + '\n' +\
traceback.format_exc())
ses.close()
conn.close()
cherrypy.response.status = 301
if id is not None:
cherrypy.response.headers['Location'] = '/' + str(id)
else:
cherrypy.response.headers['Location'] = '/'
return ''
def s(self, sid):
conn = orm.q_engine.connect()
ses = orm.sescls(bind=conn)
id = None
try:
ss = ses.query(orm.uatrains.E).filter(
orm.and_(orm.uatrains.E.oid == int(sid),
orm.uatrains.E.etype == etype.station)).all()
s = ss[0]
for station in ss:
if s.id < station.id:
s = station
id = s.id
except:
nlog.info('Uatrains error', 'Can\'t find station by sid = ' + str(sid) + '\n' +\
traceback.format_exc())
ses.close()
conn.close()
cherrypy.response.status = 301
if id is not None:
cherrypy.response.headers['Location'] = '/' + str(id)
else:
cherrypy.response.headers['Location'] = '/'
return ''
ses = None
try:
start_dt = datetime.datetime.now()
start_time = time.time()
conn = orm.null_engine.connect()
ses = orm.sescls(bind=conn)
trains = ses.query(orm.uatrains.E).\
filter(orm.or_(orm.uatrains.E.etype == 1, orm.uatrains.E.etype == 4,
orm.uatrains.E.etype == 5)).all()
for train in trains:
# Clear halts of train for the same s_id
halt_ids = ses.query(orm.distinct(orm.uatrains.TrainStation.s_id)).\
filter(orm.uatrains.TrainStation.t_id == train.id).all()
for halt_id in halt_ids:
halts = ses.query(orm.uatrains.TrainStation).\
filter(orm.and_(orm.uatrains.TrainStation.t_id == train.id,
orm.uatrains.TrainStation.s_id == halt_id[0])).all()
if len(halts) > 1:
newest_halt = halts[0]
for halt in halts:
if halt.c_date > newest_halt.c_date:
newest_halt = halt
for halt in halts:
if halt.id != newest_halt.id:
ses.delete(halt)
ses.commit()
#halts = ses.query(orm.uatrains.TrainStation).\
# filter(orm.uatrains.TrainStation.t_id == train.id).all()
#halts_to_delete = []
#for halt in halts:
# for h in halts:
# if halt.order == h.order and halt.c_date >= h.c_date and h not in halts_to_delete:
start_dt = datetime.datetime.now()
start_time = time.time()
conn = orm.null_engine.connect()
ses = orm.sescls(bind=conn)
etrains = ses.query(orm.uatrains.E).filter(
orm.or_(orm.uatrains.E.etype == 1, orm.uatrains.E.etype == 4)).all()
for et in etrains:
if et.ref_id is None:
if (et.from_date is None and et.to_date is None) or \
(et.from_date is not None and et.to_date is not None and not (\
et.from_date <= datetime.datetime.now() and \
et.to_date >= datetime.datetime.now())):
last_etrain = et
similar_etrains = ses.query(orm.uatrains.E).\
filter(orm.and_(orm.or_(orm.uatrains.E.value == et.value, orm.uatrains.E.oid == et.oid),
orm.uatrains.E.ua_title == et.ua_title,
orm.or_(orm.uatrains.E.etype == 1, orm.uatrains.E.etype == 4))).all()
if len(similar_etrains) > 1:
for similar_et in similar_etrains:
if (similar_et.from_date is not None and similar_et.to_date is not None and \
similar_et.from_date <= datetime.datetime.now() and \
similar_et.to_date >= datetime.datetime.now()) and \
((last_etrain.from_date is None and last_etrain.to_date is None) or \
(last_etrain.to_date <= similar_et.from_date) or \
(last_etrain.id < similar_et.id)):
last_etrain = similar_et
if last_etrain.id > et.id:
et.ref_id = last_etrain.id
ses.commit()
ses.close()
conn.close()
try:
start_dt = datetime.datetime.now()
start_time = time.time()
conn = orm.null_engine.connect()
ses = orm.sescls(bind=conn)
etrains = ses.query(orm.uatrains.E).filter(orm.uatrains.E.etype == 5).all()
for et in etrains:
if et.ref_id is None:
if (et.from_date is None and et.to_date is None) or \
(et.from_date is not None and et.to_date is not None and not (\
et.from_date <= datetime.datetime.now() and \
et.to_date >= datetime.datetime.now())):
last_etrain = et
similar_etrains = ses.query(orm.uatrains.E).\
filter(orm.and_(orm.uatrains.E.oid == et.oid,
orm.uatrains.E.ua_title == et.ua_title, orm.uatrains.E.ru_title == et.ru_title,
orm.uatrains.E.etype == 5)).all()
if len(similar_etrains) > 1:
for similar_et in similar_etrains:
if similar_et.c_date > last_etrain.c_date:
last_etrain = similar_et
if last_etrain.id > et.id:
et.ref_id = last_etrain.id
ses.commit()
ses.close()
conn.close()
end_time = time.time()
exec_delta = datetime.timedelta(seconds=int(end_time - start_time))
exec_log.info('uatrains bot ptrain referer %s %s' % (str(start_dt), str(exec_delta)))
except:
if ses is not None:
for ip_dn_div in ip_dn_divs:
raw_ip = raw_ip.replace(ip_dn_div, '')
ip_dn_spans = re.findall(
'<span\s*style="display:\s*none">\d*</span>', raw_ip)
for ip_dn_span in ip_dn_spans:
raw_ip = raw_ip.replace(ip_dn_span, '')
ip_parts = re.findall('>\s*([\.0-9]+)\s*<', raw_ip)
fp = orm.FreeProxy()
fp.ip = ''.join(ip_parts)
fp.port = raw_proxy[2].text.strip()
fp.protocol = raw_proxy[6].text.lower().strip()
if fp.protocol in ('socks4/5', 'https', 'http'):
try:
ses.query(orm.FreeProxy).filter(
orm.and_(
orm.and_(orm.FreeProxy.ip == fp.ip,
orm.FreeProxy.port == fp.port),
orm.FreeProxy.protocol == fp.protocol)).one()
except orm.NoResultFound:
ses.add(fp)
ses.commit()
added_fps.append((fp.ip, fp.port, fp.protocol))
except:
froxly_grabber_log.exception(
traceback.format_exc() + '\n\n' +
etree.tostring(raw_proxy).decode('utf-8'))
end_time = time.time()
exec_delta = datetime.timedelta(seconds=int(end_time -
start_time))
froxly_grabber_log.info('Finished, duration is %s \n\n' %
(str(exec_delta), ))
ses.close()
