def registerPlugin(self, FLPluginType):
plugin = FLPluginType(self.db, self.cfg, self.asyncDlMgr)
assert isinstance(plugin, FLEngineCreator), "{} is not type of FLEngineCreator.".format(plugin)
lm.debug("New plugin registered: {}".format(FLPluginType))
self.plugins.append(plugin)
return plugin.getAirlineCode()
def run(self, paths, connectionList, config):
# assert self.airportsReady == True, "Airports were not initialized!"
# assert self.connectionsReady == True, "Connections were not initialized!"
assert config['date_from'] != -1 and config['date_to'] != -1, "Wrong config!"
if not self.flight_detail_bypass and self.asyncMode and self.asyncDlMgr is not None:
self.log("Starting thread")
self.receiver = threading.Thread(target=self.__asyncReceiver)
self.receiver.start()
self.connections = []
self.proceedList = []
date_from = config['date_from'] - datetime.timedelta(days=6) #self.currentDate + delay_data;
date_to = config['date_to'] + datetime.timedelta(days=6) #date_from + monthdelta(self.month_delta)
self.log("Flights update between: {} - {}".format(date_from, date_to))
self.connections.extend(self.__prepareConnectionsQuery(paths, connectionList))
if len(self.connections) > 0:
if self.dump_restore_session and self.session.isSaved():
self.connections.extend(self.session.restoreSession())
self.session.close()
oneWayIdxList = self.__getOneWayConnectionIndexList(self.connections);
self.log("Connections: {}".format(len(self.connections)))
self.log("OneWayIdx: {}".format(len(oneWayIdxList)))
assert (len(oneWayIdxList)*2) == len(self.connections)
if not self.flight_detail_bypass:
try:
self.__getFlightDetails(date_from, date_to, oneWayIdxList)
except:
if self.dump_restore_session:
lm.debug("Session dumped!")
connectionsLeftList = []
if len(self.proceedList) % 2 != 0:
self.proceedList.pop()
connectionsLeftList.extend([x for x in connections if x not in self.proceedList])
if(len(connectionsLeftList)):
self.session.save(connectionsLeftList)
print("Unexpected error:", sys.exc_info()[0])
lm.exception()
raise
self.return_q.join()
self.asyncDlMgr.finished(self.return_q)
if not self.flight_detail_bypass and self.asyncMode:
self.receiver.join()
self.log("Total time: {}".format(datetime.datetime.now() - self.currentDate))
def dumpToFile(self, fileName, data):
if lm.dump_files():
with open(os.path.join(self.log_dir, fileName),
'w',
encoding="utf-8") as f:
json.dump(data, f, cls=tools.FLJsonEncoder, indent=4)
f.close()
def __getFlightDetails(self, date_from, date_to, oneWayIdxList):
for i, idx in enumerate(oneWayIdxList):
if date_to.year == date_from.year:
months = int(((date_to - date_from).days)/30) + 1
else:
months = int(((date_to - date_from).days)/30) + 2
for delta in range(months):
time = date_from + monthdelta(delta)
timeTableList = self.__getTimeTable(time, self.connections[idx]) #get timetable for given year-month
for flight in timeTableList:
if flight.date >= date_from and flight.date <= date_to: #datetime.datetime(2017,2,18):
if self.asyncMode:
self.__scheduleFlightDetails(flight)
else:
list = self.__fetchFlightDetails(flight)
self.__handleFlightDetails(list)
self.proceedList.append(self.connections[idx])
lm.debug("Status: {} of {} - ({}%)".format(i + 1, len(oneWayIdxList), (i+1)/len(oneWayIdxList)*100))
def __getFlightDetails(self, date_from, date_to, oneWayIdxList):
for i, idx in enumerate(oneWayIdxList):
if date_to.year == date_from.year:
months = int(((date_to - date_from).days) / 30) + 1
else:
months = int(((date_to - date_from).days) / 30) + 2
for delta in range(months):
time = date_from + monthdelta(delta)
timeTableList = self.__getTimeTable(
time,
self.connections[idx]) #get timetable for given year-month
for flight in timeTableList:
if flight.date >= date_from and flight.date <= date_to: #datetime.datetime(2017,2,18):
if self.asyncMode:
self.__scheduleFlightDetails(flight)
else:
list = self.__fetchFlightDetails(flight)
self.__handleFlightDetails(list)
self.proceedList.append(self.connections[idx])
lm.debug("Status: {} of {} - ({}%)".format(
i + 1, len(oneWayIdxList), (i + 1) / len(oneWayIdxList) * 100))
def log(self, message):
lm.debug("FlightScheduler: {0}".format(message))
def log(self, message=''):
lm.debug("WizzairDl: {0}".format(message))
def log(self, message):
lm.debug("WizzairParser: {0}".format(message))
def log(self, message):
lm.debug("SessionManager: {0}".format(message))
def debug(msg):
lm.debug("HttpManager: {0}".format(msg))
def log(self, message):
lm.debug("FLPluginManager: {0}".format(message))
def log(self, message=''):
lm.debug("WizzairDl: {0}".format(message))
def log(self, message):
lm.debug("FLSearchEngine: {0}".format(message))
def log(self, message):
lm.debug("{}Plugin: {}".format(self.airline_name, message))
def setupLogger(self, cfg, dump_files):
LogMgr.init(dump_files,
dirname=cfg['config_dir'],
configFileName=cfg['config_name'])
def log(self, message):
lm.debug("DatabaseManager: {0}".format(message))
def log(self, message=''):
lm.debug("GeoNameProvider: {0}".format(message))
def log(self, message):
lm.debug("{}Plugin: {}".format(self.airline_name, message))
def run(self, paths, connectionList, config):
# assert self.airportsReady == True, "Airports were not initialized!"
# assert self.connectionsReady == True, "Connections were not initialized!"
assert config['date_from'] != -1 and config[
'date_to'] != -1, "Wrong config!"
if not self.flight_detail_bypass and self.asyncMode and self.asyncDlMgr is not None:
self.log("Starting thread")
self.receiver = threading.Thread(target=self.__asyncReceiver)
self.receiver.start()
self.connections = []
self.proceedList = []
date_from = config['date_from'] - datetime.timedelta(
days=6) #self.currentDate + delay_data;
date_to = config['date_to'] + datetime.timedelta(
days=6) #date_from + monthdelta(self.month_delta)
self.log("Flights update between: {} - {}".format(date_from, date_to))
self.connections.extend(
self.__prepareConnectionsQuery(paths, connectionList))
if len(self.connections) > 0:
if self.dump_restore_session and self.session.isSaved():
self.connections.extend(self.session.restoreSession())
self.session.close()
oneWayIdxList = self.__getOneWayConnectionIndexList(
self.connections)
self.log("Connections: {}".format(len(self.connections)))
self.log("OneWayIdx: {}".format(len(oneWayIdxList)))
assert (len(oneWayIdxList) * 2) == len(self.connections)
if not self.flight_detail_bypass:
try:
self.__getFlightDetails(date_from, date_to, oneWayIdxList)
except:
if self.dump_restore_session:
lm.debug("Session dumped!")
connectionsLeftList = []
if len(self.proceedList) % 2 != 0:
self.proceedList.pop()
connectionsLeftList.extend([
x for x in connections if x not in self.proceedList
])
if (len(connectionsLeftList)):
self.session.save(connectionsLeftList)
print("Unexpected error:", sys.exc_info()[0])
lm.exception()
raise
self.return_q.join()
self.asyncDlMgr.finished(self.return_q)
if not self.flight_detail_bypass and self.asyncMode:
self.receiver.join()
self.log("Total time: {}".format(datetime.datetime.now() -
self.currentDate))
def finished(self, return_fifo):
lm.debug("Send finished")
self.q.join();
task = { "M": None, "return": return_fifo }
self.q.put(task)
def log(self, message):
lm.debug("SessionManager: {0}".format(message))
def run(self):
lm.debug("Waiting for task_queue")
self.task_queue.join()
def debug(msg):
lm.debug("HttpManager: {0}".format(msg))
def worker(self, i, q):
lm.debug("Created {}".format(i))
limit = 5
while True:
counter = 0
task = q.get()
method = task['M']
ret_q = task['return']
if method is None:
lm.debug("Method NONE.")
while not q.empty() and not ret_q.empty():
lm.debug("Queue is not empty")
sleep(1)
lm.debug("Queue is empty")
ret_q.put( {'data':None} )
else:
while True:
if counter == limit:
lm.debug("No response from server.")
ret_q.put( {'data':None} )
proxy = self.proxyList[random.randrange(0,len(self.proxyList))]
lm.debug("Worker: {} Method {} PROXY: {}".format(i, task['M'], proxy))
if method == 0:
# time.sleep(random.randrange(3, 12))
httpContent = HttpManager.getMethod(task['url'], proxy)
if httpContent is not None:
ret_q.put({'data':json.loads(httpContent.text), 'url': task['url'] } )
break
else:
lm.debug("Request error for: {} {}".format(task['url'], proxy))
else:
# time.sleep(random.randrange(3, 12))
httpContent = HttpManager.postMethod(task['url'], task['params'], proxy)
if httpContent is not None:
ret_q.put({'data':json.loads(httpContent.text), 'url': task['url'] } )
break
else:
lm.debug("Request error for: {} {}".format(task['url'], proxy))
counter = counter + 1
sleep(counter * 2)
q.task_done()
def log(self, message):
lm.debug("FlightScheduler: {0}".format(message))
def __createWorkers(self, num_workers):
lm.debug("Creating {}".format(num_workers))
for i in range(num_workers):
w = Thread(target=self.worker, args=(i, self.task_queue, ))
w.setDaemon(True)
w.start()
def setupLogger(self, cfg, dump_files):
LogMgr.init(dump_files, dirname=cfg['config_dir'], configFileName=cfg['config_name'])
def log(self, message):
lm.debug("DatabaseManager: {0}".format(message))
def log(self, message=''):
lm.debug("GeoNameProvider: {0}".format(message))
def dumpToFile(self, fileName, data):
if lm.dump_files():
with open(os.path.join(self.log_dir, fileName),'w', encoding="utf-8") as f:
json.dump(data, f, cls=tools.FLJsonEncoder, indent=4)
f.close()