def searchByName(self, url): resp = HttpManager.getMethod(url).text json_data = json.loads(resp) if int(json_data['totalResultsCount']) > 0: return json_data['geonames'][0] else: return {'countryCode': "N/A", "countryName": "N/A"}
def searchByName(self, url): resp = HttpManager.getMethod(url).text json_data = json.loads(resp) if int(json_data['totalResultsCount']) > 0: return json_data['geonames'][0] else: return {'countryCode':"N/A", "countryName": "N/A"}
def __fetchAirportAndConnections(self): httpcontent = HttpManager.getMethod(CommonData.AIRPORTS).text json_data = json.loads(httpcontent) self._airports = json_data['destinations'] self._connections = self.__checkConnectionsConsistency( self._airports, json_data['relations'])
def getTimeTable(self, details = { "src_iata": "", "dst_iata":"", "year":"", "month":"" }): """ Downloads monthly timetable for connection details :details: connection details with src_iata, dst_iata, year and month :returns: content in json format from service """ src_iata = details['src_iata'] dst_iata = details['dst_iata'] year = details['year'] month = details['month'] date = max(datetime.date( year, month, 1), datetime.date.today()) date_from = date.strftime("%Y-%m-%d") #{}-{}-01".format(year, month) date_to = "{}-{}-{}".format(year, month, monthrange(year, month)[1]) url = CommonData.TimeTable.format(self.api_version, src_iata, dst_iata, date_from, date_to) filePath = os.path.join(self.base, '{0}_{1}_{2}_{3}.json'.format(src_iata, dst_iata, month, year)) if self.cfg['DEBUGGING']['state'] == 'online': self.log("Get from: " + url) httpContent = HttpManager.getMethod(url).text # self.writeToFile(filePath, httpContent) else: self.log("Get from: " + filePath) with open(filePath, 'r') as f: httpContent = f.read() f.close() json_data = { 'Dates': json.loads(httpContent)['flightDates'], 'DepartureStationCode': src_iata, 'ArrivalStationCode': dst_iata } return json_data
def __updateApiVersion(self): content = HttpManager.getMethod(CommonData.API).text jsonData = json.loads(content) api_version = jsonData['apiUrl'] #re.findall(r'var apiUrl =\s*(.*?);', content, re.DOTALL | re.MULTILINE)[0].replace('"','') CommonData.Search = CommonData.Search.format(api_version) self.log("Search api: {}".format(CommonData.Search)) return api_version
def getFlightDetails(self, flight): """TODO: Docstring for __searchFlight. :options: TODO : {'src_iata':'': TODO :'dst_iata':'': TODO :'date':''}: TODO :returns: TODO """ if type(flight) is not TimeTable: raise TypeError('flight is not a TimeTable type') params = self.packParamsToJSON(flight) filePath = os.path.join(self.base, '{0}_{1}_{2}.json'.format(flight.src, flight.dst, flight.date)) if self.cfg['DEBUGGING']['state'] == 'online': httpContent = HttpManager.postMethod(CommonData.Search, params).text # self.writeToFile(filePath, httpContent) else: filePath = os.path.join(self.base, '{0}_{1}_{2}.json'.format(flight.src, flight.dst, flight.date)) self.log("Get from: " + filePath) with open(filePath, 'r') as f: httpContent = f.read() f.close() return json.loads(httpContent)
def getFlightDetails(self, flight): """TODO: Docstring for __searchFlight. :options: TODO : {'src_iata':'': TODO :'dst_iata':'': TODO :'date':''}: TODO :returns: TODO """ if type(flight) is not TimeTable: raise TypeError('flight is not a TimeTable type') params = self.packParamsToJSON(flight) filePath = os.path.join( self.base, '{0}_{1}_{2}.json'.format(flight.src, flight.dst, flight.date)) if self.cfg['DEBUGGING']['state'] == 'online': httpContent = HttpManager.postMethod(CommonData.Search, params).text # self.writeToFile(filePath, httpContent) else: filePath = os.path.join( self.base, '{0}_{1}_{2}.json'.format(flight.src, flight.dst, flight.date)) self.log("Get from: " + filePath) with open(filePath, 'r') as f: httpContent = f.read() f.close() return json.loads(httpContent)
def __updateApiVersion(self): content = HttpManager.getMethod(CommonData.API).text jsonData = json.loads(content) api_version = jsonData[ 'apiUrl'] #re.findall(r'var apiUrl =\s*(.*?);', content, re.DOTALL | re.MULTILINE)[0].replace('"','') CommonData.Search = CommonData.Search.format(api_version) self.log("Search api: {}".format(CommonData.Search)) return api_version
def worker(self, i, q): lm.debug("Created {}".format(i)) limit = 5 while True: counter = 0 task = q.get() method = task['M'] ret_q = task['return'] if method is None: lm.debug("Method NONE.") while not q.empty() and not ret_q.empty(): lm.debug("Queue is not empty") sleep(1) lm.debug("Queue is empty") ret_q.put( {'data':None} ) else: while True: if counter == limit: lm.debug("No response from server.") ret_q.put( {'data':None} ) proxy = self.proxyList[random.randrange(0,len(self.proxyList))] lm.debug("Worker: {} Method {} PROXY: {}".format(i, task['M'], proxy)) if method == 0: # time.sleep(random.randrange(3, 12)) httpContent = HttpManager.getMethod(task['url'], proxy) if httpContent is not None: ret_q.put({'data':json.loads(httpContent.text), 'url': task['url'] } ) break else: lm.debug("Request error for: {} {}".format(task['url'], proxy)) else: # time.sleep(random.randrange(3, 12)) httpContent = HttpManager.postMethod(task['url'], task['params'], proxy) if httpContent is not None: ret_q.put({'data':json.loads(httpContent.text), 'url': task['url'] } ) break else: lm.debug("Request error for: {} {}".format(task['url'], proxy)) counter = counter + 1 sleep(counter * 2) q.task_done()
def getData(self, url): tries = 10 counter = 0 while True: r = HttpMgr.getMethod(url, tries=0) if (r != None): return r.text counter += 1 time.sleep(counter * 2) if counter > tries: break
def getData(self, url): tries = 10 counter = 0 while True: r = HttpMgr.getMethod(url, tries = 0) if (r != None): return r.text; counter += 1 time.sleep( counter * 2) if counter > tries: break
def getCurrencyExchangeRate(self, currencySymbol="", baseCurrencySymbol="PLN"): """TODO: Docstring for getCurrencyRatio. :returns: TODO """ url = "http://api.fixer.io/latest?symbols={}&base={}".format( currencySymbol, baseCurrencySymbol) url2 = \ "http://apilayer.net/api/live?access_key=ffa47d756dc89a895e23a7afa65b49b6¤cies={},PLN&format=1".format(currencySymbol) resp = HttpManager.getMethod(url) if resp is not None: json_data = json.loads(resp.text)['rates'] else: resp = HttpManager.getMethod(url2) json_data = { currencySymbol: self.parseApiLayer(resp, currencySymbol) } return json_data
def __fetchAirportAndConnections(self): self.log("fetchAirportAndConnections") if self.cfg['DEBUGGING']['state'] == 'online': httpcontent = HttpManager.getMethod(CommonData.AIRPORTS).text else: filePath = os.path.join(self.base, 'Markets.json') with open(filePath, 'r') as f: httpcontent = f.read() f.close() self._airports = self.__getAirports(httpcontent) self._connections = self.__getAirportConnections(httpcontent)
def getTimeTable(self, details={ "src_iata": "", "dst_iata": "", "year": "", "month": "" }): """ Downloads monthly timetable for connection details :details: connection details with src_iata, dst_iata, year and month :returns: content in json format from service """ src_iata = details['src_iata'] dst_iata = details['dst_iata'] year = details['year'] month = details['month'] date = max(datetime.date(year, month, 1), datetime.date.today()) date_from = date.strftime("%Y-%m-%d") #{}-{}-01".format(year, month) date_to = "{}-{}-{}".format(year, month, monthrange(year, month)[1]) url = CommonData.TimeTable.format(self.api_version, src_iata, dst_iata, date_from, date_to) filePath = os.path.join( self.base, '{0}_{1}_{2}_{3}.json'.format(src_iata, dst_iata, month, year)) if self.cfg['DEBUGGING']['state'] == 'online': self.log("Get from: " + url) httpContent = HttpManager.getMethod(url).text # self.writeToFile(filePath, httpContent) else: self.log("Get from: " + filePath) with open(filePath, 'r') as f: httpContent = f.read() f.close() json_data = { 'Dates': json.loads(httpContent)['flightDates'], 'DepartureStationCode': src_iata, 'ArrivalStationCode': dst_iata } return json_data
def __fetchAirportAndConnections(self): httpcontent = HttpManager.getMethod(CommonData.AIRPORTS).text json_data = json.loads(httpcontent) self._airports = json_data['destinations'] self._connections = self.__checkConnectionsConsistency(self._airports, json_data['relations'])
def searchByGeo(self, url): resp = HttpManager.getMethod(url).text json_data = json.loads(resp) return json_data
def getFlightDetails(self, url, params = {}, proxy = {}, headers = {}): return HttpManager.getMethod(url)
def getFlightDetails(self, url, params={}, proxy={}, headers={}): return HttpManager.getMethod(url)