def test_echo_response():
"""
Test echo response, it should return the same data being sent
"""
transport_proxy = YandexTransportProxy(SERVER_HOST, SERVER_PORT)
data = transport_proxy.get_echo('Test-Message-1234')
assert data == 'Test-Message-1234'
def test_get_line_input():
"""
Test results for various URLs
"""
transport_proxy = YandexTransportProxy(SERVER_HOST, SERVER_PORT)
# URL is None
with pytest.raises(Exception):
url = None
transport_proxy.get_line(url)
# URL is Gibberish
with pytest.raises(Exception):
url = '52086gfdgfd86534'
transport_proxy.get_line(url)
# URL is non-related to Yandex
with pytest.raises(Exception):
url = 'https://en.wikipedia.org/wiki/Taiwan'
transport_proxy.get_line(url)
# URL is for stop, not route
with pytest.raises(Exception):
# Остановка Туберкулёзный диспансер № 18
url = 'https://yandex.ru/maps/213/moscow/?ll=37.583033%2C55.815337&masstransit%5BstopId%5D=stop__9642178&mode=stop&z=17'
transport_proxy.get_line(url)
wait_random_time()
def test_count_vehicles_on_route_no_data():
"""
Count vehicles with no data provided, should return None
"""
transport_proxy = YandexTransportProxy(SERVER_HOST, SERVER_PORT)
result = transport_proxy.count_vehicles_on_route(None)
assert result is None
def test_get_stop_info_input():
"""
Test results for various URLs
"""
transport_proxy = YandexTransportProxy(SERVER_HOST, SERVER_PORT)
# URL is None
with pytest.raises(Exception):
url = None
transport_proxy.get_stop_info(url)
# URL is Gibberish
with pytest.raises(Exception):
url = 'gsdiutre4326hder'
transport_proxy.get_stop_info(url)
# URL is non-related to Yandex
with pytest.raises(Exception):
url = 'https://varlamov.ru'
transport_proxy.get_stop_info(url)
# URL is for Route, not stops
with pytest.raises(Exception):
# Route #33, Dolgoprudniy
url = 'https://yandex.ru/maps/213/moscow/?ll=37.537247%2C55.938577&masstransit%5BlineId%5D=6f6f_33_bus_default&masstransit%5BstopId%5D=stop__9686981&masstransit%5BthreadId%5D=6f6fB_33_bus_default&mode=stop&z=13'
transport_proxy.get_stop_info(url)
wait_random_time()
def test_connection_no_route():
"""
Try to connect to server to which no route exists
"""
transport_proxy = YandexTransportProxy('10.100.100.100', 65432)
# Should raise socket.error exception.
sock, error = transport_proxy._connect()
assert (sock is None) and (error == "[Errno 113] No route to host")
def test_connection_to_working_server():
"""
Try to connect to a working server
"""
transport_proxy = YandexTransportProxy(SERVER_HOST, SERVER_PORT)
# Should raise socket.error exception.
sock, error = transport_proxy._connect()
assert (sock is not None) and (error == "OK")
def test_connection_refused():
"""
Try to connect to server which will definitely refuse a connection
"""
transport_proxy = YandexTransportProxy('127.0.0.1', 65432)
# Should raise socket.error exception.
sock, error = transport_proxy._connect()
assert (sock is None) and (error == "[Errno 111] Connection refused")
def test_get_vehicles_info_from_stop():
# URL is for stop, not route
transport_proxy = YandexTransportProxy(SERVER_HOST, SERVER_PORT)
try:
# Остановка Метро Тимирязевская
url = 'https://yandex.ru/maps/213/moscow/?ll=37.575338%2C55.818374&masstransit%5BstopId%5D=stop__9639793&mode=stop&z=18'
transport_proxy.get_vehicles_info(url)
except Exception as e:
pytest.fail("An exception " + str(e) + " is raised.")
wait_random_time()
def test_get_vehicles_info_with_region_input():
"""
Test results for various URLs
"""
transport_proxy = YandexTransportProxy(SERVER_HOST, SERVER_PORT)
# URL is None
with pytest.raises(Exception):
url = None
transport_proxy.get_vehicles_info_with_region(url)
# URL is Gibberish
with pytest.raises(Exception):
url = '52086gfdgfd86534'
transport_proxy.get_vehicles_info_with_region(url)
# URL is non-related to Yandex
with pytest.raises(Exception):
url = 'https://realpython.com/python-exceptions/'
transport_proxy.get_vehicles_info_with_region(url)
# URL is for stop, not route
# UPD: 15-05-2019 Apparently, the stop URL now returns getVehiclesInfoWithRegion
#with pytest.raises(Exception):
# # Остановка Станция ЗИЛ
# url = 'https://yandex.ru/maps/213/moscow/?ll=37.649233%2C55.698713&masstransit%5BstopId%5D=stop__9711712&mode=stop&z=17'
# transport_proxy.get_vehicles_info_with_region(url)
wait_random_time()
def test_get_vehicles_info_with_region_on_stop_url():
"""
Test results for various URLs
"""
transport_proxy = YandexTransportProxy(SERVER_HOST, SERVER_PORT)
# URL is for stop, not route
# 15-05-2019 Apparently, the stop URL now returns getVehiclesInfoWithRegion
try:
# Остановка Метро Марьина Роща
url = 'https://yandex.ru/maps/213/moscow/?ll=37.613299%2C55.797208&masstransit%5BstopId%5D=stop__9639993&mode=stop&z=13'
transport_proxy.get_vehicles_info_with_region(url)
except Exception as e:
pytest.fail("An exception " + str(e) + " is raised.")
wait_random_time()
def set_proxy():
global result
result = []
print("STARTING FUNCTION TESTS")
print()
global proxy
proxy = YandexTransportProxy(PROXY_HOST, PROXY_PORT)
yield None
with open(RESULT_FILENAME, "w") as f:
f.write(json.dumps(result))
def test_get_vehicles_info_input():
"""
Test results for various URLs
"""
transport_proxy = YandexTransportProxy(SERVER_HOST, SERVER_PORT)
# URL is None
with pytest.raises(Exception):
url = None
transport_proxy.get_vehicles_info(url)
# URL is Gibberish
with pytest.raises(Exception):
url = '52086gfdgfd86534'
transport_proxy.get_vehicles_info(url)
# URL is non-related to Yandex
with pytest.raises(Exception):
url = 'https://habr.com/en/all/'
transport_proxy.get_vehicles_info(url)
def test_get_all_info_input():
"""
Test results for various URLs
"""
transport_proxy = YandexTransportProxy(SERVER_HOST, SERVER_PORT)
# URL is None
with pytest.raises(Exception):
url = None
transport_proxy.get_all_info(url)
# URL is Gibberish
with pytest.raises(Exception):
url = '52086gfdgfd86534'
transport_proxy.get_all_info(url)
# URL is non-related to Yandex
with pytest.raises(Exception):
url = 'https://www.random.org/'
transport_proxy.get_all_info(url)
# NOTE: Stop in Yakutsk (Школа №7) now returns getLayerRegions method
# Last test is removed from here, it wasn't meant to be here anyway.
wait_random_time()
def perform_data_collection():
"""
Data collection test, every single request should return valid JSON object.
This test can be switched off, and data can be loaded from files instead during development.
This takes a huge amount of time to process, by the way, due to wait times between queries
(We don't want Yandex to get angry due to frequent queries, so we're playing safe here).
Expect about 40-60 minutes of data collection.
"""
global query_results
global do_data_collection
global do_stations_collection
global do_routes_collection
global data_collection_passed
if not do_data_collection:
return
if data_collection_passed:
return
print()
proxy = YandexTransportProxy(SERVER_HOST, SERVER_PORT)
if do_stations_collection:
for station, url in station_urls.items():
print("Collecting station: " + station + "... ", end='')
result = ''
try:
result = proxy.get_all_info(url)
for entry in result:
query_results.append({
"success": True,
"station": station,
"url": url,
"method": entry['method'],
"data": entry['data']
})
print(entry['method'], end=' ')
print("[OK]")
except Exception as e:
query_results.append({
"success": False,
"station": station,
"url": url,
"method": "getAllInfo (failed)",
"data": ""
})
print("[FAILED]")
print("Exception (station): ", str(e))
f = open('tests/testdata/output/station_' +
station.replace('/', '-') + '.json.pretty',
'w',
encoding='utf-8')
f.write(
json.dumps(result,
ensure_ascii=False,
indent=4,
separators=(',', ': ')))
f.close()
f = open('tests/testdata/output/station_' +
station.replace('/', '-') + '.json',
'w',
encoding='utf-8')
f.write(json.dumps(result, ensure_ascii=False))
f.close()
wait_random_time()
if do_routes_collection:
for route, url in routes_urls.items():
print("Collecting route: " + route + "... ", end='')
result = ''
try:
result = proxy.get_all_info(url)
for entry in result:
query_results.append({
"success": True,
"route": route,
"url": url,
"method": entry['method'],
"data": entry['data']
})
print(entry['method'], end=' ')
print("[OK]")
except Exception as e:
query_results.append({
"success": False,
"route": route,
"url": url,
"method": "getAllInfo (failed)",
"data": ""
})
print("[FAILED]")
print("Exception (route): ", str(e))
f = open('tests/testdata/output/route_' + route.replace('/', '-') +
'.json.pretty',
'w',
encoding='utf-8')
f.write(
json.dumps(result,
ensure_ascii=False,
indent=4,
separators=(',', ': ')))
f.close()
f = open('tests/testdata/output/route_' + route.replace('/', '-') +
'.json',
'w',
encoding='utf-8')
f.write(json.dumps(result, ensure_ascii=False))
f.close()
wait_random_time()
# Saving data to files
f = open('test_data.json', 'w', encoding='utf-8')
f.write(json.dumps(query_results, ensure_ascii=False))
f.close()
# Setting "data collection passed" flag.
data_collection_passed = True
# Basically, always succeeds
assert True == True
def __init__(self, parent, host, port):
super().__init__()
self.parent = parent
self.host = host
self.port = port
self.proxy = YandexTransportProxy(host, port)
class ExecutorThread(threading.Thread):
"""
Executor Thread class, will periodically poll Yandex Transport Proxy server.
"""
def __init__(self, parent, host, port):
super().__init__()
self.parent = parent
self.host = host
self.port = port
self.proxy = YandexTransportProxy(host, port)
def run(self):
while self.parent.is_running:
self.parent.display_error = ""
json_data = []
if self.parent.data_source == self.parent.DATA_SOURCE_FILE:
try:
json_data = self.parent.load_data_from_file(
self.parent.source_url)
self.parent.data_collection_status = self.parent.DATA_COLLECTION_OK
except Exception as e:
self.parent.display_error = "Exception (data load from file)" + str(
e)
self.parent.data_collection_status = self.parent.DATA_COLLECTION_FAILED
else:
try:
json_data = self.proxy.get_stop_info(
self.parent.source_url, timeout=self.parent.timeout)
self.parent.data_collection_status = self.parent.DATA_COLLECTION_OK
except Exception as e:
self.parent.display_error = str(e)
self.parent.data_collection_status = self.parent.DATA_COLLECTION_FAILED
self.parent.update_time = str(
datetime.datetime.now().time().strftime('%H:%M:%S'))
# Storing data to file if log_dir was specified
if self.parent.log_dir != '':
filename = self.parent.log_dir + '/' + \
str(datetime.datetime.now()).replace(':', '_') + \
'.json'
try:
f = open(filename, 'w', encoding='utf-8')
f.write(
json.dumps(json_data,
ensure_ascii=False,
indent=4,
separators=(',', ': ')))
f.close()
except Exception as e:
self.parent.display_error = str(e)
try:
self.parent.yandex_timestamp, _ = self.parent.get_yandex_timestamp(
json_data)
except Exception as e:
self.parent.display_error = "Exception (getting Yandex timestamp)" + str(
e)
# Copy data to parent
self.parent.data_lock.acquire()
self.parent.data = json_data.copy()
self.parent.data_lock.release()
# Wait for some time
for _ in range(0, self.parent.wait_time):
if not self.parent.is_running:
break
time.sleep(1)
print("EXECUTOR THREAD TERMINATED!")
def parse_route(yandex_route_id, yandex_thread_id, db_settings, ytproxy_host, ytproxy_port, timeout, force_overwrite=False):
route_id = yandex_route_id
thread_id = yandex_thread_id
url = form_route_url(route_id, thread_id)
print("Time:", str(datetime.datetime.now()))
print()
print("ID:", route_id)
try:
conn = psycopg2.connect(dbname=db_settings['db_name'],
host=db_settings['db_host'],
user=db_settings['db_user'],
port=db_settings['db_port'],
password=db_settings['db_password'])
except Exception as e:
print("Exception (connect to database in parse_route):" + str(e))
return 1
cur = conn.cursor()
sql_query = "SELECT * FROM routes WHERE route_id='" + route_id + "'"
try:
cur.execute(sql_query)
rows = cur.fetchall()
except Exception as e:
print("Exception (query existing stop):" + str(e))
return 1
if not rows:
do_insert_route = True
else:
do_insert_route = False
print("Route already in database")
if not force_overwrite:
return 2
try:
print("Getting data...")
proxy = YandexTransportProxy(ytproxy_host, ytproxy_port)
print("URL:", url)
data = proxy.line(url, timeout=timeout)
#data = json.load(open('route_troll_10_nsk.json', 'r', encoding='utf-8'))
except Exception as e:
print("Exception (obtain data):" + str(e))
return 1
try:
route_id = data['data']['features'][0]['properties']['ThreadMetaData']['lineId']
except:
pass
try:
thread_id = data['data']['features'][0]['properties']['ThreadMetaData']['id']
except Exception as e:
print("Exception (thread_id):" + str(e))
return 0
route_type = ""
try:
route_type = data['data']['features'][0]['properties']['ThreadMetaData']['type']
except:
pass
route_name = ""
try:
route_name = data['data']['features'][0]['properties']['ThreadMetaData']['name']
except:
pass
print("ID :", route_id)
print("ThreadID:", thread_id)
print("Type :", route_type)
print("Name :", route_name)
print("")
if do_insert_route:
print("Inserting stop data into database...")
sql_query = "INSERT INTO routes(route_id, name, type, timestamp, data) " \
"VALUES (" + \
"'" + route_id.translate(str.maketrans({"'":r"''"})) + "'" + "," + \
"'" + route_name.translate(str.maketrans({"'":r"''"})) + "'" + "," + \
"'" + route_type.translate(str.maketrans({"'":r"''"})) + "'" + "," + \
"TIMESTAMP '" + str(datetime.datetime.now()) + "'," \
"'" + json.dumps(data).translate(str.maketrans({"'":r"''"})) + "'" + \
")"
try:
cur.execute(sql_query)
except Exception as e:
print("Exception (insert stop):" + str(e))
return 1
print("Done")
print("")
queue_values = []
for i, feature in enumerate(data['data']['features'], start=1):
print("Segment", i)
for j, segment in enumerate(feature['features'], start=1):
# This is a point stop
if 'properties' in segment:
print(str(j).ljust(3), end='')
print(segment['properties']['name'].ljust(30), end="")
if 'StopMetaData' in segment['properties']:
print(segment['properties']['StopMetaData']['id'].ljust(20), end="")
sql_query_x = "SELECT data_id FROM queue WHERE data_id='" + segment['properties']['StopMetaData']['id'] + "' AND type='stop'"
cur.execute(sql_query_x)
rows = cur.fetchall()
if not rows:
print("NEW")
queue_values.append(segment['properties']['StopMetaData']['id'])
else:
print("QUEUED")
queue_str = ""
for i in range(0, len(queue_values) - 1):
queue_str += "(" + "'stop'," + "'" + queue_values[i] + "'" + ")" + ","
if queue_values:
queue_str += "(" + "'stop'," + "'" + queue_values[-1] + "'" + ")"
if queue_values:
try:
sql_query = "INSERT INTO queue(type, data_id) VALUES " + queue_str
cur.execute(sql_query)
except Exception as e:
print("Exception (insert into queue):" + str(e))
return 1
conn.commit()
cur.close()
conn.close()
print("ROUTE " + route_id + ", " + route_type + " " + route_name, ": PARSING COMPLETE!")
return 0
def create_if_not_exists(path):
if not os.path.exists(path):
print('Creating ', path)
os.mkdir(path)
def remove_if_exists(path):
if os.path.exists(path):
os.remove(path)
GENERATED_DIR = "generated_files/"
PROJECT_PREFIX = str(Path.home()) + "/TransportMonitoring/"
proxy = YandexTransportProxy('127.0.0.1', 25555)
create_if_not_exists(PROJECT_PREFIX + GENERATED_DIR)
DATABASE_PATH = PROJECT_PREFIX + "buses.db"
MY_DATABASE = SqliteDatabase(DATABASE_PATH)
PROXY_CONNECT_TIMEOUT = 5
ROUTES_FIELDS = {
'732': {
'line_id': "213_732_bus_mosgortrans",
'thread_id': "213A_732_bus_mosgortrans",
'stop_id': '9644642'
}
def parse_stop(yandex_stop_id,
db_settings,
ytproxy_host,
ytproxy_port,
timeout,
force_overwrite=False):
url = form_stop_url(yandex_stop_id)
stop_id = yandex_stop_id
print("Time:", str(datetime.datetime.now()))
print()
print("ID: " + yandex_stop_id)
#stop_id = 'stop__9647487' # Get from CLI instead, or make it a function.
try:
conn = psycopg2.connect(dbname=db_settings['db_name'],
host=db_settings['db_host'],
user=db_settings['db_user'],
port=db_settings['db_port'],
password=db_settings['db_password'])
except Exception as e:
print("Exception (connect to database in parse_stop):" + str(e))
return 1
cur = conn.cursor()
sql_query = "SELECT * FROM stops WHERE stop_id='" + stop_id + "'"
try:
cur.execute(sql_query)
rows = cur.fetchall()
except Exception as e:
print("Exception (query existing stop):" + str(e))
return 1
if not rows:
do_insert_stop = True
else:
do_insert_stop = False
print("Stop already in database")
if not force_overwrite:
return 2
try:
print("Getting data...")
proxy = YandexTransportProxy(ytproxy_host, ytproxy_port)
data = proxy.get_stop_info(url, timeout=timeout)
#data = json.load(open('stop_maryino.json', 'r', encoding='utf-8'))
except Exception as e:
print("Exception (obtain data):" + str(e))
return 1
print("")
# IMPORTANT! For each step check if failed!
try:
stop_id = data['data']['properties']['StopMetaData']['id']
except Exception as e:
print("Exception (get stop_id):" + str(e))
return 1
# Step 1: Check if current stop is in database,
# If not - save current stop data to Database
# Step 2: Get list of route IDs passing through this stop
try:
routes = data['data']['properties']['StopMetaData']['Transport']
except Exception as e:
print("Exception (get routes) : ")
return 1
stop_name = ''
try:
stop_name = data['data']['properties']['StopMetaData']['name']
except Exception as e:
print("No stop name in data!")
print("Stop name:", stop_name)
region = ''
try:
region = data['data']['properties']['StopMetaData']['region']['name']
except:
print("No region name in data!")
print("Region :", region)
print("")
if do_insert_stop:
print("Inserting stop data into database...")
sql_query = "INSERT INTO stops(stop_id, name, region, timestamp, data) " \
"VALUES (" + \
"'" + stop_id.translate(str.maketrans({"'":r"''"}))+ "'" + "," + \
"'" + stop_name.translate(str.maketrans({"'":r"''"})) + "'" + "," + \
"'" + region.translate(str.maketrans({"'":r"''"})) + "'" + "," + \
"TIMESTAMP '" +str(datetime.datetime.now()) + "'," \
"'" + json.dumps(data).translate(str.maketrans({"'":r"''"})) + "'" + \
")"
try:
cur.execute(sql_query)
except Exception as e:
print("Exception (insert stop):" + str(e))
return 1
print("Done")
print("")
print("Found routes:")
print("routeId".ljust(25), "threadId".ljust(25), "type".ljust(10),
"name".ljust(18), "status")
queue_values = []
for i, route in enumerate(routes):
if not 'name' in route:
raise Exception('No name in route!')
if not 'id' in route:
raise Exception('No id in route')
if not 'lineId' in route:
raise Exception('No lineId in route')
if not 'type' in route:
raise Exception('No type in route')
print(route['lineId'].ljust(25),
route['id'].ljust(25),
route['type'].ljust(10),
route['name'].ljust(18),
end=' ')
sql_query = "SELECT route_id FROM routes WHERE route_id='" + route[
'lineId'] + "'"
try:
cur.execute(sql_query)
except Exception as e:
print("Exception (query routes):" + str(e))
return 1
rows = cur.fetchall()
if not rows:
sql_query_x = "SELECT data_id FROM queue WHERE data_id='" + route[
'lineId'] + "' AND type='route'"
try:
cur.execute(sql_query_x)
except Exception as e:
print("Exception (query queue):" + str(e))
return 1
rows_x = cur.fetchall()
if not rows_x:
print("NEW")
queue_values.append([route['lineId'], route['id']])
else:
print("QUEUED")
else:
print("")
queue_str = ""
for i in range(0, len(queue_values) - 1):
queue_str += "(" + "'route'," + "'" + queue_values[i][
0] + "', '" + queue_values[i][1] + "')" + ","
if queue_values:
queue_str += "(" + "'route'," + "'" + queue_values[-1][
0] + "', '" + queue_values[-1][1] + "')"
if queue_values:
try:
sql_query = "INSERT INTO queue(type, data_id, thread_id) VALUES " + queue_str
cur.execute(sql_query)
except Exception as e:
print("Exception (insert into queue):" + str(e))
return 1
print("")
toponyms = []
# Old toponym search response object
try:
toponyms = data['data']['toponymSearchResponse']['items']
except:
print("No 'toponymSearchResponse' field.")
# New toponym search response object
try:
toponyms.append(data['data']['searchResult'])
except:
print("No 'searchResult' field.")
# Nearest stops: METRO
nearest_stops = []
queue_values = []
try:
for value in toponyms:
for stop in value['metro']:
nearest_stops.append(stop)
for stop in value['stops']:
nearest_stops.append(stop)
except:
pass
print("Nearest Stops:")
for entry in nearest_stops:
try:
print(entry['id'].ljust(30), entry['name'].ljust(50), end=' ')
except:
pass
sql_query = "SELECT data_id FROM queue WHERE data_id='" + entry[
'id'] + "' AND type='stop'"
cur.execute(sql_query)
rows = cur.fetchall()
if not rows:
print("NEW")
queue_values.append(entry['id'])
else:
print("QUEUED")
queue_str = ""
for i in range(0, len(queue_values) - 1):
queue_str += "(" + "'stop'," + "'" + queue_values[i] + "'" + ")" + ","
if queue_values:
queue_str += "(" + "'stop'," + "'" + queue_values[-1] + "'" + ")"
if queue_values:
try:
sql_query = "INSERT INTO queue(type, data_id) VALUES " + queue_str
cur.execute(sql_query)
except:
print("Exception (insert into queue):" + str(e))
return 1
print("")
conn.commit()
cur.close()
conn.close()
print("STOP " + stop_id + ", " + stop_name + " : PARSING COMPLETE!")
return 0
#!/usr/bin/env python3
"""
Get all available info for all masstransit methods and save them to the file (data.json)
"""
import json
from yandex_transport_webdriver_api import YandexTransportProxy
proxy = YandexTransportProxy('127.0.0.1', 25555)
# The stop is this one: Метро Марьино (северная)
url = "https://yandex.ru/maps/213/moscow/stops/stop__9647487/?ll=37.742975%2C55.651185&z=18"
print(
"This will take a while, about 30 secs (dirty hack to make getStopInfo appear)"
)
data = proxy.get_all_info(url)
print("")
print(data)
# Saving result to output file
with open('data.json', 'w') as file:
file.write(json.dumps(data, indent=4, separators=(',', ': ')))