def test_unavailable_run_script_during_add_tasks(self):
"""
- add tasks
Expected: selenium log was saved and endpoint was deleted
"""
from vmpool.artifact_collector import ArtifactCollector
with patch(
'core.db.Database', DatabaseMock()
):
from core.db.models import Session, Endpoint, Provider
session = Session("origin_1")
session.id = self.session_id
log_path = os.sep.join([self.artifact_dir, 'selenium_server.log'])
provider = Provider(name='noname', url='nourl')
endpoint = Endpoint(Mock(), '', provider)
endpoint.ip = '127.0.0.1'
endpoint.name = 'test_endpoint'
endpoint.ports = {'selenium': '4455', 'agent': '9000', 'vnc': '5900'}
session.endpoint = endpoint
self.app.sessions.get_session = Mock(return_value=session)
art_collector = ArtifactCollector(database=Mock())
in_queue = art_collector.save_artifact(session, 'selenium_server.log', '/var/log/selenium_server.log')
self.assertTrue(in_queue)
self.assertTrue(wait_for(
lambda: len(art_collector.get_queue()) == 0))
with open(log_path, 'r') as f:
text = f.read()
self.assertIn('Connection refused', text)
art_collector.stop()
def setUp(self):
self.ctx = self.app.test_request_context()
self.ctx.push()
with patch('flask.current_app.database',
DatabaseMock()), patch('flask.current_app.sessions',
Mock()):
from core.db.models import Session, Provider, Endpoint
self.session = Session('origin_1')
self.session.name = "session1"
provider = Provider(name='noname', url='nourl')
vm = Endpoint(Mock(), '', provider)
vm.name = 'vm1'
vm.ip = self.host
vm.ports = {
'selenium': self.webdriver_server.port,
'agent': self.vmmaster_agent.port,
'vnc': self.vnc_server.port
}
self.session.endpoint = vm
self.session.run()
from vmmaster.webdriver import commands
self.commands = commands
def test_when_during_proxy_starting_already_started_from_other_thread(
self):
from core.db.models import Session, Endpoint, Provider
provider = Provider(name='noname', url='nourl')
endpoint = Endpoint(Mock(), '', provider)
endpoint.ip = '127.0.0.1'
endpoint.name = 'test_endpoint'
endpoint.ports = {'4455': 4455, '9000': 9000, '5900': 5900}
session = Session("some_platform")
session.id = 1
session.name = "session1"
session.status = "running"
type(session).vnc_proxy_port = PropertyMock(
side_effect=[None, 55555, 55555])
session.vnc_proxy_pid = 55555
session.created = session.modified = datetime.now()
session.endpoint = endpoint
session.stop_vnc_proxy = Mock()
with patch('flask.current_app.sessions.get_session',
Mock(return_value=session)):
response = self.vmmaster_client.get(
'/api/session/{}/vnc_info'.format(session.id))
session._close()
body = json.loads(response.data)
self.assertEqual(200, response.status_code)
self.assertDictEqual({'vnc_proxy_port': 55555}, body['result'])
self.assertEqual(200, body['metacode'])
self.assertTrue(session.stop_vnc_proxy.called)
def test_500_code_run_script_during_add_tasks(self):
"""
- add tasks
Expected: selenium log was saved and endpoint was deleted
"""
from vmpool.artifact_collector import ArtifactCollector
with patch(
'core.db.Database', DatabaseMock()
):
from core.db.models import Session, Endpoint, Provider
session = Session("origin_1")
session.id = self.session_id
provider = Provider(name='noname', url='nourl')
endpoint = Endpoint(Mock(), '', provider)
endpoint.ip = '127.0.0.1'
endpoint.name = 'test_endpoint'
endpoint.ports = {'selenium': '4455', 'agent': '9000', 'vnc': '5900'}
session.endpoint = endpoint
art_collector = ArtifactCollector(database=Mock())
in_queue = art_collector.save_artifact(session, 'selenium_server.log', '/var/log/selenium_server.log')
self.assertTrue(in_queue)
self.assertTrue(wait_for(
lambda: len(art_collector.get_queue()) == 0))
art_collector.stop()
def __init__(self):
self.session = Session()
self.ruz_api = RuzApi()
self.nvr_api = Nvr_Api()
self.calendar_api = GCalendar()
self.ruz = self.session.query(OnlineRoom).filter_by(name="РУЗ").first()
self.jitsi = self.session.query(OnlineRoom).filter_by(
name="Jitsi").first()
def test_get_vnc_proxy_port_if_session_is_waiting(self):
from core.db.models import Session, Endpoint, Provider
provider = Provider(name='noname', url='nourl')
endpoint = Endpoint(Mock(), '', provider)
endpoint.ip = '127.0.0.1'
endpoint.name = 'test_endpoint'
endpoint.ports = {'4455': 4455, '9000': 9000, '5900': 5900}
session = Session("some_platform")
session.name = "session1"
session.id = 1
session.status = "waiting"
session.vnc_proxy_port = None
session.vnc_proxy_pid = None
session.created = session.modified = datetime.now()
session.endpoint = endpoint
with patch('flask.current_app.sessions.get_session',
Mock(return_value=session)):
response = self.vmmaster_client.get(
'/api/session/{}/vnc_info'.format(session.id))
body = json.loads(response.data)
self.assertEqual(200, response.status_code)
self.assertDictEqual({}, body['result'])
self.assertEqual(500, body['metacode'])
def test_api_stop_session(self):
from core.db.models import Session
session = Session("some_platform")
session.failed = Mock()
with patch('flask.current_app.sessions.get_session',
Mock(return_value=session)):
response = self.vmmaster_client.get("/api/session/{}/stop".format(
session.id))
body = json.loads(response.data)
self.assertEqual(200, body['metacode'])
session.failed.assert_any_call(
reason=constants.SESSION_CLOSE_REASON_API_CALL)
def test_endpoints_cleanup(self):
"""
- endpoint1 linked with session
- endpoint2 not linked with session
- both endpoints mark as 'deleted'
expected: endpoint1 deleted, endpoint2 not deleted
"""
class FakeOrigin(str):
short_name = 'fake_short_name'
from core.db.models import Session, Endpoint, Provider
provider = Provider('name', 'url')
endpoint1 = Endpoint(origin=FakeOrigin('fake'),
prefix='prefix',
provider=provider)
endpoint2 = Endpoint(origin=FakeOrigin('fake'),
prefix='prefix',
provider=provider)
endpoint1.deleted, endpoint2.deleted = True, True
endpoint1.save(), endpoint2.save()
session = Session(platform='some_platform',
name='__test_keep_forever_sessions_1')
session.refresh()
session.endpoint = endpoint1
session.save()
endpoints_to_delete = [
e.id for e in self.cleanup.endpoints_to_delete()
]
self.assertNotIn(endpoint1.id, endpoints_to_delete)
self.assertIn(endpoint2.id, endpoints_to_delete)
def test_api_sessions(self):
from core.db.models import Session
session = Session(self.platform, "session1",
self.desired_caps["desiredCapabilities"])
session.created = session.modified = datetime.now()
with patch('flask.current_app.sessions.active',
Mock(return_value=[session])):
response = self.vmmaster_client.get('/api/sessions')
body = json.loads(response.data)
self.assertEqual(200, response.status_code)
sessions = body['result']['sessions']
self.assertEqual(1, len(sessions))
self.assertEqual(self.platform, session.platform)
self.assertEqual(200, body['metacode'])
session.failed()
def setUp(self):
setup_config('data/config_openstack.py')
self.host = "localhost"
self.port = config.PORT
self.address = (self.host, self.port)
self.vmmaster = vmmaster_server_mock(self.port)
self.assertTrue(server_is_up(self.address))
self.free_port = get_free_port()
self.ctx = self.vmmaster.app.app_context()
self.ctx.push()
from core.db.models import Session, Provider, Endpoint
self.session = Session('some_platform')
provider = Provider(name='noname', url='nourl')
endpoint = Endpoint(Mock(), '', provider)
endpoint.ip = 'localhost'
self.session.endpoint = endpoint
def test_sessions_overflow(self):
user = Mock(id=1, max_stored_sessions=0)
from core.db.models import Session
session = Session('some_platform')
session.status = 'unknown'
session.closed = True
session.name = '__test_outdated_sessions'
session.save()
session_ids_to_delete = [
p.id for p in self.cleanup.sessions_overflow(user)
]
self.assertIn(session.id, session_ids_to_delete)
self.cleanup.delete_session_data([session])
def test_file_deletion(self):
from core.db.models import Session
session = Session('some_platform')
session.status = 'unknown'
session.name = '__test_file_deletion'
session.save()
session_dir = os.path.join(config.SCREENSHOTS_DIR, str(session.id))
system_utils.run_command(["mkdir", config.SCREENSHOTS_DIR],
silent=True)
system_utils.run_command(["mkdir", session_dir], silent=True)
system_utils.run_command(
["touch", os.path.join(session_dir, "file_for_deletion")],
silent=True)
self.cleanup.delete_session_data([session])
self.assertEqual(os.path.isdir(session_dir), 0)
system_utils.run_command(["rm", "-rf", config.SCREENSHOTS_DIR],
silent=True)
def get_session():
profiler.register_get_session_call()
dc = commands.get_desired_capabilities(request)
matched_platform, provider_id = current_app.get_matched_platforms(dc)
if not matched_platform:
raise SessionException("Cannot match platform for DesiredCapabilities: {}".format(dc))
session = Session(platform=matched_platform, dc=dc, provider_id=provider_id)
request.session = session
log.info("New session {}({}) on provider {} with dc: {}".format(session.id, session.name, provider_id, dc))
yield session
start_time = time.time()
while not session.endpoint_id:
time.sleep(constants.GET_SESSION_SLEEP_TIME)
if time.time() - start_time >= config.GET_VM_TIMEOUT:
raise CreationException("Timeout getting endpoint for {}".format(session))
session.refresh()
yield session
session.run()
yield session
class TestHttpProxy(BaseTestCase):
def setUp(self):
setup_config('data/config_openstack.py')
self.host = "localhost"
self.port = config.PORT
self.address = (self.host, self.port)
self.vmmaster = vmmaster_server_mock(self.port)
self.assertTrue(server_is_up(self.address))
self.free_port = get_free_port()
self.ctx = self.vmmaster.app.app_context()
self.ctx.push()
from core.db.models import Session, Provider, Endpoint
self.session = Session('some_platform')
provider = Provider(name='noname', url='nourl')
endpoint = Endpoint(Mock(), '', provider)
endpoint.ip = 'localhost'
self.session.endpoint = endpoint
def tearDown(self):
self.session._close()
self.ctx.pop()
self.vmmaster.app.sessions.kill_all()
self.vmmaster.app.cleanup()
self.vmmaster.stop_services()
self.assertTrue(server_is_down(self.address))
def test_proxy_successful(self):
server = ServerMock(self.host, self.free_port)
server.start()
with patch(
'core.sessions.Sessions.get_session', Mock(
return_value=self.session)
):
response = requests.get(
"http://%s:%s/proxy/session/%s/port/%s/" %
(self.host, self.port, self.session.id, self.free_port)
)
server.stop()
self.assertEqual("ok", response.content)
def test_proxy_responses_when_trying_to_connect_failed(self):
with patch(
'core.sessions.Sessions.get_session', Mock(
return_value=self.session)
):
response = requests.get(
"http://%s:%s/proxy/session/%s/port/%s/" %
(self.host, self.port, self.session.id, self.free_port)
)
self.assertEqual(
"Request forwarding failed:\n"
"Connection was refused by other side: 111: Connection refused.",
response.content)
def test_proxy_to_session_that_was_closed(self):
self.session.succeed()
with patch(
'flask.current_app.database.get_session',
Mock(return_value=self.session)
):
response = requests.get(
"http://%s:%s/proxy/session/%s/port/%s/" %
(self.host, self.port, self.session.id, self.free_port)
)
self.assertIn(
"Session {}(Success) already closed earlier".format(self.session.id),
response.content
)
def test_proxy_with_wrong_path(self):
response = requests.get(
"http://%s:%s/proxy/asdf/%s/" %
(self.host, self.port, self.free_port)
)
self.assertEqual(
"Couldn't parse request uri, "
"make sure you request uri has "
"/proxy/session/<session_id>/port/<port_number>/<destination>",
response.content)
class CommonCommandsTestCase(BaseTestCase):
webdriver_server = None
vmmaster_agent = None
vnc_server = None
host = 'localhost'
@classmethod
def setUpClass(cls):
setup_config("data/config_openstack.py")
body = {
"sessionId": None,
"desiredCapabilities": {
"platform": "some_platform",
"browserName": "firefox",
"version": "",
"javascriptEnabled": True
}
}
session_request_body = json.dumps(body)
session_request_headers = {
'content-length': '%s' % len(session_request_body),
'accept-encoding': 'identity',
'Connection': 'close',
'accept': 'application/json',
'user-agent': 'Python-urllib/2.7',
'host': '127.0.0.1:9000',
'content-type': 'application/json;charset=UTF-8',
}
cls.request = Mock()
cls.request.method = "POST"
cls.request.path = "/wd/hub/session"
cls.request.headers = dict()
cls.request.headers.update(session_request_headers)
cls.request.data = session_request_body
cls.webdriver_server = ServerMock(cls.host, get_free_port())
cls.webdriver_server.start()
cls.vmmaster_agent = ServerMock(cls.host, get_free_port())
cls.vmmaster_agent.start()
cls.vnc_server = ServerMock(cls.host, get_free_port())
cls.vnc_server.start()
cls.app = Flask(__name__)
cls.app.database = None
cls.app.sessions = None
cls.app.database_task_queue = Mock()
cls.app.pool = Mock()
def setUp(self):
self.ctx = self.app.test_request_context()
self.ctx.push()
with patch('flask.current_app.database',
DatabaseMock()), patch('flask.current_app.sessions',
Mock()):
from core.db.models import Session, Provider, Endpoint
self.session = Session('origin_1')
self.session.name = "session1"
provider = Provider(name='noname', url='nourl')
vm = Endpoint(Mock(), '', provider)
vm.name = 'vm1'
vm.ip = self.host
vm.ports = {
'selenium': self.webdriver_server.port,
'agent': self.vmmaster_agent.port,
'vnc': self.vnc_server.port
}
self.session.endpoint = vm
self.session.run()
from vmmaster.webdriver import commands
self.commands = commands
def tearDown(self):
with patch('flask.current_app.sessions',
Mock()), patch('flask.current_app.database', Mock()):
self.session._close()
self.ctx.pop()
@classmethod
def tearDownClass(cls):
cls.webdriver_server.stop()
cls.vmmaster_agent.stop()
cls.vnc_server.stop()
del cls.app
def test_get_vnc_port_if_running_proxy(self):
from core.db.models import Session, Endpoint, Provider
provider = Provider(name='noname', url='nourl')
endpoint = Endpoint(Mock(), '', provider)
endpoint.ip = '127.0.0.1'
endpoint.name = 'test_endpoint'
endpoint.ports = {'4455': 4455, '9000': 9000, '5900': 5900}
session = Session("some_platform")
session.id = 1
session.name = "session1"
session.status = "running"
session.vnc_proxy_port = 55555
session.vnc_proxy_pid = 55555
session.created = session.modified = datetime.now()
session.endpoint = endpoint
session.stop_vnc_proxy = Mock()
with patch('flask.current_app.sessions.get_session',
Mock(return_value=session)), patch(
'core.utils.kill_process', Mock(return_value=True)):
response = self.vmmaster_client.get('/api/session/%s/vnc_info' %
session.id)
session._close()
body = json.loads(response.data)
self.assertEqual(200, response.status_code)
self.assertDictEqual({'vnc_proxy_port': 55555}, body['result'])
self.assertEqual(200, body['metacode'])
self.assertTrue(session.stop_vnc_proxy.called)
def test_session_keep_forever(self):
user = Mock(id=1, max_stored_sessions=0)
from core.db.models import Session
session1 = Session(platform='some_platform',
name='__test_keep_forever_sessions_1')
session1.closed = True
session1.keep_forever = True
session1.save()
session2 = Session(platform='some_platform',
name='__test_keep_forever_sessions_2')
session2.closed = True
session2.keep_forever = False
session2.save()
session_ids_to_delete = [
p.id for p in self.cleanup.sessions_overflow(user)
]
self.assertNotIn(session1.id, session_ids_to_delete)
self.assertIn(session2.id, session_ids_to_delete)
self.cleanup.delete_session_data([session1, session2])
class CalendarManager:
def __init__(self):
self.session = Session()
self.ruz_api = RuzApi()
self.nvr_api = Nvr_Api()
self.calendar_api = GCalendar()
self.ruz = self.session.query(OnlineRoom).filter_by(name="РУЗ").first()
self.jitsi = self.session.query(OnlineRoom).filter_by(
name="Jitsi").first()
def __del__(self):
self.session.close()
async def get_rooms(self):
offline_rooms = [room.name for room in self.session.query(Room).all()]
rooms = await self.ruz_api.get_auditoriumoid()
tasks = [
self.synchronize_lessons_in_room(room["auditoriumOid"],
offline_rooms, room["number"])
for room in rooms
]
await asyncio.gather(*tasks)
logger.info(
f"Created events for {datetime.today().date() + timedelta(days=1)} - {datetime.today().date() + timedelta(days=self.ruz_api.period)}"
)
async def synchronize_lessons_in_room(self, room_id: str,
offline_rooms: list, room_name: str):
lessons = await self.get_lessons_from_room(room_id)
if lessons:
logger.info(f"""
Successfully got lessons for room {room_name}
Synchronizing lessons in calendar and Erudite
""")
# Deletes lessons from Erudite if it doesn't exist in Ruz
await self.nvr_api.check_delete_Erudite_lessons(lessons, room_id)
time.sleep(0.2)
for i in range(0, len(lessons), 10):
chunk = lessons[i:i + 10]
tasks = [
self.synchronize_lesson(room_id, lesson, offline_rooms)
for lesson in chunk
]
await asyncio.gather(*tasks)
async def get_lessons_from_room(self, room_id: str) -> list:
""" Get lessons in room from ruz """
try:
lessons = await self.ruz_api.get_lessons(room_id)
except Exception as err:
lessons = None
logger.error(err)
return lessons
async def synchronize_lesson(
self,
room_id: str,
lesson: dict,
offline_rooms: list,
):
check_data = await self.nvr_api.check_lesson(lesson)
status = check_data[0]
# Lesson not found in Erudite, so we add it
if status == "Not found":
await self.add_lesson(lesson, offline_rooms)
# Lesson found in Erudite, but the data of this lesson has to be updated
elif status == "Update":
lesson_id = check_data[1]
event_id = check_data[2]
await self.update_lesson(lesson, offline_rooms, lesson_id,
event_id)
time.sleep(0.6)
async def add_lesson(self, lesson: dict, offline_rooms: list):
""" Adds lesson to Erudite and Google Calendar """
if lesson["ruz_url"] is None or "meet.miem.hse.ru" not in lesson[
"ruz_url"]:
logger.info("Adding ruz lesson")
data = await self.test_post_lesson(lesson)
code = data[0]
erudite_lesson = data[1]
event = None
if code == 201 or code == 409:
try:
event = await self.post_lesson(lesson,
erudite_lesson["id"],
self.ruz.calendar)
time.sleep(0.6)
except:
logger.warning(f"Erudite returned - {erudite_lesson}")
if lesson["ruz_auditorium"] in offline_rooms and event:
room = self.session.query(Room).filter_by(
name=lesson["ruz_auditorium"]).first()
self.create_record(room, event)
elif lesson["ruz_url"] is not None and "meet.miem.hse.ru" in lesson[
"ruz_url"]:
logger.info("Adding jitsi lesson")
data = await self.test_post_lesson(lesson)
code = data[0]
erudite_lesson = data[1]
if code == 201 or code == 409:
try:
event = await self.post_lesson(lesson,
erudite_lesson["id"],
self.jitsi.calendar)
except:
logger.warning(f"Erudite returned - {erudite_lesson}")
async def update_lesson(self, lesson: dict, offline_rooms: list,
lesson_id: str, event_id: str):
""" Updates lesson in Erudite and Google Calendar """
if lesson["ruz_url"] is None or "meet.miem.hse.ru" not in lesson[
"ruz_url"]:
logger.info("Updating ruz lesson")
try:
event = await self.calendar_api.update_event(
self.ruz.calendar, event_id, lesson)
lesson["gcalendar_event_id"] = event["id"]
lesson["gcalendar_calendar_id"] = self.ruz.calendar
except:
logger.warning(
"Lesson could not be updated succesfully in calendar")
await self.nvr_api.update_lesson(lesson_id, lesson)
# if lesson["ruz_auditorium"] in offline_rooms:
# room = self.session.query(Room).filter_by(name=lesson["ruz_auditorium"]).first()
# self.create_record(room, event)
elif lesson["ruz_url"] is not None and "meet.miem.hse.ru" in lesson[
"ruz_url"]:
logger.info("Updating jitsi lesson")
try:
event = await self.calendar_api.update_event(
self.jitsi.calendar, event_id, lesson)
lesson["gcalendar_event_id"] = event["id"]
lesson["gcalendar_calendar_id"] = self.jitsi.calendar
except:
logger.warning(
"Lesson could not be updated succesfully in calendar")
await self.nvr_api.update_lesson(lesson_id, lesson)
async def test_post_lesson(self, lesson: dict):
""" Post a lesson with empty event_id """
lesson = dict(lesson) # make copy
lesson["gcalendar_event_id"] = ""
lesson["gcalendar_calendar_id"] = ""
data = await self.nvr_api.add_lesson(lesson)
return data
async def post_lesson(self, lesson: dict, lesson_id: str,
calendar_id: str):
""" Posts event to Google calendar and updates lesson in Erudite """
event = await self.calendar_api.create_event(self.ruz.calendar, lesson)
lesson["gcalendar_event_id"] = event["id"]
lesson["gcalendar_calendar_id"] = calendar_id
await self.nvr_api.update_lesson(lesson_id, lesson)
return event
def create_record(self, room: Room, event: dict):
start_date = event["start"]["dateTime"].split("T")[0]
end_date = event["end"]["dateTime"].split("T")[0]
if start_date != end_date:
return
creator = self.session.query(User).filter_by(
email=event["creator"]["email"]).first()
if not creator:
return
new_record = Record()
new_record.room = room
new_record.update_from_calendar(**event)
self.session.add(new_record)
self.session.commit()
user_record = UserRecord(user_id=creator.id, record_id=new_record.id)
self.session.add(user_record)
self.session.commit()
async def delete_online_events(self):
while True:
events = await self.calendar_api.get_events(self.jitsi.calendar)
if len(events) == 0:
break
for event in events:
await self.calendar_api.delete_event(self.jitsi.calendar,
event["id"])
while True:
events = await self.calendar_api.get_events(self.ruz.calendar)
if len(events) == 0:
break
for event in events:
await self.calendar_api.delete_event(self.ruz.calendar,
event["id"])