def __init__(self, name, url, queue): """ Args: name (str): name of the subject. url (str): url of the subject. queue (Queue): queue to controll threads. """ name = name.capitalize().replace("\\", "").replace("/", "").strip() self.name = Alias.id_to_alias( sha1(url.encode()).hexdigest(), settings.root_folder / name ).name self.url = url self.connection = Connection() self.queue = queue self.enable_section_indexing = self.url in settings.section_indexing_urls self.response: Response = None self.soup: BeautifulSoup = None self.notes_links = [] self.folder_lock = Lock() self.hasfolder = False # self.folder = settings.root_folder / secure_filename(self.name) self.folder = settings.root_folder / self.name self.logger = logging.getLogger(__name__) self.logger.debug( "Created %s(name=%r, url=%r)", type(self).__name__, self.name, self.url )
def get_subjects(queue): logger = logging.getLogger(__name__) connection = Connection() request = connection.get(connection.user_url) soup = BeautifulSoup(request.text, "html.parser") primary_li = soup.find_all("li", class_="contentnode")[3] lis = primary_li.find_all("li") logger.debug("Found %d potential subjects", len(lis)) subjects = [] for li in lis: course_id = int(re.search(r"course=(\d+)", li.a["href"]).group(1)) subject_url = "https://campusvirtual.uva.es/course/view.php?id=%d" % course_id name = re.search(r"^([\w\/\sáéíóúÁÉÍÓÚ]+?)\s?\(", li.text).group(1) if course_id in settings.exclude_subjects_ids: logger.info("Excluding subject %s (%d)", name, course_id) continue # Don't consider subject if 'grado' is in the name (it is the degree itself) if "grado" in name.lower(): continue logger.debug("Assembling subject %r", name) _subject = Subject(name, subject_url, queue) subjects.append(_subject) subjects.sort(key=lambda x: x.name) return subjects
def test_get_login_token(self, glp_m, scc_m, data, get_test_data, caplog): caplog.set_level(10) is_ok = data == "ok" real_token = "SCPBUp0HZE9nAvhdjfwzFxu8ESsZ0jtE" if data == "null": glp_m.return_value.text = "" else: glp_m.return_value.text = get_test_data(f"login-{data}.html.example") connection = Connection() if is_ok: token = connection.get_login_token() assert token == real_token assert not hasattr(connection, "login-token-response") scc_m.assert_not_called() glp_m.assert_called_once_with() assert caplog.record_tuples == [ ("vcm.core.networking", 10, f"Login token: {real_token}") ] return with pytest.raises(LoginError, match="Can't find token"): connection.get_login_token() assert hasattr(connection, "login-token-response") assert getattr(connection, "login-token-response") == glp_m.return_value scc_m.assert_called_once_with(connection, mock.ANY, mock.ANY) assert caplog.record_tuples == [("vcm.core.networking", 50, "Can't find token")]
def __init__(self, name, section, url, icon_url, subject, parent=None): """ Args: name (str): name of the url. url (str): URL of the url. icon_url (str or None): URL of the icon. subject (vcm.subject.Subject): subject of the url. parent (BaseLink): object that created self. """ self.name = name.strip() self.section = section self.url = url self.icon_url = icon_url self.subject = subject self.connection = Connection() self.parent = parent self.response: Response = None self.soup: BeautifulSoup = None self.filepath: Path = None self.redirect_url = None self.response_name = None self.subfolders = [] self.logger = logging.getLogger(__name__) self.logger.debug( "Created %s(name=%r, url=%r, subject=%r)", self.__class__.__name__, self.name, self.url, self.subject.name, )
def test_login_fatal_error(self, inner_login_m, scc_m, fssauu_m, caplog, has_lr): caplog.set_level(10) inner_login_m.side_effect = [LoginError("message")] * self.login_retries + [ None ] conn = Connection() if has_lr: # For historical reasons. conn._login_response = mock.MagicMock() with pytest.raises(LoginError, match="unknown error"): conn.login() assert inner_login_m.call_count == self.login_retries scc_m.assert_called_once_with(conn, mock.ANY, mock.ANY) fssauu_m.assert_not_called() expected = [] for i in range(self.login_retries): retries_left = self.login_retries - i expected.append((10, "Logging in (%d retries left)" % (retries_left))) expected.append((30, "Trying to log again due to LoginError('message')")) expected = [(self.logger_name,) + x for x in expected] assert caplog.record_tuples == expected
def test_singleton(self): conn1 = Connection() conn2 = Connection() conn3 = Connection() assert conn1 is conn2 assert conn2 is conn3 assert conn3 is conn1
def test_make_logout_request(self): conn = Connection() conn._sesskey = "<sesskey>" response = conn.make_logout_request() assert response == self.downloader_m.return_value.post.return_value self.downloader_m.return_value.post.assert_called_once_with( conn.logout_url, {"sesskey": "<sesskey>"} )
def test_inner_logout_exception(self, mlr_m): mlr_m.side_effect = DownloaderError("message") conn = Connection() with pytest.raises(LogoutError, match="Logout request raised DownloaderError"): conn.inner_logout() mlr_m.assert_called_once_with() self.scc_m.assert_not_called()
def test_handle_maintenance_mode(self, caplog): caplog.set_level(10) conn = Connection() with pytest.raises(SystemExit): conn.handle_maintenance_mode(404, "Not Found") assert caplog.record_tuples == [ (self.logger_name, 50, "Moodle under maintenance (404 - Not Found)") ]
def test_logout_ok(self, logout_m, caplog): caplog.set_level(10) conn = Connection() conn.logout() logout_m.assert_called_once_with() assert caplog.record_tuples == [ (self.logger_name, 10, "Logging out (%s retries)" % self.logout_retries), (self.logger_name, 20, "Logged out"), ]
def test_inner_logout_http_status_error(self, mlr_m, status_code): response = mock.MagicMock(ok=False, status_code=status_code) mlr_m.return_value = response conn = Connection() with pytest.raises(LogoutError, match=f"Logout returned {status_code}"): conn.inner_logout() mlr_m.assert_called_once_with() self.scc_m.assert_not_called()
def test_get_login_page(self): conn = Connection() response = conn.get_login_page() assert response == self.downloader_m.return_value.get.return_value self.downloader_m.return_value.get.assert_called_once_with(conn.login_url) # Test lru_cache response = conn.get_login_page() assert response == self.downloader_m.return_value.get.return_value self.downloader_m.return_value.get.assert_called_once_with(conn.login_url)
def test_inner_logout_ok(self, mlr_m): # TODO: put real text text = "Usted no se ha identificado. Pulse aquí para hacerlo." response = mock.MagicMock(ok=True, status_code=200, text=text) mlr_m.return_value = response conn = Connection() conn.inner_logout() mlr_m.assert_called_once_with() self.scc_m.assert_not_called()
def test_inner_login_already_logged_in(self, cali_m, glt_m, mlr_m, caplog): caplog.set_level(10) cali_m.return_value = True conn = Connection() assert conn.inner_login() is None cali_m.assert_called_once_with() glt_m.assert_not_called() mlr_m.assert_not_called() assert caplog.record_tuples == []
def test_inner_logout_fatal_logout(self, mlr_m): # TODO: put real text text = "Identificado como FEDERICO GARCÍA LORCA." response = mock.MagicMock(ok=True, status_code=200, text=text) mlr_m.return_value = response conn = Connection() with pytest.raises(LogoutError, match="Unkown error during logout"): conn.inner_logout() mlr_m.assert_called_once_with() self.scc_m.assert_called_once_with(conn, mock.ANY, mock.ANY)
def test_find_sesskey_and_user_url(self, get_test_data): login_response = mock.MagicMock() login_response.text = get_test_data("logged-in.html.example") expected_url = ( "https://campusvirtual.uva.es/user/profile.php?id=6737&showallcourses=1" ) connection = Connection() connection._login_response = login_response connection.find_sesskey_and_user_url() assert connection.sesskey == "1Yjk995Su9" assert connection.user_url == expected_url
def test_check_already_logged_in_maintenance(self, hmm_m, glp_m, caplog): caplog.set_level(10) response = mock.MagicMock( ok=False, reason="Moodle is under maintenance until Monday", status_code=501 ) glp_m.return_value = response conn = Connection() assert conn.check_already_logged_in() == hmm_m.return_value glp_m.assert_called_once_with() hmm_m.assert_called_once_with(501, response.reason) assert caplog.record_tuples == []
def test_check_already_logged_in_server_error(self, hmm_m, glp_m, caplog): caplog.set_level(10) response = mock.MagicMock(ok=False, reason="<reason>", status_code=501) glp_m.return_value = response conn = Connection() with pytest.raises(MoodleError, match=r"Moodle error \(501 - <reason>\)"): conn.check_already_logged_in() glp_m.assert_called_once_with() hmm_m.assert_not_called() assert caplog.record_tuples == [ (self.logger_name, 50, "Moodle error (501 - <reason>)") ]
def test_make_login_request(self, vc_creds_m): conn = Connection() response = conn.make_login_request(login_token="<login-token>") assert response == self.downloader_m.return_value.post.return_value data = { "anchor": "", "username": vc_creds_m.username, "password": vc_creds_m.password, "logintoken": "<login-token>", } self.downloader_m.return_value.post.assert_called_once_with( conn.login_url, data )
def test_login_ok(self, inner_login_m, scc_m, fsauu_m, caplog): caplog.set_level(10) conn = Connection() conn.login() inner_login_m.assert_called_once_with() scc_m.assert_not_called() fsauu_m.assert_called_once_with() assert caplog.record_tuples == [ (self.logger_name, 10, "Logging in (10 retries left)"), (self.logger_name, 20, "Logged in"), ]
def notify(send_to: _A, use_icons=True, nthreads=20, status_server=False): """Launches notify scanner. Args: send_to (_A): address or list of address to send the report to (email address). use_icons (bool, optional): if true, icons will be included in email. Defaults to True. nthreads (int, optional): number of threads to use. Defaults to 20. status_server (bool, optional): if true, a http server will be opened in port 80 to show the status of each thread. Defaults to False. """ logger = logging.getLogger(__name__) logger.info( "Launching notify(send_to=%r, use_icons=%s, nthreads=%s, status_server=%s)", send_to, use_icons, nthreads, status_server, ) Printer.silence() queue = Queue() threads = start_workers(queue, nthreads, killer=False) if status_server: runserver(queue, threads) with Connection(): subjects = find_subjects(queue) queue.join() send_report(subjects, use_icons, send_to)
def test_init(self): conn = Connection() assert conn._downloader is self.downloader_m.return_value assert conn._logout_response is None assert conn._login_response is None assert conn._sesskey is None assert conn._user_url is None assert conn._login_attempts == 0
def test_check_already_logged_in_server_false(self, hmm_m, glp_m, caplog): caplog.set_level(10) # TODO: put real text response = mock.MagicMock( ok=True, reason="<reason>", status_code=200, text="Identificado como FEDERICO GARCÍA LORCA", ) glp_m.return_value = response conn = Connection() assert conn.check_already_logged_in() is False glp_m.assert_called_once_with() hmm_m.assert_not_called() assert caplog.record_tuples == []
def test_check_already_logged_in_server_true(self, hmm_m, glp_m, caplog): caplog.set_level(10) # TODO: put real text response = mock.MagicMock( ok=True, reason="<reason>", status_code=200, text="Usted ya está en el sistema como FEDERICO GARCÍA LORCA", ) glp_m.return_value = response conn = Connection() assert conn.check_already_logged_in() is True glp_m.assert_called_once_with() hmm_m.assert_not_called() assert caplog.record_tuples == [ (self.logger_name, 20, "User already logged in") ]
def test_logout_some_errors(self, logout_m, caplog, nerrors): caplog.set_level(10) logout_m.side_effect = [LogoutError("message")] * nerrors + [None] conn = Connection() conn._logout_response = mock.MagicMock(status_code=501, ok=False) conn.logout() logout_m.assert_called() assert logout_m.call_count == nerrors + 1 records_expected: Any = [(10, "Logging out (%s retries)" % self.logout_retries)] for i in range(1, nerrors + 1): retries_left = self.logout_retries - i records_expected.append( (30, "Error during logout [501], %d retries left" % retries_left) ) records_expected.append((20, "Logged out")) records_expected = [(self.logger_name,) + x for x in records_expected] assert caplog.record_tuples == records_expected
def test_inner_login_server_error(self, cali_m, glt_m, mlr_m, fsauu_m, caplog): caplog.set_level(10) cali_m.return_value = False glt_m.return_value = "<token>" response = mock.MagicMock(ok=False, status_code=501, reason="<reason>") mlr_m.return_value = response conn = Connection() with pytest.raises(LoginError, match=r"Server returned 501 \(<reason>\)"): conn.inner_login() cali_m.assert_called_once_with() glt_m.assert_called_once_with() mlr_m.assert_called_once_with("<token>") mlr_m.cache_clear.assert_not_called() fsauu_m.assert_not_called() assert caplog.record_tuples == [ (self.logger_name, 20, "Logging in with user '<username>'") ]
def test_logout_fatal(self, logout_m, caplog): caplog.set_level(10) logout_m.side_effect = [LogoutError("message")] * self.logout_retries + [None] conn = Connection() conn._logout_response = mock.MagicMock(status_code=501, ok=False) with pytest.raises(LogoutError, match="Logout retries expired"): conn.logout() logout_m.assert_called() assert logout_m.call_count == self.logout_retries records_expected: Any = [(10, "Logging out (%s retries)" % self.logout_retries)] for i in range(1, self.logout_retries + 1): retries_left = self.logout_retries - i records_expected.append( (30, "Error during logout [501], %d retries left" % retries_left) ) records_expected.append((50, "Logout retries expired")) records_expected = [(self.logger_name,) + x for x in records_expected] assert caplog.record_tuples == records_expected
def test_inner_login_error(self, cali_m, glt_m, mlr_m, caplog): caplog.set_level(10) cali_m.return_value = False glt_m.return_value = "<token>" # TODO: put real text response = mock.MagicMock( ok=True, status_code=200, reason="OK", text="Usted no se ha identificado" ) mlr_m.return_value = response conn = Connection() with pytest.raises(LoginError, match="Unsuccessfull login"): conn.inner_login() cali_m.assert_called_once_with() glt_m.assert_called_once_with() mlr_m.assert_called_once_with("<token>") assert caplog.record_tuples == [ (self.logger_name, 20, "Logging in with user '<username>'") ]
def test_context_manager(self, login_m, logout_m): login_m.assert_not_called() logout_m.assert_not_called() # disable false positive pylint: disable=not-context-manager with Connection() as conn: login_m.assert_called_once_with() logout_m.assert_not_called() assert conn login_m.assert_called_once_with() logout_m.assert_called_once_with()
def test_inner_login_ok(self, cali_m, glt_m, mlr_m, caplog): caplog.set_level(10) cali_m.return_value = False glt_m.return_value = "<token>" # TODO: put real text response = mock.MagicMock( ok=True, status_code=200, reason="OK", text="Usted se ha identificado como FEDERICO GARCÍA LORCA", ) mlr_m.return_value = response conn = Connection() conn.inner_login() cali_m.assert_called_once_with() glt_m.assert_called_once_with() mlr_m.assert_called_once_with("<token>") assert caplog.record_tuples == [ (self.logger_name, 20, "Logging in with user '<username>'") ]