def __init__(self, name, url, queue):
"""
Args:
name (str): name of the subject.
url (str): url of the subject.
queue (Queue): queue to controll threads.
"""
name = name.capitalize().replace("\\", "").replace("/", "").strip()
self.name = Alias.id_to_alias(
sha1(url.encode()).hexdigest(), settings.root_folder / name
).name
self.url = url
self.connection = Connection()
self.queue = queue
self.enable_section_indexing = self.url in settings.section_indexing_urls
self.response: Response = None
self.soup: BeautifulSoup = None
self.notes_links = []
self.folder_lock = Lock()
self.hasfolder = False
# self.folder = settings.root_folder / secure_filename(self.name)
self.folder = settings.root_folder / self.name
self.logger = logging.getLogger(__name__)
self.logger.debug(
"Created %s(name=%r, url=%r)", type(self).__name__, self.name, self.url
)
def get_subjects(queue):
logger = logging.getLogger(__name__)
connection = Connection()
request = connection.get(connection.user_url)
soup = BeautifulSoup(request.text, "html.parser")
primary_li = soup.find_all("li", class_="contentnode")[3]
lis = primary_li.find_all("li")
logger.debug("Found %d potential subjects", len(lis))
subjects = []
for li in lis:
course_id = int(re.search(r"course=(\d+)", li.a["href"]).group(1))
subject_url = "https://campusvirtual.uva.es/course/view.php?id=%d" % course_id
name = re.search(r"^([\w\/\sáéíóúÁÉÍÓÚ]+?)\s?\(", li.text).group(1)
if course_id in settings.exclude_subjects_ids:
logger.info("Excluding subject %s (%d)", name, course_id)
continue
# Don't consider subject if 'grado' is in the name (it is the degree itself)
if "grado" in name.lower():
continue
logger.debug("Assembling subject %r", name)
_subject = Subject(name, subject_url, queue)
subjects.append(_subject)
subjects.sort(key=lambda x: x.name)
return subjects
def test_get_login_token(self, glp_m, scc_m, data, get_test_data, caplog):
caplog.set_level(10)
is_ok = data == "ok"
real_token = "SCPBUp0HZE9nAvhdjfwzFxu8ESsZ0jtE"
if data == "null":
glp_m.return_value.text = ""
else:
glp_m.return_value.text = get_test_data(f"login-{data}.html.example")
connection = Connection()
if is_ok:
token = connection.get_login_token()
assert token == real_token
assert not hasattr(connection, "login-token-response")
scc_m.assert_not_called()
glp_m.assert_called_once_with()
assert caplog.record_tuples == [
("vcm.core.networking", 10, f"Login token: {real_token}")
]
return
with pytest.raises(LoginError, match="Can't find token"):
connection.get_login_token()
assert hasattr(connection, "login-token-response")
assert getattr(connection, "login-token-response") == glp_m.return_value
scc_m.assert_called_once_with(connection, mock.ANY, mock.ANY)
assert caplog.record_tuples == [("vcm.core.networking", 50, "Can't find token")]
def __init__(self, name, section, url, icon_url, subject, parent=None):
"""
Args:
name (str): name of the url.
url (str): URL of the url.
icon_url (str or None): URL of the icon.
subject (vcm.subject.Subject): subject of the url.
parent (BaseLink): object that created self.
"""
self.name = name.strip()
self.section = section
self.url = url
self.icon_url = icon_url
self.subject = subject
self.connection = Connection()
self.parent = parent
self.response: Response = None
self.soup: BeautifulSoup = None
self.filepath: Path = None
self.redirect_url = None
self.response_name = None
self.subfolders = []
self.logger = logging.getLogger(__name__)
self.logger.debug(
"Created %s(name=%r, url=%r, subject=%r)",
self.__class__.__name__,
self.name,
self.url,
self.subject.name,
)
def test_login_fatal_error(self, inner_login_m, scc_m, fssauu_m, caplog, has_lr):
caplog.set_level(10)
inner_login_m.side_effect = [LoginError("message")] * self.login_retries + [
None
]
conn = Connection()
if has_lr: # For historical reasons.
conn._login_response = mock.MagicMock()
with pytest.raises(LoginError, match="unknown error"):
conn.login()
assert inner_login_m.call_count == self.login_retries
scc_m.assert_called_once_with(conn, mock.ANY, mock.ANY)
fssauu_m.assert_not_called()
expected = []
for i in range(self.login_retries):
retries_left = self.login_retries - i
expected.append((10, "Logging in (%d retries left)" % (retries_left)))
expected.append((30, "Trying to log again due to LoginError('message')"))
expected = [(self.logger_name,) + x for x in expected]
assert caplog.record_tuples == expected
def test_singleton(self):
conn1 = Connection()
conn2 = Connection()
conn3 = Connection()
assert conn1 is conn2
assert conn2 is conn3
assert conn3 is conn1
def test_make_logout_request(self):
conn = Connection()
conn._sesskey = "<sesskey>"
response = conn.make_logout_request()
assert response == self.downloader_m.return_value.post.return_value
self.downloader_m.return_value.post.assert_called_once_with(
conn.logout_url, {"sesskey": "<sesskey>"}
)
def test_inner_logout_exception(self, mlr_m):
mlr_m.side_effect = DownloaderError("message")
conn = Connection()
with pytest.raises(LogoutError, match="Logout request raised DownloaderError"):
conn.inner_logout()
mlr_m.assert_called_once_with()
self.scc_m.assert_not_called()
def test_handle_maintenance_mode(self, caplog):
caplog.set_level(10)
conn = Connection()
with pytest.raises(SystemExit):
conn.handle_maintenance_mode(404, "Not Found")
assert caplog.record_tuples == [
(self.logger_name, 50, "Moodle under maintenance (404 - Not Found)")
]
def test_logout_ok(self, logout_m, caplog):
caplog.set_level(10)
conn = Connection()
conn.logout()
logout_m.assert_called_once_with()
assert caplog.record_tuples == [
(self.logger_name, 10, "Logging out (%s retries)" % self.logout_retries),
(self.logger_name, 20, "Logged out"),
]
def test_inner_logout_http_status_error(self, mlr_m, status_code):
response = mock.MagicMock(ok=False, status_code=status_code)
mlr_m.return_value = response
conn = Connection()
with pytest.raises(LogoutError, match=f"Logout returned {status_code}"):
conn.inner_logout()
mlr_m.assert_called_once_with()
self.scc_m.assert_not_called()
def test_get_login_page(self):
conn = Connection()
response = conn.get_login_page()
assert response == self.downloader_m.return_value.get.return_value
self.downloader_m.return_value.get.assert_called_once_with(conn.login_url)
# Test lru_cache
response = conn.get_login_page()
assert response == self.downloader_m.return_value.get.return_value
self.downloader_m.return_value.get.assert_called_once_with(conn.login_url)
def test_inner_logout_ok(self, mlr_m):
# TODO: put real text
text = "Usted no se ha identificado. Pulse aquí para hacerlo."
response = mock.MagicMock(ok=True, status_code=200, text=text)
mlr_m.return_value = response
conn = Connection()
conn.inner_logout()
mlr_m.assert_called_once_with()
self.scc_m.assert_not_called()
def test_inner_login_already_logged_in(self, cali_m, glt_m, mlr_m, caplog):
caplog.set_level(10)
cali_m.return_value = True
conn = Connection()
assert conn.inner_login() is None
cali_m.assert_called_once_with()
glt_m.assert_not_called()
mlr_m.assert_not_called()
assert caplog.record_tuples == []
def test_inner_logout_fatal_logout(self, mlr_m):
# TODO: put real text
text = "Identificado como FEDERICO GARCÍA LORCA."
response = mock.MagicMock(ok=True, status_code=200, text=text)
mlr_m.return_value = response
conn = Connection()
with pytest.raises(LogoutError, match="Unkown error during logout"):
conn.inner_logout()
mlr_m.assert_called_once_with()
self.scc_m.assert_called_once_with(conn, mock.ANY, mock.ANY)
def test_find_sesskey_and_user_url(self, get_test_data):
login_response = mock.MagicMock()
login_response.text = get_test_data("logged-in.html.example")
expected_url = (
"https://campusvirtual.uva.es/user/profile.php?id=6737&showallcourses=1"
)
connection = Connection()
connection._login_response = login_response
connection.find_sesskey_and_user_url()
assert connection.sesskey == "1Yjk995Su9"
assert connection.user_url == expected_url
def test_check_already_logged_in_maintenance(self, hmm_m, glp_m, caplog):
caplog.set_level(10)
response = mock.MagicMock(
ok=False, reason="Moodle is under maintenance until Monday", status_code=501
)
glp_m.return_value = response
conn = Connection()
assert conn.check_already_logged_in() == hmm_m.return_value
glp_m.assert_called_once_with()
hmm_m.assert_called_once_with(501, response.reason)
assert caplog.record_tuples == []
def test_check_already_logged_in_server_error(self, hmm_m, glp_m, caplog):
caplog.set_level(10)
response = mock.MagicMock(ok=False, reason="<reason>", status_code=501)
glp_m.return_value = response
conn = Connection()
with pytest.raises(MoodleError, match=r"Moodle error \(501 - <reason>\)"):
conn.check_already_logged_in()
glp_m.assert_called_once_with()
hmm_m.assert_not_called()
assert caplog.record_tuples == [
(self.logger_name, 50, "Moodle error (501 - <reason>)")
]
def test_make_login_request(self, vc_creds_m):
conn = Connection()
response = conn.make_login_request(login_token="<login-token>")
assert response == self.downloader_m.return_value.post.return_value
data = {
"anchor": "",
"username": vc_creds_m.username,
"password": vc_creds_m.password,
"logintoken": "<login-token>",
}
self.downloader_m.return_value.post.assert_called_once_with(
conn.login_url, data
)
def test_login_ok(self, inner_login_m, scc_m, fsauu_m, caplog):
caplog.set_level(10)
conn = Connection()
conn.login()
inner_login_m.assert_called_once_with()
scc_m.assert_not_called()
fsauu_m.assert_called_once_with()
assert caplog.record_tuples == [
(self.logger_name, 10, "Logging in (10 retries left)"),
(self.logger_name, 20, "Logged in"),
]
def notify(send_to: _A, use_icons=True, nthreads=20, status_server=False):
"""Launches notify scanner.
Args:
send_to (_A): address or list of address to send the report to
(email address).
use_icons (bool, optional): if true, icons will be included in
email. Defaults to True.
nthreads (int, optional): number of threads to use. Defaults to 20.
status_server (bool, optional): if true, a http server will be opened
in port 80 to show the status of each thread. Defaults to False.
"""
logger = logging.getLogger(__name__)
logger.info(
"Launching notify(send_to=%r, use_icons=%s, nthreads=%s, status_server=%s)",
send_to,
use_icons,
nthreads,
status_server,
)
Printer.silence()
queue = Queue()
threads = start_workers(queue, nthreads, killer=False)
if status_server:
runserver(queue, threads)
with Connection():
subjects = find_subjects(queue)
queue.join()
send_report(subjects, use_icons, send_to)
def test_init(self):
conn = Connection()
assert conn._downloader is self.downloader_m.return_value
assert conn._logout_response is None
assert conn._login_response is None
assert conn._sesskey is None
assert conn._user_url is None
assert conn._login_attempts == 0
def test_check_already_logged_in_server_false(self, hmm_m, glp_m, caplog):
caplog.set_level(10)
# TODO: put real text
response = mock.MagicMock(
ok=True,
reason="<reason>",
status_code=200,
text="Identificado como FEDERICO GARCÍA LORCA",
)
glp_m.return_value = response
conn = Connection()
assert conn.check_already_logged_in() is False
glp_m.assert_called_once_with()
hmm_m.assert_not_called()
assert caplog.record_tuples == []
def test_check_already_logged_in_server_true(self, hmm_m, glp_m, caplog):
caplog.set_level(10)
# TODO: put real text
response = mock.MagicMock(
ok=True,
reason="<reason>",
status_code=200,
text="Usted ya está en el sistema como FEDERICO GARCÍA LORCA",
)
glp_m.return_value = response
conn = Connection()
assert conn.check_already_logged_in() is True
glp_m.assert_called_once_with()
hmm_m.assert_not_called()
assert caplog.record_tuples == [
(self.logger_name, 20, "User already logged in")
]
def test_logout_some_errors(self, logout_m, caplog, nerrors):
caplog.set_level(10)
logout_m.side_effect = [LogoutError("message")] * nerrors + [None]
conn = Connection()
conn._logout_response = mock.MagicMock(status_code=501, ok=False)
conn.logout()
logout_m.assert_called()
assert logout_m.call_count == nerrors + 1
records_expected: Any = [(10, "Logging out (%s retries)" % self.logout_retries)]
for i in range(1, nerrors + 1):
retries_left = self.logout_retries - i
records_expected.append(
(30, "Error during logout [501], %d retries left" % retries_left)
)
records_expected.append((20, "Logged out"))
records_expected = [(self.logger_name,) + x for x in records_expected]
assert caplog.record_tuples == records_expected
def test_inner_login_server_error(self, cali_m, glt_m, mlr_m, fsauu_m, caplog):
caplog.set_level(10)
cali_m.return_value = False
glt_m.return_value = "<token>"
response = mock.MagicMock(ok=False, status_code=501, reason="<reason>")
mlr_m.return_value = response
conn = Connection()
with pytest.raises(LoginError, match=r"Server returned 501 \(<reason>\)"):
conn.inner_login()
cali_m.assert_called_once_with()
glt_m.assert_called_once_with()
mlr_m.assert_called_once_with("<token>")
mlr_m.cache_clear.assert_not_called()
fsauu_m.assert_not_called()
assert caplog.record_tuples == [
(self.logger_name, 20, "Logging in with user '<username>'")
]
def test_logout_fatal(self, logout_m, caplog):
caplog.set_level(10)
logout_m.side_effect = [LogoutError("message")] * self.logout_retries + [None]
conn = Connection()
conn._logout_response = mock.MagicMock(status_code=501, ok=False)
with pytest.raises(LogoutError, match="Logout retries expired"):
conn.logout()
logout_m.assert_called()
assert logout_m.call_count == self.logout_retries
records_expected: Any = [(10, "Logging out (%s retries)" % self.logout_retries)]
for i in range(1, self.logout_retries + 1):
retries_left = self.logout_retries - i
records_expected.append(
(30, "Error during logout [501], %d retries left" % retries_left)
)
records_expected.append((50, "Logout retries expired"))
records_expected = [(self.logger_name,) + x for x in records_expected]
assert caplog.record_tuples == records_expected
def test_inner_login_error(self, cali_m, glt_m, mlr_m, caplog):
caplog.set_level(10)
cali_m.return_value = False
glt_m.return_value = "<token>"
# TODO: put real text
response = mock.MagicMock(
ok=True, status_code=200, reason="OK", text="Usted no se ha identificado"
)
mlr_m.return_value = response
conn = Connection()
with pytest.raises(LoginError, match="Unsuccessfull login"):
conn.inner_login()
cali_m.assert_called_once_with()
glt_m.assert_called_once_with()
mlr_m.assert_called_once_with("<token>")
assert caplog.record_tuples == [
(self.logger_name, 20, "Logging in with user '<username>'")
]
def test_context_manager(self, login_m, logout_m):
login_m.assert_not_called()
logout_m.assert_not_called()
# disable false positive pylint: disable=not-context-manager
with Connection() as conn:
login_m.assert_called_once_with()
logout_m.assert_not_called()
assert conn
login_m.assert_called_once_with()
logout_m.assert_called_once_with()
def test_inner_login_ok(self, cali_m, glt_m, mlr_m, caplog):
caplog.set_level(10)
cali_m.return_value = False
glt_m.return_value = "<token>"
# TODO: put real text
response = mock.MagicMock(
ok=True,
status_code=200,
reason="OK",
text="Usted se ha identificado como FEDERICO GARCÍA LORCA",
)
mlr_m.return_value = response
conn = Connection()
conn.inner_login()
cali_m.assert_called_once_with()
glt_m.assert_called_once_with()
mlr_m.assert_called_once_with("<token>")
assert caplog.record_tuples == [
(self.logger_name, 20, "Logging in with user '<username>'")
]
