def test_release_date():
# Neither a date object nor a string
with pytest.raises(AttributeError):
compatibility.Check(
package_name='test',
package_version='0.1',
release_date=(2021, 1, 1))
# valid date object
assert compatibility.Check(
package_name='test',
package_version='0.1',
release_date=date(2021, 1, 1))
# valid string
assert compatibility.Check(
package_name='test',
package_version='0.1',
release_date='2021-01-01')
# malformed date string
with pytest.raises(ValueError):
compatibility.Check(
package_name='test',
package_version='0.1',
release_date='2021-Jan-10')
# valid string format, but invalid date
with pytest.raises(ValueError):
compatibility.Check(
package_name='test',
package_version='0.1',
release_date='2021-13-01')
def test_check_system_exceptions():
# not a dictionary
with pytest.raises(ValueError) as excinfo:
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
system_support='Linux')
assert 'must be a dictionary' in str(excinfo.value)
# unknown key in dict
with pytest.raises(ValueError) as excinfo:
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
system_support={'typo': {'foo'}})
assert 'Unknown key' in str(excinfo.value)
# value for key is not a set
with pytest.raises(ValueError) as excinfo:
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
system_support={'full': ['Linux']})
assert 'Use a set to hold values' in str(excinfo.value)
# Unknown system
with pytest.raises(ValueError) as excinfo:
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
system_support={'full': {'foo'}})
assert 'Invalid system' in str(excinfo.value)
def test_missing_or_empty_paramameters():
"3 parameters are required, the other 3 have defaults."
# package name missing
with pytest.raises(ValueError) as excinfo:
compatibility.Check(
package_name='',
package_version='1',
release_date=date(2021, 1, 1))
assert 'Missing package name!' in str(excinfo.value)
# package name whitespace only
with pytest.raises(ValueError) as excinfo:
compatibility.Check(
package_name=' ',
package_version='1',
release_date=date(2021, 1, 1))
assert 'Missing package name!' in str(excinfo.value)
# missing version
with pytest.raises(ValueError) as excinfo:
compatibility.Check(
package_name='test',
package_version='',
release_date=date(2021, 1, 1))
assert 'Missing package version!' in str(excinfo.value)
# missing release date
with pytest.raises(ValueError) as excinfo:
compatibility.Check(
package_name='test',
package_version='1',
release_date='')
def test_check_system_incompatible_systems():
with patch('platform.system') as system:
system.return_value = 'Linux'
with pytest.raises(RuntimeError) as excinfo:
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
system_support={'incompatible': {'Linux'}}
)
assert 'is incompatible' in str(excinfo.value)
def test_check_system_partial(caplog):
# platform is listed under partial
with patch('platform.system') as system:
system.return_value = 'Linux'
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
system_support={'partial': {'Linux'}}
)
assert 'has only partial support' in caplog.text
def test_check_version_age_logging(caplog):
caplog.set_level(logging.INFO)
# always nag
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
nag_over_update={
'nag_days_after_release': 3,
'nag_in_hundred': 100
})
assert 'There could be updates and security fixes' in caplog.text
def test_check_system_UNKNOWN_SUPPORT(caplog):
caplog.set_level(logging.DEBUG)
# platform support unknown
with patch('platform.system') as system:
system.return_value = 'Linux'
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
system_support={'full': {'Windows'}}
)
assert 'support for Linux is unknown' in caplog.text
def test_languages():
# not supported language
with pytest.raises(ValueError) as excinfo:
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
language_messages='not-a-language')
assert 'Invalid value for language_messages!' in str(excinfo.value)
# supported language: en
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
language_messages='en')
# supported language: de
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
language_messages='de')
def test_check_system_CONTRADICTIONS():
with patch('platform.system') as system:
system.return_value = 'Windows'
# Cannot be incompatible and have full support
with pytest.raises(ValueError) as excinfo:
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
system_support={'full': {'Windows'},
'incompatible': {'Windows'}}
)
assert 'support AND be incompatible' in str(excinfo.value)
# cannot be fully and partialy supported
with pytest.raises(ValueError) as excinfo:
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
system_support={'full': {'Windows'},
'partial': {'Windows'}}
)
assert 'fully AND only partially supported' in str(excinfo.value)
def test_check_system(caplog):
caplog.set_level(logging.DEBUG)
# supported platform
with patch('platform.system') as system:
system.return_value = 'Linux'
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
system_support={'full': {'Linux'},
'partial': set(),
'incompatible': {'MacOS', 'Windows'}}
)
assert 'fully supports Linux' in caplog.text
def __init__(self, mail_settings: Dict[str, Any]):
"""Check the mail settings for plausibility and set
missing values to their default. """
compatibility.Check(
package_name='bote',
package_version=version.__version__,
release_date=version.release_date,
python_version_support={
'min_version': '3.6',
'incompatible_versions': [],
'max_tested_version': '3.10'
},
nag_over_update={
'nag_days_after_release': 365,
'nag_in_hundred': 100
},
language_messages='en',
system_support={'full': {'Linux', 'Windows', 'MacOS'}})
userprovided.parameters.validate_dict_keys(
dict_to_check=mail_settings,
allowed_keys={
'server', 'server_port', 'encryption', 'username',
'passphrase', 'recipient', 'sender', 'wrap_width'
},
necessary_keys={'recipient', 'sender'},
dict_name='mail_settings')
# Not all keys must be there.
# Provide default values for missing ones with defaultdict:
self.server: str = mail_settings.get('server', 'localhost')
self.is_local = bool(self.server in ('localhost', '127.0.0.1', '::1'))
# Encryption defaults to 'off' as the default for server is localhost.
self.encryption: str = mail_settings.get('encryption', 'off')
if self.encryption not in ('off', 'starttls', 'ssl'):
raise ValueError('Invalid value for the encryption parameter!')
# Enforce encryption if the connection is not to localhost:
if not self.is_local and self.encryption == 'off':
raise err.UnencryptedRemoteConnection(
'Connection is not local, but unencrypted!')
self.server_port = mail_settings.get('server_port', None)
if self.server_port:
if not userprovided.parameters.is_port(self.server_port):
raise ValueError('Port must be integer (0 to 65535)')
elif not self.is_local:
raise ValueError(
'Provide a port if you connect to a remote SMTP server.')
self.username = mail_settings.get('username', None)
self.passphrase = mail_settings.get('passphrase', None)
# Even for a remote connection username and passphrase might be
# not necessary - for example if the identification is host based.
# Therfore no exception is thrown.
if not self.username:
logging.debug('Parameter username is empty.')
if not self.passphrase:
logging.debug('Parameter passphrase is empty.')
self.default_recipient: str = ''
self.recipient: Union[str, dict] = mail_settings['recipient']
if type(self.recipient) == dict:
if len(self.recipient) == 0:
raise ValueError('Dictionary recipient is empty.')
# Warn if there is no default key
try:
self.default_recipient = self.recipient['default']
except KeyError:
logging.warning("No default key in recipient dictionary!")
# TO DO: check for all recipient keys if mailadresses are valid
elif type(self.recipient) == str:
if not userprovided.mail.is_email(str(self.recipient)):
raise err.NotAnEmail('recipient is not a valid email!')
self.default_recipient = self.recipient
else:
raise ValueError(
'Parameter recipient must be either string or dictionary.')
self.sender = mail_settings['sender']
if not userprovided.mail.is_email(self.sender):
raise err.NotAnEmail('sender is not a valid email!')
self.wrap_width = mail_settings.get('wrap_width', 80)
if not isinstance(self.wrap_width, int):
raise ValueError('wrap_width is not an integer!')
# Create SSL context. According to the docs this will:
# * load the system’s trusted CA certificates,
# * enable certificate validation and hostname checking,
# * try to choose reasonably secure protocol and cipher settings.
# see:
# https://docs.python.org/3/library/ssl.html#ssl-security
self.context = ssl.create_default_context()
def test_check_version_age():
# Test *temporarily* disabled because if the guard clause is there, the mypy unreachable
# code check, cannot be silenced and there is always an error.
# value of nag_over_update is None
# my_check = compatibility.Check(
# package_name='test',
# package_version='1',
# release_date=date(2021, 1, 1),
# nag_over_update=None)
# my_check.check_version_age(None)
# nag_in_hundred is 0
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
nag_over_update={
'nag_days_after_release': 1,
'nag_in_hundred': 0
})
# negative value
with pytest.raises(ValueError) as excinfo:
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
nag_over_update={
'nag_days_after_release': -42,
'nag_in_hundred': 100
})
assert 'nag_days_after_release must not be negative.' in str(excinfo.value)
# non integer value
with pytest.raises(ValueError) as excinfo:
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
nag_over_update={
'nag_days_after_release': 'foo',
'nag_in_hundred': 100
})
assert 'Some key im nag_over_update has wrong type!' in str(excinfo.value)
# Note: Directly mocking datetime will fail, because it is C-Code !
# Solution could be partial mocking, see.
# https://docs.python.org/3/library/unittest.mock-examples.html#partial-mocking
# However, it is simpler to calculate the release date:
a_week_ago = date.today() - timedelta(days=7)
# days since release below threshold
compatibility.Check(
package_name='test',
package_version='1',
release_date=a_week_ago,
nag_over_update={
'nag_days_after_release': 100,
'nag_in_hundred': 100
})
# days since release above threshold
compatibility.Check(
package_name='test',
package_version='1',
release_date=a_week_ago,
nag_over_update={
'nag_days_after_release': 3,
'nag_in_hundred': 100
})
# nag_in_hundred negative
with pytest.raises(ValueError) as excinfo:
compatibility.Check(
package_name='test',
package_version='1',
release_date=a_week_ago,
nag_over_update={
'nag_days_after_release': 3,
'nag_in_hundred': -100
})
assert 'must be int between 0 and 100' in str(excinfo.value)
# nag_in_hundred above 100
with pytest.raises(ValueError) as excinfo:
compatibility.Check(
package_name='test',
package_version='1',
release_date=a_week_ago,
nag_over_update={
'nag_days_after_release': 3,
'nag_in_hundred': 101
})
assert 'must be int between 0 and 100' in str(excinfo.value)
def test_running_wrong_python():
# Instead of mocking, create a version string
# relative to the one running this test:
major = sys.version_info.major
minor = sys.version_info.minor
releaselevel = sys.version_info.releaselevel
running_version_short = f"{major}.{minor}"
running_version_long = f"{major}.{minor}.{releaselevel}"
version_minor_above = f"{major}.{minor + 1}"
version_major_above = f"{major + 1}.{minor}"
# running version is above max tested version
# TO Do : check if logging is called
# Minimal test version is 3.6, so 3.0
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
python_version_support={
'min_version': '3.0',
'incompatible_versions': [],
'max_tested_version': '3.0'})
# major version required is larger than version running
with pytest.raises(RuntimeError) as excinfo:
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
python_version_support={
'min_version': version_major_above,
'incompatible_versions': [],
'max_tested_version': '9.100'})
# minor version above is required
with pytest.raises(RuntimeError) as excinfo:
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
python_version_support={
'min_version': version_minor_above,
'incompatible_versions': [],
'max_tested_version': '9.100'})
# short form of running version is in list of incompatible versions
with pytest.raises(RuntimeError) as excinfo:
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
python_version_support={
'min_version': '0.0',
'incompatible_versions': [running_version_short],
'max_tested_version': '9.100'})
# long form of running version is in list of incompatible versions
with pytest.raises(RuntimeError):
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
python_version_support={
'min_version': '0.0',
'incompatible_versions': [running_version_long],
'max_tested_version': '9.100'})
def test_python_versions_as_parameters():
# python_version_support: missing key
with pytest.raises(ValueError) as excinfo:
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
python_version_support={
'min_version': '3.7',
'incompatible_versions': []
})
assert 'Parameter python_version_support incomplete!' in str(excinfo.value)
# python_version_support: additional key
with pytest.raises(ValueError) as excinfo:
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
python_version_support={
'min_version': '3.8',
'incompatible_versions': [],
'max_tested_version': '3.9',
'additional_key': '1.2'
})
assert 'Parameter python_version_support: too many keys!' in str(excinfo.value)
# python_version_support: right number of keys but contains unknown key
with pytest.raises(ValueError) as excinfo:
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
python_version_support={
'min_version': '3.8',
'incompatible_versions': [],
'unknown_key': '3.9'
})
assert 'Parameter python_version_support contains unknown keys.' in str(excinfo.value)
# python_version_support: wrong value for min_version
with pytest.raises(ValueError) as excinfo:
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
python_version_support={
'min_version': 'x.y',
'incompatible_versions': [],
'max_tested_version': '3.9'
})
assert 'Value for key min_version incorrect.' in str(excinfo.value)
# python_version_support: wrong value for max_tested_version
with pytest.raises(ValueError) as excinfo:
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
python_version_support={
'min_version': '3.8',
'incompatible_versions': [],
'max_tested_version': '3.x'
})
assert 'Value for key max_tested_version incorrect.' in str(excinfo.value)
# python_version_support: wrong version strings in incompatible_versions
with pytest.raises(ValueError) as excinfo:
compatibility.Check(
package_name='test',
package_version='1',
release_date=date(2021, 1, 1),
python_version_support={
'min_version': '3.6',
'incompatible_versions': ['100.7.alpha', '100.8', 'foo'],
'max_tested_version': '3.9'
})
assert 'cannot be parsed.' in str(excinfo.value)
def __init__(self,
database_settings: dict,
target_directory: str,
filename_prefix: str = '',
project_name: str = 'Bot',
bot_user_agent: str = 'Bot',
bot_behavior: Union[dict, None] = None,
mail_settings: Union[dict, None] = None,
mail_behavior: Union[dict, None] = None,
chrome_name: str = ''):
"Set defaults, create instances, ..."
compatibility.Check(
package_name='exoskeleton',
package_version=version.__version__,
release_date=version.release_date,
python_version_support={
'min_version': '3.8',
'incompatible_versions': [],
'max_tested_version': '3.9'},
nag_over_update={
'nag_days_after_release': 120,
'nag_in_hundred': 100},
language_messages='en',
system_support={
'full': {'Linux', 'MacOS', 'Windows'}
}
)
self.project: str = project_name.strip()
self.user_agent: str = bot_user_agent.strip()
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# INIT: Database Setup / Establish a Database Connection
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Init database Connection
self.db = database_connection.DatabaseConnection(database_settings)
self.cur = self.db.get_cursor()
self.db_check = database_schema_check.DatabaseSchemaCheck(self.db)
self.stats = statistics_manager.StatisticsManager(self.db)
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# INIT: Mail / Notification Setup
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
mail_settings = dict() if not mail_settings else mail_settings
mail_behavior = dict() if not mail_behavior else mail_behavior
self.milestone: Optional[int] = mail_behavior.get('milestone_num',
None)
if self.milestone and not isinstance(self.milestone, int):
raise ValueError('milestone_num must be integer!')
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# INIT: Bot Behavior
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
if bot_behavior:
userprovided.parameters.validate_dict_keys(
dict_to_check=bot_behavior,
allowed_keys={'connection_timeout',
'queue_max_retries',
'queue_revisit',
'rate_limit_wait',
'stop_if_queue_empty',
'wait_min',
'wait_max'},
necessary_keys=None,
dict_name='bot_behavior')
else:
bot_behavior = dict()
# Seconds until a connection times out:
self.connection_timeout: int = userprovided.parameters.int_in_range(
"self.connection_timeout",
bot_behavior.get('connection_timeout', 60),
1, 60, 50)
# Init Classes
self.time = time_manager.TimeManager(
bot_behavior.get('wait_min', 5),
bot_behavior.get('wait_max', 30))
self.notify = notification_manager.NotificationManager(
self.project, mail_settings, mail_behavior, self.time, self.stats, self.milestone)
self.labels = label_manager.LabelManager(self.db)
self.file = file_manager.FileManager(
self.db,
target_directory,
filename_prefix)
self.errorhandling = error_manager.CrawlingErrorManager(
self.db,
bot_behavior.get('queue_max_retries', 3),
bot_behavior.get('rate_limit_wait', 1860)
)
self.controlled_browser = remote_control_chrome.RemoteControlChrome(
chrome_name,
self.errorhandling,
self.stats)
self.action = actions.ExoActions(
self.db,
self.stats,
self.file,
self.time,
self.errorhandling,
self.controlled_browser,
self.user_agent,
self.connection_timeout)
self.blocklist = blocklist_manager.BlocklistManager(self.db)
self.queue = queue_manager.QueueManager(
self.db,
self.blocklist,
self.time,
self.stats,
self.action,
self.notify,
self.labels,
bot_behavior)
self.jobs = job_manager.JobManager(self.db)
# Create other objects
self.cnt: Counter = Counter()