def connect_to_database():
'''
Connects to the postgresql database, and returns the connection.
'''
try:
return psycopg2.connect(user=get_string_property("database", "user"),
password=get_string_property("database", "password"),
host=get_string_property("database", "host"),
port=get_string_property("database", "port"),
database=get_string_property("database", "database"))
except psycopg2.Error as error:
error_message = "Could not connect to the database, the following error was thrown: {}".format(error)
LOG.critical(error_message)
sys.exit(error_message)
def _get_soup_of_page(self, url: str) -> BeautifulSoup:
conf_username = get_string_property("confluence", "username")
conf_password = get_string_property("confluence", "password")
req = urllib.request.Request(url)
credentials = ('%s:%s' % (conf_username, conf_password))
encoded_credentials = base64.b64encode(credentials.encode('ascii'))
req.add_header('Authorization',
'Basic %s' % encoded_credentials.decode("ascii"))
try:
page = urllib.request.urlopen(req)
except HTTPError:
# If the page for some reason can't load, it's better to continue and not use edit history, than
# to return nothing.
page = ""
return BeautifulSoup(page, 'html.parser')
def __init__(self):
datasource_switcher = {
"CONFLUENCE": self._init_confluence_loader,
"BBC": self._init_bbc_loader
}
datasource = get_string_property('datasource', 'data_source')
datasource_switcher.get(datasource, self._init_default)()
self.LOG.info(
"the DataLoader has been created for the datasource: {}".format(
datasource))
def test_get_string_property_success(self, mocker):
# Setup
input = "string"
expected = input
read_property_mock = mocker.patch(
"helper.ConfigReader._read_property",
return_value = input
)
# Run
actual = get_string_property("test", "parameters")
# Check
assert actual == expected
read_property_mock.assert_called_once_with("test", "parameters")
def test_get_property_success(self):
# setup
expected = "completedness"
with open(self.ini_location, "a") as config:
config.write("\n[test]\nvalue={}".format(expected))
# run
actual = get_string_property("test", "value")
# breakdown
file = None
with open(self.ini_location, "r") as config:
file = config.readlines()
with open(self.ini_location, "w") as config:
[config.write(line) for line in file[:-2]]
# check
assert actual == expected
def retrieve_history(self, username: str) -> List[str]:
# Because I was unable to get an API endpoint working for confluence, we take a somewhat unorthodox approach
# and scrape the page. This is very much intended as a temporary measure.
base_url = get_string_property("confluence", "location")
full_url = f"{base_url}/confluence/display/~{username}"
soup = self._get_soup_of_page(full_url)
# Find all update items
update_items = soup.find_all("div", {"class": "update-item-details"})
history_urls = []
for update_item in update_items:
item_url_end = update_item.find("a")["href"]
history_urls.append(f"{base_url}{item_url_end}")
# We need to visit the page of the item in history to get the id, I can't get it in the url, see
# https://jira.atlassian.com/browse/CONFSERVER-11285
found_ids: List[str] = []
for url in history_urls:
history_soup = self._get_soup_of_page(url)
meta_object = history_soup.find("meta", {"name": "ajs-page-id"})
if meta_object is not None:
found_ids.append(meta_object["content"])
return found_ids
def test():
get_string_property("test", "value")
def test_get_property_nonexistent_property(self):
# run
with pytest.raises(NoSectionError):
get_string_property("test", "value")
def __init__(self):
self.LOG = logging.getLogger(__name__)
self._confluence_url = get_string_property("confluence", "api_base") + \
get_string_property("confluence", "api_recent_subpath")
def _setup_logger():
logformat = get_string_property("logging", "format")
loglevel = logging.getLevelName(get_string_property("logging", "level"))
logging.basicConfig(format=logformat, level=loglevel)
logging.info("Logger initiated")
def _setup_fast_api():
ip = get_string_property("api", "ip")
port = get_int_property("api", "port")
uvicorn.run(app, host=ip, port=port)
def __init__(self):
self.LOG = logging.getLogger(__name__)
self._bbc_path = get_root_dir() / get_string_property("path", "BBC_documents")
def _init_default(self):
raise ValueError(
"The datasource {} is invalid. Please consult the manual for possible loaders."
.format(get_string_property('datasource', 'data_source')))
