def app():
app = get_test_app()
with app.app_context():
server_config = ConfigManager.get_instance().get_server_config()
application_root = ConfigManager.get_instance().get_application_root()
app.register_blueprint(incidence_controller,
url_prefix=application_root)
app.register_blueprint(municipality_controller,
url_prefix=application_root)
app.register_blueprint(swaggerui_controller,
url_prefix=application_root)
app.config['DEVELOPMENT'] = server_config["development"]
with app.open_resource(
"tests/api_blackbox/testdata/dump_municipalities.sql"
) as f_municipalities:
with app.open_resource(
"tests/api_blackbox/testdata/dump_incidences.sql"
) as f_incidences:
engine = db.get_engine()
with engine.connect() as con:
create_incidence = '''
CREATE TABLE incidence (
"incidencesId" integer NOT NULL,
"bfsNr" integer NOT NULL,
date date NOT NULL,
incidence double precision NOT NULL,
cases integer NOT NULL,
cases_cumsum_14d integer NOT NULL
);
'''
create_municipality = '''
CREATE TABLE municipality (
"bfsNr" integer NOT NULL,
name character varying(256) NOT NULL,
canton character varying(2) NOT NULL,
area double precision NOT NULL,
population integer NOT NULL,
region character varying(256) NOT NULL
);
'''
con.execute(create_municipality)
con.execute(create_incidence)
query_municipalities = text(
f_municipalities.read().decode("utf8"))
con.execute(query_municipalities)
query_incidences = text(f_incidences.read().decode("utf8"))
con.execute(query_incidences)
yield app
def __init__(self, chatId):
self.chat_id = chatId
authenticator = IAMAuthenticator(ConfigManager.get_ibm_token())
self.assistant_id = ConfigManager.get_ibm_assistent_id()
self.assistant = AssistantV2(version='2020-02-05',
authenticator=authenticator)
self.assistant.set_service_url(ConfigManager.get_ibm_assistent_url())
self.validate_session()
def app():
app = get_app()
server_config = ConfigManager.get_instance().get_server_config()
application_root = ConfigManager.get_instance().get_application_root()
app.register_blueprint(waypoint_controller, url_prefix=application_root)
app.register_blueprint(incidence_controller, url_prefix=application_root)
app.register_blueprint(municipality_controller,
url_prefix=application_root)
app.config['DEVELOPMENT'] = server_config["development"]
yield app
class ConnectionManager:
def __init__(self):
self.configMgr = None
self.clientConnections = []
self.initialize()
def initialize(self):
self.configMgr = ConfigManager()
# for i in range(0, self.configMgr.getNumConnections() - 1):
self.clientConnections.append(SimpleClient())
self.clientConnections[0].connect(self.configMgr.getNodes())
keyspace = "balajiSpace"
replication = "3"
tableName = "expData"
how_many_at_a_time = 500 # How many records to insert in a batch.
howLong = 10 # Run for 120 seconds.
NUM_REQUESTS = 24
sleep_interval = 1.0/NUM_REQUESTS # control distribution of a second
file_seek_pos = 0
#self.clientConnections[0].create_schema(keyspace, replication)
#self.clientConnections[0].create_column_family(keyspace, tableName)
#self.clientConnections[0].load_data(keyspace, tableName, how_many_at_a_time, sleep_interval, NUM_REQUESTS, file_seek_pos)
#self.clientConnections[0].query_schema(keyspace, tableName, how_many_at_a_time, sleep_interval, NUM_REQUESTS, file_seek_pos)
self.clientConnections[0].getAndWrite(keyspace, tableName, how_many_at_a_time, sleep_interval, NUM_REQUESTS, file_seek_pos)
self.clientConnections[0].close()
def check_date_format(date):
if not date:
return True
try:
datetime.strptime(
date, ConfigManager.get_instance().get_required_date_format())
return True
except ValueError:
return False
def startup(self):
"""
startup routine
Loads from cloned state
"""
# clean working dir and extract config, creds, and state
log.info("Cleaning working directory...")
command = "rm config.ini && rm -rf data && rm -rf creds"
shell(command)
log.info("Extracting cloned state...")
command = "tar xzf clone.tar.gz"
shell(command)
# load config
log.info("Loading configuration...")
self.cm = ConfigManager()
self.conf = self.cm.load_config(configFile)
# load state
log.info("Loading state...")
self.state = StateManager(self.conf)
self.state.load_state()
class ConnectionManager:
def __init__(self):
self.configMgr = None
self.clientConnections = []
self.initialize()
def initialize(self):
self.configMgr = ConfigManager()
for i in range(0, self.configMgr.getNumConnections() - 1):
self.clientConnections.append(SimpleClient())
self.clientConnections[0].connect(self.configMgr.getNodes())
keyspace = "finSpa"
replication = "2"
tableName = "finData"
how_many_at_a_time = 100 # How many records to insert in a batch.
howLong = 120 # Run for 120 seconds.
sleep_interval = 3 # Sleep for 3 seconds.
first_N_rows = 500 # how many rows to fetch at a time.
#self.clientConnections[0].create_schema(keyspace, replication)
#self.clientConnections[0].create_column_family(keyspace, tableName)
self.clientConnections[0].load_data(keyspace, tableName, how_many_at_a_time, sleep_interval)
self.clientConnections[0].query_schema(keyspace, tableName, first_N_rows, howLong, sleep_interval)
self.clientConnections[0].close()
def startup(self):
"""
startup routine
Loads from cloned state
"""
# clean working dir and extract config, creds, and state
log.info("Cleaning working directory...")
command = "rm config.ini && rm -rf data && rm -rf creds"
shell(command)
log.info("Extracting cloned state...")
command = "tar xzf clone.tar.gz"
shell(command)
# load config
log.info("Loading configuration...")
self.cm = ConfigManager()
self.conf = self.cm.load_config(configFile)
# load state
log.info("Loading state...")
self.state = StateManager(self.conf)
self.state.load_state()
def get_municipalities_df_with_file_check(
forceUrlDownload: bool = False) -> pd.DataFrame:
"""Returns pandas DataFrame
All municipalities of GR with info about bfsNr, municipality name, area name and canton name.
Arguments:
forceUrlDownload (default = False): force a fresh download of the municipalities data from the source web page.
"""
print(
f'get_municipalities_df_with_file_check: forceUrlDownload is set to {forceUrlDownload}.'
)
# get configuration for download urls and filenames
cfgm = ConfigManager.get_instance()
bfs_cf = cfgm.get_bfs_community_booth_configuration()
# declare df_municipalities for later assignment
df_municipalities = None
# excel_file_path = os.path.abspath(os.path.join('resources','municipality_data', bfs_cf['excelFileName']))
excel_file_path = os.path.join(cfgm.ROOT_DIR, 'resources',
'municipality_data',
bfs_cf['excelFileName'])
# if excel file does not exist, download and save it previously
if os.path.isfile(excel_file_path) and forceUrlDownload is False:
print(f"{excel_file_path} exists and no datasource reload forced.")
print(f"Reading {excel_file_path} now...")
dfs = pd.read_excel(excel_file_path)
df_municipalities = dfs
print(f"Done reading {excel_file_path}.")
else:
if forceUrlDownload is False:
print(
f"{excel_file_path} does not exist. Excel needs to be downloaded..."
)
print(f"Fetching data from {bfs_cf['dataSourcePostUrl']}.")
response = requests.post(bfs_cf['dataSourcePostUrl'])
excelData = response.content
dfs = pd.read_excel(excelData)
df_municipalities = dfs
# write file down to disk
with pd.ExcelWriter(excel_file_path) as writer:
df_municipalities.to_excel(writer)
print(
f'Excel has been downloaded and persisted to {excel_file_path}.'
)
print(
f'Preprocessing the output DF now: Renaming columns and removing other cantons than GR...'
)
df_municipalities.rename(columns={
'BFS Gde-nummer': 'BFS_Nr',
'Bezirks-nummer': 'Bezirks_Nr',
'Datum der Aufnahme': 'Aufnahmedatum',
'Hist.-Nummer': 'Hist_Nr'
},
inplace=True)
df_municipalities = df_municipalities[df_municipalities['Kanton'] == 'GR']
df_municipalities = df_municipalities[[
'BFS_Nr', 'Gemeindename', 'Bezirksname', 'Kanton'
]]
df_municipalities.set_index('BFS_Nr', inplace=True)
print(f'Finished preprocessing. Resulting dataframe:')
print(f"{df_municipalities}")
return df_municipalities
from datetime import datetime
import json
import logging
import pandas as pd
import pytest
from configManager import ConfigManager
from services.incidence_service import IncidenceService
application_root = ConfigManager.get_instance().get_application_root()
# Enforce trailing slash
application_root = application_root if application_root.endswith(
'/') else f'{application_root}/'
testdata_path = 'tests/api_blackbox/testdata/'
df = ConfigManager.get_instance().get_required_date_format()
MOCK_CANTON = 'GR'
NUMBER_OF_MOCKED_INCIDENCES = 1111
NUMBER_OF_MOCKED_INCIDENCES_BFSNR = 11
default_language = 'de-DE'
class MockIncidenceServiceResponse:
@staticmethod
def get_incidences(canton, dateFrom, dateTo, bfs_nr=None):
assert canton == 'GR'
incidences = json.load(
open(f'{testdata_path}test_incidences_GR_20210420_20210430.json',
'r'))
import logging
import pytest
from configManager import ConfigManager
from services.municipality_service import MunicipalityService
application_root = ConfigManager.get_instance().get_application_root()
# Enforce trailing slash
application_root = application_root if application_root.endswith(
'/') else f'{application_root}/'
MOCK_CANTON = 'GR'
NUMBER_OF_MOCKED_MUNICIPALITIES = 2
default_language = 'de-DE'
class MockMunicipalityServiceResponse:
@staticmethod
def get_municipalities(canton):
# Mocking only (parts of) canton GR
assert canton == MOCK_CANTON
return [{"area": 42.51, "bfsNr": 3506, "canton": "GR", "name": "Vaz/Obervaz", "population": 2780},
{"area": 190.14, "bfsNr": 3544, "canton": "GR", "name": "Berg\u00fcn Filisur", "population": 905}], 200
@staticmethod
def get_municipality(canton, bfs_nr) -> object:
switcher = {
3506: {"area": 42.51, "bfsNr": 3506, "canton": "GR", "name": "Vaz/Obervaz", "population": 2780},
3544: {"area": 190.14, "bfsNr": 3544, "canton": "GR", "name": "Berg\u00fcn Filisur", "population": 905}
}
return switcher.get(int(bfs_nr), []), 200
import redis
from app import app
from configManager import ConfigManager
from controllers.waypoint_controller import waypoint_controller
from controllers.incidence_controller import incidence_controller
from controllers.municipality_controller import municipality_controller
from flask_cors import CORS
import logging
from werkzeug.exceptions import InternalServerError, NotFound
logger = logging.getLogger(__name__)
server_config = ConfigManager.get_instance().get_server_config()
application_root = ConfigManager.get_instance().get_application_root()
app.register_blueprint(waypoint_controller, url_prefix=application_root)
app.register_blueprint(incidence_controller, url_prefix=application_root)
app.register_blueprint(municipality_controller, url_prefix=application_root)
@app.errorhandler(Exception)
def handle_excpetion(e):
if isinstance(e, NotFound):
# Not found exception also contains automatic calls from browsers, e.g. to /favicon.ico
logger.debug('A NotFound exception occurred.', exc_info=e)
return e
elif isinstance(e, redis.exceptions.ConnectionError):
logger.critical(
from data_access.canton_data_access import CantonDataAccess
from data_access.geo_data_access import GeoDataAccess
from datetime import timedelta
from configManager import ConfigManager
import logging
import pandas as pd
import math
import matplotlib
import matplotlib.pyplot as pyplot
logger = logging.getLogger(__name__)
df = ConfigManager.get_instance().get_required_date_format()
incidence_retry_days: int = ConfigManager.get_instance(
).get_incidence_retry_days()
search_radius: int = ConfigManager.get_instance().get_geoservice_search_radius(
)
no_incidence_color: str = ConfigManager.get_instance().get_no_incidence_color()
min_incidence_normalize_value = ConfigManager.get_instance(
).get_min_incidence_normalize_value()
max_incidence_normalize_value = ConfigManager.get_instance(
).get_max_incidence_normalize_value()
class WaypointService:
@staticmethod
def get_waypoints_data(waypoints) -> (dict, set):
municipalities_geo_data = []
timedout_cantons = set()
municipalities_geo_data = GeoDataAccess.get_geodata(waypoints)
def __init__(self, master):
master.minsize(width=300, height=80)
self.savedRealmName = []
self.savedRealmAddress = []
self.selectRealmButton = []
self.removeRealmButton = []
self.rlmManager = RealmManager()
self.cfgManager = ConfigManager()
if not self.cfgManager.hasGameFolder():
newGameFolder = chooseGameFolder()
# Force user to choose a game folder
while newGameFolder is None:
isCanceled = simpledialog.askretrycancel(
'Question',
'You need to select a game folder',
parent=root)
if isCanceled:
root.destroy()
return
newGameFolder = chooseGameFolder()
self.cfgManager.updateGameFolder(newGameFolder)
self.mainFrame = Frame(master, width=300, height=100)
self.mainFrame.grid(row=0, column=0, padx=20)
self.title = Message(self.mainFrame,
text='WoW Realm Changer',
width=400,
font=('Helvetica', 18, 'bold'))
self.title.grid()
inputFrame = Frame(self.mainFrame)
inputFrame.grid(row=1, column=0)
self.realmInputLabel = Message(inputFrame,
text='set realmlist:',
width=100,
font=('Helvetica', 11))
self.realmInputLabel.grid(row=1, column=1)
self.realmAddressInput = Entry(inputFrame)
self.realmAddressInput.grid(row=1, column=2)
self.saveButton = Button(inputFrame,
text='SAVE',
command=self.addRealm)
self.saveButton.grid(row=1, column=4)
self.currentRealmFrame = Frame(self.mainFrame)
self.currentRealmFrame.grid(row=2, column=0)
self.realmInitText = Message(self.currentRealmFrame,
text=self.rlmManager.currentRealm(),
width=300,
font=('Helvetica', 11))
self.realmInitText.grid(row=2, column=0)
self.buildSavedRealmsList()
def check_supported_language(language):
return language and language.lower() in [lng.lower() for lng in ConfigManager.get_instance().get_languages() if lng]
class BPConDemo:
def __init__(self):
try:
self.startup()
self.state = StateManager(self.conf)
self.loop = asyncio.get_event_loop()
self.bpcon = BPConProtocol(self.conf, self.state)
self.paxos_server = websockets.serve(self.bpcon.main_loop,
self.conf['ip_addr'],
self.conf['port'],
ssl=self.conf['ssl'])
self.loop.run_until_complete(self.paxos_server)
log.info("Started BPCon on port {}".format(self.conf['port']))
if self.conf['is_client']:
log.debug("is client. making test requests")
for x in range(1):
self.commit("P,{},hello{}".format(x, x))
self.commit("P,test,value")
self.commit("P,test{},value{}".format(x, x))
self.commit("P,test2,value2")
self.commit("P,test,value3")
self.commit("D,test2,")
log.debug("requests complete")
except Exception as e:
log.info(e)
def commit(self, msg):
self.loop.run_until_complete(self.bpcon_request(msg))
def got_commit_result(self, future):
if future.done():
if not future.cancelled():
self.log.info("commit result: {}".format(future.result()))
else:
self.log.info("future cancelled")
else:
self.log.info("future not done")
@asyncio.coroutine
def bpcon_request(self, msg):
log.debug("making request: {}".format(msg))
bpcon_task = asyncio.Future()
bpcon_task.add_done_callback(self.got_commit_result)
try:
timer_result = asyncio.wait_for(bpcon_task,
3.0) # timer possibly unneccessary
commit_result = yield from self.bpcon.request(
msg, bpcon_task) # returns boolean
log.info("bpcon request result: {}".format(commit_result))
return commit_result
except asyncio.TimeoutError:
log.info("bpcon commit timed out")
except asyncio.CancelledError:
log.info("bpcon commit future cancelled")
except Exception as e:
log.debug(e)
def startup(self):
"""
startup routine
Loads from cloned state
"""
# clean working dir and extract config, creds, and state
log.info("Cleaning working directory...")
command = "rm config.ini && rm -rf data && rm -rf creds"
shell(command)
log.info("Extracting cloned state...")
command = "tar xzf clone.tar.gz"
shell(command)
# load config
log.info("Loading configuration...")
self.cm = ConfigManager()
self.conf = self.cm.load_config(configFile)
# load state
log.info("Loading state...")
self.state = StateManager(self.conf)
self.state.load_state()
"""
def clone(self):
#create a copy of db, peers, peer creds, and config
#save to compressed archive
#used to add new nodes to system
try:
self.state.image_state()
self.cm.save_config()
backupdir = "data/"
cfile = "config.ini"
command = "tar czf clone.tar.gz {} {} creds/".format(cfile,backupdir)
shell(command)
log.info("clone of state successfully created")
except Exception as e:
log.info("clone of state failed")
def handle_reconfig_request(self, epoch=0):
toreturn = self.bpcon.bmsgs
if epoch != 0:
self.clone()
with open('clone.tar.gz', 'r') as fh:
toreturn += "<>{}".format(fh.read())
log.debug("cloned state added successfully")
return toreturn
def make_reconfig_request(self, wss):
# epoch = current epoch
pass
"""
def shutdown(self):
print("\nShutdown initiated...")
print("\nDatabase contents:\n{}".format(
self.bpcon.state.db.kvstore)) # save state here
self.paxos_server.close()
from services.municipality_service import MunicipalityService
from flask import jsonify, request, Blueprint
from configManager import ConfigManager
from app import app
import logging
from services.errorhandler_service import ErrorHandlerService
from flask_cors import cross_origin
logger = logging.getLogger(__name__)
municipality_controller = Blueprint('municipality_controller', __name__)
df = ConfigManager.get_instance().get_required_date_format()
default_language = ConfigManager.get_instance().get_languages()[0]
@app.route('/cantons/<canton>/municipalities/')
@cross_origin()
@municipality_controller.route('/cantons/<canton>/municipalities/',
methods=['GET'])
def get_municipalities_for_canton(canton):
"""
Returns municipalities of the given canton
---
produces:
- application/json
parameters:
- in: path
name: canton
type: string
required: true
from flask import jsonify, request, Blueprint
from configManager import ConfigManager
from app import app
import logging
from flask_cors import cross_origin
from services.waypoint_service import WaypointService
from services.errorhandler_service import ErrorHandlerService
from flask import make_response
logger = logging.getLogger(__name__)
waypoint_controller = Blueprint('waypoint_controller', __name__)
df = ConfigManager.get_instance().get_required_date_format()
search_radius: int = ConfigManager.get_instance().get_geoservice_search_radius(
)
default_language = ConfigManager.get_instance().get_languages()[0]
@app.route('/waypoints/', methods=['POST'])
@cross_origin()
@waypoint_controller.route('/waypoints/', methods=['POST'])
def post_waypoints():
"""
Returns municipalities and their corona- and geo-information where the given waypoints lay in.
---
description: Municipalities with corona and geo-information
produces:
- application/json
consumes:
- application/json
from telegram.ext import Updater, CommandHandler, MessageHandler, Filters
from session_manager import SessionManager
from io import BytesIO
from assistant import IBMAssistant
import voice
from configManager import ConfigManager
import logging
logging.basicConfig(
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
level=logging.INFO)
logger = logging.getLogger('TelegramBot')
updater = Updater(token=ConfigManager.get_telegram_token(), use_context=True)
dispatcher = updater.dispatcher
def start(update, context):
assistant = IBMAssistant(update.effective_chat.id)
response_text = assistant.send_message('Olá')
context.bot.send_message(chat_id=update.effective_chat.id,
text=response_text)
def message(update, context):
message_received = update.message.text
assistant = IBMAssistant(update.effective_chat.id)
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
from configManager import ConfigManager
import logging
import logging.config
from os import path
app = Flask(__name__)
log_file_path = path.join(path.dirname(path.abspath(__file__)), 'log.conf')
logging.config.fileConfig(log_file_path, disable_existing_loggers=True)
app.config['SQLALCHEMY_DATABASE_URI'] = 'postgresql://%(user)s:\
%(pw)s@%(host)s:%(port)s/%(db)s' % ConfigManager.get_instance().get_postgres_config()
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024
app.config['JSON_SORT_KEYS'] = False
db = SQLAlchemy(app)
def create_app():
return app
def get_test_app():
global db, app
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///:memory:'
# overwrite the postgresql settings, as db gets referenced by services
db = SQLAlchemy(app)
from datetime import datetime
import pickle
import pytest
from configManager import ConfigManager
from services.waypoint_service import WaypointService
from data_access.canton_data_access import CantonDataAccess
testdata_path = 'tests/services/testdata/'
df = ConfigManager.get_instance().get_required_date_format()
class MockCantonDataAccessResponse:
@staticmethod
def get_incidences(canton, dateFrom, dateTo, bfs_nr=None):
# Mocking only canton GR
assert canton == 'GR'
with open(f'{testdata_path}GR_get_incidences_20210420_20210427.pkl',
'rb') as fh:
result = pickle.load(fh)
dateFrom_date = datetime.strptime(dateFrom, df)
dateTo_date = datetime.strptime(dateTo, df)
filtered_result = [
i for i in result if (i['bfsNr'] == bfs_nr or bfs_nr is None)
and datetime.strptime(i['date'], df) >= dateFrom_date
and datetime.strptime(i['date'], df) <= dateTo_date
]
return filtered_result, None
def get_cantons_metadata_df_with_file_check(
forceUrlDownload: bool = False) -> pd.DataFrame:
"""Internal method which decides whether to use a local copy of the excel or fetching a new one.
Returns pandas DataFrame
All municipalities of Switzerland with info about bfsNr, municipality name, area in km2 and population.
Arguments:
forceUrlDownload (default = False): force a fresh download of the cantons meta data from the source web page.
"""
print(
f'get_cantons_metadata_df_with_file_check: forceUrlDownload is set to {forceUrlDownload}.'
)
# get configuration for download urls and filenames
cfgm = ConfigManager.get_instance()
cf_cm = cfgm.get_canton_metadata_configuration()
# declare df_municipalities for later assignment
df_canton_metadata = None
excel_file_path = os.path.join(cfgm.ROOT_DIR, 'resources',
'municipality_data', cf_cm['excelFileName'])
# if excel file does not exist, download and save it previously
if os.path.isfile(excel_file_path) and forceUrlDownload is False:
print(f"{excel_file_path} exists and no datasource reload forced.")
print(f"Reading {excel_file_path} now...")
dfs = pd.read_excel(excel_file_path)
df_canton_metadata = dfs
print(f"Done reading {excel_file_path}.")
else:
if forceUrlDownload is False:
print(
f"{excel_file_path} does not exist. Excel needs to be downloaded..."
)
print(f"Fetching data from {cf_cm['dataSourceGetUrl']}.")
response = requests.get(cf_cm['dataSourceGetUrl'])
excelData = response.content
dfs = pd.read_excel(excelData)
df_canton_metadata = dfs
# write file down to disk
with pd.ExcelWriter(excel_file_path) as writer:
df_canton_metadata.to_excel(writer)
print(
f'Excel has been downloaded and persisted to {excel_file_path}.'
)
df_canton_metadata = df_canton_metadata.dropna()
df_canton_metadata.columns = df_canton_metadata.loc[4].values
df_canton_metadata = df_canton_metadata.iloc[1:]
df_population_for_bfsNr = df_canton_metadata[[
'Gemeindecode', 'Gemeindename', 'Einwohner', 'Gesamtfläche in km²'
]]
df_population_for_bfsNr.reset_index(inplace=True)
df_population_for_bfsNr = df_population_for_bfsNr.drop(['index'], axis=1)
df_population_for_bfsNr.rename(columns={
'Gemeindecode':
'BFS_Nr',
'Gesamtfläche in km²':
'Gesamtflaeche_in_km2'
},
inplace=True)
df_population_for_bfsNr.set_index('BFS_Nr', inplace=True)
# There is no new dataset of municipality metadata provided which contains merged municipalities per 01.01.2021
# Merge municipality 'Muntogna da Schons' (BFS 3715) from municipalities:
# Casti-Wergenstein (3703)
# Donat (3705)
# Lohn (GR) (3707)
# Mathon (3708)
# and add a new series to the df
series_muntogna_da_schons = df_population_for_bfsNr.loc[[
3703, 3705, 3707, 3708
]].sum()
series_muntogna_da_schons['Gemeindename'] = 'Muntogna da Schons'
series_muntogna_da_schons.name = 3715
df_population_for_bfsNr = df_population_for_bfsNr.append(
series_muntogna_da_schons)
return df_population_for_bfsNr
class CantonDataAccess:
canton_api_urls = ConfigManager.get_instance().get_cantonservice_urls()
@staticmethod
def get_default_date():
return datetime.now()
@staticmethod
def get_municipalities(canton):
logger.info(
f'CantonDataAccess.get_municipalities(canton) with canton={canton}'
)
if not CantonDataAccess.__is_canton_available(canton):
return [], 404
try:
municipalities = CantonDataAccess.__get_municipalities(
canton, None)
logger.debug(
f'Got {len(municipalities)} municipalities from Canton Service {canton}.'
)
result = [Municipality(**m).as_dict for m in municipalities]
return result, 200
except requests.exceptions.Timeout as errh:
logger.warning(
'Timeout when calling CantonDataAccess.__get_municipalities or processing its response'
)
return None, 408
except requests.exceptions.HTTPError as errh:
log_msg = f'HTTPError when calling CantonDataAccess.__get_municipalities: {errh.response.status_code} - {errh.response.reason}'
if errh.response.status_code < 500:
logger.warning(log_msg)
else:
logger.exception(log_msg)
return None, errh.response.status_code
except requests.exceptions.RequestException:
logger.exception(
'Exception when calling CantonDataAccess.__get_municipalities or processing its response.'
)
return None, None
@staticmethod
def get_municipality(canton, bfs_nr):
logger.info(
f'CantonDataAccess.get_municipalitiy(canton, bfs_nr) with canton={canton}, bfs_nr={bfs_nr}'
)
if not CantonDataAccess.__is_canton_available(canton):
return None, 404
try:
municipality = CantonDataAccess.__get_municipalities(
canton, bfs_nr)
return Municipality(**municipality).as_dict, 200
except requests.exceptions.Timeout as errh:
logger.warning(
'Timeout when calling CantonDataAccess.__get_municipalities or processing its response'
)
return None, 408
except requests.exceptions.HTTPError as errh:
log_msg = f'HTTPError when calling CantonDataAccess.__get_municipalities: {errh.response.status_code} - {errh.response.reason}'
if errh.response.status_code < 500:
logger.warning(log_msg)
else:
logger.exception(log_msg)
return None, errh.response.status_code
except requests.exceptions.RequestException:
logger.exception(
'Exception when calling CantonDataAccess.__get_municipalities or processing its response.'
)
return None, None
@staticmethod
def get_incidences(canton, dateFrom, dateTo, bfs_nr=None):
logger.info(
f'CantonDataAccess.get_incidences(canton, dateFrom, dateTo, bfs_nr=None) with canton={canton}, dateFrom={dateFrom}, dateTo={dateTo}, bfs_nr={bfs_nr}'
)
if not CantonDataAccess.__is_canton_available(canton):
return None, 404
try:
if bfs_nr is None:
incidences = CantonDataAccess.__get_incidences(
canton, dateFrom, dateTo)
else:
incidences = CantonDataAccess.__get_incidences(
canton, dateFrom, dateTo, bfs_nr)
logger.debug(
f'Got {len(incidences)} incidences from Canton Service {canton}.'
)
result = []
for i in incidences:
result.append(Incidence(**i).as_dict)
return result, 200
except requests.exceptions.Timeout as errh:
logger.warning(
'Timeout when calling CantonDataAccess.__get_incidences or processing its response'
)
return None, 408
except requests.exceptions.HTTPError as errh:
log_msg = f'HTTPError when calling CantonDataAccess.__get_incidences: {errh.response.status_code} - {errh.response.reason}'
if errh.response.status_code < 500:
logger.warning(log_msg)
else:
logger.exception(log_msg)
return None, errh.response.status_code
except requests.exceptions.RequestException:
logger.exception(
'Exception when calling CantonDataAccess.__get_incidences or processing its response'
)
return None, None
@staticmethod
def __get_incidences(canton: str, dateFrom, dateTo, bfs_nr=None):
resource_path = '/incidences/'
query_params = {'dateFrom': dateFrom, 'dateTo': dateTo}
response = CantonDataAccess.__get_response(canton, resource_path,
bfs_nr, query_params)
return response.json()
@staticmethod
def __get_municipalities(canton: str, bfs_nr=None):
resource_path = '/municipalities/'
response = CantonDataAccess.__get_response(canton, resource_path,
bfs_nr)
return response.json()
@staticmethod
def __get_response(canton: str,
resource_path: str,
path_param: str = None,
query_params: dict = None):
host, ssl_cert_path = CantonDataAccess.__get_request_info(canton)
url = f'{host}{resource_path}' + \
(f'{path_param}/' if path_param is not None else '')
logger.debug(f'Going to call url: {url}')
if ssl_cert_path != '':
response = requests.get(url,
verify=ssl_cert_path,
params=query_params,
timeout=(3.05, 15))
else:
response = requests.get(url,
params=query_params,
timeout=(3.05, 15))
response.raise_for_status()
logger.debug(
f'Got response from Canton Service {canton}. (url: {url}, from_cache: {response.from_cache if hasattr(response, "from_cache") else "nocache"}, has SSL cert file: {(ssl_cert_path != "")})'
)
return response
@staticmethod
def __get_request_info(canton: str):
host: str = ''
canton_ssl_cert_path: str = ''
if CantonDataAccess.__is_canton_available(canton):
host = CantonDataAccess.canton_api_urls[canton]['url']
canton_ssl_cert_name = CantonDataAccess.canton_api_urls[canton][
'ssl_ca_cert']
if canton_ssl_cert_name != '':
canton_ssl_cert_path = f'certificates/{canton_ssl_cert_name}'
return host, canton_ssl_cert_path
@staticmethod
def __is_canton_available(canton: str):
if canton in CantonDataAccess.canton_api_urls and CantonDataAccess.canton_api_urls[
canton]['url'] != '':
return True
else:
logger.warning(f'Canton not available: {canton}')
return False
app = Flask(__name__)
# Setup swagger
swagger = Swagger(app,
template=template,
config={"specs_route": "/"},
merge=True)
# Setup logging
log_file_path = path.join(path.dirname(path.abspath(__file__)), 'log.conf')
logging.config.fileConfig(log_file_path, disable_existing_loggers=True)
logger = logging.getLogger(__name__)
redis_server = ConfigManager.get_instance().get_redis_server_config()
secret_key = ConfigManager.get_instance().get_secret()
redis_caching_hours: float = ConfigManager.get_instance(
).get_redis_caching_hours()
redis_conn = redis.Redis(host=redis_server['host'],
port=redis_server['port'],
password=redis_server['password'])
# Setup request_cache with redis backend
requests_cache.install_cache(cache_name='cantonservice_cache',
backend='redis',
connection=redis_conn,
secret_key=secret_key,
expire_after=3600 *
redis_caching_hours) # in s, default 4h
from services.incidence_service import IncidenceService
from services.municipality_service import MunicipalityService
from data_access.canton_data_access import CantonDataAccess
from configManager import ConfigManager
from datetime import timedelta
from app import requests_cache
# Clear the cache before warmup
# Otherwise we wouldn't really warmup up the cache because expire time of
# already cached requests will not be reset...
if requests_cache.is_installed():
requests_cache.clear()
# Get all configured cantons from config
cantonsservice_urls = ConfigManager.get_instance().get_cantonservice_urls()
# Get date format from config
df = ConfigManager.get_instance().get_required_date_format()
# Set the date_from and date_to dates for warmup
date_from = date_to = CantonDataAccess.get_default_date().strftime(df)
print(f'Fetching for date_from {date_from} to date_to {date_to}')
# Loop through all the cantons
for canton, canton_data in cantonsservice_urls.items():
# Only process canton if a server is configured
if canton_data['url'] != '':
print(f"Current canton {canton} with URL {canton_data['url']}")
# Fetch all the municipalities for current canton
class App:
def __init__(self, master):
master.minsize(width=300, height=80)
self.savedRealmName = []
self.savedRealmAddress = []
self.selectRealmButton = []
self.removeRealmButton = []
self.rlmManager = RealmManager()
self.cfgManager = ConfigManager()
if not self.cfgManager.hasGameFolder():
newGameFolder = chooseGameFolder()
# Force user to choose a game folder
while newGameFolder is None:
isCanceled = simpledialog.askretrycancel(
'Question',
'You need to select a game folder',
parent=root)
if isCanceled:
root.destroy()
return
newGameFolder = chooseGameFolder()
self.cfgManager.updateGameFolder(newGameFolder)
self.mainFrame = Frame(master, width=300, height=100)
self.mainFrame.grid(row=0, column=0, padx=20)
self.title = Message(self.mainFrame,
text='WoW Realm Changer',
width=400,
font=('Helvetica', 18, 'bold'))
self.title.grid()
inputFrame = Frame(self.mainFrame)
inputFrame.grid(row=1, column=0)
self.realmInputLabel = Message(inputFrame,
text='set realmlist:',
width=100,
font=('Helvetica', 11))
self.realmInputLabel.grid(row=1, column=1)
self.realmAddressInput = Entry(inputFrame)
self.realmAddressInput.grid(row=1, column=2)
self.saveButton = Button(inputFrame,
text='SAVE',
command=self.addRealm)
self.saveButton.grid(row=1, column=4)
self.currentRealmFrame = Frame(self.mainFrame)
self.currentRealmFrame.grid(row=2, column=0)
self.realmInitText = Message(self.currentRealmFrame,
text=self.rlmManager.currentRealm(),
width=300,
font=('Helvetica', 11))
self.realmInitText.grid(row=2, column=0)
self.buildSavedRealmsList()
def buildSavedRealmsList(self):
self.savedRealmsFrame = Frame(self.mainFrame)
self.savedRealmsFrame.grid(row=3, pady=(20, 10))
self.savedRealmsMessage = Message(self.savedRealmsFrame,
text='Saved Realms',
width=300,
font=('Helvetica', 15, 'bold'))
self.savedRealmsMessage.grid()
for idx, realm in enumerate(self.cfgManager.savedRealms()):
self.addRealmRow(realm)
def setActiveRealm(self, realm):
self.rlmManager.changeActiveRealm(realm)
self.realmInitText.configure(text='Realm changed to: ' + realm['name'])
def removeSavedRealm(self, idx):
self.cfgManager.removeRealm(idx)
self.removeRealmRow(idx)
def addRealm(self):
realmName = simpledialog.askstring('Input',
'What is the name of the realm?',
parent=root)
realmAddress = self.realmAddressInput.get()
realm = {'address': realmAddress, 'name': realmName}
self.cfgManager.addRealm(realm)
self.addRealmRow(realm)
def removeRealmRow(self, index):
self.savedRealmName[index].destroy()
self.savedRealmAddress[index].destroy()
self.selectRealmButton[index].destroy()
self.removeRealmButton[index].destroy()
def addRealmRow(self, realm):
if not realm:
return
index = len(self.savedRealmName)
name = realm['name']
address = realm['address']
self.savedRealmName.append(
Message(self.savedRealmsFrame, text=name, width=250))
self.savedRealmName[index].grid(column=0, row=index + 1)
self.savedRealmAddress.append(
Message(self.savedRealmsFrame, text=address, width=250))
self.savedRealmAddress[index].grid(column=1, row=index + 1)
self.selectRealmButton.append(
Button(self.savedRealmsFrame,
text='SET',
command=lambda rlm=realm: self.setActiveRealm(rlm)))
self.selectRealmButton[index].grid(column=2, row=index + 1)
self.removeRealmButton.append(
Button(self.savedRealmsFrame,
text='REMOVE',
command=lambda idx=index: self.removeSavedRealm(idx)))
self.removeRealmButton[index].grid(column=3, row=index + 1)
self.totalRealmItems = index + 1
class BPConDemo:
def __init__(self):
try:
self.startup()
self.state = StateManager(self.conf)
self.loop = asyncio.get_event_loop()
self.bpcon = BPConProtocol(self.conf, self.state)
self.paxos_server = websockets.serve(self.bpcon.main_loop, self.conf['ip_addr'], self.conf['port'], ssl=self.conf['ssl'])
self.loop.run_until_complete(self.paxos_server)
log.info("Started BPCon on port {}".format(self.conf['port']))
if self.conf['is_client']:
log.debug("is client. making test requests")
for x in range(1):
self.commit("P,{},hello{}".format(x,x))
self.commit("P,test,value")
self.commit("P,test{},value{}".format(x,x))
self.commit("P,test2,value2")
self.commit("P,test,value3")
self.commit("D,test2,")
log.debug("requests complete")
except Exception as e:
log.info(e)
def commit(self,msg):
self.loop.run_until_complete(self.bpcon_request(msg))
def got_commit_result(self, future):
if future.done():
if not future.cancelled():
self.log.info("commit result: {}".format(future.result()))
else:
self.log.info("future cancelled")
else:
self.log.info("future not done")
@asyncio.coroutine
def bpcon_request(self, msg):
log.debug("making request: {}".format(msg))
bpcon_task = asyncio.Future()
bpcon_task.add_done_callback(self.got_commit_result)
try:
timer_result = asyncio.wait_for(bpcon_task, 3.0) # timer possibly unneccessary
commit_result = yield from self.bpcon.request(msg, bpcon_task) # returns boolean
log.info("bpcon request result: {}".format(commit_result))
return commit_result
except asyncio.TimeoutError:
log.info("bpcon commit timed out")
except asyncio.CancelledError:
log.info("bpcon commit future cancelled")
except Exception as e:
log.debug(e)
def startup(self):
"""
startup routine
Loads from cloned state
"""
# clean working dir and extract config, creds, and state
log.info("Cleaning working directory...")
command = "rm config.ini && rm -rf data && rm -rf creds"
shell(command)
log.info("Extracting cloned state...")
command = "tar xzf clone.tar.gz"
shell(command)
# load config
log.info("Loading configuration...")
self.cm = ConfigManager()
self.conf = self.cm.load_config(configFile)
# load state
log.info("Loading state...")
self.state = StateManager(self.conf)
self.state.load_state()
"""
def clone(self):
#create a copy of db, peers, peer creds, and config
#save to compressed archive
#used to add new nodes to system
try:
self.state.image_state()
self.cm.save_config()
backupdir = "data/"
cfile = "config.ini"
command = "tar czf clone.tar.gz {} {} creds/".format(cfile,backupdir)
shell(command)
log.info("clone of state successfully created")
except Exception as e:
log.info("clone of state failed")
def handle_reconfig_request(self, epoch=0):
toreturn = self.bpcon.bmsgs
if epoch != 0:
self.clone()
with open('clone.tar.gz', 'r') as fh:
toreturn += "<>{}".format(fh.read())
log.debug("cloned state added successfully")
return toreturn
def make_reconfig_request(self, wss):
# epoch = current epoch
pass
"""
def shutdown(self):
print("\nShutdown initiated...")
print("\nDatabase contents:\n{}".format(self.bpcon.state.db.kvstore)) # save state here
self.paxos_server.close()
from ibm_cloud_sdk_core.authenticators import IAMAuthenticator
from ibm_watson import TextToSpeechV1, SpeechToTextV1
from io import BytesIO
import logging
from configManager import ConfigManager
logger = logging.getLogger('TelegramBot')
t2sauth = IAMAuthenticator(ConfigManager.get_t2s_token())
s2tauth = IAMAuthenticator(ConfigManager.get_s2t_token())
text2speech = TextToSpeechV1(authenticator=t2sauth)
text2speech.set_service_url(ConfigManager.get_t2s_url())
speech2text = SpeechToTextV1(authenticator=s2tauth)
speech2text.set_service_url(ConfigManager.get_s2t_url())
def convert_voice(audio_file):
response = speech2text.recognize(audio=audio_file,
content_type='audio/ogg',
model='pt-BR_NarrowbandModel')
result = response.get_result()
logger.info('Detectada frase: ' +
result['results'][0]['alternatives'][0]['transcript'])
return result['results'][0]['alternatives'][0]['transcript']
import math
import requests
import json
import pandas as pd
import time
from datetime import datetime, timedelta
from configManager import ConfigManager
import logging
logger = logging.getLogger(__file__)
arcgis_config = ConfigManager.get_instance(
).get_arcgis_rest_services_cases_per_region_configuration()
# The covid-19 API allows only sending 2000 data rows per request
# One date contains 12 rows. Therefore we can request up to 166 days per request (166 * 12 = 1992)
MAX_DAYS_PER_REQUEST: int = math.floor(
int(arcgis_config['max_data_rows_per_request']) /
int(arcgis_config['number_of_regions'])) # floor(2000 / 12) = 166
MAX_RESULT_ROWS: int = int(arcgis_config['number_of_regions']
) * MAX_DAYS_PER_REQUEST # 166 * 12 = 1992
def fetch_corona_data(dateFrom, dateTo) -> str:
'''
Performs a query on the canton endpoint for receiving covid-19 data. See https://curl.trillworks.com/ for request usage.
'''
endpoint_url = arcgis_config['endpoint_url']
# params = (
import sys import uuid import os import datetime import argparse import requests import json import time import serial from functools import wraps from configManager import ConfigManager credentials = ConfigManager() ############################# Configuration ########################## port = "/dev/tty.usbserial-FT99JMNX" #Please Change COM Port Here baudrate = 9600 ###################################################################### delay = 0.5 userName = credentials.configData[ConfigManager.USERNAME_KEY] #print userName password = credentials.configData[ConfigManager.PASSWORD_KEY] #print password pilot = credentials.configData[ConfigManager.PILOT_KEY]
#!/bin/python
from client import start_client
from configManager import ConfigManager
if __name__ == '__main__':
cm = ConfigManager()
mysql_db_server_ip = cm.getConfigValue('mysql_db_server_ip')
mysql_user = cm.getConfigValue('mysql_user')
mysql_password = cm.getConfigValue('mysql_password')
mysql_db_name = cm.getConfigValue('mysql_db_name')
REDIS_HOST = cm.getConfigValue('REDIS_HOST')
REDIS_PORT = cm.getConfigValue('REDIS_PORT')
REDIS_db = cm.getConfigValue('REDIS_db')
if mysql_db_server_ip is None or mysql_db_server_ip == '':
raise Exception('mysql_db_server_ip is not configured')
if mysql_user is None or mysql_db_server_ip == '':
raise Exception('mysql_user is not configured')
if mysql_password is None or mysql_db_server_ip == '':
raise Exception('mysql_password is not configured')
if mysql_db_name is None or mysql_db_server_ip == '':
raise Exception('mysql_db_name is not configured')
if REDIS_HOST is None or mysql_db_server_ip == '':
raise Exception('REDIS_HOST is not configured')
if REDIS_PORT is None or mysql_db_server_ip == '':
raise Exception('REDIS_PORT is not configured')
if REDIS_db is None or mysql_db_server_ip == '':
raise Exception('REDIS_db is not configured')
start_client(mysql_db_server_ip, mysql_user, mysql_password, mysql_db_name,
REDIS_HOST, REDIS_PORT, REDIS_db)