def load_into_csv_list(self):
session = self.connector.get_session()
query = "SELECT Wahlkreis.wahlkreisnr, Bezirk.bezirknr, Sprengel.sprengelnr, Sprengel.wahlberechtigte, " \
"Sprengel.abgegebene, Sprengel.ungueltige, Stimmabgabe.abkuerzung, Stimmabgabe.anzahl " \
"FROM Wahlkreis " \
"INNER JOIN Bezirk ON Wahlkreis.wahlkreisnr = Bezirk.wahlkreisnr " \
"INNER JOIN Sprengel ON Bezirk.bezirknr = Sprengel.bezirknr " \
"AND Sprengel.termin = '" + self.wahltermin + "' " \
"INNER JOIN Stimmabgabe ON Stimmabgabe.termin = '" + self.wahltermin + "' " \
"AND Stimmabgabe.Bezirknr = Bezirk.bezirknr " \
"AND Stimmabgabe.sprengelnr = Sprengel.sprengelnr;"
result = session.execute(query).fetchall()
header = OrderedSet(["WK", "BZ", "SPR", "WBER", "ABG.", "UNG."])
datalist = []
line = {}
first_party = None
for i in range(0, len(result)):
current_party = result[i]["abkuerzung"]
if first_party is None or current_party == first_party:
if line:
datalist.append(line)
line = {}
first_party = current_party
line["WK"] = result[i]["wahlkreisnr"]
line["BZ"] = result[i]["bezirknr"]
line["SPR"] = result[i]["sprengelnr"]
line["WBER"] = result[i]["wahlberechtigte"]
line["ABG."] = result[i]["abgegebene"]
line["UNG."] = result[i]["ungueltige"]
line[current_party] = result[i]["anzahl"]
header.add(current_party)
return datalist, list(header)
def read_data(self, termin):
query = "SELECT wahlkreis.id, bezirk.bnr, sprengel.snr, sprengel.berechtigte, " \
"sprengel.abgegeben, sprengel.ungueltige, parteistimmen.pbez, parteistimmen.stimmanzahl " \
"FROM wahlkreis " \
"INNER JOIN bezirk ON wahlkreis.id = bezirk.wkid " \
"INNER JOIN sprengel ON bezirk.bnr = sprengel.bnr " \
"AND sprengel.termin = '" + termin + "' " \
"INNER JOIN parteistimmen ON parteistimmen.termin = '" + termin + "' " \
"AND parteistimmen.bnr = bezirk.bnr " \
"AND parteistimmen.snr = sprengel.snr;"
r = self.s.execute(query).fetchall()
header = OrderedSet(["WK", "BZ", "SPR", "WBER", "ABG", "UNG"])
data = []
l = {}
first_party = None
for i in range(0, len(r)):
current_party = r[i]["pbez"]
if first_party is None or current_party == first_party:
l = {}
first_party = current_party
l["WK"] = r[i]["id"]
l["BZ"] = r[i]["bnr"]
l["SPR"] = r[i]["snr"]
l["WBER"] = r[i]["berechtigte"]
l["ABG"] = r[i]["abgegeben"]
l["UNG"] = r[i]["ungueltige"]
data.append(l)
l[current_party] = r[i]["stimmanzahl"]
header.add(current_party)
return data, list(header)
def load(self):
session = self.connector.get_session()
query = "SELECT Wahlkreis.wahlkreisnr, Bezirk.bezirknr, Sprengel.sprengelnr, Sprengel.wahlberechtigte, Sprengel.abgegebene, Sprengel.ungueltige, Stimmabgabe.abkuerzung, Stimmabgabe.anzahl FROM Wahlkreis INNER JOIN Bezirk ON Wahlkreis.wahlkreisnr = Bezirk.wahlkreisnr INNER JOIN Sprengel ON Bezirk.bezirknr = Sprengel.bezirknr AND Sprengel.termin = '" + self.wahltermin + "' INNER JOIN Stimmabgabe ON Stimmabgabe.termin = '" + self.wahltermin + "' AND Stimmabgabe.Bezirknr = Bezirk.bezirknr AND Stimmabgabe.sprengelnr = Sprengel.sprengelnr;"
result = session.execute(query).fetchall()
header = OrderedSet(["WK", "BZ", "SPR", "WBER", "ABG", "UNG"])
rawdata = []
line = {}
erste_partei = None
for i in range(0, len(result)):
aktuelle_partei = result[i]["abkuerzung"]
if erste_partei is None or aktuelle_partei == erste_partei:
if line:
rawdata.append(line)
line = {}
erste_partei = aktuelle_partei
line["WK"] = result[i]["wahlkreisnr"]
line["BZ"] = result[i]["bezirknr"]
line["SPR"] = result[i]["sprengelnr"]
line["WBER"] = result[i]["wahlberechtigte"]
line["ABG"] = result[i]["abgegebene"]
line["UNG"] = result[i]["ungueltige"]
line[aktuelle_partei] = result[i]["anzahl"]
header.add(aktuelle_partei)
return rawdata, list(header)
def reset_hosts_file(password):
reset_hosts_set = OrderedSet()
reading_addeter_lines = False
hosts_file_text = read_hosts_file(password)
if (ADDETER_START in hosts_file_text):
for line in hosts_file_text.splitlines():
if (reading_addeter_lines):
if (line == ADDETER_END):
reading_addeter_lines = False
elif (line == ADDETER_START):
reading_addeter_lines = True
else:
reset_hosts_set.add(line + "\n")
write_hosts_file(password, reset_hosts_set)
def update_hosts_file(hosts_urls, password):
update_hosts_set = OrderedSet()
reset_hosts_file(password)
hosts_file_text = read_hosts_file(password)
for line in hosts_file_text.splitlines():
update_hosts_set.add(line + "\n")
update_hosts_set.add(ADDETER_START + "\n")
update_hosts_set.update(load_remote_hosts(hosts_urls))
update_hosts_set.add(ADDETER_END + "\n")
write_hosts_file(password, update_hosts_set)
class Propagator:
MAX_REQUESTED_KEYS_TO_KEEP = 1000
def __init__(self):
self.requests = Requests()
self.requested_propagates_for = OrderedSet()
# noinspection PyUnresolvedReferences
def propagate(self, request: Request, clientName):
"""
Broadcast a PROPAGATE to all other nodes
:param request: the REQUEST to propagate
"""
if self.requests.has_propagated(request, self.name):
logger.trace("{} already propagated {}".format(self, request))
else:
self.requests.add_propagate(request, self.name)
propagate = self.createPropagate(request, clientName)
logger.info("{} propagating request {} from client {}".format(
self, (request.identifier, request.reqId), clientName),
extra={
"cli": True,
"tags": ["node-propagate"]
})
self.send(propagate)
@staticmethod
def createPropagate(request: Union[Request, dict],
client_name) -> Propagate:
"""
Create a new PROPAGATE for the given REQUEST.
:param request: the client REQUEST
:return: a new PROPAGATE msg
"""
if not isinstance(request, (Request, dict)):
logger.error(
"{}Request not formatted properly to create propagate".format(
THREE_PC_PREFIX))
return
logger.trace("Creating PROPAGATE for REQUEST {}".format(request))
request = request.as_dict if isinstance(request, Request) else \
request
if isinstance(client_name, bytes):
client_name = client_name.decode()
return Propagate(request, client_name)
# noinspection PyUnresolvedReferences
def canForward(self, request: Request):
"""
Determine whether to forward client REQUESTs to replicas, based on the
following logic:
- If exactly f+1 PROPAGATE requests are received, then forward.
- If less than f+1 of requests then probably there's no consensus on the
REQUEST, don't forward.
- If more than f+1 then already forwarded to replicas, don't forward
Even if the node hasn't received the client REQUEST itself, if it has
received enough number of PROPAGATE messages for the same, the REQUEST
can be forwarded.
:param request: the client REQUEST
"""
if self.requests.forwarded(request):
return 'already forwarded'
# If not enough Propogates, don't bother comparing
if not self.quorums.propagate.is_reached(self.requests.votes(request)):
return 'not finalised'
req = self.requests.req_with_acceptable_quorum(request,
self.quorums.propagate)
if req:
self.requests.set_finalised(req)
return None
else:
return 'not finalised'
# noinspection PyUnresolvedReferences
def forward(self, request: Request):
"""
Forward the specified client REQUEST to the other replicas on this node
:param request: the REQUEST to propagate
"""
key = request.key
logger.debug('{} forwarding request {} to {} replicas'.format(
self, key, self.replicas.sum_inbox_len))
self.replicas.pass_message(ReqKey(*key))
self.monitor.requestUnOrdered(*key)
self.requests.mark_as_forwarded(request, self.replicas.num_replicas)
# noinspection PyUnresolvedReferences
def recordAndPropagate(self, request: Request, clientName):
"""
Record the request in the list of requests and propagate.
:param request:
:param clientName:
"""
self.requests.add(request)
self.propagate(request, clientName)
self.tryForwarding(request)
def tryForwarding(self, request: Request):
"""
Try to forward the request if the required conditions are met.
See the method `canForward` for the conditions to check before
forwarding a request.
"""
cannot_reason_msg = self.canForward(request)
if cannot_reason_msg is None:
# If haven't got the client request(REQUEST) for the corresponding
# propagate request(PROPAGATE) but have enough propagate requests
# to move ahead
self.forward(request)
else:
logger.debug("{} not forwarding request {} to its replicas "
"since {}".format(self, request, cannot_reason_msg))
def request_propagates(self, req_keys):
"""
Request PROPAGATEs for the given request keys. Since replicas can
request PROPAGATEs independently of each other, check if it has
been requested recently
:param req_keys:
:return:
"""
i = 0
for (idr, req_id) in req_keys:
if (idr, req_id) not in self.requested_propagates_for:
self.request_msg(PROPAGATE, {
f.IDENTIFIER.nm: idr,
f.REQ_ID.nm: req_id
})
self._add_to_recently_requested((idr, req_id))
i += 1
else:
logger.debug(
'{} already requested PROPAGATE recently for {}'.format(
self, (idr, req_id)))
return i
def _add_to_recently_requested(self, key):
while len(self.requested_propagates_for
) > self.MAX_REQUESTED_KEYS_TO_KEEP:
self.requested_propagates_for.pop(last=False)
self.requested_propagates_for.add(key)