def get_statistics_string(self, storage=None):
if storage is None:
storage = factory.get_incident_storage()
events = storage.get_events()
providers = []
for idx, event in enumerate(events):
for incident in storage.get_incidents_from_event(event):
if incident["provider_info"]["name"] not in providers:
providers.append(incident["provider_info"]["name"])
providers = sorted(providers)
buffer = ";".join(storage._collection_names.values()) + "\n"
def list_all(event, call, buffer, providers, one_line=None):
if one_line is None:
if event.get(call, None) is not None:
buffer = buffer + (" " + call)
if event[call].get("status", None) is not None:
buffer = buffer + (
" " + json.dumps(event[call]["status"])) + "\n"
if event[call].get("incidents", None) is not None:
for incident in event[call]["incidents"]:
buffer = buffer + (
" " +
incident["provider_info"]["pushed"] + " - " +
incident["provider_info"]["name"] + " - " +
json.dumps(incident["arguments"])) + "\n"
else:
providers_found = []
for unused in providers:
providers_found.append("-")
if event.get(call, None) is not None:
if event[call].get("incidents", None) is not None:
for idx, provider in enumerate(providers):
for incident in event[call]["incidents"]:
if provider in incident["provider_info"][
"name"]:
providers_found[idx] = "x"
buffer = buffer + one_line + " " + "".join(
providers_found) + " "
return buffer
# for idx, event in enumerate(events):
# buffer = buffer + (str(idx) + ": " + event["id_string"]) + "\n"
buffer = buffer + ("Providers: " + " ".join(providers) + "\n")
buffer = buffer + (
"event number; create; in_progress; finish; result; \n")
for idx, event in enumerate(events):
buffer = buffer + (event["id_string"] + " ")
buffer = list_all(event, "create", buffer, providers, ';')
buffer = list_all(event, "in_progress", buffer, providers, ';')
buffer = list_all(event, "finish", buffer, providers, ';')
buffer = list_all(event, "result", buffer, providers, ';')
buffer = buffer + ("\n")
return buffer
def resend(url, unique_string, provider):
""" Resend one or more incidents from the store
"""
from bos_incidents import factory
storage = factory.get_incident_storage()
incident = storage.get_incident_by_unique_string_and_provider(
unique_string, provider)
pprint(incident)
incident.update(dict(skip_storage=True))
resend_incidents(url, incident)
def setUp(self):
super(TestWithSampleIncidents, self).setUp()
self.storage = factory.get_incident_storage("mongodbtest", purge=True)
# iterate the sampledata and insert all incidents
sampledata_path = join("dump", "sampledata", "incidents")
onlyfiles = [f for f in listdir(sampledata_path) if isfile(join(sampledata_path, f))]
for _file in onlyfiles:
with open(join(sampledata_path, _file)) as json_file:
data = json.load(json_file)
self.storage.insert_incident(data)
def __init__(self,
message,
lookup_instance,
config,
clear_caches=True,
**kwargs):
self.message = message
self.lookup = lookup_instance
self.config = config
# Obtain data for unique key
# The "id" contains everything we need to identify an individual event
# which itself contains at least the sport, and the teams
# Get the id (internally used only)
self.id = message.get("id")
# Incident Storage
if "storage" in kwargs and kwargs["storage"]:
self.storage = kwargs["storage"]
else:
self.storage = factory.get_incident_storage(
kwargs.get("mongodb", None), purge=kwargs.get("purge", False))
# Normalize incident
self.normalizer = IncidentsNormalizer(
chain=lookup_instance._network_name)
self.normalize(message)
# Let's clear the caches for Proposals and events
# We need to do so because of the internal pypeerplays cache.
# The cache reduces the API calls to the backend and thus latency.
# However, to be sure the event hasn't been created since before the
# cache has expired, we force a refresh from the blockchain.
if clear_caches:
Events.clear_cache()
Proposals.clear_cache()
# Try obtain the sport
self.sport = LookupSport(self.id.get("sport"))
# Given the sport, try to obtain the league (event group)
self.eventgroup = LookupEventGroup(self.sport,
self.id.get("event_group_name"))
self.event = None # Will be filled in after receiving a trigger
# Get Teams from query
self.teams = [self.id.get("home"), self.id.get("away")]
# Get start time from query
self.start_time = parse(self.id.get("start_time", ""))
def show_incidents_per_id(incident_id=None, call=None, use="mongodb"):
if use == "bos-auto":
use = "mongodb"
store = factory.get_incident_storage(use=use)
try:
event = store.get_event_by_id(incident_id, resolve=True)
except EventNotFoundException:
return jsonify("Event not found")
for key in list(event.keys()):
if not (key == call or key == "id" or key == "id_string"):
event.pop(key)
return jsonify(event)
def worker(queue):
""" Start the (redis queue) worker to deal with the received messages
"""
from .redis_con import get_redis
from . import work
work.unlock()
# Let's drop the redis database and refill it from incident store
with Connection(get_redis()):
def retrigger_from_events(events, call):
for event in events:
for incidentid in event.get(call, {}).get("incidents", []):
incident = storage.resolve_to_incident(incidentid)
q.enqueue(work.process,
args=(incident, ),
kwargs=dict(
proposer=config.get("BOOKIE_PROPOSER"),
approver=config.get("BOOKIE_APPROVER")))
# only reply the first incident!
break
q = Queue("default")
# Empty queue!
q.empty()
log.info("Redis Queue cleared")
log.info("Refilling redis queue from incident store")
storage = factory.get_incident_storage()
for call in INCIDENT_CALLS:
# "postponed", "unhandled exception, retrying soon" and "unknown" handled by scheduler
for status_name in [
"undecided", "connection lost", "related object not found",
"event missing in bos_incidents"
]:
events = list(
storage.get_events_by_call_status(call=call,
status_name=status_name))
if len(events):
log.info("Retriggering {} {}:{} incidents".format(
len(events), call, status_name))
retrigger_from_events(events, call)
# This runs the Worker as thread
with Connection(get_redis()):
w = Worker([queue])
w.work()
def ensure_consistency(filter, test):
""" Delete references to deleted incidents
"""
from bos_incidents import factory
storage = factory.get_incident_storage()
if filter:
incidents = storage.get_incidents(
dict(unique_string={"$regex": ".*" + filter + ".*"}))
else:
incidents = storage.get_incidents()
for incident in incidents:
if "id" not in incident:
if test:
print(" > " + incident["unique_string"] + "-" +
incident["provider_info"]["name"])
else:
storage.delete_incident(incident)
def list(call, status):
""" List events
"""
from bos_incidents import factory
t = PrettyTable(["identifier", "Incidents", "status"], hrules=ALLBORDERS)
t.align = 'l'
storage = factory.get_incident_storage()
if not call:
events = storage.get_events(resolve=False)
else:
events = storage.get_events_by_call_status(call=call, status_name=status)
for event in events:
t.add_row([
event["id_string"],
format_event_incidents(event),
format_event_incident_statuses(event)
])
click.echo(str(t))
def list_providers():
""" List events
"""
from bos_incidents import factory
t = PrettyTable(["Provider", "Incident Count"], hrules=ALLBORDERS)
t.align = 'l'
storage = factory.get_incident_storage()
providers = storage.get_distinct("provider_info.name")
for provider in providers:
t.add_row([
provider,
str(
storage.get_incidents_count(
{"provider_info.name": provider}))
])
click.echo(t)
def get_statistics(self, storage=None):
if storage is None:
storage = factory.get_incident_storage()
events = storage.get_events()
providers = []
for event in events:
for incident in storage.get_incidents_from_event(event):
if incident["provider_info"]["name"] not in providers:
providers.append(incident["provider_info"]["name"])
return_dict = {}
return_dict["providers"] = providers
return_dict["storage_information"] = storage._mongodb_config
return_dict["events"] = events
return json.dumps(return_dict)
def show_incidents_per_id(incident_id=None, call=None, use="mongodb"):
if use == "bos-auto":
use = "mongodb"
store = factory.get_incident_storage(use=use)
try:
event = store.get_event_by_id(incident_id, resolve=True)
except EventNotFoundException:
return jsonify("Event not found")
# fetch them seperately, we want raw data
incidents = store.get_incidents_by_id(incident_id, call)
return jsonify(
{
"status": event.get(call, {"status": "empty"})["status"],
"incidents": list(incidents)
}
)
def status(status_name):
""" Show events that have a status 'status'
"""
import builtins
from bos_incidents import factory
t = PrettyTable(["identifier", "Incidents", "Status"],
hrules=ALLBORDERS)
t.align = 'l'
storage = factory.get_incident_storage()
for call in INCIDENT_CALLS:
events = storage.get_events_by_call_status(call=call,
status_name=status_name)
for event in events:
full_event = storage.get_event_by_id(event["id_string"])
t.add_row([
full_event["id_string"],
format_event_incidents(full_event),
format_event_incident_statuses(full_event)
])
click.echo(t)
def rm(unique_string, provider, filter, test):
""" Remove an incident from the store
"""
from bos_incidents import factory
storage = factory.get_incident_storage()
if unique_string and provider:
incidents = [
storage.get_incident_by_unique_string_and_provider(
unique_string, provider)
]
elif filter:
incidents = storage.get_incidents(
dict(unique_string={"$regex": ".*" + filter + ".*i"}))
if test:
print("To be deleted: ")
for incident in incidents:
if test:
print(" > " + incident["unique_string"] + "-" +
incident["provider_info"]["name"])
else:
storage.delete_incident(incident)
def show(unique_string, provider, filter):
""" Show the content of a specific incidents
"""
from bos_incidents import factory
storage = factory.get_incident_storage()
if provider is not None:
incident = [
storage.get_incident_by_unique_string_and_provider(
unique_string, provider)
]
else:
if filter is not None:
incidents = storage.get_incidents(
dict(unique_string={"$regex": ".*" + filter + ".*i"}))
elif unique_string is not None:
incidents = storage.get_incidents(
dict(unique_string=unique_string))
else:
incidents = storage.get_incidents()
for incident in incidents:
pprint(incident)
def __init__(self, message, lookup_instance, config, **kwargs):
self.message = message
self.lookup = lookup_instance
self.config = config
# Obtain data for unique key
# The "id" contains everything we need to identify an individual event
# which itself contains at least the sport, and the teams
# Get the id (internally used only)
self.id = message.get("id")
# Incident Storage
if "storage" in kwargs and kwargs["storage"]:
self.storage = kwargs["storage"]
else:
self.storage = factory.get_incident_storage(
kwargs.get("mongodb", None), purge=kwargs.get("purge", False))
# Normalize incident
self.normalizer = IncidentsNormalizer(
chain=lookup_instance._network_name)
self.normalize(message)
# Try obtain the sport
self.sport = LookupSport(self.id.get("sport"))
# Given the sport, try to obtain the league (event group)
self.eventgroup = LookupEventGroup(self.sport,
self.id.get("event_group_name"))
self.event = None # Will be filled in after receiving a trigger
# Get Teams from query
self.teams = [self.id.get("home"), self.id.get("away")]
# Get start time from query
self.start_time = parse(self.id.get("start_time", ""))
def show(identifier):
""" Show event
"""
from bos_incidents import factory
t = PrettyTable(["identifier", "Incidents"], hrules=ALLBORDERS)
t.align = 'l'
storage = factory.get_incident_storage()
event = storage.get_event_by_id(identifier)
incidents = format_incidents(event)
id = event["id"]
id["start_time"] = parser.parse(id["start_time"]).replace(tzinfo=None)
t.add_row([
"\n".join([
id["sport"],
id["event_group_name"],
id["start_time"].strftime("%Y/%m/%d"),
"home: {}".format(id["home"]),
"away: {}".format(id["away"]),
]),
str(incidents)
])
click.echo(t)
def list(begin, end, filter):
""" List incidents from the bos-incidents store
"""
from bos_incidents import factory
t = PrettyTable(["identifier", "Incidents"], hrules=ALLBORDERS)
t.align = 'l'
storage = factory.get_incident_storage()
for event in storage.get_events(resolve=True):
# pprint(event)
if not ("id" in event and event["id"]):
continue
id = event["id"]
id["start_time"] = parser.parse(
id["start_time"]).replace(tzinfo=None)
# Limit time
if begin and end and (id["start_time"] < begin
or id["start_time"] > end):
continue
incidents = format_incidents(event)
t.add_row([
"\n".join([
id["sport"],
id["event_group_name"],
id["start_time"].strftime("%Y/%m/%d"),
"home: {}".format(id["home"]),
"away: {}".format(id["away"]),
]),
str(incidents)
])
click.echo(t)
def replay(identifier, call, status_name, url):
""" replay from event
"""
from bos_incidents import factory
storage = factory.get_incident_storage()
event = storage.get_event_by_id(identifier, resolve=False)
for incident_call, content in event.items():
if not content or "incidents" not in content:
continue
if call and call != "*" and incident_call != call:
continue
if status_name and content["status"]["name"] != status_name:
continue
for _incident in content["incidents"]:
incident = storage.resolve_to_incident(_incident)
pprint(incident)
incident.update(dict(skip_storage=True))
resend_incidents(url, incident)
def finalize_purge():
storage = factory.get_incident_storage(purge=True)
return "success"
def purge():
""" Purge the entire store
"""
from bos_incidents import factory
factory.get_incident_storage(purge=True)
def replay(filter=None, use="mongodb"):
if use == "bos-auto":
use = "mongodb"
if not Config.get("advanced_features", False):
abort(404)
form = ReplayForm()
form.chain.data = Config.get("connection", "use")
formTitle = "Replay incidents"
formMessages = []
proxy_incidents = None
if not form.back.data and form.validate_on_submit():
if form.check.data:
from .dataproxy_link import control
# query dataproxy
try:
proxy_incidents = control.get_replayable_incidents(form.unique_string.data, form.dataproxy.data, form.chain.data, form.witness.data)
form.replay.render_kw = {'disabled': False}
del form.check
formMessages.append(str(len(proxy_incidents)) + " incidents found on dataproxy")
for _tmp in proxy_incidents:
if type(_tmp) == dict:
formMessages.append(" - " + _tmp["unique_string"] + " provider: " + _tmp["provider_info"]["name"])
else:
formMessages.append(" - " + _tmp)
except Exception as e:
del form.back
flash(str(e), category="error")
if form.replay.data:
del form.back
from .dataproxy_link import control
try:
proxy_incidents = control.replay_incidents(form.unique_string.data, form.dataproxy.data, form.chain.data, form.witness.data)
formMessages = ["Replay has been triggered"]
formMessages.append("")
formMessages.append(str(len(proxy_incidents)) + " incidents found on dataproxy")
for _tmp in proxy_incidents:
if type(_tmp) == dict:
formMessages.append(" - " + _tmp["unique_string"] + " provider: " + _tmp["provider_info"]["name"])
else:
formMessages.append(" - " + _tmp)
except Exception as e:
flash(str(e), category="error")
else:
if filter is not None:
form.unique_string.data = filter
form.witness.data = "All"
del form.back
if form.unique_string.data is not None and not form.unique_string.data.strip() == "":
store = factory.get_incident_storage(use=use)
incidents = list(store.get_incidents(filter_dict=dict(
unique_string={"$regex": ".*" + form.unique_string.data + ".*"}
)))
if len(formMessages) > 0:
formMessages.append("")
formMessages.append(str(len(incidents)) + " incidents found locally for " + str(form.unique_string.data))
for _tmp in incidents:
formMessages.append(" - " + _tmp["unique_string"] + " provider: " + _tmp["provider_info"]["name"])
return render_template_menuinfo('generic.html', **locals())
def check_scheduled(storage=None,
func_callback=None,
proposer=None,
approver=None): # pragma: no cover
"""
"""
log.info("Scheduler checking incidents database ... "
"(approver: {}/ proposer: {})".format(approver, proposer))
# Invident Storage
if not storage:
storage = factory.get_incident_storage()
if not proposer:
proposer = config.get("BOOKIE_PROPOSER")
if not approver:
approver = config.get("BOOKIE_APPROVER")
push_to_queue = []
for call in INCIDENT_CALLS:
log.info("- querying call {}".format(call))
events = []
for status_name in ["postponed", "unhandled exception, retrying soon"]:
for event in storage.get_events_by_call_status(
call=call,
status_name=status_name,
status_expired_before=datetime.utcnow(),
):
events.append(event)
for status_name in ["connection lost", "unknown"]:
for event in storage.get_events_by_call_status(
call=call, status_name=status_name):
events.append(event)
ids = list()
log.info("Scheduler retriggering " + str(len(events)) +
" incident ...")
for event in events:
for incidentid in event.get(call, {}).get("incidents", []):
incident = storage.resolve_to_incident(incidentid)
if func_callback:
push_to_queue.append(incident)
# it is enough to trigger one incident, worker will check the whole call
break
# Flask queue
q = Queue(connection=get_redis())
# only push into the queue if it's somewhat empty (with 10% buffer), otherwise wait
if q.count + 2 < len(push_to_queue):
for incident in push_to_queue:
job = q.enqueue(
func_callback,
args=(incident, ),
kwargs=dict(proposer=proposer, approver=approver),
)
ids.append(job.id)
return ids
def setUp(self):
super(TestMongoOperationStorage, self).setUp()
self.storage = factory.get_incident_storage("mongodbtest", purge=True)
from bos_incidents import factory, exceptions
from bos_incidents.validator import IncidentValidator, InvalidIncidentFormatException
from bookiesports.normalize import IncidentsNormalizer, NotNormalizableException
from .log import log
from . import INCIDENT_CALLS
config = loadConfig()
# Flask app and parameters
app = Flask(__name__)
redis = get_redis()
use_connection(redis)
# Invident Storage
storage = factory.get_incident_storage()
validator = IncidentValidator()
# API whitelist
api_whitelist = resolve_hostnames(config.get("api_whitelist", ["0.0.0.0"]))
background_threads = []
@app.route("/")
def home():
""" Let's not expose that this is a bos-auto endpoint
"""
return "", 404
def __init__(self):
self.storage = factory.get_incident_storage()
self.providers = self.storage.get_distinct("provider_info.name")
pass
def show_incidents(from_date=None, to_date=None, matching=None, use="mongodb"):
if request.args.get("matching_today", None) is not None:
return redirect(url_for("show_incidents", matching=utils.date_to_string()[0:10]))
if matching is not None:
try:
match_date = utils.string_to_date(matching[0:20])
if from_date is None:
from_date = match_date - timedelta(days=3)
if to_date is None:
to_date = match_date + timedelta(days=3)
except InvalidRFC3339Error:
pass
if type(matching) == str:
matching = matching.split(",")
if from_date is None:
from_date = request.args.get("from_date", None)
if from_date is None:
from_date = utils.date_to_string(-7)
if type(from_date) == str:
from_date = utils.string_to_date(from_date)
if to_date is None:
to_date = request.args.get("to_date", None)
if to_date is None:
to_date = utils.date_to_string(21)
if type(to_date) == str:
to_date = utils.string_to_date(to_date)
store = None
unresolved_events = None
try:
store = factory.get_incident_storage(use=use)
unresolved_events = store.get_events(resolve=False)
except IncidentStorageLostException:
flash("BOS-mint could not find an incident store, or connection failed. Is a BOS-auto instance running alongside that grants access?")
return redirect(url_for('overview'))
events = []
# resort for provider view
for event in unresolved_events:
try:
event_scheduled = utils.string_to_date(event["id_string"][0:20])
except InvalidRFC3339Error:
event_scheduled = utils.string_to_date(event["id_string"][0:23])
if event_scheduled <= to_date and event_scheduled >= from_date and\
(matching is None or all([x.lower() in event["id_string"].lower() for x in matching])):
store.resolve_event(event)
else:
continue
for call in ["create", "in_progress", "finish", "result", "dynamic_bmgs", "canceled"]:
try:
incident_provider_dict = {}
for incident in event[call]["incidents"]:
provider = incident["provider_info"]["name"]
try:
incident_dict = incident_provider_dict[provider]
except KeyError:
incident_provider_dict[provider] = {"incidents": [],
"replay_links": {}}
incident_dict = incident_provider_dict[provider]
incident_provider_dict[provider]["incidents"].append(incident)
try:
replay_url = Ping().get_replay_url(provider, incident, call)
if replay_url is not None:
incident_dict["replay_links"][incident["unique_string"]] = replay_url
except Exception as e:
pass
event[call]["incidents_per_provider"] = incident_provider_dict
except KeyError:
pass
event["id_string"] = get_id_as_string(event["id"])
events.append(event)
from_date = utils.date_to_string(from_date)
to_date = utils.date_to_string(to_date)
if use == "mongodb":
use = "bos-auto"
if use == "auto":
use = "bos-auto"
return render_template_menuinfo('showIncidents.html', **locals())
def reset_storage():
return factory.get_incident_storage("mongodbtest", purge=True)
# Setup base lookup
lookup = Lookup(
proposer="init0",
blockchain_instance=peerplays,
network="unittests",
sports_folder=os.path.join(os.path.dirname(os.path.realpath(__file__)),
"bookiesports"),
)
lookup.set_approving_account("init0")
lookup.set_proposing_account("init0")
# ensure lookup isn't broadcasting either
assert lookup.blockchain.nobroadcast
# Storage
storage = factory.get_incident_storage("mongodbtest", purge=True)
# Incident validator
validator = IncidentValidator()
normalizer = IncidentsNormalizer(chain="unittests")
def reset_storage():
return factory.get_incident_storage("mongodbtest", purge=True)
def lookup_test_event(id):
return LookupEvent(
**{
"id": "1.22.2242",
"teams": ["Atlanta Hawks", "Boston Celtics"],
def check_scheduled(storage=None,
func_callback=None,
proposer=None,
approver=None): # pragma: no cover
"""
"""
log.info("Scheduler checking incidents database ... "
"(approver: {}/ proposer: {})".format(approver, proposer))
# Invident Storage
if not storage:
storage = factory.get_incident_storage()
if not proposer:
proposer = config.get("BOOKIE_PROPOSER")
if not approver:
approver = config.get("BOOKIE_APPROVER")
push_to_queue = []
for call in INCIDENT_CALLS:
log.info("- querying call {}".format(call))
events = []
# midway is the status of the incidents which are terminated intentionally as creating bms took time.
for status_name in ["midway"]:
for event in storage.get_events_by_call_status(
call=call, status_name=status_name):
events.append(event)
for status_name in ["postponed", "unhandled exception, retrying soon"]:
for event in storage.get_events_by_call_status(
call=call,
status_name=status_name,
status_expired_before=datetime.utcnow(),
):
events.append(event)
for status_name in ["connection lost", "unknown"]:
for event in storage.get_events_by_call_status(
call=call, status_name=status_name):
events.append(event)
ids = list()
log.info("Scheduler retriggering " + str(len(events)) +
" incident ...")
for event in events:
for incidentid in event.get(call, {}).get("incidents", []):
incident = storage.resolve_to_incident(incidentid)
if func_callback:
push_to_queue.append(incident)
# it is enough to trigger one incident, worker will check the whole call
break
# Flask queue
q = Queue(connection=get_redis())
# If push_to_queue has events, then empty queue and fill it again with push_to_queue
# This approach ensures that events which are on the chain but not all bms are created shall be processed first.
if len(push_to_queue) > 0:
q.empty()
for incident in push_to_queue:
job = q.enqueue(
func_callback,
args=(incident, ),
kwargs=dict(proposer=proposer, approver=approver),
)
ids.append(job.id)
return ids
