async def main():
seria_bot = SeriaBot()
sentry_client = SentryClient(config.SENTRY_DSN, transport=AioHttpTransport)
future_tasks = [
seria_bot.start(config.DISCORD_TOKEN),
sentry_client.remote.get_transport().close()
]
return await asyncio.gather(*future_tasks)
async def _startup(app: web.Application) -> None:
dsn = os.environ["SENTRY_DSN"]
environment = app["config"]["app"]["environment"]
sentry_client = SentryClient(dsn=dsn,
transport=AioHttpTransport,
environment=environment)
sentry_logging_handler = SentryHandler(sentry_client)
sentry_logging_handler.setLevel(logging.ERROR)
setup_logging(sentry_logging_handler)
app["sentry"] = sentry_client
def __init__(self, config_path=None):
self._needcleanup = True
# logging will be configured in Cfg constructor
self.config = Config(config_path)
self.sentry = None
_sentry_dsn = self.config.get('sentry.dsn')
if _sentry_dsn:
log.info(
'Sentry\'s raven error-grabber connected for app release=%s',
__release__)
self.sentry = SentryClient(
dsn=_sentry_dsn,
release=__release__,
)
try:
log.info('Starting %s [%s]', __title__, __release__)
self._connect_notifiers(self.config)
self.web = OurFlask(self.config)
self.web.app = self
log.info('* WEB SERVER ready')
db = FlaskSQLAlchemy(model_class=Model)
self.db = db
self.web.db = db
db.init_app(self.web) # connect Flask and Flask-SqlAlchemy
db.create_all(app=self.web) # create DB schema
log.info('* DB ready')
self.scheduler = ScheduledWorker(self.config)
log.info('* SCHEDULER ready')
blinker.signal('app.initialized').send(self)
log.info('Entire application initialized')
except: # noqa: E722
if self.sentry:
self.sentry.captureException()
# MUST BE RE-THROWN!
# if you don't want to re-raise the exception - don't use
# bare EXCEPT instruction!
raise
def __init__(self,
slugs,
table,
time_field,
symbol_tickets,
periods,
dt1=None,
dt2=None,
max_last=None,
strategy_run_ids=None):
"""
:param slugs: dict {`name`: `select`} defining what to select from `table`
:param table: table to select from
:param time_field: specific field for pagination
:param symbol_tickets: list: symbol sid to get sequence for
:param periods: list: period (minutes) to get sequence for
:param dt1: min datetime to search for
:param dt2: max datetime to search for
"""
if not symbol_tickets:
raise KeyError('Symbol ticket must be defined')
if not isinstance(symbol_tickets, (list, set, tuple)):
symbol_tickets = [symbol_tickets]
if not isinstance(periods, (list, set, tuple)):
periods = [periods]
self.sentry = SentryClient(settings.SENTRY_DSN)
self.slugs = slugs
self.index = 0
self.table = table
self.time_field = time_field
self.__symbol_tickets = symbol_tickets
self.__periods = periods
self.max_last = max_last
self.strategy_run_ids = strategy_run_ids or []
self.dt1 = dt1 or datetime(2000, 1, 1)
self.dt2 = dt2 or datetime(2030, 1, 1)
self.data_size = 0
data = self.load_data()
if data:
self.data_size = len(data[self.time_field])
for slug in self.slugs:
self[slug] = MQBSequence(slug, data[slug], max_last=max_last)
async def set_sentry(self, ctx, dsn=None):
"""Sets the DSN for Sentry."""
if self.client is None:
self.client = SentryClient(site=self.liara.user.id)
try:
self.client.set_dsn(dsn)
except InvalidDsn:
await ctx.send('That DSN is invalid.')
return
await self.settings.set('dsn', dsn)
if dsn is None:
await ctx.send('DSN cleared.')
else:
await ctx.send(
'DSN successfully set! All your exceptions will now be logged to Sentry.'
)
self.client.captureMessage('Sentry for Liara successfully set up.')
def send_android_error_report(user_id, error_report):
# Encountered a corrupted (write error) error report upload on Apr 30 2017, adding error sentry
# so that we get *some* report of the error occuring but also delete that file from the device.
with ErrorSentry(SENTRY_DSN, sentry_client_kwargs={'transport': HTTPTransport}):
#get all non-empty lines in the error report
contents = [line for line in error_report.splitlines() if line.strip() ]
# the first line contains a unix millisecond timestamp, construct a datetime
# The printed value in the crash report is in UTC
try: #Beiwe version greater than 4
timestamp = datetime.fromtimestamp(float(contents[0]) / 1000)
contents.pop(0) #remove timestamp from message text
except ValueError: #Beiwe version 4
timestamp = datetime.fromtimestamp(float(request.values['file_name'].split("_")[1]) / 1000)
device_identifiers = contents[0].split(',')
contents.pop(0) # remove device identifiers from message text
# Insert the actual error message as the first line
report_title = contents[0].split(":", 1)[1].strip()
if "}" in report_title: #cut title at end of file name
report_title = report_title.split("}", 1)[0] + "}"
contents.insert(0, "Android Error: %s" % report_title)
# the second line contains all the identifiers. Clean it up and parse into a dictionary.
device_identifiers = {ID.strip().split(":",1)[0] : ID.strip().split(":",1)[1]
for ID in device_identifiers}
#get a useful timestamp...
eastern_time = timestamp.replace(tzinfo=tz.gettz('UTC')).astimezone(tz.gettz('America/New_York'))
#construct some useful tags for this error report, add all identifiers as tags.
tags = {"Android_Error": "android error",
"user_id":user_id,
"date": str(timestamp.date()),
"time": str(timestamp).split(" ")[1].split(".")[0],
"eastern_date": str(eastern_time.date()),
"eastern_time": str(eastern_time).split(" ")[1].split(".")[0]
}
tags.update(device_identifiers)
sentry_client = SentryClient(dsn=SENTRY_DSN,
tags=tags,
transport=HTTPTransport )
sentry_client.captureMessage("\n".join(contents))
def __init__(self, strategy_run):
"""
:param strategy_run: {
_id: str,
symbol_ticket: str,
period: int,
iteration_time: datetime,
iteration_number: int,
context: Context,
}
"""
self.name = self.__class__.__name__
self.strategy_run = strategy_run
self.sentry = SentryClient(settings.SENTRY_DSN)
self.mqb = None
if not self.strategy_run.get('context'):
self.strategy_run['context'] = dill.dumps(Context())
if not self.strategy_run.get('iteration_number'):
self.strategy_run['iteration_number'] = 0
def setup_logging(level=None, logfile=None):
"""
Setups cchloader logging system.
It will setup sentry logging if SENTRY_DSN environment is defined
:param level: logging.LEVEL to set to logger (defaults INFO)
:param logfile: File to write the log
:return: logger
"""
stream = logging.StreamHandler()
stream.setFormatter(logging.Formatter(LOG_FORMAT))
logger = logging.getLogger('cchloader')
del logger.handlers[:]
logger.addHandler(stream)
if logfile:
hdlr = logging.FileHandler(logfile)
formatter = logging.Formatter(LOG_FORMAT)
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
if sentry_sdk:
sentry_sdk.init(release=VERSION)
else:
sentry = SentryClient()
sentry.tags_context({'version': VERSION})
sentry_handler = SentryHandler(sentry, level=logging.ERROR)
logger.addHandler(sentry_handler)
if isinstance(level, basestring):
level = getattr(logging, level.upper(), None)
if level is None:
level = logging.INFO
logger.setLevel(level)
return logger
def __init__(self, service_name: str, *, api: dict, sentry_uri: str = None, threads_number: int = 10, **kwargs):
"""
Parameters
----------
service_name : str
...
api : dict
...
sentry_uri : str, optional
...
# TODO: See base
"""
super(WorkerService, self).__init__(**kwargs)
self._service_name = service_name
self._api = api
self._sentry_client = None
if sentry_uri is not None:
self._sentry_client = SentryClient(sentry_uri, transport=AioHttpTransport)
self._threads_number = threads_number
self._queue = asyncio.Queue(1)
async def on_command_error(self, context, exception):
# raven setup
if self.client is None:
self.client = SentryClient(site=self.liara.user.id)
dsn = await self.settings.get('dsn', None)
if dsn is None:
return
try:
self.client.set_dsn(dsn)
except InvalidDsn:
await self.settings.delete('dsn')
return
if isinstance(exception, commands_errors.MissingRequiredArgument):
return
if isinstance(exception, commands_errors.CommandNotFound):
return
if isinstance(exception, commands_errors.BadArgument):
return
_exception = exception.original
if isinstance(_exception, discord.Forbidden):
return # not my problem
message = context.message
async with self.client_lock:
self.client.user_context({'id': message.author.id})
# noinspection PyBroadException
try:
raise _exception
except Exception:
self.client.captureException(data={'message': message.content},
extra={
'guild_id': message.guild.id,
'channel_id':
message.channel.id,
'message_id': message.id
})
def make_sentry_client(sentry_type, tags=None):
dsn = get_dsn_from_string(sentry_type)
tags = tags or {}
return SentryClient(dsn=dsn, tags=tags, transport=HTTPTransport)
def watch_loop():
v1 = client.CoreV1Api()
w = watch.Watch()
sentry = SentryClient(
dsn=DSN,
install_sys_hook=False,
install_logging_hook=False,
include_versions=False,
capture_locals=False,
context={},
environment=ENV,
release=RELEASE,
transport=ThreadedRequestsHTTPTransport,
)
# try:
# resource_version = v1.list_event_for_all_namespaces().items[-1].metadata.resource_version
# except:
# resource_version = 0
if EVENT_NAMESPACE:
stream = w.stream(v1.list_namespaced_event, EVENT_NAMESPACE)
else:
stream = w.stream(v1.list_event_for_all_namespaces)
for event in stream:
logging.debug("event: %s" % event)
event_type = event['type'].lower()
event = event['object']
meta = {
k: v
for k, v in event.metadata.to_dict().items() if v is not None
}
creation_timestamp = meta.pop('creation_timestamp', None)
level = (event.type and event.type.lower())
level = LEVEL_MAPPING.get(level, level)
component = source_host = reason = namespace = name = short_name = kind = None
if event.source:
source = event.source.to_dict()
if 'component' in source:
component = source['component']
if 'host' in source:
source_host = source['host']
if event.reason:
reason = event.reason
if event.involved_object and event.involved_object.namespace:
namespace = event.involved_object.namespace
elif 'namespace' in meta:
namespace = meta['namespace']
if event.involved_object and event.involved_object.kind:
kind = event.involved_object.kind
if event.involved_object and event.involved_object.name:
name = event.involved_object.name
if not MANGLE_NAMES or kind in MANGLE_NAMES:
bits = name.split('-')
if len(bits) in (1, 2):
short_name = bits[0]
else:
short_name = "-".join(bits[:-2])
else:
short_name = name
message = event.message
if namespace and short_name:
obj_name = "(%s/%s)" % (namespace, short_name)
else:
obj_name = "(%s)" % (namespace, )
if level in ('warning', 'error') or event_type in ('error', ):
if event.involved_object:
meta['involved_object'] = {
k: v
for k, v in event.involved_object.to_dict().items()
if v is not None
}
fingerprint = []
tags = {}
if component:
tags['component'] = component
if reason:
tags['reason'] = event.reason
fingerprint.append(event.reason)
if namespace:
tags['namespace'] = namespace
fingerprint.append(namespace)
if short_name:
tags['name'] = short_name
fingerprint.append(short_name)
if kind:
tags['kind'] = kind
fingerprint.append(kind)
data = {
'sdk': SDK_VALUE,
'server_name': source_host or 'n/a',
'culprit': "%s %s" % (obj_name, reason),
}
sentry.captureMessage(
message,
# culprit=culprit,
data=data,
date=creation_timestamp,
extra=meta,
fingerprint=fingerprint,
level=level,
tags=tags,
)
data = {}
if name:
data['name'] = name
if namespace:
data['namespace'] = namespace
breadcrumbs.record(
data=data,
level=level,
message=message,
timestamp=time.mktime(creation_timestamp.timetuple()),
)
__contact__ = "https://www.metachris.com"
__copyright__ = "Copyright 2015 Chris Hager"
# Whether to show debug output
DEBUG = True
IS_FROZEN = False
if getattr(sys, 'frozen', False):
# we are running in a bundle
IS_FROZEN = True
BUNDLE_DIR = sys._MEIPASS
else:
# we are running in a normal Python environment
BUNDLE_DIR = os.path.dirname(os.path.abspath(__file__))
SENTRY = SentryClient(settings.SENTRY_CLIENT_URI)
ERROR_FILE_NOT_FOUND = 1
ERROR_DOWNLOAD = 2
ERROR_PDF_INVALID = 4
REF_TEST = namedtuple('ref', ['ref', 'reftype'])
REFS_TEST = [
REF_TEST("http://doc.qt.io/qt-5/qml-qtquick-controls-tableview.html",
"url"),
REF_TEST(
"https://docs.python.org/2/library/collections.html#collections.namedtuple",
"url"),
REF_TEST(
"http://nxr.northwestern.edu/sites/default/files/publications/BROKENSefa12a.pdf",
"pdf"),
from algoliasearch import algoliasearch
from elasticsearch import Elasticsearch
from elasticsearch import client as es_client
from mixpanel import Mixpanel
# Loads .env into environment variables
from pathlib import Path # python3 only
env_path = Path('.') / '.env'
load_dotenv(dotenv_path=env_path)
mp = Mixpanel('e3b4939c1ae819d65712679199dfce7e')
pp = pprint.PrettyPrinter(indent=4) #, width=160)
sentry = SentryClient(
'https://*****:*****@sentry.io/1004428',
environment='local' if 'HOME' in os.environ
and os.environ['HOME'] == '/Users/jeremy' else 'production')
es = Elasticsearch(
# https://elastic:z5X0jw5hyJRzpEIOvFQmWwxN@ad56f010315f958f3a4d179dc36e6554.us-east-1.aws.found.io:9243/
['https://ad56f010315f958f3a4d179dc36e6554.us-east-1.aws.found.io:9243/'],
http_auth=('elastic', os.getenv('ELASTIC_PWD')),
scheme='https',
port=9243,
)
UsingAlgolia = False
class Client:
"""docstring for Client"""
def upload(OS_API=""):
""" Entry point to upload GPS, Accelerometer, Audio, PowerState, Calls Log, Texts Log,
Survey Response, and debugging files to s3.
Behavior:
The Beiwe app is supposed to delete the uploaded file if it receives an html 200 response.
The API returns a 200 response when the file has A) been successfully handled, B) the file it
has been sent is empty, C) the file did not decrypt properly. We encountered problems in
production with incorrectly encrypted files (as well as Android generating "rList" files
under unknown circumstances) and the app then uploads them. The source of encryption errors
is not well understood and could not be tracked down. In order to salvage partial data the
server decrypts files to the best of its ability and uploads it to S3. In order to delete
these files we still send a 200 response.
(The above about encryption is awful, in a theoretical version 2.0 the 200 response would be
replaced with a difference response code to allow for better debugging and less/fewer ... hax.)
A 400 error means there is something is wrong with the uploaded file or its parameters,
administrators will be emailed regarding this upload, the event will be logged to the apache
log. The app should not delete the file, it should try to upload it again at some point.
If a 500 error occurs that means there is something wrong server side, administrators will be
emailed and the event will be logged. The app should not delete the file, it should try to
upload it again at some point.
Request format:
send an http post request to studies.beiwe.org/upload, remember to include security
parameters (see user_authentication for documentation). Provide the contents of the file,
encrypted (see encryption specification) and properly converted to Base64 encoded text,
as a request parameter entitled "file".
Provide the file name in a request parameter entitled "file_name". """
patient_id = request.values['patient_id']
user = User(patient_id)
#Slightly different values for iOS vs Android behavior.
#Android sends the file data as standard form post parameter (request.values)
#iOS sends the file as a multipart upload (so ends up in request.files)
#if neither is found, consider the "body" of the post the file
#("body" post is not currently used by any client, only here for completeness)
if "file" in request.files:
uploaded_file = request.files['file']
elif "file" in request.values:
uploaded_file = request.values['file']
else:
uploaded_file = request.data
if isinstance(uploaded_file, FileStorage):
uploaded_file = uploaded_file.read()
file_name = request.values['file_name']
# print "uploaded file name:", file_name, len(uploaded_file)
if "crashlog" in file_name.lower():
send_android_error_report(user._id, uploaded_file)
return render_template('blank.html'), 200
if file_name[:6] == "rList-":
return render_template('blank.html'), 200
client_private_key = get_client_private_key(patient_id, user['study_id'])
try:
uploaded_file = decrypt_device_file(patient_id, uploaded_file,
client_private_key, user)
except HandledError as e:
# when decrypting fails, regardless of why, we rely on the decryption code
# to log it correctly and return 200 OK to get the device to delete the file.
# We do not want emails on these types of errors, so we use log_error explicitly.
print "the following error was handled:"
log_error(e, "%s; %s; %s" % (patient_id, file_name, e.message))
return render_template('blank.html'), 200
# except DecryptionKeyInvalidError:
# return render_template('blank.html'), 200
except OurBase64Error:
if IS_STAGING:
print "decryption problems" + "#" * 200
print
print patient_id
print
print file_name
print uploaded_file
print
raise
print "decryption success:", file_name
#if uploaded data a) actually exists, B) is validly named and typed...
if uploaded_file and file_name and contains_valid_extension(file_name):
s3_upload(file_name.replace("_", "/"), uploaded_file, user["study_id"])
FileToProcess.append_file_for_processing(file_name.replace("_", "/"),
user["study_id"], patient_id)
UploadTracking.create(
{
"file_path": file_name.replace("_", "/"),
"timestamp": datetime.utcnow(),
"user_id": patient_id,
"file_size": len(uploaded_file)
},
random_id=True)
return render_template('blank.html'), 200
else:
error_message = "an upload has failed " + patient_id + ", " + file_name + ", "
if not uploaded_file:
#it appears that occasionally the app creates some spurious files
#with a name like "rList-org.beiwe.app.LoadingActivity"
error_message += "there was no/an empty file, returning 200 OK so device deletes bad file."
log_error(Exception("upload error"), error_message)
return render_template('blank.html'), 200
elif not file_name:
error_message += "there was no provided file name, this is an app error."
elif file_name and not contains_valid_extension(file_name):
error_message += "contains an invalid extension, it was interpretted as "
error_message += grab_file_extension(file_name)
else:
error_message += "AN UNKNOWN ERROR OCCURRED."
sentry_client = SentryClient(dsn=SENTRY_DSN,
tags={
"upload_error": "upload error",
"user_id": user._id
},
transport=HTTPTransport)
sentry_client.captureMessage(error_message)
# log_and_email_500_error(Exception("upload error"), error_message)
return abort(400)
def watch_loop():
logging.info("Starting Kubernetes watcher")
v1 = client.CoreV1Api()
w = watch.Watch()
logging.info("Initializing Sentry client")
sentry = SentryClient(
dsn=DSN,
install_sys_hook=False,
install_logging_hook=False,
include_versions=False,
capture_locals=False,
context={},
environment=ENV,
release=RELEASE,
transport=ThreadedRequestsHTTPTransport,
)
# try:
# resource_version = v1.list_event_for_all_namespaces().items[-1].metadata.resource_version
# except:
# resource_version = 0
if EVENT_NAMESPACES and len(EVENT_NAMESPACES) == 1:
stream = w.stream(v1.list_namespaced_event, EVENT_NAMESPACES[0])
else:
stream = w.stream(v1.list_event_for_all_namespaces)
for event in stream:
logging.debug("event: %s" % event)
event_type = event["type"].lower()
event = event["object"]
meta = {
k: v
for k, v in event.metadata.to_dict().items() if v is not None
}
creation_timestamp = meta.pop("creation_timestamp", None)
level = event.type and event.type.lower()
level = LEVEL_MAPPING.get(level, level)
component = source_host = reason = namespace = name = short_name = kind = None
if event.source:
source = event.source.to_dict()
if "component" in source:
component = source["component"]
if COMPONENTS_EXCLUDED and component in COMPONENTS_EXCLUDED:
continue
if "host" in source:
source_host = source["host"]
if event.reason:
reason = event.reason
if REASONS_EXCLUDED and reason in REASONS_EXCLUDED:
continue
if event.involved_object and event.involved_object.namespace:
namespace = event.involved_object.namespace
elif "namespace" in meta:
namespace = meta["namespace"]
if namespace and EVENT_NAMESPACES and namespace not in EVENT_NAMESPACES:
continue
if namespace and EVENT_NAMESPACES_EXCLUDED and namespace in EVENT_NAMESPACES_EXCLUDED:
continue
if event.involved_object and event.involved_object.kind:
kind = event.involved_object.kind
if event.involved_object and event.involved_object.name:
name = event.involved_object.name
if not MANGLE_NAMES or kind in MANGLE_NAMES:
bits = name.split("-")
if len(bits) in (1, 2):
short_name = bits[0]
else:
short_name = "-".join(bits[:-2])
else:
short_name = name
message = event.message
if namespace and short_name:
obj_name = "(%s/%s)" % (namespace, short_name)
else:
obj_name = "(%s)" % (namespace, )
if level in EVENT_LEVELS or event_type in ("error", ):
if event.involved_object:
meta["involved_object"] = {
k: v
for k, v in event.involved_object.to_dict().items()
if v is not None
}
fingerprint = []
tags = {}
if CLUSTER_NAME:
tags["cluster"] = CLUSTER_NAME
if component:
tags["component"] = component
if reason:
tags["reason"] = event.reason
fingerprint.append(event.reason)
if namespace:
tags["namespace"] = namespace
fingerprint.append(namespace)
if short_name:
tags["name"] = short_name
fingerprint.append(short_name)
if kind:
tags["kind"] = kind
fingerprint.append(kind)
data = {
"sdk": SDK_VALUE,
"server_name": source_host or "n/a",
"culprit": "%s %s" % (obj_name, reason),
}
logging.debug("Sending event to Sentry:\n{}".format(data))
sentry.captureMessage(
message,
# culprit=culprit,
data=data,
date=creation_timestamp,
extra=meta,
fingerprint=fingerprint,
level=level,
tags=tags,
)
data = {}
if name:
data["name"] = name
if namespace:
data["namespace"] = namespace
breadcrumbs.record(
data=data,
level=level,
message=message,
timestamp=time.mktime(creation_timestamp.timetuple()),
)
import config
import pytz
from datastore import redis_db
from discord import Client as DiscordClient
from discord import Game, Message, Server
from google_service_account import spreadsheet
from pygsheets import Cell
from raven import Client as SentryClient
from raven_aiohttp import AioHttpTransport
TZINFO = pytz.timezone('Asia/Taipei')
player_with_job_pattern = re.compile('(?P<player>.*)\((?P<job>.*)\)')
sentry_client = SentryClient(config.SENTRY_DSN, transport=AioHttpTransport)
class SeriaBot(DiscordClient):
discord_server: Server = None
async def on_ready(self):
"""Get server and channel from discord connection"""
self.discord_server = self.get_server(config.DISCORD_SERVER_ID)
async def on_error(self, event_method, *args, **kwargs):
await super().on_error(event_method, *args, **kwargs)
sentry_client.captureException()
async def on_message(self, message: Message):
"""Listener about message"""
def __init__(self, liara):
self.liara = liara
self.settings = dataIO.load_json('sentry')
if 'dsn' not in self.settings:
self.settings['dsn'] = None
self.client = SentryClient(site=liara.user.id)
logger.setLevel(logging.DEBUG)
logger.addHandler(ch)
logger.addHandler(fh)
elasticsearch_logger = logging.getLogger('elasticsearch')
elasticsearch_logger.setLevel(logging.ERROR)
# --------------------------------
# Backend services
# --------------------------------
es = Elasticsearch(hosts=[{
'host': ES_HOST,
'port': ES_PORT
}],
serializer=OrderedDictJsonSerializer())
sentry_client = SentryClient(dsn=SENTRY_URL, )
# --------------------------------
# Initialization
# --------------------------------
logger.info('---------------------------------------------------------')
logger.info('Initialization path: : ' + script_path)
logger.info('Root path: ' + ROOT_PATH)
logger.info('Working dir path: : ' + os.getcwd())
logger.info('Log file: : ' + LOG_FILE_PATH)
logger.info('---------------------------------------------------------')
def bootstrap():
"""
Prepare the database backend