def __init__(self, bencoded_msg): try: self._msg_dict = bencode.decode(bencoded_msg) except (bencode.DecodeError): log.exception('invalid bencode') raise MsgError, 'invalid bencode' # Make sure the decoded data is a dict and has a TID key try: self.tid = self._msg_dict[TID] except (TypeError): raise MsgError, 'decoded data is not a dictionary' except (KeyError): raise MsgError, 'key TID not found' # Sanitize TID if not (isinstance(self.tid, str) and self.tid): raise MsgError, 'TID must be a non-empty binary string' # Sanitize TYPE try: self.type = self._msg_dict[TYPE] except (KeyError): raise MsgError, 'key TYPE not found' if not self.type in (QUERY, RESPONSE, ERROR): raise MsgError, 'Unknown TYPE value' if self.type == QUERY: self._sanitize_query() elif self.type == ERROR: self._sanitize_error() return
def run(self): try: self.extract() self.transform() self.load() except Exception as e: log.exception(e)
def handle_error(e): if not isinstance(e, dbutils.DBConnectionError): log.exception(e) code = 500 if isinstance(e, HTTPException): code = e.code response = flask.make_response('Unknown exception: {}'.format(str(e)), code) _add_cors_headers(response) # even if we fail, we should still add CORS headers, or browsers won't display real error status return response
def encode(data): output = cStringIO.StringIO() try: encode_f[type(data)](data, output) except (KeyError): log.exception('Data: %s' % data) raise EncodeError, 'see ERROR log' result = output.getvalue() output.close() return result
def __loadSystemSetting(self): """ Load system settings and restore these settings. """ try: pos = consttype.SysSetting.value("position") size = consttype.SysSetting.value("size") self.resize(size) self.move(pos) except Exception as e: log.exception(e)
def __finiModules(self): """ Finish module operation. """ for subWin in self.mdiArea.subWindowList(): module = subWin.widget() if isinstance(module, AppMod): try: module.fini() except Exception as e: log.exception(e)
def report(roster, cli, config): '''Print a human-friendly report about a time-slice of the roster.''' time_zone = config['roster.time_zone'] # We use datetimes even if the ultimate goal is operate at date level as # we need to preserve the timezone information all along fuzzy = cli['<fuzzy>'] # Fuzzy should be interpreted as always indicating a month. if fuzzy: try: start = fuzzy.replace(day=1) end = start + relativedelta(months=1, days=-1) except Exception as e: log.critical('Cannot parse <fuzzy> parameter "{}"'.format(fuzzy)) log.exception(e.message) raise # A range can be whatever elif cli['<start>']: start = cli['<start>'] end = cli['<end>'] if start > end: msg = 'Tried to generate a report for negative timespan ({} to {})' log.critical(msg.format(start, end)) exit(os.EX_DATAERR) else: now = datetime.datetime.now(tz=pytz.timezone(time_zone)) start = now.replace(day=1) + relativedelta(months=-1) end = start + relativedelta(months=1, days=-1) data = roster.report(start, end) weekdays = defaultdict(int) weekends = defaultdict(int) for day, people in data: target = weekdays if day.weekday() < 5 else weekends for person in people: target[person] += 1 print('\n O N - C A L L R O S T E R') print('=====================================================') print(' {} - {}\n\n'.format(start.strftime('%d %b %Y'), end.strftime('%d %b %Y'))) for row in data: print(' {:<20}{}'.format(row[0].strftime('%d %b %Y, %a'), ', '.join(row[1]))) print('\n\n SUMMARY') print('-----------------------------------------------------') print(' Name Weekdays Weekends Total') print('-----------------------------------------------------') names = sorted(list(set(weekends.keys() + weekdays.keys()))) template = ' {:<26}{:>3}{:>10}{:>8}' for name in names: wd = weekdays[name] we = weekends[name] print(template.format(name, wd or '-', we or '-', wd + we)) print('-----------------------------------------------------\n')
def on_query_received(self, query_msg, addr): log.debug('query received\n%s\nSource: %s' % (`query_msg`, `addr`)) try: handler = self.query_handler[query_msg.query] except (KeyError, ValueError): log.exception('Invalid QUERY') return # ignore query #TODO2: send error back? response_msg = handler(query_msg) self.notify_routing_m_on_query(node.Node(addr, query_msg.sender_id, query_msg.ns_node)) return response_msg
def loadConf(self): """ @override Load persisted config values """ with open(self._filename, 'rb') as f: try: conf = pickle.load(f) except EOFError as e: log.exception(e) else: self._conf = conf self._mergeDefConf()
def on_response_received(self, response_msg): try: response_msg.sanitize_response(self.query) except (message.MsgError): log.exception( "We don't like dirty reponses: %r|nresponse ignored" % response_msg) return # Response ignored self.node.is_ns = response_msg.ns_node if self.node.id: if response_msg.sender_id != self.node.id: return # Ignore response else: self.node.id = response_msg.sender_id #TODO2: think whether late responses should be accepted if self.timeout_task.cancelled: log.warning( "Response recevived but it's too late!!\n%r, %r" % (response_msg, self.timeout_task)) return # Ignore response self.timeout_task.cancel() nodes = [] try: nodes.extend(response_msg.nodes) except (AttributeError): pass try: nodes.extend(response_msg.nodes2) except (AttributeError): pass # Notify routing manager (if nodes found). # Do not notify when the query was a GET_PEERS because # the lookup is in progress and the routing_m shouldn't # generate extra traffic. if self.query == message.FIND_NODE and \ nodes and self.notify_routing_m_on_nodes_found_f: self.notify_routing_m_on_nodes_found_f(nodes) # Notify routing manager (response) self.node.is_ns = response_msg.ns_node if self.notify_routing_m_on_response_f: self.notify_routing_m_on_response_f(self.node) # Do callback to whomever did the query self.on_response_f(response_msg, self.node) return True # the response was fine
def run(self): """Main loop activated by calling self.start()""" last_task_run = time.time() stop_flag = self.stop_flag while not stop_flag: timeout_raised = False try: data, addr = self.s.recvfrom(BUFFER_SIZE) except (AttributeError): log.warning('udp_listen has not been called') time.sleep(self.task_interval) #TODO2: try using Event and wait timeout_raised = True except (socket.timeout): timeout_raised = True except (socket.error), e: log.critical( 'Got socket.error when receiving (more info follows)') log.exception('See critical log above') else: ip_is_blocked = self.floodbarrier_active and \ self.floodbarrier.ip_blocked(addr[0]) if ip_is_blocked: log.warning('%s blocked' % `addr`) else: self.datagram_received_f(data, addr) if timeout_raised or \ time.time() - last_task_run > self.task_interval: #with self._lock: self._lock.acquire() try: while True: task = self.tasks.consume_task() if task is None: break task.fire_callbacks() stop_flag = self.stop_flag finally: self._lock.release()
def sendto(self, data, addr): """Send data to addr using the UDP port used by listen_udp.""" #with self._lock: self._lock.acquire() try: try: bytes_sent = self.s.sendto(data, addr) if bytes_sent != len(data): log.critical( 'Just %d bytes sent out of %d (Data follows)' % ( bytes_sent, len(data))) log.critical('Data: %s' % data) except (socket.error): log.critical( 'Got socket.error when sending (more info follows)') log.critical('Sending data to %r\n%r' % (addr, data)) log.exception('See critical log above') finally: self._lock.release()
def execute_endpoint(*args, **kwargs): log.debug("{0} {1}{2}".format( flask.request.method, flask.request.url_rule.rule, "?" + flask.request.query_string if flask.request.query_string else "" )) try: obj = self.execute_endpoint_inner(endpoint, *args, **kwargs) if not obj: obj = ApiObject() obj.code = httplib.OK obj.msg = None except self.Error as error: obj = ApiObject() obj.code = error.code obj.msg = error.msg log.exception(obj.msg) except: obj = ApiObject() obj.code = httplib.INTERNAL_SERVER_ERROR obj.msg = None log.exception(obj.msg) obj.msg = obj.msg if obj.msg else httplib.responses[obj.code] data = obj.encode() self.dump("response:\n{0}".format(data)) response = flask.Response( data, obj.code, headers=None, mimetype="application/json" ) return response
def __createAppMod(self, modCls): """ Create module. """ if issubclass(modCls, AppQtMod): # AppQtMod instance if modCls.__name__ in self.__modSubWin.keys(): subWin = self.__modSubWin[modCls.__name__] else: try: module = modCls() if module.init(): subWin = self.__addSubWin(module) except Exception as e: log.exception(e) self.__delSubWin(modCls) return if subWin != self.mdiArea.activeSubWindow(): self.mdiArea.setActiveSubWindow(subWin) subWin.showMaximized()
def on_message(client, userdata, message): try: m = MqttMessage(message.topic, message.payload) q.put(m) except: log.exception("Error putting mqtt message to queue!")
def before_request(): # http://flask.pocoo.org/docs/1.0/api/#application-globals flask.g.grafolean_data = {} if not flask.request.endpoint in app.view_functions: # Calling /api/admin/migratedb with GET (instead of POST) is a common mistake, so it deserves a warning in the log: if flask.request.path == '/api/admin/migratedb' and flask.request.method == 'GET': log.warning("Did you want to use POST instead of GET?") return "Resource not found", 404 # Browser might (if frontend and backend are not on the same origin) send a pre-flight OPTIONS request to get the # CORS settings. In this case 'Authorization' header will not be set, which could lead to 401 response, which browser # doesn't like. So let's just return 200 on all OPTIONS: if flask.request.method == 'OPTIONS': # we need to set 'Allow' header to notify caller which methods are available: methods = set() for rule in app.url_map.iter_rules(): if flask.request.url_rule == rule: methods |= rule.methods response = flask.make_response('', 200) response.headers['Allow'] = ",".join(sorted(methods)) return response if flask.request.method in ['GET', 'HEAD', 'POST']: # While it is true that CORS is client-side protection, the rules about preflights allow these 3 types of requests # to be sent to the server without OPTIONS preflight - which means that browser will learn about violation too late. # To combat this, we still check Origin header and explicitly deny non-whitelisted requests: origin_header = flask.request.headers.get('Origin', None) if origin_header: # is it a cross-origin request? # still, we sometimes get origin header even if it is not a cross-origin request, so let's double check that we # indeed are doing CORS: if flask.request.url_root.rstrip('/') != origin_header: if origin_header not in CORS_DOMAINS and flask.request.path != '/api/status/info': # this path is an exception return 'CORS not allowed for this origin', 403 if dbutils.db is None: dbutils.db_connect() if dbutils.db is None: # oops, DB error... we should return 5xx: return 'Service unavailable', 503 view_func = app.view_functions[flask.request.endpoint] # unless we have explicitly used @noauth decorator, do authorization check here: if not hasattr(view_func, '_noauth'): try: user_id = None user_is_bot = False authorization_header = flask.request.headers.get('Authorization') query_params_bot_token = flask.request.args.get('b') if authorization_header is not None: received_jwt = JWT.forge_from_authorization_header(authorization_header, allow_leeway=0) flask.g.grafolean_data['jwt'] = received_jwt user_id = received_jwt.data['user_id'] elif query_params_bot_token is not None: user_id = Bot.authenticate_token(query_params_bot_token) user_is_bot = True if user_id is None: log.info("Authentication failed (no such user)") return "Access denied", 401 # check permissions: resource = flask.request.path[len('/api/'):] resource = resource.rstrip('/') is_allowed = Permission.is_access_allowed( user_id=user_id, resource=resource, method=flask.request.method, ) if not is_allowed: log.info("Access denied (permissions check failed) {} {} {}".format(user_id, resource, flask.request.method)) return "Access to resource denied, insufficient permissions", 403 flask.g.grafolean_data['user_id'] = user_id flask.g.grafolean_data['user_is_bot'] = user_is_bot except AuthFailedException as ex: log.info(f"Authentication failed: {str(ex)}") return "Access denied", 401 except: log.exception("Exception while checking access rights") return "Could not validate access", 500
def admin_mqttauth_plug(check_type): """ --- post: summary: Authorization for Mosquitto with mosquitto-auth-plug plugin tags: - Admin description: > If using MQTT (with iegomez/mosquitto-go-auth plugin), it should be configured to ask this endpoint about access rights via JWT tokens (Authorization header). The JWT token is supplied to MQTT by frontend via websockets through username (password is not used). See [mosquitto-go-auth](https://github.com/iegomez/mosquitto-go-auth) for more info. parameters: - name: check_type in: path description: "One of the 3 modes of calling this endpoint" required: true schema: type: "string" enum: - "getuser" - "superuser" - "aclcheck" - name: Authorization in: header description: "JWT token (in other words: MQTT username)" schema: type: string required: true responses: 200: description: Access allowed 401: description: Access denied """ # mqtt-auth-plug urlencodes JWT tokens, so we must decode them here: authorization_header = flask.request.headers.get('Authorization') authorization_header = urllib.parse.unquote(authorization_header, encoding='utf-8') # debugging: # if authorization_header == 'Bearer secret': # log.info('--- secret account authenticated ---') # return "", 200 # log_received_jwt = JWT.forge_from_authorization_header(authorization_header, allow_leeway=3600*24*365*10) # log.info('mqtt-auth {}: {}, {}'.format(check_type.upper(), log_received_jwt.data, flask.request.form.to_dict())) try: if check_type == 'getuser': # we don't complicate about newly expired tokens here - if they are at all valid, browser will refresh them anyway. received_jwt = JWT.forge_from_authorization_header(authorization_header, allow_leeway=JWT.TOKEN_CAN_BE_REFRESHED_FOR) # jwt token was successfully decoded, so we can allow for the fact that this is a valid user - we'll still see about # access rights though (might be superuser, in which case everything goes, or it might be checked via aclcheck) return "", 200 elif check_type == 'superuser': # we don't complicate about newly expired tokens here - if they are at all valid, browser will refresh them anyway. received_jwt = JWT.forge_from_authorization_header(authorization_header, allow_leeway=JWT.TOKEN_CAN_BE_REFRESHED_FOR) # is this our own attempt to publish something to MQTT, and the mosquitto auth plugin is asking us to authenticate ourselves? is_superuser = bool(received_jwt.data.get('superuser', False)) if is_superuser: return "", 200 log.info("Access denied (not a superuser)") return "Access denied", 401 elif check_type == 'aclcheck': params = flask.request.form.to_dict() # When client connects, username is jwt token. However subscribing to topics doesn't necessarily reconnect so # fresh JWT token is not sent and we are getting the old one. This is OK though - if user kept the connection # we can assume that they would just keep refreshing the token. So we allow for some large leeway (10 years) received_jwt = JWT.forge_from_authorization_header(authorization_header, allow_leeway=3600*24*365*10) # superusers can do whatever they want to: is_superuser = bool(received_jwt.data.get('superuser', False)) if is_superuser: return "", 200 # From https://github.com/iegomez/mosquitto-go-auth#acl-access-values: # ACL access values: Mosquitto 1.5 introduced a new ACL access value, MOSQ_ACL_SUBSCRIBE, which is similar # to the classic MOSQ_ACL_READ value but not quite the same: # # * MOSQ_ACL_SUBSCRIBE when a client is asking to subscribe to a topic string. # * This differs from MOSQ_ACL_READ in that it allows you to # * deny access to topic strings rather than by pattern. For # * example, you may use MOSQ_ACL_SUBSCRIBE to deny # * subscriptions to '#', but allow all topics in # * MOSQ_ACL_READ. This allows clients to subscribe to any # * topic they want, but not discover what topics are in use # * on the server. # * MOSQ_ACL_READ when a message is about to be sent to a client (i.e. whether # * it can read that topic or not). # # The main difference is that subscribe is checked at first, when a client connects and tells the broker it # wants to subscribe to some topic, while read is checked when an actual message is being published to that # topic, which makes it particular. So in practice you could deny general subscriptions such as # by returning # false from the acl check when you receive MOSQ_ACL_SUBSCRIBE, but allow any particular one by returning true # on MOSQ_ACL_READ. Please take this into consideration when designing your ACL records on every backend. # - from now on we only allow READ access, all other access levels are denied for non-superusers # - subscribing is only allowed for resources to which user has read access too requested_access = int(params['acc']) if requested_access not in [1, 4] : # NONE = 0, READ = 1, WRITE = 2, SUBSCRIBE = 4 instead_got = {0: "0/NONE", 2: "2/WRITE"}.get(requested_access, requested_access) log.info(f"Access denied (only 1/READ or 4/SUBSCRIBE allowed, requested access: {instead_got})") return "Access denied", 401 # only 'changed/#' can actually be read by normal users: if params['topic'][:8] != 'changed/': log.info("Access denied (wrong topic)") return "Access denied", 401 resource = params['topic'][8:] # remove 'changed/' from the start of the topic to get the resource resource = resource.rstrip('/') # finally, make sure user has access rights: user_id = received_jwt.data['user_id'] is_allowed = Permission.is_access_allowed( user_id=user_id, resource=resource, method='GET', # users can only request read access (apart from backend, which is superuser anyway) ) if is_allowed: return "", 200 log.info("Access denied (permissions check failed for user '{}', url '{}', method 'GET')".format(user_id, resource)) return "Access denied", 401 return "Invalid endpoint", 404 except AuthFailedException as ex: log.info(f"Authentication failed: {str(ex)}") return "Access denied", 401 except: log.exception("Exception while checking access rights") return "Access denied", 401