def process(self, data): #print data try: #print 'before loads', type(data), data data = jsonloads(data) #print 'after loads', type(data), data, data.keys() except ValueError: self.transport.write('{"error_msg": "json format error"}') getlogger(logger_path).error('json format error') return #self.transport.write('test') #print data try: dtype = str(data.pop('type')) except KeyError: return if dtype == '1': self.query(data) elif dtype == '2': self.propelling(data) elif dtype == '3': self.commit(data) elif dtype == '4': self.reset_alarm(data) else: return
def save2logger(self, data): #print data try: datetime = data.pop('datetime') content = jsondumps(data.pop('error'), False) item_id = int(data.pop('id')) except KeyError as e: getlogger(logger_path).error('has not key:%s' % e.message) return #content = json.dumps(data) self.p.rpush(item_id, datetime) self.p.hset('%s.%s' % (item_id, datetime), 0, content) self.p.expire('%s.%s' % (item_id, datetime), EXPIRE_TIME) self.p.set('%s.l_last_time', datetime) self.p.incr('%s.l_count' % item_id) self.p.expire('%s.l_last_time', EXPIRE_TIME) self.p.expire('%s.l_count', EXPIRE_TIME) self.p.execute() self._remlist(item_id, datetime) sql = "insert into ma_log_data(item_id, log_time, content) \ values('%s', '%s', '%s')" % (item_id, s2datetime(datetime), content) execute_sql(sql, True)
def query(self, transport, data): try: table = data['table'] oid = data['oid'] time = data.get('time') num = data.get('num') except KeyError as e: transport.write('{"error_msg": "KeyError %s"}' % e.message) getlogger(logger_path).error('has not key:%s' % e.message) return item_ids = [] oids = [] for k, v in oid.iteritems(): if len(v) > 0: item_ids.extend(v) else: oids.append(k) if oids: item_ids.extend(get_item_ids(oids)) if not item_ids: transport.write(json2proxy({"error_msg": "no items"})) print json2proxy({"error_msg": "no items"}) else: contents = get_contents(table, item_ids, time, num) contents = json2proxy(contents) print item_ids, contents transport.write(contents)
def save2monitor(self, data): try: datetime = data.pop('datetime') is_attention = data.pop('is_attention', 0) item_id = int(data.pop('id')) except KeyError as e: getlogger(logger_path).error('has not key:%s' % e.message) return content = jsondumps(data, False) #monitor self.p.rpush(item_id, datetime) self.p.hset('%s.%s' % (item_id, datetime), is_attention, content) self.p.expire('%s.%s' % (item_id, datetime), EXPIRE_TIME) #item self.p.set('%s.m_last_time' % item_id, datetime) self.p.set('%s.m_last_data' % item_id, content) self.p.set('%s.m_is_attention' % item_id, is_attention) self.p.expire('%s.m_last_time' % item_id, EXPIRE_TIME) self.p.expire('%s.m_last_data' % item_id, EXPIRE_TIME) self.p.expire('%s.m_is_attention' % item_id, EXPIRE_TIME) print 'save2alarm redis' self.p.execute() self._remlist(item_id, datetime) sql = u"insert into ma_monitor_data_YYMMDD(item_id,\ date_time, is_attention, content) values('%s', '%s',\ '%s', '%s')" % (item_id, s2datetime(datetime), is_attention, content) execute_sql(sql, True)
def __init__(self, filename): """ filedb storage content format: {'filename_pattern': ('filename', pos)} """ self.lock.acquire() #print filename, self._shared if filename in self._shared: self.__dict__ = self._shared[filename]['dict'] self._shared[filename]['ref'] += 1 else: self._shared[filename] = {'dict': self.__dict__, 'ref': 1} self.filename = filename if not os.path.isfile(filename): self.filedb = {} else: try: f = open(filename) try: self.filedb = eval(f.read()) except SyntaxError: self.filedb = {} f.close() except IOError: getlogger(configure.logger_path).error( 'no such a record file:%s' % filename) #import time #time.sleep(1) self.lock.release()
def load(self, cmds): """Parse config and load into memory""" self.configs = {} self.min_param = 4 self.max_param = 5 for k, v in cmds.iteritems(): try: self.configs[k] = self._resolve(v) except AssertionError: getlogger(configure.logger_path).error('config %s:%s error' % (k, str(v)))
def _load_func(self, cmd): last_point = 0 try: last_point = cmd.rindex('.') except ValueError: getlogger(configure.logger_path).error( '%s should be added with module name' % cmd) raise ValueError('%s should be added with module name' % cmd) module = cmd[:last_point] func = cmd[last_point + 1:] exec "import %s" % module return eval("%s.%s" % (module, func))
def create(self, family, types): s = socket.socket(family, types) try: #print "连接地址==",self.addr s.connect(self.addr) except socket.error: getlogger(configure.logger_path).error("connection refuse:%s" % str(self.addr)) #raise socket.error("connection refuse:%s" % str(self.addr)) #print "连接失败,等待重试" return s
def commit(self, data): try: datatype = data.pop('filter') except KeyError: getlogger(logger_path).error('has not key:%s' % KeyError.message) return fastdb = FastDb() if datatype == 'monitor_filter': fastdb.save2monitor(data) elif datatype == 'alarm_filter': fastdb.save2alarm(data) elif datatype == 'logger_filter': fastdb.save2logger(data)
def check_is_alive2(self, msg2): #print "创建连接22" #print "msf2===============self.addr",msg2,self.addr # msg2数据格式 # 176${"filter": "monitor_filter", "\u5185\u5b58\u5269\u4f59": [3934, "MB"], "datetime": 1332291290, "\u5185\u5b58\u4f7f\u7528": [161, "MB"], "type": 3, "id": 180, "is_attention": 1} #socket.setdefaulttimeout(1000000) conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: #print "连接地址22==",self.addr conn.connect(self.addr) conn.send(msg2) except socket.error: getlogger(configure.logger_path).error("connection refuse:%s" % str(self.addr))
def discord_tokenthings(dn, discordtokens): logger = getlogger('maint.tokens.discord') old_rtoken = discordtokens.get('rtoken') expires = discordtokens.get('expires') charid = discordtokens.get('uid') discorduid = discordtokens.get('discorduid') if not old_rtoken: return True if expires: current_time = time.time() if expires - current_time > 86400: return True result, code = _discordrefresh.refresh_token(old_rtoken) if code is not True: # broken token, or broken oauth? # the distinction matters. # see env/lib/python3.5/site-packages/oauthlib/oauth2/rfc6749/errors.py msg = 'unable to refresh discord token for {0}: {1}'.format(dn, result) logger.info(msg) # only these exception types are valid reasons to purge a token purgetype = [ 'InvalidGrantError', 'UnauthorizedClientError', 'InvalidClientError', 'InvalidTokenError' ] if result in purgetype: # purge the entry from the ldap user _ldaphelpers.update_singlevalue(dn, 'discordRefreshToken', None) _ldaphelpers.update_singlevalue(dn, 'discordAccessToken', None) _ldaphelpers.update_singlevalue(dn, 'discordAccessTokenExpires', None) msg = 'invalid discord token entries purged for user {}'.format(dn) logger.info(msg) return True # either way, this has failed in an unrecoverable way return True atoken = result.get('access_token') rtoken = result.get('refresh_token') expires = result.get('expires_at') # store the updated token result, value = storetokens(charid, atoken, rtoken, expires, token_type='discord') if result == False: msg = 'unable to store discord tokens for user {}'.format(dn) logger.error(msg) return False return True
def _logger(args): """ args must be dict, and one of it's key must be 'filename_pattern' """ try: filename_pattern = args.get('filename_pattern').strip() except AttributeError: getlogger( configure.logger_path).error('logger params should be dict and\ contains key:filename_pattern') raise AttributeError('logger params should be dict and\ contains key:filename_pattern') record = _get_last_record(filename_pattern) content, pos = _get_content(record) _update_record(filename_pattern, pos) return fun({'filename_pattern': filename_pattern, 'content': content})
def _get_last_record(filename_pattern): """ if record is EOF of file and have more files then update record else keep the latest record """ import glob filedb = FileDB(logger_position) record = filedb.get(filename_pattern) files = glob.glob(filename_pattern) if not files: getlogger(configure.logger_path).error( 'logger file pattern:%s is not correct' % (filename_pattern)) raise NotExitError('filename pattern:%s is not correct' % filename_pattern) files.sort() def is_EOF(record): f = open(record[0]) f.seek(0, 2) flag = (f.tell() == record[1]) f.close() return flag if not os.path.isfile(logger_position) or record is None and files: record = (files[0], 0) elif is_EOF(record): if record[0] in files: try: record = files[files.index(record[0]) + 1], 0 except IndexError: #getlogger().error('no more logger generator') record = filedb.get(filename_pattern) return record
def _resolve(self, data): if not len(data): return config = {} assert len(data[0]) == 2 try: conn = None for d in data: if len(d) == 2: conn = self._load_func(d[0])(d[1]) else: assert len(d) >= self.min_param and len( d) <= self.max_param param = None if len(d) == self.max_param: param = d[4] try: assert type(param) is dict except AssertionError: getlogger(configure.logger_path).error( "type of param:%s is not dict" % param) raise TypeError try: assert len(d[2]) >= 1 assert type(d[3]) is dict and len(d[3]) == 1 except AssertionError: getlogger(configure.logger_path).error( ('configure file is not' 'correct(format error:should be tuple or' ' dict): %s' % (str(data)))) config[d[0]] = { 'fun': self._load_func(d[1]), 'rate': d[2], 'id': d[3]['id'], 'param': param, 'inet': conn } except IndexError: getlogger(configure.logger_path).error( 'parameters %s format is not correct') raise IndexError('parameters %s format is not correct') except AssertionError: raise AssertionError return config
def tokenthings(dn, evetokens, discordtokens): logger = getlogger('maint.tokens.tokenthings') # retries retry_max = 5 sleep = 1 function = __name__ tokens = ['eve', 'discord'] for token_type in tokens: retry_count = 0 done = False while (retry_count < retry_max and not done): if retry_count > 0: msg = '{0} token update retry {1} of {2}'.format(token_type, retry_count, retry_max) logger.warning(msg) if token_type == 'eve': result = eve_tokenthings(dn, evetokens) if token_type == 'discord': result = discord_tokenthings(dn, discordtokens) retry_count += 1 if result: # success, all done. done = True else: msg = '{0} token update failed. sleeping {1} seconds before retrying'.format(token_type, sleep) logger.warning(msg) time.sleep(sleep) if retry_count == retry_max: msg = '{0} token update failed {1} times. giving up. '.format(token_type, retry_max) logger.warning(msg)
def core_trisupers(): # logging logger = getlogger('core.trisupers') ipaddress = request.args.get('log_ip') if ipaddress is None: ipaddress = request.headers['X-Real-Ip'] charid = request.args.get('charid') if charid is None: error = 'need a charid to authenticate with' js = json.dumps({'error': error}) resp = Response(js, status=405, mimetype='application/json') return resp securitylog('supers audit', ipaddress=ipaddress, charid=charid) # check for auth groups allowed_roles = ['tsadmin'] dn = 'ou=People,dc=triumvirate,dc=rocks' code, result = _ldaphelpers.ldap_search(__name__, dn, '(uid={})'.format(charid), ['authGroup']) if code == 'error': error = 'unable to check auth groups roles for {0}: ({1}) {2}'.format( charid, code, result) logger.error(error) js = json.dumps({'error': error}) resp = Response(js, status=500, mimetype='application/json') return resp if result == None: msg = 'charid {0} not in ldap'.format(charid) logger.error(msg) js = json.dumps({'error': msg}) return Response(js, status=404, mimetype='application/json') (_, result), = result.items() if not "command" in result['authGroup']: error = 'insufficient authgroup privileges to access this endpoint.' logger.error(error) js = json.dumps({'error': error}) resp = Response(js, status=403, mimetype='application/json') return resp # get people in trisupers pilots = [] filterstr = '(&(esiAccessToken=*)(alliance=933731581)(authGroup=trisupers))' attrlist = ['uid', 'corporation', 'characterName', 'altOf'] code, result = _ldaphelpers.ldap_search(__name__, dn, filterstr, attrlist) for _, info in result.items(): pilots.append(info['uid']) result_supers = result # iterate through and add each charid from registered alts for charid in pilots: filterstr = 'altOf={}'.format(charid) attrlist = ['uid', 'corporation', 'characterName', 'altOf'] code, result = _ldaphelpers.ldap_search(__name__, dn, filterstr, attrlist) if result: result_supers = {**result_supers, **result} supers = dict() with ThreadPoolExecutor(40) as executor: futures = { executor.submit(audit_pilot, result_supers[cn]): cn for cn in result_supers } for future in as_completed(futures): data = future.result() try: supers.update(data) except Exception as err: msg = 'super audit for failed: {0}'.format(err) logger.error(msg) supers_cleaned = {} for key in supers: supers_cleaned[supers[key]['item_id']] = supers[key] js = json.dumps(supers_cleaned) return Response(js, status=200, mimetype='application/json')
def core_corpcapitals(): dn = 'ou=People,dc=triumvirate,dc=rocks' # logging logger = getlogger('core.corp_capitals') ipaddress = request.args.get('log_ip') if ipaddress is None: ipaddress = request.headers['X-Real-Ip'] charid = request.args.get('charid') if charid is None: error = 'need a charid to authenticate with' js = json.dumps({'error': error}) resp = Response(js, status=405, mimetype='application/json') return resp securitylog('capitals audit', ipaddress=ipaddress, charid=charid) # check for auth groups allowed_roles = ['Director', 'Personnel_Manager'] roles = check_role(charid, allowed_roles) if not roles: return Response(json.dumps({'error': "forbidden"}), status=403, mimetype='application/json') # get corp code, result = _ldaphelpers.ldap_search(__name__, dn, '(uid={})'.format(charid), ['corporation']) if code == 'error': error = 'unable to check auth groups roles for {0}: ({1}) {2}'.format( charid, code, result) logger.error(error) js = json.dumps({'error': error}) resp = Response(js, status=500, mimetype='application/json') return resp if result == None: msg = 'charid {0} not in ldap'.format(charid) logger.error(msg) js = json.dumps({'error': msg}) return Response(js, status=404, mimetype='application/json') (_, result), = result.items() # get super pilots filterstr = '(&(esiAccessToken=*)(corporation={0}))'.format( result['corporation']) attrlist = ['uid', 'corporation', 'characterName', 'altOf'] code_capitals, result_capitals = _ldaphelpers.ldap_search( __name__, dn, filterstr, attrlist) supers = dict() with ThreadPoolExecutor(75) as executor: futures = { executor.submit(audit_pilot_capitals, result_capitals[cn]): cn for cn in result_capitals } for future in as_completed(futures): data = future.result() try: supers.update(data) except Exception as err: msg = 'capital audit for failed: {0}'.format(err) logger.error(msg) supers_cleaned = {} for key in supers: supers_cleaned[supers[key]['item_id']] = supers[key] js = json.dumps(supers_cleaned) return Response(js, status=200, mimetype='application/json')
def core_srp_requests_past(charid): logger = getlogger('core.srp.post') ipaddress = request.args.get('log_ip') if ipaddress is None: ipaddress = request.headers['X-Real-Ip'] securitylog('SRP post', ipaddress=ipaddress, charid=charid) kill_url = request.form['url'] fleetfc = request.form['fleetfc'] notes = request.form['notes'] # only people in tri can get SRP affiliations = _esihelpers.esi_affiliations(charid) if affiliations['allianceid'] != 933731581: response = {'error': 'not eligible for SRP'} return Response(json.dumps(response), status=401, mimetype='application/json') # use regex to get the zkill kill ID pattern = re.compile('(.*)zkillboard.com/kill/(\d+)(.*)') match = re.match(pattern, kill_url) if match: killid = match.group(2) else: response = {'error': 'unable to parse zkill url: {0}'.format(kill_url)} return Response(json.dumps(response), status=400, mimetype='application/json') # fetch zkill data request_url = 'killID/{}/'.format(killid) code, result = _esi.esi(__name__, request_url, base='zkill') try: if result.get('error'): msg = 'zkill error {0}: {1}'.format(request_url, result['error']) logger.error(msg) response = {'error': msg} return Response(json.dumps(response), status=400, mimetype='application/json') except Exception as e: # f****d up zkill api spews different datatypes and doesn't use http return codes right pass killhash = result[0]['zkb']['hash'] value = result[0]['zkb']['totalValue'] # fetch actual kill data from ESI request_url = 'killmails/{0}/{1}/'.format(killid, killhash) code, result = _esi.esi(__name__, request_url, version='v1') if code != 200: msg = 'ESI error {0}: {1}'.format(request_url, result) response = {'error': msg} logger.error(msg) return Response(json.dumps(response), status=500, mimetype='application/json') killtime = result['killmail_time'] victim = result['victim'] shipid = result['victim']['ship_type_id'] victimid = result['victim']['character_id'] victim = _esihelpers.esi_affiliations(victimid) request_url = 'universe/types/{}/'.format(shipid) code, result = _esi.esi(__name__, request_url) if code != 200: msg = 'ESI error {0}: {1}'.format(request_url, result) response = {'error': msg} logger.error(msg) return Response(json.dumps(response), status=500, mimetype='application/json') shipname = result['name'] killtime = killtime.replace('T', ' ') killtime = killtime.replace('Z', '') # start doing db checks try: sql_conn = mysql.connect(database=_database.DB_DATABASE, user=_database.DB_USERNAME, password=_database.DB_PASSWORD, host=_database.DB_HOST) except mysql.Error as err: msg = 'mysql error: {0}'.format(err) logger.error(msg) js = json.dumps({'error': msg}) resp = Response(js, status=500, mimetype='application/json') return resp cursor = sql_conn.cursor() # check that there is no duplicate killmail query = 'SELECT killID from SRP where killID=%s' try: rowcount = cursor.execute(query, (killid, )) except mysql.Error as err: msg = 'mysql error: {0}'.format(err) logger.error(msg) js = json.dumps({'error': msg}) cursor.close() return Response(js, status=500, mimetype='application/json') if rowcount > 0: msg = 'kill ID {0} already submitted for SRP'.format(killid) js = json.dumps({'error': msg}) cursor.close() return Response(js, status=400, mimetype='application/json') # fetch estimated payout query = 'SELECT value FROM CalcSRP WHERE shipTypeID=%s' try: rowcount = cursor.execute(query, (shipid, )) rows = cursor.fetchall() except mysql.Error as err: msg = 'mysql error: {0}'.format(err) logger.error(msg) js = json.dumps({'error': msg}) cursor.close() return Response(js, status=500, mimetype='application/json') if rowcount == 0: msg = 'ship type {0} not eligible for SRP'.format(shipname) js = json.dumps({'error': msg}) cursor.close() return Response(js, status=400, mimetype='application/json') payout, = rows payout = payout[0] # map the charid to a charnme for payments paychar = _esihelpers.esi_affiliations(charid) paychar = paychar['charname'] # insert data into SRP table query = 'INSERT into SRP (RequestTime, RequestedByCharID, LossTime, Shiptype, shipTypeID, charID, ' query += 'charName, zkbLink, killID, srpStatus, payChar, fleetFC, estPayout, obs)' query += ' VALUES (FROM_UNIXTIME(%s), %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)' try: result = cursor.execute( query, (time.time(), charid, killtime, shipname, shipid, victimid, victim['charname'], kill_url, killid, 0, paychar, fleetfc, payout, notes)) except mysql.Error as err: msg = 'mysql error inserting SRP: {0}'.format(err) logger.error(msg) js = json.dumps({'error': msg}) cursor.close() return Response(js, status=500, mimetype='application/json') sql_conn.commit() cursor.close() return Response({}, status=200, mimetype='application/json')
def registeruser(charid, atoken, rtoken, isalt=False, altof=None, tempblue=False, renter=False): # put the barest skeleton of information into ldap/mysql logger = getlogger('core.sso.registeruser') # get character affiliations headers = { 'Content-Type': 'application/json', 'Accept': 'application/json' } if isalt: msg = 'registering user {0} (alt of {1})'.format(charid, altof) else: msg = 'registering user {}'.format(charid) logger.info(msg) # affiliations affiliations = _esihelpers.esi_affiliations(charid) charname = affiliations.get('charname') corpid = affiliations.get('corpid') corpname = affiliations.get('corporation_name') allianceid = affiliations.get('allianceid') alliancename = affiliations.get('alliancename') # sort out basic auth groups if tempblue: # default level of access for vanguard blues authgroups = ['public', 'vanguardBlue'] accountstatus = 'blue' if renter: # renter groups authgroups = ['public', 'renters'] accountstatus = 'public' else: # default level of access authgroups = ['public'] accountstatus = 'public' if allianceid == 933731581: # tri specific authgroup authgroups.append('triumvirate') accountstatus = 'blue' # setup the service user/pass cn, dn = _ldaphelpers.ldap_normalize_charname(charname) dn = "cn={},ou=People,dc=triumvirate,dc=rocks".format(cn) # create the stub result, code = _ldaphelpers.ldap_create_stub(charid=charid, charname=charname, isalt=isalt, altof=altof, accountstatus=accountstatus, authgroups=authgroups, rtoken=rtoken, atoken=atoken) # if result: # # if isalt: # msg = 'new user {0} (alt of {1} registered'.format(charname, altof) # else: # msg = 'new user {0} registered'.format(charname) return
def audit_pilot(entry): ships = dict() basic_pilot = dict() uid = entry['uid'] logger = getlogger('core.audit_pilot.{0}.supers'.format(uid)) msg = 'auditing character {0} for supers'.format(uid) logger.debug(msg) corpid = entry['corporation'] charname = entry['characterName'] altOf = entry['altOf'] corpid = entry['corporation'] # data pollution fix if altOf == 'None': altOf = None basic_pilot['uid'] = uid basic_pilot['pilot'] = charname basic_pilot['valid'] = False basic_pilot['main_charid'] = altOf corp_info = _esihelpers.corporation_info(corpid) if corp_info is not None: basic_pilot['corporation'] = corp_info['name'] else: basic_pilot['corporation'] = "Unknown" # hardcoded data for asset typeids titans = { 11567: 'Avatar', 671: 'Erebus', 45649: 'Komodo', 3764: 'Leviathan', 42241: 'Molok', 23773: 'Ragnarok', } supers = { 23919: 'Aeon', 22852: 'Hel', 23913: 'Nyx', 3514: 'Revenant', 42125: 'Vendetta', 23917: 'Wyvern' } # fetch current ship code, current_ship = _esihelpers.current_ship(uid) if code == False: return ships # fetch super/titan typeids out of char assets code, char_assets = _esihelpers.find_types(uid, list(titans) + list(supers)) if code == False: return ships # fetch character location code, location = _esihelpers.char_location(uid) if code == False: return ships # fetch main charname, if it exists dn = 'ou=People,dc=triumvirate,dc=rocks' filterstr = 'uid={}'.format(altOf) attrlist = ['uid', 'characterName'] if not altOf == None: result = _ldaphelpers.ldap_uid2name(altOf) if result == None: msg = 'failed to find main for {0}'.format(altOf) logger.warning(msg) try: main = result except: main = "Unknown" else: main = charname # does this character have a titan/super in assets? for asset in char_assets: asset_typeid = asset.get('type_id') asset_id = asset.get('item_id') if asset_typeid in list(titans) + list(supers): ships[asset_id] = basic_pilot ships[asset_id]['typeid'] = asset_typeid ships[asset_id]['item_id'] = asset_id ships[asset_id]['main_charname'] = main ships[asset_id]['active'] = False ships[asset_id]['location_id'] = asset['location_id'] if asset['location_type'] == 'station': request_url = 'universe/stations/{}/'.format( asset['location_id']) code, result = common.request_esi.esi(__name__, request_url, method='get', version='latest') if code == 404: ships[asset_id]['location_name'] = 'STATION NOT FOUND' elif code == 200: ships[asset_id]['location_name'] = result['name'] else: ships[asset_id]['location_name'] = "STATION ERROR" elif asset['location_type'] == 'other': request_url = 'universe/structures/{}/'.format( asset['location_id']) code, result = common.request_esi.esi(__name__, request_url, method='get', version='v2', charid=uid) if code == 200: ships[asset_id]['location_name'] = result['name'] elif code == 403 or code == 401: ships[asset_id]['location_name'] = "CITADEL FORBIDDEN" elif code == 404: ships[asset_id]['location_name'] = 'CITADEL NOT FOUND' else: ships[asset_id]['location_name'] = "CITADEL ERROR" else: ships[asset_id]['location_name'] = 'TYPE UNKNOWN' if asset_typeid in list(supers): ships[asset_id]['type'] = supers[asset_typeid] ships[asset_id]['class'] = "Supercarrier" if asset_typeid in list(titans): ships[asset_id]['type'] = titans[asset_typeid] ships[asset_id]['class'] = "Titan" # is this character flying a titan/super? # this is last to override the asset search with the active super (if any) active_typeid = current_ship.get('ship_type_id') active_id = current_ship.get('ship_item_id') if active_typeid in list(titans) + list(supers): # setup basics ships[active_id] = basic_pilot ships[active_id]['typeid'] = active_typeid ships[active_id]['item_id'] = active_id ships[active_id]['main_charname'] = main ships[active_id]['active'] = True ships[active_id]['location_name'] = location['location'] # more complex location ids and names can be filled in later? # actual ship specific shit if active_typeid in list(titans): ships[active_id]['type'] = titans[active_typeid] ships[active_id]['class'] = "Titan" if active_typeid in list(supers): ships[active_id]['type'] = supers[active_typeid] ships[active_id]['class'] = "Supercarrier" return ships
def auth_evesso_callback(): logger = getlogger('core.sso.callback') client_id = _eve.client_id client_secret = _eve.client_secret redirect_url = _eve.redirect_url base_url = 'https://login.eveonline.com' token_url = base_url + '/v2/oauth/token' # the user has (ostensibly) authenticated with the application, now # the access token can be fetched altof = session.get('altof') isalt = session.get('isalt') tempblue = session.get('tempblue') renter = session.get('renter') state = session.get('oauth2_state') ipaddress = request.headers['X-Real-Ip'] # security logging if isalt == True: detail = 'alt of {}'.format(altof) auth_scopes = scope elif tempblue == True: detail = 'temp blue' # make sure we only check for the blue scope list auth_scopes = blue_scope elif renter == True: detail = 'renter' auth_scopes = renter_scope else: detail = None auth_scopes = scope action = 'SSO callback' securitylog(action=action, ipaddress=ipaddress, detail=detail) # handle oauth token manipulation oauth_session = OAuth2Session( client_id=client_id, state=state, redirect_uri=redirect_url, auto_refresh_kwargs={ 'client_id': client_id, 'client_secret': client_secret, }, auto_refresh_url=token_url, ) headers = {'Accept': 'application/json', 'Content-Type': 'application/x-www-form-urlencoded' } try: atoken = oauth_session.fetch_token( token_url, client_secret=client_secret, authorization_response=request.url, headers=headers, ) except Exception as error: msg = 'unable to fetch eve sso access token: {0}'.format(error) logger.error(msg) return('ERROR: ' + str(error)) access_token = atoken['access_token'] refresh_token = atoken['refresh_token'] expires_at = atoken['expires_at'] try: charid, charname, scopes = verify(access_token) except Exception as e: # this ought to never happen msg = 'unable to verify eve sso access token: {0}'.format(error) logger.error(msg) message = 'SORRY, internal error. Try again.' response = make_response(message) return response # full ESI affiliations affilliations = _esihelpers.esi_affiliations(charid) if affilliations.get('error'): msg = 'error in fetching affiliations for {0}: {1}'.format(charid, affilliations.get('error')) logger.error(msg) message = 'SORRY, internal error. Try again.' response = make_response(message) return response allianceid = affilliations.get('allianceid') alliancename = affilliations.get('alliancename') corpid = affilliations.get('corpid') # ldap, if any userinfo = _ldaphelpers.ldap_userinfo(charid) # get alt status, if any, from ldap if userinfo and not isalt: altof = userinfo.get('altOf') if altof is not None: isalt = True # what the f**k is going on # this is a check that _shouldnt_ trigger anymore if isalt: if altof == None or altof == 'None': msg = 'is an alt but altof = None? wtf. charid {0} altof {1} {2}'.format(charid, altof, type(altof)) logger.error(msg) msg = 'error in fetching alt information. please poke saeka.' response = make_response(msg) return response # fix authgroup to an empty array in case nothing if not userinfo: authgroups = [] else: authgroups = userinfo.get('authGroup') if authgroups is None: authgroups = [] # verify that the atoken we get actually has the correct scopes that we requested # just in case someone got cute and peeled some off. if not check_scope(charid, auth_scopes, atoken=access_token): # the user peeled something off the scope list. naughty. msg = 'user {0} modified scope list'.format(charid) logger.warning(msg) securitylog(action='core login scope modification', charid=charid, ipaddress=ipaddress) message = "Don't tinker with the scope list, please.<br>" message += "If you have an issue with it, talk to triumvirate leadership." response = make_response(message) return response else: # scopes validate msg = 'user {0} has expected scopes'.format(charid) logger.debug(msg) # register the user, store the tokens registeruser(charid, access_token, refresh_token, tempblue=tempblue, isalt=isalt, altof=altof, renter=renter) storetokens(charid, access_token, refresh_token, expires=expires_at) ## TESTS ## ## check affiliations and for bans # check to see if the user is banned if 'banned' in authgroups: # banned users not allowed under any conditions message = 'nope.avi' if isalt == True: msg = 'banned user {0} ({1}) tried to register alt {2}'.format(charid, charname, altof) logger.warning(msg) securitylog(action='banned user tried to register', charid=charid, ipaddress=ipaddress, detail='alt of {0}'.format(altof)) else: msg = 'banned user {0} ({1}) tried to register'.format(charid, charname) logger.warning(msg) securitylog(action='banned user tried to register', charid=charid, ipaddress=ipaddress) return make_response(message) # only tri & blues are allowed to use auth if allianceid not in vg_blues() and allianceid not in vg_alliances() and allianceid not in vg_renters(): if not isalt: # not an alt, not a blue. not a renter. go away. msg = 'please contact a recruiter if you are interested in joining triumvirate' logmsg = 'non-blue user {0} ({1}) tried to register'.format(charid, charname) logger.warning(logmsg) securitylog(action='non-blue user tried to register', charid=charid, ipaddress=ipaddress) return make_response(msg) else: # someone is registering a non-blue alt, nbd pass # make sure the temp blue endpoint not being used by tri proper if tempblue: # this is a tri blue, but not tri proper. # ...or at least ought to be. if allianceid in vg_alliances(): # naughty! but not worth logging msg = 'please use the other login endpoint. <br>' msg += 'this is a lower privileged one for blues <b>ONLY</b>' return make_response(msg) # is this a temp blue trying to login with the wrong endpoint? if allianceid in vg_blues(): if not tempblue: # no big deal. we got extra scopes for it. tempblue = True # the user has passed the various exclusions, gg # security logging action = 'SSO callback completed' detail = None if isalt == True: detail='alt of {0}'.format(altof) elif tempblue == True: detail='blue from {0}'.format(alliancename) elif renter == True: detail='renter from {0}'.format(alliancename) securitylog(action=action, charid=charid, ipaddress=ipaddress, detail=detail) expire_date = datetime.datetime.now() + datetime.timedelta(days=14) # build the cookie and construct the http response if isalt == True: # if the character being logged in is an alt, make a session for the main. if userinfo: # the alt is alredy registered. go to homepage. response = make_response(redirect('https://www.triumvirate.rocks')) else: # go to alt registration page to show update. response = make_response(redirect('https://www.triumvirate.rocks/altregistration')) cookie = _session.makesession(altof) msg = 'created session for user: {0} (alt of {1})'.format(charname, altof) logger.info(msg) else: # proceed normally otherwise response = make_response(redirect('https://www.triumvirate.rocks')) cookie = _session.makesession(charid) msg = 'created session for user: {0} (charid {1})'.format(charname, charid) logger.info(msg) response.set_cookie('tri_core', cookie, domain='.triumvirate.rocks', expires=expire_date) if cookie == False: # unable to construct session cookie msg = 'error in creating session cookie for user {0}'.format(charid) logger.error(msg) message = 'SORRY, internal error. Try again.' return make_response(message) # handle registered users if userinfo is not None: # already in ldap, and not banned action = 'core login' if isalt: # is a registered alt msg = 'alt user {0} (alt of {1}) already registered'.format(charname, altof) logger.info(msg) securitylog(action=action, charid=charid, ipaddress=ipaddress, detail='via alt {0}'.format(altof)) code, result = _ldaphelpers.ldap_altupdate(__name__, altof, charid) return response else: if tempblue: # registered blue main. msg = 'user {0} ({1}) already registered'.format(charid, charname) logger.info(msg) securitylog(action=action, charid=charid, ipaddress=ipaddress, detail='blue from {0}'.format(alliancename)) code, result = _ldaphelpers.ldap_altupdate(__name__, altof, charid) return response else: # registered character msg = 'user {0} ({1}) already registered'.format(charid, charname) logger.info(msg) securitylog(action=action, charid=charid, ipaddress=ipaddress) code, result = _ldaphelpers.ldap_altupdate(__name__, altof, charid) return response # after this point, the only folks that are left are unregistered users # handle new temp blues if tempblue: msg = 'user {0} ({1}) not registered'.format(charid, charname) logger.info(msg) securitylog(action='core user registered', charid=charid, ipaddress=ipaddress, detail='blue from {0}'.format(alliancename)) return response # handle new renters if renter: msg = 'user {0} ({1}) not registered'.format(charid, charname) logger.info(msg) securitylog(action='core user registered', charid=charid, ipaddress=ipaddress, detail='renter from {0}'.format(alliancename)) return response # handle new alts if isalt: msg = 'alt user {0} (alt of {1}) not registered'.format(charname, altof) logger.info(msg) securitylog(action='alt user registered', charid=charid, ipaddress=ipaddress, detail='alt of {0}'.format(altof)) return response else: msg = 'user {0} ({1}) not registered'.format(charid, charname) logger.info(msg) securitylog(action='core user registered', charid=charid, ipaddress=ipaddress) return response
def _get_others(table, ids, interval, num): """get contents from ma_monitor_data_YYMMDD, ma_alarm_data, ma_log_data""" contents = [] conn = redis_conn() try: begin = interval[0] end = interval[1] except TypeError as e: getlogger(configure.logger_path).error('time:%s should be list' % e.message) except IndexError as e: getlogger(configure.logger_path).error('time:%s length is not correct' % e.message) return contents = [] queue = Queue() gens = [] #ps = [] size = 0 for i in ids: #print begin, conn.hkeys('m.%s' % i), end if table == 'ma_monitor_data_YYMMDD': fields = filter(lambda x: begin <= int(float(x)) <= end, conn.lrange(i, 0, -1)) size += 1 #p = Process(target=_reduce_data, args=(i, fields, num, table, #queue)) #p.start() r = reduce_data(i, fields, num, table, queue) gens.append(r) elif table == 'ma_alarm_data': #fields = filter(lambda x: begin <= int(float(x)) <= end, #conn.lrange(i, 0, -1)) fields = conn.lrange(i, 0, -1) #fields = _pick_data(fields, num) for f in fields: c = {'item_id': i,} c['begin_time'] = f solved_level = current_level = content = end_time = None if conn.exists('%s.%s.solved'): try: end_time, content = conn.hgetall('%s.%s.solved' % (i, f)) current_level = 0 solved_level = conn.hkeys('%s.%s' % (i, f))[-1] except IndexError: pass else: r = conn.hgetall('%s.%s' % (i, f)) try: k = sorted(r.keys())[-1] except IndexError: pass else: current_level, content = k, r[k] c['end_time'] = end_time c['current_level'] = current_level c['content'] = content c['solved_level'] = solved_level contents.append(c) #elif table == 'ma_alarm_data': #fields = filter(lambda x: begin <= int(float(x)) <= end, #conn.lrange(i, 0, -1)) #fields = _pick_data(fields, num) #for f in fields: #c = {'item_id': i,} #c['begin_time'] = f #c['solved'] = conn.exists('%s.%s.solved') #contents.append(c) elif table == 'ma_log_data': fields = filter(lambda x: begin <= int(float(x)) <= end, conn.lrange(i, 0, -1)) #fields = _pick_data(fields, num) for f in fields: c = {'item_id': i,} c['datetime'] = f c['content'] = conn.hget('%s.%s' % (i, f), 0) contents.append(c) if table == 'ma_monitor_data_YYMMDD': scheme(gens) for i in range(size): item = queue.get() contents.extend(item) return contents
def eve_tokenthings(dn, evetokens): logger = getlogger('maint.tokens.eve') charid = evetokens.get('uid') roles = evetokens.get('roles') ldap_scopes = evetokens.get('scopes') old_rtoken = evetokens.get('rtoken') if not old_rtoken: return True if roles is None: roles = [] if ldap_scopes is None: ldap_scopes = [] msg = 'updating eve token for charid {0}'.format(charid) logger.debug(msg) result, code = _everefresh.refresh_token(old_rtoken) if code is not True: # broken token, or broken oauth? # the distinction matters. # see env/lib/python3.5/site-packages/oauthlib/oauth2/rfc6749/errors.py msg = 'unable to refresh token for charid {0}: {1}'.format(charid, result) logger.error(msg) # only these exception types are valid reasons to purge a token purgetype = [ 'InvalidGrantError', 'UnauthorizedClientError', 'InvalidClientError', 'InvalidTokenError' ] if result in purgetype: # purge the entry from the ldap user _ldaphelpers.update_singlevalue(dn, 'esiRefreshToken', None) _ldaphelpers.update_singlevalue(dn, 'esiAccessToken', None) _ldaphelpers.update_singlevalue(dn, 'esiAccessTokenExpires', None) # corp roles and esi scopes now serve no purpose since the tokens are gone _ldaphelpers.update_singlevalue(dn, 'esiScope', None) _ldaphelpers.update_singlevalue(dn, 'corporationRole', None) msg = 'invalid token entries purged for user {}'.format(dn) logger.info(msg) # either way, this has failed in an unrecoverable way return True atoken = result.get('access_token') rtoken = result.get('refresh_token') expires = result.get('expires_at') # store the updated token result, value = storetokens(charid, atoken, rtoken, expires, token_type='esi') if result == False: msg = 'unable to store tokens for user {}'.format(dn) logger.error(msg) return False # the updated token is now in LDAP # fetch all corporation roles for the updated token if the scope allows that if 'esi-characters.read_corporation_roles.v1' in ldap_scopes: request_url = 'characters/{0}/roles/'.format(charid) code, result = common.request_esi.esi(__name__, request_url, method='get', charid=charid, version='v2') if code == 403: error = 'no perms to read roles for {0}: ({1}) {2}'.format(charid, code, result) logger.debug(error) elif not code == 200: error = 'unable to get character roles for {0}: ({1}) {2}'.format(charid, code, result) logger.error(error) else: # figure out what needs to be added and removed from ldap result = result.get('roles') missing_roles = set(result) - set(roles) extra_roles = set(roles) - set(result) for missing in list(missing_roles): _ldaphelpers.add_value(dn, 'corporationRole', missing) for extra in list(extra_roles): _ldaphelpers.update_singlevalue(dn, 'corporationRole', extra, delete=True) else: # legacy token that doesn't have this scope. probably not getting updates on these lol pass try: token_charid, charname, token_scopes = verify(atoken) except Exception as e: # this ought to never happen msg = 'unable to verify eve sso access token: {0}'.format(error) logger.error(msg) return if not token_charid == charid: # ought to never happen, just a safety check from a scare once msg = 'stored token for charid {0} belongs to charid {1}'.format(charid, token_charid) logger.critical(msg) return # so given an array of scopes, let's check that what we want is in the list of scopes the character's token has missing_scopes = set(token_scopes) - set(ldap_scopes) extra_scopes = set(ldap_scopes) - set(token_scopes) for missing in list(missing_scopes): _ldaphelpers.add_value(dn, 'esiScope', missing) for extra in list(extra_scopes): _ldaphelpers.update_singlevalue(dn, 'esiScope', extra, delete=True) return True
def fetch_chardetails(charid): chardetails = dict() logger = getlogger('core.corpaudit.chardetails.{0}'.format(charid)) dn = 'ou=People,dc=triumvirate,dc=rocks' filterstr='(uid={})'.format(charid) attrlist=['characterName', 'authGroup', 'teamspeakdbid', 'esiAccessToken', 'altOf', 'corporation', 'lastKill', 'lastKillTime','corporationName'] code, result = _ldaphelpers.ldap_search(__name__, dn, filterstr, attrlist) if result is None or not code: # no result? simple response. chardetails['location'] = 'Unknown' chardetails['corporation'] = 'Unknown' chardetails['online'] = 'Unknown' chardetails['last_online'] = 'Unknown' chardetails['token_status'] = False chardetails['teamspeak_status'] = False chardetails['isalt'] = 'Unknown' chardetails['lastKill'] = None chardetails['lastKillTime'] = None chardetails['altof'] = None # fetch affiliations affiliations = _esihelpers.esi_affiliations(charid) chardetails['corporation'] = affiliations.get('corpname') chardetails['charname'] = affiliations.get('charname') else: try: (dn, info), = result.items() except ValueError: print("error: {}".format(charid)) chardetails['charname'] = info['characterName'] # convert last kill time into human readable try: chardetails['lastKill'] = info['lastKill'] except Exception as e: chardetails['lastKill'] = None try: killtime = int(info['lastKillTime']) killtime = time.strftime("%Y/%m/%d, %H:%M:%S", time.localtime(killtime)) chardetails['lastKillTime'] = killtime except Exception as e: chardetails['lastKillTime'] = None corp_id = info['corporation'] if corp_id is None: print("error: {} no corp".format(charid)) chardetails['corporation'] = info['corporationName'] # does the char have a token? try: detail = info['esiAccessToken'] if len(detail) > 0: chardetails['token_status'] = True else: chardetails['token_status'] = False except Exception as e: chardetails['token_status'] = False # teamspeak registration? try: detail = info['teamspeakdbid'] if len(detail) > 0: chardetails['teamspeak_status'] = True else: chardetails['teamspeak_status'] = False except Exception as e: chardetails['teamspeak_status'] = False # is this an alt? # cast the altof detail to something useful try: detail = info['altOf'] except Exception as e: detail = None # str(None) == False if str(detail).isdigit(): chardetails['isalt'] = True request_url = 'characters/{0}/'.format(detail) code, result = common.request_esi.esi(__name__, request_url, 'get') if not code == 200: msg = '/characters/{0}/ API error {1}: {2}'.format(detail, code, result) logger.warning(msg) try: chardetails['altof'] = result['name'] except KeyError as error: msg = 'User does not exist: {0})'.format(charid) logger.error(msg) chardetails['altof'] = 'Unknown' else: chardetails['altof'] = None chardetails['isalt'] = False ## start fetching character-specific information # request_url = 'characters/{0}/location/'.format(charid) code, result = common.request_esi.esi(__name__, request_url, method='get', charid=charid, version='v1') if not code == 200: # it doesn't really matter msg = 'characters loction API error {0}: {1}'.format(code, result) logger.debug(msg) location = None chardetails['location_id'] = location chardetails['location'] = 'Unknown' else: # can include either station_id or structure_id location = result['solar_system_id'] chardetails['location_id'] = location request_url = 'characters/{0}/location/'.format(charid) code, result = common.request_esi.esi(__name__, request_url, method='get', charid=charid, version='v1') if not code == 200: # it doesn't really matter msg = 'characters loction API error {0}: {1}'.format(code, result) logger.debug(msg) location = None else: # can include either station_id or structure_id location = result['solar_system_id'] chardetails['location_id'] = location # map the location to a name if location == None: chardetails['location'] = 'Unknown' else: request_url = 'universe/systems/{0}/'.format(location) code, result = common.request_esi.esi(__name__, request_url, 'get', version='v4') if not code == 200: msg = '{0} API error: {1}'.format(request_url, result) logger.error(msg) chardetails['location'] = 'Unknown' else: chardetails['location'] = result['name'] # get online status request_url = 'characters/{0}/online/'.format(charid) code, result = common.request_esi.esi(__name__, request_url, method='get', charid=charid, version='v2') if not code == 200: # it doesn't really matter msg = '{0} API error: {1}'.format(request_url, result) logger.debug(msg) location = None chardetails['online'] = 'Unknown' chardetails['last_online'] = 'Unknown' else: chardetails['online'] = result['online'] chardetails['last_online'] = result['last_login'] try: request_url_corp = 'corporations/{0}/'.format(corp_id) code_corp, result_corp = common.request_esi.esi(__name__, request_url_corp, 'get') if not code_corp == 200: _logger.log('[' + __name__ + '] /corporations API error {0}: {1}'.format(code_corp, result_corp['error']), _logger.LogLevel.WARNING) msg = '{0} API error: {1}'.format(request_url_corp, result) logger.warning(msg) else: chardetails['corporation'] = result_corp['name'] except KeyError as error: msg = 'corporation id does not exist: {0}'.format(corp_id) logger.error(msg) charname = None return chardetails
def audit_discord_perms(): logger = getlogger('discord.audit') msg = 'discord auditing' logger.info(msg) bot_token = _discord.social_token server = 358117641724100609 logging.getLogger('discord.audit').setLevel(logging.DEBUG) # these discord roles are manually assigned or everyone gets ignore_roles = [ 'Admin', 'IT Manager', 'TRI Friends', '@everyone', 'production', 'rorqcoordination' ] # old and new discord bots ignore_uids = [534463865744916480, 345393204784267265] # these roles are ones the bot can't manage above_bot = ['Admin', 'BOT'] # redis r = redis.StrictRedis(host='localhost', port=6379, db=0) try: r.client_list() except redis.exceptions.ConnectionError as err: msg = 'Redis connection error: ' + str(err) logger.error(msg) except Exception as err: msg = 'Redis generic error: ' + str(err) logger.error(msg) # fetch all the members from the social discord users = discord_allmembers(token=bot_token, target_server=server) # trim to one item userdata = users[server] # iterate through and populate with core ldap data if possible for user in userdata: # don't try to audit self if user['discorduid'] in ignore_uids: continue dn = 'ou=People,dc=triumvirate,dc=rocks' filterstr = '(discorduid={0})'.format(user['discorduid']) attrlist = [ 'authGroup', 'characterName', 'corporation', 'corporationName', 'discorduid', 'discordRefreshToken', 'discord2fa', 'esiRefreshToken', 'uid', 'alliance' ] code, result = _ldaphelpers.ldap_search(__name__, dn, filterstr, attrlist) if result is None: # unregisterds lose roles immediately, then are kicked "later" msg = 'discord user {0} ({1}) unregistered in core'.format( user['name'], user['display_name']) message = 'You are unregistered on TRI CORE, but you have meaningful roles.\n' message += 'These roles are being removed until you REGISTER on https://www.triumvirate.rocks/comms\n' # purge the user's roles purge_roles = set(user['roles']) - set(ignore_roles) purge_roles = list(purge_roles) if len(purge_roles) > 0: # no point in making noises for users that won't be affected logger.info(msg) # warn the user discord_message_user(token=bot_token, member=user['object'], message=message) # purge the roles discord_changeroles(token=bot_token, target_server=server, member=user['object'], roles_add=list(missing_roles), roles_del=purge_roles) pass # purge if they have no roles whatsoever: # don't want to purge people with manual roles (tri friends) roles = set(user['roles']) - set(['@everyone']) if not roles: # start the clock that the character has been seen # a cache time of 10 days lets this get purged from redis well after i'm done with it timestamp = r.get('discordaudit:unregistered:{0}'.format( user['discorduid'])) if timestamp is None: r.setex( 'discordaudit:unregistered:{0}'.format( user['discorduid']), 864000, time.time()) message = 'Unregistered and role-free idlers are not allowed on the TRI discord.\n' message += 'You have 24 hours before being purged.\n' discord_message_user(token=bot_token, member=user['object'], message=message) continue difference = time.time() - float(timestamp) if difference >= 86400: message = "You've been kicked from the TRI discord. Feel free to re-register on CORE\n" message += "...or get an invite + roles from leadership" discord_message_user(token=bot_token, member=user['object'], message=message) msg = "kicking unregistered user {0}".format(user['name']) logger.info(msg) discord_kick_user(token=bot_token, member=user['object']) continue else: try: (dn, info), = result.items() except Exception as e: msg = "discord uid {0} has {1} total instances registered".format( user['discorduid'], len(result)) logger.warning(msg) continue corpid = info['corporation'] # see if this user can be managed by the bot managable = True if set(user['roles']).intersection(set(above_bot)): # check a nonzero intersection of the user roles and any "Above the bot" roles managable = False # fetch corp ticker request_url = 'corporations/{0}/'.format(corpid) code, result = common.request_esi.esi(__name__, request_url, 'get', version='v4') if code != 200: # something broke severely msg = 'corporations API error {0}: {1}'.format(code, result) logger.error(msg) # can't process without the ticker. continue else: ticker = result.get('ticker') # construct appropriate user name if ticker is None: correct_name = '{0}'.format(info['characterName']) else: correct_name = '[{0}] {1}'.format(ticker, info['characterName']) if user['display_name'] != correct_name and managable: msg = 'wrong name. using: {0}, should have: {1}'.format( user['display_name'], correct_name) discord_changenick(token=bot_token, target_server=server, member=user['object'], nickname=correct_name) # if the discord or ESI tokens have been removed, remove their roles. if info['discordRefreshToken'] is None: message = "Your discord token is no longer valid.\n" message += "Roles are being removed until you re-register discord on TRI CORE\n" purge_roles = set(user['roles']) - set(ignore_roles) purge_roles = list(purge_roles) # only take action on actual roles to remove if purge_roles: msg = "character {0} no longer has a discord refresh token - removing roles".format( info['characterName']) logger.info(msg) discord_message_user(token=bot_token, member=user['object'], message=message) discord_changeroles(token=bot_token, target_server=server, member=user['object'], roles_del=purge_roles) continue if info['esiRefreshToken'] is None: message = "Your ESI token is no longer valid.\n" message += "Roles are being removed until you reconnect to TRI CORE\n" purge_roles = set(user['roles']) - set(ignore_roles) purge_roles = list(purge_roles) # only take action on actual roles to remove if purge_roles: msg = "character {0} no longer has a ESI refresh token - removing roles".format( info['characterName']) logger.info(msg) discord_message_user(token=bot_token, member=user['object'], message=message) discord_changeroles(token=bot_token, target_server=server, member=user['object'], roles_del=purge_roles) continue # construct the correct discord roles for this user # this will be absolutely correct as far as ldap is concerned correct_roles = [] # should maybe add something for banned users maybe, but they # will get purged of all meaningful roles anyway if info['alliance'] == 933731581: # tri role and corp tags only meaningful if in tri correct_roles.append('[{0}]'.format(ticker)) # 2factor secured group membership # this is for tri only if info['discord2fa']: correct_roles.append('Triumvirate') # map ldap authgroups to discord roles # only people with 2FA get privileged roles for authgroup in info['authGroup']: mapping = authgroup_mapping(authgroup) if mapping is not None: correct_roles.append(mapping) else: correct_roles.append('NO 2FA') missing_roles = set(correct_roles) - set(user['roles']) extra_roles = set(user['roles']) - set(correct_roles) extra_roles = extra_roles - set(ignore_roles) if extra_roles: msg = 'user {0} has extra roles: {1}'.format( user['name'], extra_roles) logger.info(msg) if missing_roles: msg = 'user {0} has missing roles: {1}'.format( user['name'], missing_roles) logger.info(msg) if extra_roles or missing_roles: if managable: discord_changeroles(token=bot_token, target_server=server, member=user['object'], roles_add=list(missing_roles), roles_del=list(extra_roles))
#!/usr/bin/python3 from tri_api import app import argparse from common.logger import getlogger_new as getlogger if __name__ == '__main__': logger = getlogger('tri_api') msg = 'tri core API online' logger.info(msg) app.run()
def core_corpaudit(charid): # do a corp level audit of who has services ipaddress = request.headers['X-Real-Ip'] log_charid = request.args.get('log_charid') logger = getlogger('core.corpaudit') securitylog('corp audit information request', ipaddress=ipaddress, charid=log_charid) try: charid = int(charid) except ValueError: msg = 'charid parameters must be integer: {0}'.format(charid) logger.warning(msg) js = json.dumps({ 'error': msg}) resp = Response(js, status=401, mimetype='application/json') return resp corporation_id_list = [] character_id_list = [] dn = 'ou=People,dc=triumvirate,dc=rocks' code, char_result = _ldaphelpers.ldap_search(__name__, dn, '(&(|(uid={0})(altOf={0}))(esiAccessToken=*))' .format(charid), ['uid', 'corporation', 'corporationName', 'characterName']) if code == False: error = 'failed to fetch characters for {0}: ({1}) {2}'.format(charid, code, char_result) logger.error(error) js = json.dumps({'error': error}) resp = Response(js, status=500, mimetype='application/json') return resp for cn in char_result: data = char_result[cn] allowed_roles = ['Director', 'Personnel_Manager'] roles = check_role(data['uid'], allowed_roles) if not roles: msg = "character {0} has insufficient roles for {1}".format(data['characterName'], data['corporationName']) logger.info(msg) continue msg = 'character {0} has sufficient roles to view corp auditing information'.format(cn) logger.debug(msg) if data['corporation'] not in corporation_id_list: request_url = 'corporations/{0}/members/'.format(data['corporation']) code, result = common.request_esi.esi(__name__, request_url, method='get', charid=data['uid'], version='v3') if not code == 200: # something broke severely msg = 'corporations API error {0}: {1}'.format(code, result) logger.error(msg) result = {'code': code, 'error': msg} return code, result character_id_list = result corporation_id_list.append(data['corporation']) # start constructing which member has what users = dict() with ThreadPoolExecutor(25) as executor: futures = { executor.submit(fetch_chardetails, user): user for user in character_id_list } for future in as_completed(futures): #charid = futures[future]['character_id'] data = future.result() if data is not None: users[data['charname']] = data js = json.dumps(users) resp = Response(js, status=200, mimetype='application/json') return resp
def audit_pilot_capitals(entry): ships = dict() basic_pilot = dict() uid = entry['uid'] logger = getlogger('core.audit_pilot.{0}.capitals'.format(uid)) msg = 'auditing character {0} for capitals'.format(uid) logger.debug(msg) corpid = entry['corporation'] charname = entry['characterName'] altOf = entry['altOf'] # data pollution fix if altOf == 'None': altOf = None basic_pilot['uid'] = uid basic_pilot['pilot'] = charname basic_pilot['valid'] = False basic_pilot['main_charid'] = altOf corp_info = _esihelpers.corporation_info(corpid) if corp_info is not None: basic_pilot['corporation'] = corp_info['name'] else: basic_pilot['corporation'] = "Unknown" # hardcoded data for asset typeids carriers = { 23757: 'Archon', 23915: 'Chimera', 24483: 'Nidhoggur', 23911: 'Thanatos' } dreads = { 19724: 'Moros', 19722: 'Naglfar', 19726: 'Phoenix', 19720: 'Revelation', } fax = {37604: 'Apostle', 37606: 'Lif', 37605: 'Minokawa', 37607: 'Ninazu'} # fetch current ship code, current_ship = _esihelpers.current_ship(uid) if code == False: return ships # fetch super/titan typeids out of char assets code, char_assets = _esihelpers.find_types( uid, list(carriers) + list(dreads) + list(fax)) if code == False: return ships # fetch character location code, location = _esihelpers.char_location(uid) if code == False: return ships if not altOf == None: result = _ldaphelpers.ldap_uid2name(altOf) if result == None: msg = 'failed to find main for {0}'.format(altOf) logger.warning(msg) try: main = result except: main = "Unknown" else: main = charname # does this character have a titan/super in assets? for asset in char_assets: asset_typeid = asset.get('type_id') asset_id = asset.get('item_id') if asset_typeid in list(carriers) + list(dreads) + list(fax): ships[asset_id] = basic_pilot ships[asset_id]['item_id'] = asset_id ships[asset_id]['typeid'] = asset_typeid ships[asset_id]['main_charname'] = main ships[asset_id]['active'] = False ships[asset_id]['location_id'] = asset['location_id'] if asset['location_type'] == 'station': request_url = 'universe/stations/{}/'.format( asset['location_id']) code, result = common.request_esi.esi(__name__, request_url, method='get', version='latest') if code == 404: ships[asset_id]['location_name'] = 'STATION NOT FOUND' elif code == 200: ships[asset_id]['location_name'] = result['name'] else: ships[asset_id]['location_name'] = "STATION ERROR" elif asset['location_type'] == 'other': request_url = 'universe/structures/{}/'.format( asset['location_id']) code, result = common.request_esi.esi(__name__, request_url, method='get', version='v2', charid=uid) if code == 200: ships[asset_id]['location_name'] = result['name'] elif code == 403 or code == 401: ships[asset_id]['location_name'] = "CITADEL FORBIDDEN" elif code == 404: ships[asset_id]['location_name'] = 'CITADEL NOT FOUND' else: ships[asset_id]['location_name'] = "CITADEL ERROR" else: ships[asset_id]['location_name'] = 'TYPE UNKNOWN' if asset_typeid in list(carriers): ships[asset_id]['type'] = carriers[asset_typeid] ships[asset_id]['class'] = "Carrier" elif asset_typeid in list(dreads): ships[asset_id]['type'] = dreads[asset_typeid] ships[asset_id]['class'] = "Dreadnought" elif asset_typeid in list(fax): ships[asset_id]['type'] = fax[asset_typeid] ships[asset_id]['class'] = "FAX" # is this character flying a titan/super? # this is last to override the asset search with the active super (if any) active_typeid = current_ship.get('ship_type_id') active_id = current_ship.get('ship_item_id') if active_typeid in list(carriers) + list(dreads) + list(fax): # setup basics ships[active_id] = basic_pilot ships[active_id]['typeid'] = active_typeid ships[active_id]['item_id'] = active_id ships[active_id]['main_charname'] = main ships[active_id]['active'] = True ships[active_id]['location_name'] = location['location'] # more complex location ids and names can be filled in later? # actual ship specific shit if active_typeid in list(carriers): ships[active_id]['type'] = carriers[active_typeid] ships[active_id]['class'] = "Carrier" elif active_typeid in list(dreads): ships[active_id]['type'] = dreads[active_typeid] ships[active_id]['class'] = "Dreadnought" elif active_typeid in list(fax): ships[active_id]['type'] = fax[active_typeid] ships[active_id]['class'] = "FAX" return ships
def maillist_forward(charid=None, mailing_list=None): logger = getlogger('ping_relay.mailing_lists') # setup redis r = redis.StrictRedis(host='localhost', port=6379, db=0) try: r.client_list() except redis.exceptions.ConnectionError as err: msg = 'redis connection error: {0}'.format(err) logger.error(msg) except redis.exceptions.ConnectionRefusedError as err: msg = 'redis connection error: {0}'.format(err) logger.error(msg) except Exception as err: msg = 'redis generic error: {0}'.format(err) logger.error(msg) # fetch mailing lists to confirm that the target is still valid request_url = 'characters/{0}/mail/lists/'.format(charid) code, result = _esi.esi(__name__, request_url, 'get', version='v1', charid=charid) if code != 200: # something broke severely msg = 'mailing list error: {0}'.format(result) logger.error(msg) # can't process without the ticker. return valid_list = False for item in result: if mailing_list == item.get('name'): valid_list = True valid_listid = item.get('mailing_list_id') if not valid_list: msg = 'character no longer subscribed to list {0}'.format(mailing_list) logger.error(msg) return # now that the mailing list id is known, character mails can be filtered request_url = 'characters/{0}/mail/'.format(charid) code, result = _esi.esi(__name__, request_url, 'get', version='v1', charid=charid) if code != 200: # something broke severely msg = 'mail endpoint error: {0}'.format(result) logger.error(msg) # can't process without the ticker. return fetch_mails = [] cutoff = 0 for mail in result: mail_id = mail.get('mail_id') recipients = mail.get('recipients')[0] # convert timestamp to epoch # example: 2018-05-08T17:30:00Z timestamp = time.strptime(mail.get('timestamp'), "%Y-%m-%dT%H:%M:%SZ") timestamp = time.mktime(timestamp) timestamp = int(timestamp) if recipients.get( 'recipient_id') == valid_listid and timestamp > cutoff: fetch_mails.append(mail_id) # parse mail # thx https://stackoverflow.com/questions/753052/strip-html-from-strings-in-python?utm_medium=organic&utm_source=google_rich_qa&utm_campaign=google_rich_qa tag_re = re.compile(r'(<!--.*?-->|<[^>]*>)') for mail in fetch_mails: request_url = 'characters/{0}/mail/{1}/'.format(charid, mail) code, result = _esi.esi(__name__, request_url, 'get', version='v1', charid=charid) if code != 200: # something broke severely msg = 'char {0} mail id {1} error: {2}'.format( charid, mail, result) logger.error(msg) # can't process without the ticker. continue subject = result.get('subject') body = tag_re.sub(' ', result.get('body')) timestamp = time.strptime(result.get('timestamp'), "%Y-%m-%dT%H:%M:%SZ") timestamp = time.mktime(timestamp) timestamp = int(timestamp) print(subject, timestamp, body)
def audit_core(): # keep the ldap account status entries in sync logger = getlogger('audit.core') logger.info('auditing CORE LDAP') # online check first # do esi status check first request_url = 'status' code, result = common.request_esi.esi(__name__, request_url, method='get', version='v1') if not code == 200: error = 'ESI offline' logger.error(error) return # fix open file limitations try: resource.setrlimit(resource.RLIMIT_NOFILE, (25000, 75000)) except Exception as e: logger.warn('unable to set nofile rlimit: {0}'.format(e)) pass # fetch all non-banned LDAP users dn = 'ou=People,dc=triumvirate,dc=rocks' filterstr = '(&(!(accountstatus=banned))(!(accountStatus=immortal)))' attributes = ['uid'] code, nonbanned_users = _ldaphelpers.ldap_search(__name__, dn, filterstr, attributes) if code == False: logger.error('unable to find ldap users') return # fetch all tri LDAP users dn = 'ou=People,dc=triumvirate,dc=rocks' filterstr = 'alliance=933731581' attributes = ['uid' ] code, tri_users = _ldaphelpers.ldap_search(__name__, dn, filterstr, attributes) if code == False: logger.error('unable to find ldap users') return # fetch all blue dn = 'ou=People,dc=triumvirate,dc=rocks' filterstr = 'accountStatus=blue' attributes = ['uid'] code, blue_users = _ldaphelpers.ldap_search(__name__, dn, filterstr, attributes) if code == False: logger.error('unable to find ldap users') return # fetch ALL LDAP users dn = 'ou=People,dc=triumvirate,dc=rocks' filterstr = '(!(accountStatus=immortal))' attributes = ['uid', 'characterName', 'accountStatus', 'authGroup', 'corporation', 'alliance', 'allianceName', 'corporationName', 'discordRefreshToken', 'discorduid', 'discord2fa', 'discordName', 'altOf', 'esiRefreshToken', 'esiScope', 'lastLogin', 'corporationRole' ] code, users = _ldaphelpers.ldap_search(__name__, dn, filterstr, attributes) if code == False: logger.error('unable to find ldap users') return logger.info('total ldap users: {}'.format(len(users))) logger.info('total non-banned ldap users: {}'.format(len(nonbanned_users))) logger.info('total blue ldap users: {}'.format(len(blue_users))) logger.info('total tri ldap users: {}'.format(len(tri_users))) # loop through each user and determine the correct status activity = dict() with ThreadPoolExecutor(30) as executor: futures = { executor.submit(user_audit, dn, users[dn]): dn for dn in users.keys() } for future in as_completed(futures): data = future.result() activity[dn] = data