def process(self, data):
#print data
try:
#print 'before loads', type(data), data
data = jsonloads(data)
#print 'after loads', type(data), data, data.keys()
except ValueError:
self.transport.write('{"error_msg": "json format error"}')
getlogger(logger_path).error('json format error')
return
#self.transport.write('test')
#print data
try:
dtype = str(data.pop('type'))
except KeyError:
return
if dtype == '1':
self.query(data)
elif dtype == '2':
self.propelling(data)
elif dtype == '3':
self.commit(data)
elif dtype == '4':
self.reset_alarm(data)
else:
return
def save2logger(self, data):
#print data
try:
datetime = data.pop('datetime')
content = jsondumps(data.pop('error'), False)
item_id = int(data.pop('id'))
except KeyError as e:
getlogger(logger_path).error('has not key:%s' % e.message)
return
#content = json.dumps(data)
self.p.rpush(item_id, datetime)
self.p.hset('%s.%s' % (item_id, datetime), 0, content)
self.p.expire('%s.%s' % (item_id, datetime), EXPIRE_TIME)
self.p.set('%s.l_last_time', datetime)
self.p.incr('%s.l_count' % item_id)
self.p.expire('%s.l_last_time', EXPIRE_TIME)
self.p.expire('%s.l_count', EXPIRE_TIME)
self.p.execute()
self._remlist(item_id, datetime)
sql = "insert into ma_log_data(item_id, log_time, content) \
values('%s', '%s', '%s')" % (item_id, s2datetime(datetime),
content)
execute_sql(sql, True)
def query(self, transport, data):
try:
table = data['table']
oid = data['oid']
time = data.get('time')
num = data.get('num')
except KeyError as e:
transport.write('{"error_msg": "KeyError %s"}' % e.message)
getlogger(logger_path).error('has not key:%s' % e.message)
return
item_ids = []
oids = []
for k, v in oid.iteritems():
if len(v) > 0:
item_ids.extend(v)
else:
oids.append(k)
if oids:
item_ids.extend(get_item_ids(oids))
if not item_ids:
transport.write(json2proxy({"error_msg": "no items"}))
print json2proxy({"error_msg": "no items"})
else:
contents = get_contents(table, item_ids, time, num)
contents = json2proxy(contents)
print item_ids, contents
transport.write(contents)
def save2monitor(self, data):
try:
datetime = data.pop('datetime')
is_attention = data.pop('is_attention', 0)
item_id = int(data.pop('id'))
except KeyError as e:
getlogger(logger_path).error('has not key:%s' % e.message)
return
content = jsondumps(data, False)
#monitor
self.p.rpush(item_id, datetime)
self.p.hset('%s.%s' % (item_id, datetime), is_attention, content)
self.p.expire('%s.%s' % (item_id, datetime), EXPIRE_TIME)
#item
self.p.set('%s.m_last_time' % item_id, datetime)
self.p.set('%s.m_last_data' % item_id, content)
self.p.set('%s.m_is_attention' % item_id, is_attention)
self.p.expire('%s.m_last_time' % item_id, EXPIRE_TIME)
self.p.expire('%s.m_last_data' % item_id, EXPIRE_TIME)
self.p.expire('%s.m_is_attention' % item_id, EXPIRE_TIME)
print 'save2alarm redis'
self.p.execute()
self._remlist(item_id, datetime)
sql = u"insert into ma_monitor_data_YYMMDD(item_id,\
date_time, is_attention, content) values('%s', '%s',\
'%s', '%s')" % (item_id, s2datetime(datetime), is_attention, content)
execute_sql(sql, True)
def __init__(self, filename):
"""
filedb storage content format:
{'filename_pattern': ('filename', pos)}
"""
self.lock.acquire()
#print filename, self._shared
if filename in self._shared:
self.__dict__ = self._shared[filename]['dict']
self._shared[filename]['ref'] += 1
else:
self._shared[filename] = {'dict': self.__dict__, 'ref': 1}
self.filename = filename
if not os.path.isfile(filename):
self.filedb = {}
else:
try:
f = open(filename)
try:
self.filedb = eval(f.read())
except SyntaxError:
self.filedb = {}
f.close()
except IOError:
getlogger(configure.logger_path).error(
'no such a record file:%s' % filename)
#import time
#time.sleep(1)
self.lock.release()
def load(self, cmds):
"""Parse config and load into memory"""
self.configs = {}
self.min_param = 4
self.max_param = 5
for k, v in cmds.iteritems():
try:
self.configs[k] = self._resolve(v)
except AssertionError:
getlogger(configure.logger_path).error('config %s:%s error' %
(k, str(v)))
def _load_func(self, cmd):
last_point = 0
try:
last_point = cmd.rindex('.')
except ValueError:
getlogger(configure.logger_path).error(
'%s should be added with module name' % cmd)
raise ValueError('%s should be added with module name' % cmd)
module = cmd[:last_point]
func = cmd[last_point + 1:]
exec "import %s" % module
return eval("%s.%s" % (module, func))
def create(self, family, types):
s = socket.socket(family, types)
try:
#print "连接地址==",self.addr
s.connect(self.addr)
except socket.error:
getlogger(configure.logger_path).error("connection refuse:%s" %
str(self.addr))
#raise socket.error("connection refuse:%s" % str(self.addr))
#print "连接失败，等待重试"
return s
def commit(self, data):
try:
datatype = data.pop('filter')
except KeyError:
getlogger(logger_path).error('has not key:%s' % KeyError.message)
return
fastdb = FastDb()
if datatype == 'monitor_filter':
fastdb.save2monitor(data)
elif datatype == 'alarm_filter':
fastdb.save2alarm(data)
elif datatype == 'logger_filter':
fastdb.save2logger(data)
def check_is_alive2(self, msg2):
#print "创建连接22"
#print "msf2===============self.addr",msg2,self.addr
# msg2数据格式
# 176${"filter": "monitor_filter", "\u5185\u5b58\u5269\u4f59": [3934, "MB"], "datetime": 1332291290, "\u5185\u5b58\u4f7f\u7528": [161, "MB"], "type": 3, "id": 180, "is_attention": 1}
#socket.setdefaulttimeout(1000000)
conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
#print "连接地址22==",self.addr
conn.connect(self.addr)
conn.send(msg2)
except socket.error:
getlogger(configure.logger_path).error("connection refuse:%s" %
str(self.addr))
def discord_tokenthings(dn, discordtokens):
logger = getlogger('maint.tokens.discord')
old_rtoken = discordtokens.get('rtoken')
expires = discordtokens.get('expires')
charid = discordtokens.get('uid')
discorduid = discordtokens.get('discorduid')
if not old_rtoken:
return True
if expires:
current_time = time.time()
if expires - current_time > 86400:
return True
result, code = _discordrefresh.refresh_token(old_rtoken)
if code is not True:
# broken token, or broken oauth?
# the distinction matters.
# see env/lib/python3.5/site-packages/oauthlib/oauth2/rfc6749/errors.py
msg = 'unable to refresh discord token for {0}: {1}'.format(dn, result)
logger.info(msg)
# only these exception types are valid reasons to purge a token
purgetype = [ 'InvalidGrantError', 'UnauthorizedClientError', 'InvalidClientError', 'InvalidTokenError' ]
if result in purgetype:
# purge the entry from the ldap user
_ldaphelpers.update_singlevalue(dn, 'discordRefreshToken', None)
_ldaphelpers.update_singlevalue(dn, 'discordAccessToken', None)
_ldaphelpers.update_singlevalue(dn, 'discordAccessTokenExpires', None)
msg = 'invalid discord token entries purged for user {}'.format(dn)
logger.info(msg)
return True
# either way, this has failed in an unrecoverable way
return True
atoken = result.get('access_token')
rtoken = result.get('refresh_token')
expires = result.get('expires_at')
# store the updated token
result, value = storetokens(charid, atoken, rtoken, expires, token_type='discord')
if result == False:
msg = 'unable to store discord tokens for user {}'.format(dn)
logger.error(msg)
return False
return True
def _logger(args):
"""
args must be dict, and one of it's key must be
'filename_pattern'
"""
try:
filename_pattern = args.get('filename_pattern').strip()
except AttributeError:
getlogger(
configure.logger_path).error('logger params should be dict and\
contains key:filename_pattern')
raise AttributeError('logger params should be dict and\
contains key:filename_pattern')
record = _get_last_record(filename_pattern)
content, pos = _get_content(record)
_update_record(filename_pattern, pos)
return fun({'filename_pattern': filename_pattern, 'content': content})
def _get_last_record(filename_pattern):
"""
if record is EOF of file and have more files then update record
else keep the latest record
"""
import glob
filedb = FileDB(logger_position)
record = filedb.get(filename_pattern)
files = glob.glob(filename_pattern)
if not files:
getlogger(configure.logger_path).error(
'logger file pattern:%s is not correct' % (filename_pattern))
raise NotExitError('filename pattern:%s is not correct' %
filename_pattern)
files.sort()
def is_EOF(record):
f = open(record[0])
f.seek(0, 2)
flag = (f.tell() == record[1])
f.close()
return flag
if not os.path.isfile(logger_position) or record is None and files:
record = (files[0], 0)
elif is_EOF(record):
if record[0] in files:
try:
record = files[files.index(record[0]) + 1], 0
except IndexError:
#getlogger().error('no more logger generator')
record = filedb.get(filename_pattern)
return record
def _resolve(self, data):
if not len(data):
return
config = {}
assert len(data[0]) == 2
try:
conn = None
for d in data:
if len(d) == 2:
conn = self._load_func(d[0])(d[1])
else:
assert len(d) >= self.min_param and len(
d) <= self.max_param
param = None
if len(d) == self.max_param:
param = d[4]
try:
assert type(param) is dict
except AssertionError:
getlogger(configure.logger_path).error(
"type of param:%s is not dict" % param)
raise TypeError
try:
assert len(d[2]) >= 1
assert type(d[3]) is dict and len(d[3]) == 1
except AssertionError:
getlogger(configure.logger_path).error(
('configure file is not'
'correct(format error:should be tuple or'
' dict): %s' % (str(data))))
config[d[0]] = {
'fun': self._load_func(d[1]),
'rate': d[2],
'id': d[3]['id'],
'param': param,
'inet': conn
}
except IndexError:
getlogger(configure.logger_path).error(
'parameters %s format is not correct')
raise IndexError('parameters %s format is not correct')
except AssertionError:
raise AssertionError
return config
def tokenthings(dn, evetokens, discordtokens):
logger = getlogger('maint.tokens.tokenthings')
# retries
retry_max = 5
sleep = 1
function = __name__
tokens = ['eve', 'discord']
for token_type in tokens:
retry_count = 0
done = False
while (retry_count < retry_max and not done):
if retry_count > 0:
msg = '{0} token update retry {1} of {2}'.format(token_type, retry_count, retry_max)
logger.warning(msg)
if token_type == 'eve':
result = eve_tokenthings(dn, evetokens)
if token_type == 'discord':
result = discord_tokenthings(dn, discordtokens)
retry_count += 1
if result:
# success, all done.
done = True
else:
msg = '{0} token update failed. sleeping {1} seconds before retrying'.format(token_type, sleep)
logger.warning(msg)
time.sleep(sleep)
if retry_count == retry_max:
msg = '{0} token update failed {1} times. giving up. '.format(token_type, retry_max)
logger.warning(msg)
def core_trisupers():
# logging
logger = getlogger('core.trisupers')
ipaddress = request.args.get('log_ip')
if ipaddress is None:
ipaddress = request.headers['X-Real-Ip']
charid = request.args.get('charid')
if charid is None:
error = 'need a charid to authenticate with'
js = json.dumps({'error': error})
resp = Response(js, status=405, mimetype='application/json')
return resp
securitylog('supers audit', ipaddress=ipaddress, charid=charid)
# check for auth groups
allowed_roles = ['tsadmin']
dn = 'ou=People,dc=triumvirate,dc=rocks'
code, result = _ldaphelpers.ldap_search(__name__, dn,
'(uid={})'.format(charid),
['authGroup'])
if code == 'error':
error = 'unable to check auth groups roles for {0}: ({1}) {2}'.format(
charid, code, result)
logger.error(error)
js = json.dumps({'error': error})
resp = Response(js, status=500, mimetype='application/json')
return resp
if result == None:
msg = 'charid {0} not in ldap'.format(charid)
logger.error(msg)
js = json.dumps({'error': msg})
return Response(js, status=404, mimetype='application/json')
(_, result), = result.items()
if not "command" in result['authGroup']:
error = 'insufficient authgroup privileges to access this endpoint.'
logger.error(error)
js = json.dumps({'error': error})
resp = Response(js, status=403, mimetype='application/json')
return resp
# get people in trisupers
pilots = []
filterstr = '(&(esiAccessToken=*)(alliance=933731581)(authGroup=trisupers))'
attrlist = ['uid', 'corporation', 'characterName', 'altOf']
code, result = _ldaphelpers.ldap_search(__name__, dn, filterstr, attrlist)
for _, info in result.items():
pilots.append(info['uid'])
result_supers = result
# iterate through and add each charid from registered alts
for charid in pilots:
filterstr = 'altOf={}'.format(charid)
attrlist = ['uid', 'corporation', 'characterName', 'altOf']
code, result = _ldaphelpers.ldap_search(__name__, dn, filterstr,
attrlist)
if result:
result_supers = {**result_supers, **result}
supers = dict()
with ThreadPoolExecutor(40) as executor:
futures = {
executor.submit(audit_pilot, result_supers[cn]): cn
for cn in result_supers
}
for future in as_completed(futures):
data = future.result()
try:
supers.update(data)
except Exception as err:
msg = 'super audit for failed: {0}'.format(err)
logger.error(msg)
supers_cleaned = {}
for key in supers:
supers_cleaned[supers[key]['item_id']] = supers[key]
js = json.dumps(supers_cleaned)
return Response(js, status=200, mimetype='application/json')
def core_corpcapitals():
dn = 'ou=People,dc=triumvirate,dc=rocks'
# logging
logger = getlogger('core.corp_capitals')
ipaddress = request.args.get('log_ip')
if ipaddress is None:
ipaddress = request.headers['X-Real-Ip']
charid = request.args.get('charid')
if charid is None:
error = 'need a charid to authenticate with'
js = json.dumps({'error': error})
resp = Response(js, status=405, mimetype='application/json')
return resp
securitylog('capitals audit', ipaddress=ipaddress, charid=charid)
# check for auth groups
allowed_roles = ['Director', 'Personnel_Manager']
roles = check_role(charid, allowed_roles)
if not roles:
return Response(json.dumps({'error': "forbidden"}),
status=403,
mimetype='application/json')
# get corp
code, result = _ldaphelpers.ldap_search(__name__, dn,
'(uid={})'.format(charid),
['corporation'])
if code == 'error':
error = 'unable to check auth groups roles for {0}: ({1}) {2}'.format(
charid, code, result)
logger.error(error)
js = json.dumps({'error': error})
resp = Response(js, status=500, mimetype='application/json')
return resp
if result == None:
msg = 'charid {0} not in ldap'.format(charid)
logger.error(msg)
js = json.dumps({'error': msg})
return Response(js, status=404, mimetype='application/json')
(_, result), = result.items()
# get super pilots
filterstr = '(&(esiAccessToken=*)(corporation={0}))'.format(
result['corporation'])
attrlist = ['uid', 'corporation', 'characterName', 'altOf']
code_capitals, result_capitals = _ldaphelpers.ldap_search(
__name__, dn, filterstr, attrlist)
supers = dict()
with ThreadPoolExecutor(75) as executor:
futures = {
executor.submit(audit_pilot_capitals, result_capitals[cn]): cn
for cn in result_capitals
}
for future in as_completed(futures):
data = future.result()
try:
supers.update(data)
except Exception as err:
msg = 'capital audit for failed: {0}'.format(err)
logger.error(msg)
supers_cleaned = {}
for key in supers:
supers_cleaned[supers[key]['item_id']] = supers[key]
js = json.dumps(supers_cleaned)
return Response(js, status=200, mimetype='application/json')
def core_srp_requests_past(charid):
logger = getlogger('core.srp.post')
ipaddress = request.args.get('log_ip')
if ipaddress is None:
ipaddress = request.headers['X-Real-Ip']
securitylog('SRP post', ipaddress=ipaddress, charid=charid)
kill_url = request.form['url']
fleetfc = request.form['fleetfc']
notes = request.form['notes']
# only people in tri can get SRP
affiliations = _esihelpers.esi_affiliations(charid)
if affiliations['allianceid'] != 933731581:
response = {'error': 'not eligible for SRP'}
return Response(json.dumps(response),
status=401,
mimetype='application/json')
# use regex to get the zkill kill ID
pattern = re.compile('(.*)zkillboard.com/kill/(\d+)(.*)')
match = re.match(pattern, kill_url)
if match:
killid = match.group(2)
else:
response = {'error': 'unable to parse zkill url: {0}'.format(kill_url)}
return Response(json.dumps(response),
status=400,
mimetype='application/json')
# fetch zkill data
request_url = 'killID/{}/'.format(killid)
code, result = _esi.esi(__name__, request_url, base='zkill')
try:
if result.get('error'):
msg = 'zkill error {0}: {1}'.format(request_url, result['error'])
logger.error(msg)
response = {'error': msg}
return Response(json.dumps(response),
status=400,
mimetype='application/json')
except Exception as e:
# f****d up zkill api spews different datatypes and doesn't use http return codes right
pass
killhash = result[0]['zkb']['hash']
value = result[0]['zkb']['totalValue']
# fetch actual kill data from ESI
request_url = 'killmails/{0}/{1}/'.format(killid, killhash)
code, result = _esi.esi(__name__, request_url, version='v1')
if code != 200:
msg = 'ESI error {0}: {1}'.format(request_url, result)
response = {'error': msg}
logger.error(msg)
return Response(json.dumps(response),
status=500,
mimetype='application/json')
killtime = result['killmail_time']
victim = result['victim']
shipid = result['victim']['ship_type_id']
victimid = result['victim']['character_id']
victim = _esihelpers.esi_affiliations(victimid)
request_url = 'universe/types/{}/'.format(shipid)
code, result = _esi.esi(__name__, request_url)
if code != 200:
msg = 'ESI error {0}: {1}'.format(request_url, result)
response = {'error': msg}
logger.error(msg)
return Response(json.dumps(response),
status=500,
mimetype='application/json')
shipname = result['name']
killtime = killtime.replace('T', ' ')
killtime = killtime.replace('Z', '')
# start doing db checks
try:
sql_conn = mysql.connect(database=_database.DB_DATABASE,
user=_database.DB_USERNAME,
password=_database.DB_PASSWORD,
host=_database.DB_HOST)
except mysql.Error as err:
msg = 'mysql error: {0}'.format(err)
logger.error(msg)
js = json.dumps({'error': msg})
resp = Response(js, status=500, mimetype='application/json')
return resp
cursor = sql_conn.cursor()
# check that there is no duplicate killmail
query = 'SELECT killID from SRP where killID=%s'
try:
rowcount = cursor.execute(query, (killid, ))
except mysql.Error as err:
msg = 'mysql error: {0}'.format(err)
logger.error(msg)
js = json.dumps({'error': msg})
cursor.close()
return Response(js, status=500, mimetype='application/json')
if rowcount > 0:
msg = 'kill ID {0} already submitted for SRP'.format(killid)
js = json.dumps({'error': msg})
cursor.close()
return Response(js, status=400, mimetype='application/json')
# fetch estimated payout
query = 'SELECT value FROM CalcSRP WHERE shipTypeID=%s'
try:
rowcount = cursor.execute(query, (shipid, ))
rows = cursor.fetchall()
except mysql.Error as err:
msg = 'mysql error: {0}'.format(err)
logger.error(msg)
js = json.dumps({'error': msg})
cursor.close()
return Response(js, status=500, mimetype='application/json')
if rowcount == 0:
msg = 'ship type {0} not eligible for SRP'.format(shipname)
js = json.dumps({'error': msg})
cursor.close()
return Response(js, status=400, mimetype='application/json')
payout, = rows
payout = payout[0]
# map the charid to a charnme for payments
paychar = _esihelpers.esi_affiliations(charid)
paychar = paychar['charname']
# insert data into SRP table
query = 'INSERT into SRP (RequestTime, RequestedByCharID, LossTime, Shiptype, shipTypeID, charID, '
query += 'charName, zkbLink, killID, srpStatus, payChar, fleetFC, estPayout, obs)'
query += ' VALUES (FROM_UNIXTIME(%s), %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)'
try:
result = cursor.execute(
query, (time.time(), charid, killtime, shipname, shipid, victimid,
victim['charname'], kill_url, killid, 0, paychar, fleetfc,
payout, notes))
except mysql.Error as err:
msg = 'mysql error inserting SRP: {0}'.format(err)
logger.error(msg)
js = json.dumps({'error': msg})
cursor.close()
return Response(js, status=500, mimetype='application/json')
sql_conn.commit()
cursor.close()
return Response({}, status=200, mimetype='application/json')
def registeruser(charid,
atoken,
rtoken,
isalt=False,
altof=None,
tempblue=False,
renter=False):
# put the barest skeleton of information into ldap/mysql
logger = getlogger('core.sso.registeruser')
# get character affiliations
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json'
}
if isalt:
msg = 'registering user {0} (alt of {1})'.format(charid, altof)
else:
msg = 'registering user {}'.format(charid)
logger.info(msg)
# affiliations
affiliations = _esihelpers.esi_affiliations(charid)
charname = affiliations.get('charname')
corpid = affiliations.get('corpid')
corpname = affiliations.get('corporation_name')
allianceid = affiliations.get('allianceid')
alliancename = affiliations.get('alliancename')
# sort out basic auth groups
if tempblue:
# default level of access for vanguard blues
authgroups = ['public', 'vanguardBlue']
accountstatus = 'blue'
if renter:
# renter groups
authgroups = ['public', 'renters']
accountstatus = 'public'
else:
# default level of access
authgroups = ['public']
accountstatus = 'public'
if allianceid == 933731581:
# tri specific authgroup
authgroups.append('triumvirate')
accountstatus = 'blue'
# setup the service user/pass
cn, dn = _ldaphelpers.ldap_normalize_charname(charname)
dn = "cn={},ou=People,dc=triumvirate,dc=rocks".format(cn)
# create the stub
result, code = _ldaphelpers.ldap_create_stub(charid=charid,
charname=charname,
isalt=isalt,
altof=altof,
accountstatus=accountstatus,
authgroups=authgroups,
rtoken=rtoken,
atoken=atoken)
# if result:
#
# if isalt:
# msg = 'new user {0} (alt of {1} registered'.format(charname, altof)
# else:
# msg = 'new user {0} registered'.format(charname)
return
def audit_pilot(entry):
ships = dict()
basic_pilot = dict()
uid = entry['uid']
logger = getlogger('core.audit_pilot.{0}.supers'.format(uid))
msg = 'auditing character {0} for supers'.format(uid)
logger.debug(msg)
corpid = entry['corporation']
charname = entry['characterName']
altOf = entry['altOf']
corpid = entry['corporation']
# data pollution fix
if altOf == 'None': altOf = None
basic_pilot['uid'] = uid
basic_pilot['pilot'] = charname
basic_pilot['valid'] = False
basic_pilot['main_charid'] = altOf
corp_info = _esihelpers.corporation_info(corpid)
if corp_info is not None:
basic_pilot['corporation'] = corp_info['name']
else:
basic_pilot['corporation'] = "Unknown"
# hardcoded data for asset typeids
titans = {
11567: 'Avatar',
671: 'Erebus',
45649: 'Komodo',
3764: 'Leviathan',
42241: 'Molok',
23773: 'Ragnarok',
}
supers = {
23919: 'Aeon',
22852: 'Hel',
23913: 'Nyx',
3514: 'Revenant',
42125: 'Vendetta',
23917: 'Wyvern'
}
# fetch current ship
code, current_ship = _esihelpers.current_ship(uid)
if code == False:
return ships
# fetch super/titan typeids out of char assets
code, char_assets = _esihelpers.find_types(uid,
list(titans) + list(supers))
if code == False:
return ships
# fetch character location
code, location = _esihelpers.char_location(uid)
if code == False:
return ships
# fetch main charname, if it exists
dn = 'ou=People,dc=triumvirate,dc=rocks'
filterstr = 'uid={}'.format(altOf)
attrlist = ['uid', 'characterName']
if not altOf == None:
result = _ldaphelpers.ldap_uid2name(altOf)
if result == None:
msg = 'failed to find main for {0}'.format(altOf)
logger.warning(msg)
try:
main = result
except:
main = "Unknown"
else:
main = charname
# does this character have a titan/super in assets?
for asset in char_assets:
asset_typeid = asset.get('type_id')
asset_id = asset.get('item_id')
if asset_typeid in list(titans) + list(supers):
ships[asset_id] = basic_pilot
ships[asset_id]['typeid'] = asset_typeid
ships[asset_id]['item_id'] = asset_id
ships[asset_id]['main_charname'] = main
ships[asset_id]['active'] = False
ships[asset_id]['location_id'] = asset['location_id']
if asset['location_type'] == 'station':
request_url = 'universe/stations/{}/'.format(
asset['location_id'])
code, result = common.request_esi.esi(__name__,
request_url,
method='get',
version='latest')
if code == 404:
ships[asset_id]['location_name'] = 'STATION NOT FOUND'
elif code == 200:
ships[asset_id]['location_name'] = result['name']
else:
ships[asset_id]['location_name'] = "STATION ERROR"
elif asset['location_type'] == 'other':
request_url = 'universe/structures/{}/'.format(
asset['location_id'])
code, result = common.request_esi.esi(__name__,
request_url,
method='get',
version='v2',
charid=uid)
if code == 200:
ships[asset_id]['location_name'] = result['name']
elif code == 403 or code == 401:
ships[asset_id]['location_name'] = "CITADEL FORBIDDEN"
elif code == 404:
ships[asset_id]['location_name'] = 'CITADEL NOT FOUND'
else:
ships[asset_id]['location_name'] = "CITADEL ERROR"
else:
ships[asset_id]['location_name'] = 'TYPE UNKNOWN'
if asset_typeid in list(supers):
ships[asset_id]['type'] = supers[asset_typeid]
ships[asset_id]['class'] = "Supercarrier"
if asset_typeid in list(titans):
ships[asset_id]['type'] = titans[asset_typeid]
ships[asset_id]['class'] = "Titan"
# is this character flying a titan/super?
# this is last to override the asset search with the active super (if any)
active_typeid = current_ship.get('ship_type_id')
active_id = current_ship.get('ship_item_id')
if active_typeid in list(titans) + list(supers):
# setup basics
ships[active_id] = basic_pilot
ships[active_id]['typeid'] = active_typeid
ships[active_id]['item_id'] = active_id
ships[active_id]['main_charname'] = main
ships[active_id]['active'] = True
ships[active_id]['location_name'] = location['location']
# more complex location ids and names can be filled in later?
# actual ship specific shit
if active_typeid in list(titans):
ships[active_id]['type'] = titans[active_typeid]
ships[active_id]['class'] = "Titan"
if active_typeid in list(supers):
ships[active_id]['type'] = supers[active_typeid]
ships[active_id]['class'] = "Supercarrier"
return ships
def auth_evesso_callback():
logger = getlogger('core.sso.callback')
client_id = _eve.client_id
client_secret = _eve.client_secret
redirect_url = _eve.redirect_url
base_url = 'https://login.eveonline.com'
token_url = base_url + '/v2/oauth/token'
# the user has (ostensibly) authenticated with the application, now
# the access token can be fetched
altof = session.get('altof')
isalt = session.get('isalt')
tempblue = session.get('tempblue')
renter = session.get('renter')
state = session.get('oauth2_state')
ipaddress = request.headers['X-Real-Ip']
# security logging
if isalt == True:
detail = 'alt of {}'.format(altof)
auth_scopes = scope
elif tempblue == True:
detail = 'temp blue'
# make sure we only check for the blue scope list
auth_scopes = blue_scope
elif renter == True:
detail = 'renter'
auth_scopes = renter_scope
else:
detail = None
auth_scopes = scope
action = 'SSO callback'
securitylog(action=action, ipaddress=ipaddress, detail=detail)
# handle oauth token manipulation
oauth_session = OAuth2Session(
client_id=client_id,
state=state,
redirect_uri=redirect_url,
auto_refresh_kwargs={
'client_id': client_id,
'client_secret': client_secret,
},
auto_refresh_url=token_url,
)
headers = {'Accept': 'application/json', 'Content-Type': 'application/x-www-form-urlencoded' }
try:
atoken = oauth_session.fetch_token(
token_url,
client_secret=client_secret,
authorization_response=request.url,
headers=headers,
)
except Exception as error:
msg = 'unable to fetch eve sso access token: {0}'.format(error)
logger.error(msg)
return('ERROR: ' + str(error))
access_token = atoken['access_token']
refresh_token = atoken['refresh_token']
expires_at = atoken['expires_at']
try:
charid, charname, scopes = verify(access_token)
except Exception as e:
# this ought to never happen
msg = 'unable to verify eve sso access token: {0}'.format(error)
logger.error(msg)
message = 'SORRY, internal error. Try again.'
response = make_response(message)
return response
# full ESI affiliations
affilliations = _esihelpers.esi_affiliations(charid)
if affilliations.get('error'):
msg = 'error in fetching affiliations for {0}: {1}'.format(charid, affilliations.get('error'))
logger.error(msg)
message = 'SORRY, internal error. Try again.'
response = make_response(message)
return response
allianceid = affilliations.get('allianceid')
alliancename = affilliations.get('alliancename')
corpid = affilliations.get('corpid')
# ldap, if any
userinfo = _ldaphelpers.ldap_userinfo(charid)
# get alt status, if any, from ldap
if userinfo and not isalt:
altof = userinfo.get('altOf')
if altof is not None:
isalt = True
# what the f**k is going on
# this is a check that _shouldnt_ trigger anymore
if isalt:
if altof == None or altof == 'None':
msg = 'is an alt but altof = None? wtf. charid {0} altof {1} {2}'.format(charid, altof, type(altof))
logger.error(msg)
msg = 'error in fetching alt information. please poke saeka.'
response = make_response(msg)
return response
# fix authgroup to an empty array in case nothing
if not userinfo:
authgroups = []
else:
authgroups = userinfo.get('authGroup')
if authgroups is None:
authgroups = []
# verify that the atoken we get actually has the correct scopes that we requested
# just in case someone got cute and peeled some off.
if not check_scope(charid, auth_scopes, atoken=access_token):
# the user peeled something off the scope list. naughty.
msg = 'user {0} modified scope list'.format(charid)
logger.warning(msg)
securitylog(action='core login scope modification', charid=charid, ipaddress=ipaddress)
message = "Don't tinker with the scope list, please.<br>"
message += "If you have an issue with it, talk to triumvirate leadership."
response = make_response(message)
return response
else:
# scopes validate
msg = 'user {0} has expected scopes'.format(charid)
logger.debug(msg)
# register the user, store the tokens
registeruser(charid, access_token, refresh_token, tempblue=tempblue, isalt=isalt, altof=altof, renter=renter)
storetokens(charid, access_token, refresh_token, expires=expires_at)
## TESTS
##
## check affiliations and for bans
# check to see if the user is banned
if 'banned' in authgroups:
# banned users not allowed under any conditions
message = 'nope.avi'
if isalt == True:
msg = 'banned user {0} ({1}) tried to register alt {2}'.format(charid, charname, altof)
logger.warning(msg)
securitylog(action='banned user tried to register', charid=charid, ipaddress=ipaddress, detail='alt of {0}'.format(altof))
else:
msg = 'banned user {0} ({1}) tried to register'.format(charid, charname)
logger.warning(msg)
securitylog(action='banned user tried to register', charid=charid, ipaddress=ipaddress)
return make_response(message)
# only tri & blues are allowed to use auth
if allianceid not in vg_blues() and allianceid not in vg_alliances() and allianceid not in vg_renters():
if not isalt:
# not an alt, not a blue. not a renter. go away.
msg = 'please contact a recruiter if you are interested in joining triumvirate'
logmsg = 'non-blue user {0} ({1}) tried to register'.format(charid, charname)
logger.warning(logmsg)
securitylog(action='non-blue user tried to register', charid=charid, ipaddress=ipaddress)
return make_response(msg)
else:
# someone is registering a non-blue alt, nbd
pass
# make sure the temp blue endpoint not being used by tri proper
if tempblue:
# this is a tri blue, but not tri proper.
# ...or at least ought to be.
if allianceid in vg_alliances():
# naughty! but not worth logging
msg = 'please use the other login endpoint. <br>'
msg += 'this is a lower privileged one for blues <b>ONLY</b>'
return make_response(msg)
# is this a temp blue trying to login with the wrong endpoint?
if allianceid in vg_blues():
if not tempblue:
# no big deal. we got extra scopes for it.
tempblue = True
# the user has passed the various exclusions, gg
# security logging
action = 'SSO callback completed'
detail = None
if isalt == True:
detail='alt of {0}'.format(altof)
elif tempblue == True:
detail='blue from {0}'.format(alliancename)
elif renter == True:
detail='renter from {0}'.format(alliancename)
securitylog(action=action, charid=charid, ipaddress=ipaddress, detail=detail)
expire_date = datetime.datetime.now() + datetime.timedelta(days=14)
# build the cookie and construct the http response
if isalt == True:
# if the character being logged in is an alt, make a session for the main.
if userinfo:
# the alt is alredy registered. go to homepage.
response = make_response(redirect('https://www.triumvirate.rocks'))
else:
# go to alt registration page to show update.
response = make_response(redirect('https://www.triumvirate.rocks/altregistration'))
cookie = _session.makesession(altof)
msg = 'created session for user: {0} (alt of {1})'.format(charname, altof)
logger.info(msg)
else:
# proceed normally otherwise
response = make_response(redirect('https://www.triumvirate.rocks'))
cookie = _session.makesession(charid)
msg = 'created session for user: {0} (charid {1})'.format(charname, charid)
logger.info(msg)
response.set_cookie('tri_core', cookie, domain='.triumvirate.rocks', expires=expire_date)
if cookie == False:
# unable to construct session cookie
msg = 'error in creating session cookie for user {0}'.format(charid)
logger.error(msg)
message = 'SORRY, internal error. Try again.'
return make_response(message)
# handle registered users
if userinfo is not None:
# already in ldap, and not banned
action = 'core login'
if isalt:
# is a registered alt
msg = 'alt user {0} (alt of {1}) already registered'.format(charname, altof)
logger.info(msg)
securitylog(action=action, charid=charid, ipaddress=ipaddress, detail='via alt {0}'.format(altof))
code, result = _ldaphelpers.ldap_altupdate(__name__, altof, charid)
return response
else:
if tempblue:
# registered blue main.
msg = 'user {0} ({1}) already registered'.format(charid, charname)
logger.info(msg)
securitylog(action=action, charid=charid, ipaddress=ipaddress, detail='blue from {0}'.format(alliancename))
code, result = _ldaphelpers.ldap_altupdate(__name__, altof, charid)
return response
else:
# registered character
msg = 'user {0} ({1}) already registered'.format(charid, charname)
logger.info(msg)
securitylog(action=action, charid=charid, ipaddress=ipaddress)
code, result = _ldaphelpers.ldap_altupdate(__name__, altof, charid)
return response
# after this point, the only folks that are left are unregistered users
# handle new temp blues
if tempblue:
msg = 'user {0} ({1}) not registered'.format(charid, charname)
logger.info(msg)
securitylog(action='core user registered', charid=charid, ipaddress=ipaddress, detail='blue from {0}'.format(alliancename))
return response
# handle new renters
if renter:
msg = 'user {0} ({1}) not registered'.format(charid, charname)
logger.info(msg)
securitylog(action='core user registered', charid=charid, ipaddress=ipaddress, detail='renter from {0}'.format(alliancename))
return response
# handle new alts
if isalt:
msg = 'alt user {0} (alt of {1}) not registered'.format(charname, altof)
logger.info(msg)
securitylog(action='alt user registered', charid=charid, ipaddress=ipaddress, detail='alt of {0}'.format(altof))
return response
else:
msg = 'user {0} ({1}) not registered'.format(charid, charname)
logger.info(msg)
securitylog(action='core user registered', charid=charid, ipaddress=ipaddress)
return response
def _get_others(table, ids, interval, num):
"""get contents from ma_monitor_data_YYMMDD, ma_alarm_data, ma_log_data"""
contents = []
conn = redis_conn()
try:
begin = interval[0]
end = interval[1]
except TypeError as e:
getlogger(configure.logger_path).error('time:%s should be list' % e.message)
except IndexError as e:
getlogger(configure.logger_path).error('time:%s length is not correct' % e.message)
return
contents = []
queue = Queue()
gens = []
#ps = []
size = 0
for i in ids:
#print begin, conn.hkeys('m.%s' % i), end
if table == 'ma_monitor_data_YYMMDD':
fields = filter(lambda x: begin <= int(float(x)) <= end,
conn.lrange(i, 0, -1))
size += 1
#p = Process(target=_reduce_data, args=(i, fields, num, table,
#queue))
#p.start()
r = reduce_data(i, fields, num, table, queue)
gens.append(r)
elif table == 'ma_alarm_data':
#fields = filter(lambda x: begin <= int(float(x)) <= end,
#conn.lrange(i, 0, -1))
fields = conn.lrange(i, 0, -1)
#fields = _pick_data(fields, num)
for f in fields:
c = {'item_id': i,}
c['begin_time'] = f
solved_level = current_level = content = end_time = None
if conn.exists('%s.%s.solved'):
try:
end_time, content = conn.hgetall('%s.%s.solved' % (i, f))
current_level = 0
solved_level = conn.hkeys('%s.%s' % (i, f))[-1]
except IndexError:
pass
else:
r = conn.hgetall('%s.%s' % (i, f))
try:
k = sorted(r.keys())[-1]
except IndexError:
pass
else:
current_level, content = k, r[k]
c['end_time'] = end_time
c['current_level'] = current_level
c['content'] = content
c['solved_level'] = solved_level
contents.append(c)
#elif table == 'ma_alarm_data':
#fields = filter(lambda x: begin <= int(float(x)) <= end,
#conn.lrange(i, 0, -1))
#fields = _pick_data(fields, num)
#for f in fields:
#c = {'item_id': i,}
#c['begin_time'] = f
#c['solved'] = conn.exists('%s.%s.solved')
#contents.append(c)
elif table == 'ma_log_data':
fields = filter(lambda x: begin <= int(float(x)) <= end,
conn.lrange(i, 0, -1))
#fields = _pick_data(fields, num)
for f in fields:
c = {'item_id': i,}
c['datetime'] = f
c['content'] = conn.hget('%s.%s' % (i, f), 0)
contents.append(c)
if table == 'ma_monitor_data_YYMMDD':
scheme(gens)
for i in range(size):
item = queue.get()
contents.extend(item)
return contents
def eve_tokenthings(dn, evetokens):
logger = getlogger('maint.tokens.eve')
charid = evetokens.get('uid')
roles = evetokens.get('roles')
ldap_scopes = evetokens.get('scopes')
old_rtoken = evetokens.get('rtoken')
if not old_rtoken:
return True
if roles is None:
roles = []
if ldap_scopes is None:
ldap_scopes = []
msg = 'updating eve token for charid {0}'.format(charid)
logger.debug(msg)
result, code = _everefresh.refresh_token(old_rtoken)
if code is not True:
# broken token, or broken oauth?
# the distinction matters.
# see env/lib/python3.5/site-packages/oauthlib/oauth2/rfc6749/errors.py
msg = 'unable to refresh token for charid {0}: {1}'.format(charid, result)
logger.error(msg)
# only these exception types are valid reasons to purge a token
purgetype = [ 'InvalidGrantError', 'UnauthorizedClientError', 'InvalidClientError', 'InvalidTokenError' ]
if result in purgetype:
# purge the entry from the ldap user
_ldaphelpers.update_singlevalue(dn, 'esiRefreshToken', None)
_ldaphelpers.update_singlevalue(dn, 'esiAccessToken', None)
_ldaphelpers.update_singlevalue(dn, 'esiAccessTokenExpires', None)
# corp roles and esi scopes now serve no purpose since the tokens are gone
_ldaphelpers.update_singlevalue(dn, 'esiScope', None)
_ldaphelpers.update_singlevalue(dn, 'corporationRole', None)
msg = 'invalid token entries purged for user {}'.format(dn)
logger.info(msg)
# either way, this has failed in an unrecoverable way
return True
atoken = result.get('access_token')
rtoken = result.get('refresh_token')
expires = result.get('expires_at')
# store the updated token
result, value = storetokens(charid, atoken, rtoken, expires, token_type='esi')
if result == False:
msg = 'unable to store tokens for user {}'.format(dn)
logger.error(msg)
return False
# the updated token is now in LDAP
# fetch all corporation roles for the updated token if the scope allows that
if 'esi-characters.read_corporation_roles.v1' in ldap_scopes:
request_url = 'characters/{0}/roles/'.format(charid)
code, result = common.request_esi.esi(__name__, request_url, method='get', charid=charid, version='v2')
if code == 403:
error = 'no perms to read roles for {0}: ({1}) {2}'.format(charid, code, result)
logger.debug(error)
elif not code == 200:
error = 'unable to get character roles for {0}: ({1}) {2}'.format(charid, code, result)
logger.error(error)
else:
# figure out what needs to be added and removed from ldap
result = result.get('roles')
missing_roles = set(result) - set(roles)
extra_roles = set(roles) - set(result)
for missing in list(missing_roles):
_ldaphelpers.add_value(dn, 'corporationRole', missing)
for extra in list(extra_roles):
_ldaphelpers.update_singlevalue(dn, 'corporationRole', extra, delete=True)
else:
# legacy token that doesn't have this scope. probably not getting updates on these lol
pass
try:
token_charid, charname, token_scopes = verify(atoken)
except Exception as e:
# this ought to never happen
msg = 'unable to verify eve sso access token: {0}'.format(error)
logger.error(msg)
return
if not token_charid == charid:
# ought to never happen, just a safety check from a scare once
msg = 'stored token for charid {0} belongs to charid {1}'.format(charid, token_charid)
logger.critical(msg)
return
# so given an array of scopes, let's check that what we want is in the list of scopes the character's token has
missing_scopes = set(token_scopes) - set(ldap_scopes)
extra_scopes = set(ldap_scopes) - set(token_scopes)
for missing in list(missing_scopes):
_ldaphelpers.add_value(dn, 'esiScope', missing)
for extra in list(extra_scopes):
_ldaphelpers.update_singlevalue(dn, 'esiScope', extra, delete=True)
return True
def fetch_chardetails(charid):
chardetails = dict()
logger = getlogger('core.corpaudit.chardetails.{0}'.format(charid))
dn = 'ou=People,dc=triumvirate,dc=rocks'
filterstr='(uid={})'.format(charid)
attrlist=['characterName', 'authGroup', 'teamspeakdbid', 'esiAccessToken', 'altOf', 'corporation', 'lastKill', 'lastKillTime','corporationName']
code, result = _ldaphelpers.ldap_search(__name__, dn, filterstr, attrlist)
if result is None or not code:
# no result? simple response.
chardetails['location'] = 'Unknown'
chardetails['corporation'] = 'Unknown'
chardetails['online'] = 'Unknown'
chardetails['last_online'] = 'Unknown'
chardetails['token_status'] = False
chardetails['teamspeak_status'] = False
chardetails['isalt'] = 'Unknown'
chardetails['lastKill'] = None
chardetails['lastKillTime'] = None
chardetails['altof'] = None
# fetch affiliations
affiliations = _esihelpers.esi_affiliations(charid)
chardetails['corporation'] = affiliations.get('corpname')
chardetails['charname'] = affiliations.get('charname')
else:
try:
(dn, info), = result.items()
except ValueError:
print("error: {}".format(charid))
chardetails['charname'] = info['characterName']
# convert last kill time into human readable
try:
chardetails['lastKill'] = info['lastKill']
except Exception as e:
chardetails['lastKill'] = None
try:
killtime = int(info['lastKillTime'])
killtime = time.strftime("%Y/%m/%d, %H:%M:%S", time.localtime(killtime))
chardetails['lastKillTime'] = killtime
except Exception as e:
chardetails['lastKillTime'] = None
corp_id = info['corporation']
if corp_id is None:
print("error: {} no corp".format(charid))
chardetails['corporation'] = info['corporationName']
# does the char have a token?
try:
detail = info['esiAccessToken']
if len(detail) > 0:
chardetails['token_status'] = True
else:
chardetails['token_status'] = False
except Exception as e:
chardetails['token_status'] = False
# teamspeak registration?
try:
detail = info['teamspeakdbid']
if len(detail) > 0:
chardetails['teamspeak_status'] = True
else:
chardetails['teamspeak_status'] = False
except Exception as e:
chardetails['teamspeak_status'] = False
# is this an alt?
# cast the altof detail to something useful
try:
detail = info['altOf']
except Exception as e:
detail = None
# str(None) == False
if str(detail).isdigit():
chardetails['isalt'] = True
request_url = 'characters/{0}/'.format(detail)
code, result = common.request_esi.esi(__name__, request_url, 'get')
if not code == 200:
msg = '/characters/{0}/ API error {1}: {2}'.format(detail, code, result)
logger.warning(msg)
try:
chardetails['altof'] = result['name']
except KeyError as error:
msg = 'User does not exist: {0})'.format(charid)
logger.error(msg)
chardetails['altof'] = 'Unknown'
else:
chardetails['altof'] = None
chardetails['isalt'] = False
## start fetching character-specific information
#
request_url = 'characters/{0}/location/'.format(charid)
code, result = common.request_esi.esi(__name__, request_url, method='get', charid=charid, version='v1')
if not code == 200:
# it doesn't really matter
msg = 'characters loction API error {0}: {1}'.format(code, result)
logger.debug(msg)
location = None
chardetails['location_id'] = location
chardetails['location'] = 'Unknown'
else:
# can include either station_id or structure_id
location = result['solar_system_id']
chardetails['location_id'] = location
request_url = 'characters/{0}/location/'.format(charid)
code, result = common.request_esi.esi(__name__, request_url, method='get', charid=charid, version='v1')
if not code == 200:
# it doesn't really matter
msg = 'characters loction API error {0}: {1}'.format(code, result)
logger.debug(msg)
location = None
else:
# can include either station_id or structure_id
location = result['solar_system_id']
chardetails['location_id'] = location
# map the location to a name
if location == None:
chardetails['location'] = 'Unknown'
else:
request_url = 'universe/systems/{0}/'.format(location)
code, result = common.request_esi.esi(__name__, request_url, 'get', version='v4')
if not code == 200:
msg = '{0} API error: {1}'.format(request_url, result)
logger.error(msg)
chardetails['location'] = 'Unknown'
else:
chardetails['location'] = result['name']
# get online status
request_url = 'characters/{0}/online/'.format(charid)
code, result = common.request_esi.esi(__name__, request_url, method='get', charid=charid, version='v2')
if not code == 200:
# it doesn't really matter
msg = '{0} API error: {1}'.format(request_url, result)
logger.debug(msg)
location = None
chardetails['online'] = 'Unknown'
chardetails['last_online'] = 'Unknown'
else:
chardetails['online'] = result['online']
chardetails['last_online'] = result['last_login']
try:
request_url_corp = 'corporations/{0}/'.format(corp_id)
code_corp, result_corp = common.request_esi.esi(__name__, request_url_corp, 'get')
if not code_corp == 200:
_logger.log('[' + __name__ + '] /corporations API error {0}: {1}'.format(code_corp, result_corp['error']), _logger.LogLevel.WARNING)
msg = '{0} API error: {1}'.format(request_url_corp, result)
logger.warning(msg)
else:
chardetails['corporation'] = result_corp['name']
except KeyError as error:
msg = 'corporation id does not exist: {0}'.format(corp_id)
logger.error(msg)
charname = None
return chardetails
def audit_discord_perms():
logger = getlogger('discord.audit')
msg = 'discord auditing'
logger.info(msg)
bot_token = _discord.social_token
server = 358117641724100609
logging.getLogger('discord.audit').setLevel(logging.DEBUG)
# these discord roles are manually assigned or everyone gets
ignore_roles = [
'Admin', 'IT Manager', 'TRI Friends', '@everyone', 'production',
'rorqcoordination'
]
# old and new discord bots
ignore_uids = [534463865744916480, 345393204784267265]
# these roles are ones the bot can't manage
above_bot = ['Admin', 'BOT']
# redis
r = redis.StrictRedis(host='localhost', port=6379, db=0)
try:
r.client_list()
except redis.exceptions.ConnectionError as err:
msg = 'Redis connection error: ' + str(err)
logger.error(msg)
except Exception as err:
msg = 'Redis generic error: ' + str(err)
logger.error(msg)
# fetch all the members from the social discord
users = discord_allmembers(token=bot_token, target_server=server)
# trim to one item
userdata = users[server]
# iterate through and populate with core ldap data if possible
for user in userdata:
# don't try to audit self
if user['discorduid'] in ignore_uids:
continue
dn = 'ou=People,dc=triumvirate,dc=rocks'
filterstr = '(discorduid={0})'.format(user['discorduid'])
attrlist = [
'authGroup', 'characterName', 'corporation', 'corporationName',
'discorduid', 'discordRefreshToken', 'discord2fa',
'esiRefreshToken', 'uid', 'alliance'
]
code, result = _ldaphelpers.ldap_search(__name__, dn, filterstr,
attrlist)
if result is None:
# unregisterds lose roles immediately, then are kicked "later"
msg = 'discord user {0} ({1}) unregistered in core'.format(
user['name'], user['display_name'])
message = 'You are unregistered on TRI CORE, but you have meaningful roles.\n'
message += 'These roles are being removed until you REGISTER on https://www.triumvirate.rocks/comms\n'
# purge the user's roles
purge_roles = set(user['roles']) - set(ignore_roles)
purge_roles = list(purge_roles)
if len(purge_roles) > 0:
# no point in making noises for users that won't be affected
logger.info(msg)
# warn the user
discord_message_user(token=bot_token,
member=user['object'],
message=message)
# purge the roles
discord_changeroles(token=bot_token,
target_server=server,
member=user['object'],
roles_add=list(missing_roles),
roles_del=purge_roles)
pass
# purge if they have no roles whatsoever:
# don't want to purge people with manual roles (tri friends)
roles = set(user['roles']) - set(['@everyone'])
if not roles:
# start the clock that the character has been seen
# a cache time of 10 days lets this get purged from redis well after i'm done with it
timestamp = r.get('discordaudit:unregistered:{0}'.format(
user['discorduid']))
if timestamp is None:
r.setex(
'discordaudit:unregistered:{0}'.format(
user['discorduid']), 864000, time.time())
message = 'Unregistered and role-free idlers are not allowed on the TRI discord.\n'
message += 'You have 24 hours before being purged.\n'
discord_message_user(token=bot_token,
member=user['object'],
message=message)
continue
difference = time.time() - float(timestamp)
if difference >= 86400:
message = "You've been kicked from the TRI discord. Feel free to re-register on CORE\n"
message += "...or get an invite + roles from leadership"
discord_message_user(token=bot_token,
member=user['object'],
message=message)
msg = "kicking unregistered user {0}".format(user['name'])
logger.info(msg)
discord_kick_user(token=bot_token, member=user['object'])
continue
else:
try:
(dn, info), = result.items()
except Exception as e:
msg = "discord uid {0} has {1} total instances registered".format(
user['discorduid'], len(result))
logger.warning(msg)
continue
corpid = info['corporation']
# see if this user can be managed by the bot
managable = True
if set(user['roles']).intersection(set(above_bot)):
# check a nonzero intersection of the user roles and any "Above the bot" roles
managable = False
# fetch corp ticker
request_url = 'corporations/{0}/'.format(corpid)
code, result = common.request_esi.esi(__name__,
request_url,
'get',
version='v4')
if code != 200:
# something broke severely
msg = 'corporations API error {0}: {1}'.format(code, result)
logger.error(msg)
# can't process without the ticker.
continue
else:
ticker = result.get('ticker')
# construct appropriate user name
if ticker is None:
correct_name = '{0}'.format(info['characterName'])
else:
correct_name = '[{0}] {1}'.format(ticker, info['characterName'])
if user['display_name'] != correct_name and managable:
msg = 'wrong name. using: {0}, should have: {1}'.format(
user['display_name'], correct_name)
discord_changenick(token=bot_token,
target_server=server,
member=user['object'],
nickname=correct_name)
# if the discord or ESI tokens have been removed, remove their roles.
if info['discordRefreshToken'] is None:
message = "Your discord token is no longer valid.\n"
message += "Roles are being removed until you re-register discord on TRI CORE\n"
purge_roles = set(user['roles']) - set(ignore_roles)
purge_roles = list(purge_roles)
# only take action on actual roles to remove
if purge_roles:
msg = "character {0} no longer has a discord refresh token - removing roles".format(
info['characterName'])
logger.info(msg)
discord_message_user(token=bot_token,
member=user['object'],
message=message)
discord_changeroles(token=bot_token,
target_server=server,
member=user['object'],
roles_del=purge_roles)
continue
if info['esiRefreshToken'] is None:
message = "Your ESI token is no longer valid.\n"
message += "Roles are being removed until you reconnect to TRI CORE\n"
purge_roles = set(user['roles']) - set(ignore_roles)
purge_roles = list(purge_roles)
# only take action on actual roles to remove
if purge_roles:
msg = "character {0} no longer has a ESI refresh token - removing roles".format(
info['characterName'])
logger.info(msg)
discord_message_user(token=bot_token,
member=user['object'],
message=message)
discord_changeroles(token=bot_token,
target_server=server,
member=user['object'],
roles_del=purge_roles)
continue
# construct the correct discord roles for this user
# this will be absolutely correct as far as ldap is concerned
correct_roles = []
# should maybe add something for banned users maybe, but they
# will get purged of all meaningful roles anyway
if info['alliance'] == 933731581:
# tri role and corp tags only meaningful if in tri
correct_roles.append('[{0}]'.format(ticker))
# 2factor secured group membership
# this is for tri only
if info['discord2fa']:
correct_roles.append('Triumvirate')
# map ldap authgroups to discord roles
# only people with 2FA get privileged roles
for authgroup in info['authGroup']:
mapping = authgroup_mapping(authgroup)
if mapping is not None:
correct_roles.append(mapping)
else:
correct_roles.append('NO 2FA')
missing_roles = set(correct_roles) - set(user['roles'])
extra_roles = set(user['roles']) - set(correct_roles)
extra_roles = extra_roles - set(ignore_roles)
if extra_roles:
msg = 'user {0} has extra roles: {1}'.format(
user['name'], extra_roles)
logger.info(msg)
if missing_roles:
msg = 'user {0} has missing roles: {1}'.format(
user['name'], missing_roles)
logger.info(msg)
if extra_roles or missing_roles:
if managable:
discord_changeroles(token=bot_token,
target_server=server,
member=user['object'],
roles_add=list(missing_roles),
roles_del=list(extra_roles))
#!/usr/bin/python3
from tri_api import app
import argparse
from common.logger import getlogger_new as getlogger
if __name__ == '__main__':
logger = getlogger('tri_api')
msg = 'tri core API online'
logger.info(msg)
app.run()
def core_corpaudit(charid):
# do a corp level audit of who has services
ipaddress = request.headers['X-Real-Ip']
log_charid = request.args.get('log_charid')
logger = getlogger('core.corpaudit')
securitylog('corp audit information request', ipaddress=ipaddress, charid=log_charid)
try:
charid = int(charid)
except ValueError:
msg = 'charid parameters must be integer: {0}'.format(charid)
logger.warning(msg)
js = json.dumps({ 'error': msg})
resp = Response(js, status=401, mimetype='application/json')
return resp
corporation_id_list = []
character_id_list = []
dn = 'ou=People,dc=triumvirate,dc=rocks'
code, char_result = _ldaphelpers.ldap_search(__name__, dn, '(&(|(uid={0})(altOf={0}))(esiAccessToken=*))'
.format(charid), ['uid', 'corporation', 'corporationName', 'characterName'])
if code == False:
error = 'failed to fetch characters for {0}: ({1}) {2}'.format(charid, code, char_result)
logger.error(error)
js = json.dumps({'error': error})
resp = Response(js, status=500, mimetype='application/json')
return resp
for cn in char_result:
data = char_result[cn]
allowed_roles = ['Director', 'Personnel_Manager']
roles = check_role(data['uid'], allowed_roles)
if not roles:
msg = "character {0} has insufficient roles for {1}".format(data['characterName'], data['corporationName'])
logger.info(msg)
continue
msg = 'character {0} has sufficient roles to view corp auditing information'.format(cn)
logger.debug(msg)
if data['corporation'] not in corporation_id_list:
request_url = 'corporations/{0}/members/'.format(data['corporation'])
code, result = common.request_esi.esi(__name__, request_url, method='get', charid=data['uid'], version='v3')
if not code == 200:
# something broke severely
msg = 'corporations API error {0}: {1}'.format(code, result)
logger.error(msg)
result = {'code': code, 'error': msg}
return code, result
character_id_list = result
corporation_id_list.append(data['corporation'])
# start constructing which member has what
users = dict()
with ThreadPoolExecutor(25) as executor:
futures = { executor.submit(fetch_chardetails, user): user for user in character_id_list }
for future in as_completed(futures):
#charid = futures[future]['character_id']
data = future.result()
if data is not None:
users[data['charname']] = data
js = json.dumps(users)
resp = Response(js, status=200, mimetype='application/json')
return resp
def audit_pilot_capitals(entry):
ships = dict()
basic_pilot = dict()
uid = entry['uid']
logger = getlogger('core.audit_pilot.{0}.capitals'.format(uid))
msg = 'auditing character {0} for capitals'.format(uid)
logger.debug(msg)
corpid = entry['corporation']
charname = entry['characterName']
altOf = entry['altOf']
# data pollution fix
if altOf == 'None':
altOf = None
basic_pilot['uid'] = uid
basic_pilot['pilot'] = charname
basic_pilot['valid'] = False
basic_pilot['main_charid'] = altOf
corp_info = _esihelpers.corporation_info(corpid)
if corp_info is not None:
basic_pilot['corporation'] = corp_info['name']
else:
basic_pilot['corporation'] = "Unknown"
# hardcoded data for asset typeids
carriers = {
23757: 'Archon',
23915: 'Chimera',
24483: 'Nidhoggur',
23911: 'Thanatos'
}
dreads = {
19724: 'Moros',
19722: 'Naglfar',
19726: 'Phoenix',
19720: 'Revelation',
}
fax = {37604: 'Apostle', 37606: 'Lif', 37605: 'Minokawa', 37607: 'Ninazu'}
# fetch current ship
code, current_ship = _esihelpers.current_ship(uid)
if code == False:
return ships
# fetch super/titan typeids out of char assets
code, char_assets = _esihelpers.find_types(
uid,
list(carriers) + list(dreads) + list(fax))
if code == False:
return ships
# fetch character location
code, location = _esihelpers.char_location(uid)
if code == False:
return ships
if not altOf == None:
result = _ldaphelpers.ldap_uid2name(altOf)
if result == None:
msg = 'failed to find main for {0}'.format(altOf)
logger.warning(msg)
try:
main = result
except:
main = "Unknown"
else:
main = charname
# does this character have a titan/super in assets?
for asset in char_assets:
asset_typeid = asset.get('type_id')
asset_id = asset.get('item_id')
if asset_typeid in list(carriers) + list(dreads) + list(fax):
ships[asset_id] = basic_pilot
ships[asset_id]['item_id'] = asset_id
ships[asset_id]['typeid'] = asset_typeid
ships[asset_id]['main_charname'] = main
ships[asset_id]['active'] = False
ships[asset_id]['location_id'] = asset['location_id']
if asset['location_type'] == 'station':
request_url = 'universe/stations/{}/'.format(
asset['location_id'])
code, result = common.request_esi.esi(__name__,
request_url,
method='get',
version='latest')
if code == 404:
ships[asset_id]['location_name'] = 'STATION NOT FOUND'
elif code == 200:
ships[asset_id]['location_name'] = result['name']
else:
ships[asset_id]['location_name'] = "STATION ERROR"
elif asset['location_type'] == 'other':
request_url = 'universe/structures/{}/'.format(
asset['location_id'])
code, result = common.request_esi.esi(__name__,
request_url,
method='get',
version='v2',
charid=uid)
if code == 200:
ships[asset_id]['location_name'] = result['name']
elif code == 403 or code == 401:
ships[asset_id]['location_name'] = "CITADEL FORBIDDEN"
elif code == 404:
ships[asset_id]['location_name'] = 'CITADEL NOT FOUND'
else:
ships[asset_id]['location_name'] = "CITADEL ERROR"
else:
ships[asset_id]['location_name'] = 'TYPE UNKNOWN'
if asset_typeid in list(carriers):
ships[asset_id]['type'] = carriers[asset_typeid]
ships[asset_id]['class'] = "Carrier"
elif asset_typeid in list(dreads):
ships[asset_id]['type'] = dreads[asset_typeid]
ships[asset_id]['class'] = "Dreadnought"
elif asset_typeid in list(fax):
ships[asset_id]['type'] = fax[asset_typeid]
ships[asset_id]['class'] = "FAX"
# is this character flying a titan/super?
# this is last to override the asset search with the active super (if any)
active_typeid = current_ship.get('ship_type_id')
active_id = current_ship.get('ship_item_id')
if active_typeid in list(carriers) + list(dreads) + list(fax):
# setup basics
ships[active_id] = basic_pilot
ships[active_id]['typeid'] = active_typeid
ships[active_id]['item_id'] = active_id
ships[active_id]['main_charname'] = main
ships[active_id]['active'] = True
ships[active_id]['location_name'] = location['location']
# more complex location ids and names can be filled in later?
# actual ship specific shit
if active_typeid in list(carriers):
ships[active_id]['type'] = carriers[active_typeid]
ships[active_id]['class'] = "Carrier"
elif active_typeid in list(dreads):
ships[active_id]['type'] = dreads[active_typeid]
ships[active_id]['class'] = "Dreadnought"
elif active_typeid in list(fax):
ships[active_id]['type'] = fax[active_typeid]
ships[active_id]['class'] = "FAX"
return ships
def maillist_forward(charid=None, mailing_list=None):
logger = getlogger('ping_relay.mailing_lists')
# setup redis
r = redis.StrictRedis(host='localhost', port=6379, db=0)
try:
r.client_list()
except redis.exceptions.ConnectionError as err:
msg = 'redis connection error: {0}'.format(err)
logger.error(msg)
except redis.exceptions.ConnectionRefusedError as err:
msg = 'redis connection error: {0}'.format(err)
logger.error(msg)
except Exception as err:
msg = 'redis generic error: {0}'.format(err)
logger.error(msg)
# fetch mailing lists to confirm that the target is still valid
request_url = 'characters/{0}/mail/lists/'.format(charid)
code, result = _esi.esi(__name__,
request_url,
'get',
version='v1',
charid=charid)
if code != 200:
# something broke severely
msg = 'mailing list error: {0}'.format(result)
logger.error(msg)
# can't process without the ticker.
return
valid_list = False
for item in result:
if mailing_list == item.get('name'):
valid_list = True
valid_listid = item.get('mailing_list_id')
if not valid_list:
msg = 'character no longer subscribed to list {0}'.format(mailing_list)
logger.error(msg)
return
# now that the mailing list id is known, character mails can be filtered
request_url = 'characters/{0}/mail/'.format(charid)
code, result = _esi.esi(__name__,
request_url,
'get',
version='v1',
charid=charid)
if code != 200:
# something broke severely
msg = 'mail endpoint error: {0}'.format(result)
logger.error(msg)
# can't process without the ticker.
return
fetch_mails = []
cutoff = 0
for mail in result:
mail_id = mail.get('mail_id')
recipients = mail.get('recipients')[0]
# convert timestamp to epoch
# example: 2018-05-08T17:30:00Z
timestamp = time.strptime(mail.get('timestamp'), "%Y-%m-%dT%H:%M:%SZ")
timestamp = time.mktime(timestamp)
timestamp = int(timestamp)
if recipients.get(
'recipient_id') == valid_listid and timestamp > cutoff:
fetch_mails.append(mail_id)
# parse mail
# thx https://stackoverflow.com/questions/753052/strip-html-from-strings-in-python?utm_medium=organic&utm_source=google_rich_qa&utm_campaign=google_rich_qa
tag_re = re.compile(r'(<!--.*?-->|<[^>]*>)')
for mail in fetch_mails:
request_url = 'characters/{0}/mail/{1}/'.format(charid, mail)
code, result = _esi.esi(__name__,
request_url,
'get',
version='v1',
charid=charid)
if code != 200:
# something broke severely
msg = 'char {0} mail id {1} error: {2}'.format(
charid, mail, result)
logger.error(msg)
# can't process without the ticker.
continue
subject = result.get('subject')
body = tag_re.sub(' ', result.get('body'))
timestamp = time.strptime(result.get('timestamp'),
"%Y-%m-%dT%H:%M:%SZ")
timestamp = time.mktime(timestamp)
timestamp = int(timestamp)
print(subject, timestamp, body)
def audit_core():
# keep the ldap account status entries in sync
logger = getlogger('audit.core')
logger.info('auditing CORE LDAP')
# online check first
# do esi status check first
request_url = 'status'
code, result = common.request_esi.esi(__name__, request_url, method='get', version='v1')
if not code == 200:
error = 'ESI offline'
logger.error(error)
return
# fix open file limitations
try:
resource.setrlimit(resource.RLIMIT_NOFILE, (25000, 75000))
except Exception as e:
logger.warn('unable to set nofile rlimit: {0}'.format(e))
pass
# fetch all non-banned LDAP users
dn = 'ou=People,dc=triumvirate,dc=rocks'
filterstr = '(&(!(accountstatus=banned))(!(accountStatus=immortal)))'
attributes = ['uid']
code, nonbanned_users = _ldaphelpers.ldap_search(__name__, dn, filterstr, attributes)
if code == False:
logger.error('unable to find ldap users')
return
# fetch all tri LDAP users
dn = 'ou=People,dc=triumvirate,dc=rocks'
filterstr = 'alliance=933731581'
attributes = ['uid' ]
code, tri_users = _ldaphelpers.ldap_search(__name__, dn, filterstr, attributes)
if code == False:
logger.error('unable to find ldap users')
return
# fetch all blue
dn = 'ou=People,dc=triumvirate,dc=rocks'
filterstr = 'accountStatus=blue'
attributes = ['uid']
code, blue_users = _ldaphelpers.ldap_search(__name__, dn, filterstr, attributes)
if code == False:
logger.error('unable to find ldap users')
return
# fetch ALL LDAP users
dn = 'ou=People,dc=triumvirate,dc=rocks'
filterstr = '(!(accountStatus=immortal))'
attributes = ['uid', 'characterName', 'accountStatus', 'authGroup', 'corporation',
'alliance', 'allianceName', 'corporationName', 'discordRefreshToken', 'discorduid',
'discord2fa', 'discordName', 'altOf', 'esiRefreshToken', 'esiScope', 'lastLogin',
'corporationRole'
]
code, users = _ldaphelpers.ldap_search(__name__, dn, filterstr, attributes)
if code == False:
logger.error('unable to find ldap users')
return
logger.info('total ldap users: {}'.format(len(users)))
logger.info('total non-banned ldap users: {}'.format(len(nonbanned_users)))
logger.info('total blue ldap users: {}'.format(len(blue_users)))
logger.info('total tri ldap users: {}'.format(len(tri_users)))
# loop through each user and determine the correct status
activity = dict()
with ThreadPoolExecutor(30) as executor:
futures = { executor.submit(user_audit, dn, users[dn]): dn for dn in users.keys() }
for future in as_completed(futures):
data = future.result()
activity[dn] = data
