def registerNewAgent():
""" Register a new agent with the commander server """
log.debug(f"<{request.remote_addr}> registering a new agent")
if missingParams := missing(request,
data=["registrationKey", "hostname", "os"]):
log.warning(f"<{request.remote_addr}> {missingParams}")
return {"error": missingParams}, 400
def newAdmin():
""" Create a new admin account using valid session """
log.debug(f"<{request.remote_addr}> creating new admin account")
if missingParams := missing(request, data=["username", "password",
"name"]):
log.warning(f"<{request.remote_addr}> {missingParams}")
return {"error": missingParams}, 400
def updateCredentials():
""" Authenticate an admin and update that admin's credentials """
log.debug(f"<{request.remote_addr}> updating credentials")
if missingParams := missing(request,
data=["username", "password", "newPassword"]):
log.warning(f"<{request.remote_addr}> {missingParams}")
return {"error": missingParams}, 400
def sendExecutable():
""" Send executable or script to the agent for execution """
log.debug(f"<{request.remote_addr}> sending executable")
if missingParams := missing(request,
headers=["Agent-ID"],
data=["filename"]):
log.warning(f"<{request.remote_addr}> {missingParams}")
return {"error": missingParams}, 400
def serve_stylesheet(stylesheet):
if stylesheet not in stylesheets:
raise Exception(
'"{}" is excluded from the allowed static files'.format(
stylesheet))
log.debug('CSS directory: {}, CSS file: {}'.format(css_directory,
stylesheet))
return flask.send_from_directory(css_directory, stylesheet)
def getJobLibrary():
""" Return simplified library overview in json format """
log.debug(f"<{request.remote_addr}> getting job library")
try:
library = Library.objects().get()
except DoesNotExist:
log.info(f"<{request.remote_addr}> returning empty library")
return "", 204
log.info(f"<{request.remote_addr}> returning library overview")
return {"library": library.to_json()}, 200
def get_data(file_id):
"""Get data for all users"""
if file_id not in FILENAMES:
raise KeyError('Unknown file ID {}'.format(file_id))
filename = FILENAMES[file_id]
df = pd.read_hdf(filename)
levels, counts = np.unique(df['ares'], return_counts=True)
counts = counts[::-1].cumsum()[::-1]
log.debug('<<< Loaded {}'.format(filename))
return df, levels, counts
def create_hb201901(chart_data, node_id, click_time):
answer = bytes(app.config['HB201901_CODE'], 'utf-8')
answer_tip = 'I also want to know'
question_tip = '2**73+37 is a prime number?'
failed_tip = '胜败乃兵家常事，少侠请从头再来'
tips = {
'-1': '好疼啊',
'0': '点疼我了~',
'1': '2019新年快乐！\n难度：Easy',
'2': '2是第一个素数',
'16': 'AES的16位key是什么？',
'37': '答案的关键',
'73': '谢耳朵最喜欢的数字',
'2019': '新年快乐~',
'9102': '我有一个疑问\n需要2019解答',
'9444732965739290427429': '恭喜你找到了！'
}
#log.debug('hb201901 tips: %s' % tips)
tip = tips.get(str(node_id))
try:
click_time = '%10.06f' % click_time
secret_key = click_time.replace('.', '')
log.debug('Crypt secret_key: %s' % secret_key)
node_id = int(node_id)
create_node(chart_data, node_id, node_id)
except Exception as e:
log.error(str(e))
node_id = 1
tip = failed_tip
if tip:
name = '0'
if node_id == 2019:
cpt = Crypt(secret_key)
name = cpt.encrypt(answer_tip)
if node_id == 9102:
cpt = Crypt(secret_key)
name = cpt.encrypt(question_tip)
elif node_id == 9444732965739290427429: # 2**73+37
cpt = Crypt(secret_key)
name = cpt.encrypt(base64.b64encode(answer).decode('utf-8'))
create_tips(chart_data, tip, name)
log.debug('create_hb201901 tip:%s' % tip)
if node_id > 0:
create_nodes(chart_data, [int(node_id)], 3)
if tip == failed_tip:
return False
return True
def getRegistrationKey():
""" Get or generate the registration key that agents need to register with commander """
log.debug(f"<{request.remote_addr}> getting registration key")
# return current key if one exists
regKeyQuery = RegistrationKey.objects()
if regKeyQuery:
return {"registration-key": regKeyQuery[0]["regKey"]}
# create registration key in db
newKey = str(uuid4())
regKey = RegistrationKey(regKey=newKey)
regKey.save()
# return new key
log.info(f"<{request.remote_addr}> successfully fetched registration key")
return {"registration-key": newKey}
def resolve_range(relayoutData, axis='x'):
"""Extract xrange from relayoutData. Return either [start, end] or 'auto'"""
log.debug(relayoutData)
assert axis in 'xy'
ax = axis
if not relayoutData or ax + 'axis.autorange' in relayoutData or 'autosize' in relayoutData:
range = 'auto'
elif ax + 'axis.range' in relayoutData:
range = relayoutData[ax + 'axis.range']
else:
range = [
relayoutData[ax + 'axis.range[0]'],
relayoutData[ax + 'axis.range[1]']
]
return range
def getLatestAgentInstallers(version):
""" Gets the latest agent installers from GitHub """
log.debug(
f"fetching latest agent installers for commander agent {version}")
# get client cert from CAPy if we don't already have it
if not path.exists("agent/certs/client.crt") or not path.exists(
"agent/certs/client.key") or not path.exists(
"agent/certs/root.crt"):
response = requests.get("http://" + app.config["CA_HOSTNAME"] +
"/ca/host-certificate",
headers={"Content-Type": "application/json"},
data={"hostname": "installer"})
if response.status_code != 200:
raise CAPyError("failed to get installer cert from CAPy")
with open("agent/certs/client.crt", "w") as f:
f.write(response.json["cert"])
with open("agent/certs/client.key", "w") as f:
f.write(response.json["key"])
with open("agent/certs/root.crt", "w") as f:
f.write(response.json["root"])
# get installers from GitHub
with requests.get(
f"https://github.com/lawndoc/commander/releases/download/{version}/windowsInstaller.zip"
) as response:
if response.status_code != 200:
raise GitHubError("failed to get windows installer from GitHub")
with open(f"agent/installers/{version}/windowsInstaller.zip",
'wb') as f:
shutil.copyfileobj(response.raw, f)
with requests.get(
f"https://github.com/lawndoc/commander/releases/download/{version}/linuxInstaller.zip"
) as response:
if response.status_code != 200:
raise GitHubError("failed to get linux installer from GitHub")
with open(f"agent/installers/{version}/linuxInstaller.zip", 'wb') as f:
shutil.copyfileobj(response.raw, f)
# add certs to installer zips
with zipfile.ZipFile(f"agent/installers/{version}/windowsInstaller.zip",
'a') as windowsZip:
# TODO: I think I need to convert line endings here before adding the files
windowsZip.write("agent/certs/client.crt", arcname="client.crt")
windowsZip.write("agent/certs/client.key", arcname="client.key")
windowsZip.write("agent/certs/root.crt", arcname="root.crt")
with zipfile.ZipFile(f"agent/installers/{version}/linuxInstaller.zip",
'a') as linuxZip:
linuxZip.write("agent/certs/client.crt", arcname="client.crt")
linuxZip.write("agent/certs/client.key", arcname="client.key")
linuxZip.write("agent/certs/root.crt", arcname="root.crt")
def updateRegistrationKey():
""" Generate and return a new registration key that agents need to register with commander """
log.debug(f"<{request.remote_addr}> updating registration key")
# make sure a current key exists
regKeyQuery = RegistrationKey.objects()
if not regKeyQuery:
regKey = RegistrationKey(regKey="placeholder")
else:
regKey = regKeyQuery[0]
# update the registration key and save to db
newKey = str(uuid4())
regKey["regKey"] = newKey
regKey.save()
# return new key
log.info(
f"<{request.remote_addr}> successfully regenerated the registration key"
)
return {"registration-key": newKey}
def douban_book_api(isbn):
url='https://api.douban.com/v2/book/isbn/'+str(isbn)
try:
socket=urllib2.urlopen(url)
json_str=socket.read()
ret=json.loads(json_str)
ret_dict = {
'flag':'true',
'isbn':ret['isbn13'],
'author':ret['author'],
'bname':ret['title'],
'byear':ret['pubdate'],
'pagination':ret['pages'],
'price':ret['price'],
'bcover':ret['image'],
'publisher':ret['publisher'],
}
ret_dict['byear']=ret_dict['byear'].replace(u'年','.')
ret_dict['byear']=ret_dict['byear'].replace(u'月','.')
ret_dict['byear']=ret_dict['byear'].replace(u'日','')
ret_dict['byear']=ret_dict['byear'].replace(u'-','.')
ret_dict['byear']=ret_dict['byear'].replace(u'/','.')
ret_dict['price']=ret_dict['price'].replace(u'元','')
ret_dict['price']=ret_dict['price'].replace(u'CNY','')
ret_dict['price']=ret_dict['price'].replace(u' ','')
ret_dict['pagination']=ret_dict['pagination'].replace(u'页','')
ret_dict['pagination']=ret_dict['pagination'].replace(u'p','')
ret_dict['pagination']=ret_dict['pagination'].replace(u'P','')
log.debug('douban_book_api::douban returned json:',json_str.decode('utf-8'))
print(ret_dict['price'])
print(ret_dict['bcover'])
try:
ret_dict['translator']=ret['translator']
ret_dict['bcover']=ret['images']['large']
log.debug('douban_book_api::before return::ret_dict->',ret_dict)
return ret_dict
except:
return ret_dict
except urllib2.HTTPError as e:
return {'flag':'false','error_code':e.code}
except Exception as e:
return {'flag':'false','error':str(e),}
def agentCheckin(ws):
""" Agent checking in -- send file to be executed if a job is waiting """
log.debug(f"<{ws.sock.getpeername()[0]}> agent checking in")
# get agent ID from socket
while True:
data = ws.receive()
# validate that json data was sent
try:
jsonData = json.loads(data)
except Exception:
log.warning(
f"[{ws.sock.getpeername()[0]}] invalid json data received from agent during checkin"
)
ws.close(
400,
json.dumps({"error": "message was not a valid json object"}))
return {"error": "message was not a valid json object"}
# convert data to request-like object for missing()
request = SimpleNamespace(**{
"headers": jsonData,
"remote_addr": ws.sock.getpeername()[0]
})
# check if the Agent ID was provided in the json data
if missingParams := missing(request, headers=["Agent-ID"]):
log.warning(f"<{request.remote_addr}> {missingParams}")
ws.close(400, json.dumps({"error": missingParams}))
return {"error": missingParams}
# make sure Agent ID exists in the DB
agentQuery = Agent.objects(agentID__exact=request.headers["Agent-ID"])
if not agentQuery:
log.warning(
f"<{request.remote_addr}> agentID not found in database")
ws.close(
400,
json.dumps({
"error":
"agent ID not found, please check ID or register"
}))
return {"error": "agent ID not found, please check ID or register"}
break
def submit_label(label=None,
xrange='auto',
comment=None,
autoscroll=False,
session_id=None,
user_id=None):
if xrange == 'auto':
xrange = cache.get(flat_key(session_id, 'auto_xrange'))
if xrange is None:
raise KeyError('auto_xrange not found in cache')
try:
if True in autoscroll:
autoscroll = True
else:
autoscroll = False
except TypeError:
autoscroll = autoscroll
if autoscroll:
log.debug('Triggering autoscroll...')
cache.set(flat_key(session_id, 'autoscroll'), True)
cache.set(flat_key(session_id, 'autoscroll_xrange'), xrange)
label_df = cache.get(flat_key(user_id, 'label_df'))
if label_df is None:
log.debug('Initialised in-memory label store')
label_df = empty_label_df()
new_entry = dict(user_id=user_id,
session_id=session_id,
file_id=cache.get(flat_key(session_id, 'file_id')),
time_added=time.time(),
start=xrange[0],
end=xrange[1],
label=label,
comment=comment)
label_df = label_df.append(new_entry, ignore_index=True)
cache.set(flat_key(user_id, 'label_df'), label_df)
log.debug('Stored new label: {}'.format(new_entry))
log.debug('All labels:\n{}'.format(label_df))
return None
def login():
""" Authenticate an admin and return a new session token if successful """
log.debug(f"<{request.remote_addr}> logging in")
if missingParams := missing(request, data=["username", "password"]):
log.warning(f"<{request.remote_addr}> {missingParams}")
return {"error": missingParams}, 400
def make_figure(file_id,
xrange='auto',
yrange='auto',
session_id=None,
level=None,
baselevel=None):
# Re-set view range on file change
if session_id:
old_filename = cache.get(flat_key(session_id, 'filename'))
if old_filename and old_filename != file_id:
log.debug('Filename changed, resetting xrange')
xrange = 'auto'
yrange = 'auto'
cache.set(flat_key(session_id, 'file_id'), file_id)
log.debug('Filename: {}'.format(file_id))
# plot data
df, levels, counts = get_data(file_id)
x = df['t']
y = df['x']
auto_xrange = [x.min(), x.max()]
cache.set(flat_key(session_id, 'auto_xrange'), auto_xrange)
duration = auto_xrange[1] - auto_xrange[0]
if xrange == 'auto':
xrange = auto_xrange
xdelta = xrange[1] - xrange[0]
preload_xstart = xrange[0] - PRELOAD_MARGIN * xdelta
preload_xend = xrange[1] + PRELOAD_MARGIN * xdelta
viewport_idx = (preload_xstart <= x) & (x <= preload_xend)
log.debug('X-axis range: {}'.format(xdelta))
# auto-choose baselevel
if baselevel is None:
for l, c in zip(levels, counts):
if c <= MAX_POINTS_TOTAL:
baselevel = l
break
else:
baselevel = levels[-1]
# auto-choose level
if level is None:
for l, c in zip(levels, counts):
if c * (xdelta / duration) <= MAX_POINTS_VIEWPORT:
level = l
break
else:
level = levels[-1]
log.debug('level: {}, baselevel: {}'.format(level, baselevel))
idx = (df['ares'] >= level) & viewport_idx # fine points inside viewport
idx = (df['ares'] >= baselevel) | idx # coarse points outside viewport
filtered_x = x[idx]
filtered_y = y[idx]
log.debug('Display points: {}'.format(len(filtered_x)))
# figure
figure = {
'data': [{
'x': filtered_x,
'y': filtered_y,
'name': 'data',
}],
'layout': {
'dragmode': 'pan',
'xaxis': {
'range': xrange,
'title': 'time',
},
'yaxis': {
'range': yrange,
'title': 'value',
}
}
}
return figure
def on_admin_push(request):
#TODO:要求用户验证
#前端传过来的是序列化后的json字符串, 需要loads一下
watcher=None
try:
if request.session['account']!=settings.SUPER_USER:
try:
Watcher.objects.get(account=request.session['account'],iswatching=True)
except:
raise Exception(unicode("非法用户尝试修改值班干事"))
push_json_str=request.POST['data']
push_json=json.loads(push_json_str)
for item in push_json['watch_list']:
#检查输入
log.debug('on_admin_push','for item start checking')
keys = ['account','watchsum','name','spnumber','iswatching','lpnumber','type']
for key in keys:
if key not in item:
raise Exception('incomplete data')
if check.is_clean_input(key,item[key]) == False:
print key, item[key]
raise Exception('unsafe data')
log.debug('on_admin_push','for item end checking')
#先删除
log.debug('on_admin_push','delete items')
if item.get('type')=='delete':
is_logined = (item['account']==request.session['account'])
log.debug('on_admin_push','%s deleted'%item['account'])
Watcher.objects.all().filter(account=item['account']).delete()
if is_logined:
del request.session['account']
else:
if item.get('type')=='new':
default_password=hashlib.md5(item['account']).hexdigest()
default_password=hashlib.sha1(default_password).hexdigest()
log.debug('on_admin_push','trying to create new watcher %s'%item['account'])
watcher=Watcher(
account=B(item['account']),
name=B(item['name']),
lpnumber=B(item['lpnumber']),
spnumber=B(item['spnumber']),
password=B(default_password),
watchsum=0,
iswatching=False)
log.debug('on_admin_push','succeed to create new watcher')
else:
log.debug('on_admin_push','trying to get item %s'%item['account'])
watcher=Watcher.objects.get(account=item['account'])
log.debug('on_admin_push','succeed to get item')
if('yes'==item['iswatching']):
watcher.iswatching=True
elif('no'==item['iswatching']):
watcher.iswatching=False
if watcher.account == request.session['account'] and watcher.account!=settings.SUPER_USER:
del request.session['account']
log.info('on_admin_push','after setting watching state')
watcher.save()
log.info('on_admin_push','after save')
return HttpResponse(json.dumps({'flag_succeed':'true',}))
except Exception as e:
log.error('exception in on_admin_push:',unicode(e))
error=Error(what=B(unicode(e)))
error.save()
return HttpResponse(json.dumps({'flag_succeed':'false',}))
def create_hb201902(chart_data, node_id, click_time):
answer = bytes(app.config['HB201902_CODE'], 'utf-8')
failed_tip = '胜败乃兵家常事，少侠请从头再来'
tips = {
'0': '点得我好疼啊~',
'1': '送分题\n难度：未知',
'2': '2是第一个素数',
'16': 'AES的16位key是什么？',
'37': '答案的关键',
'73': '谢耳朵最喜欢的数字',
'2019': '新年快乐~',
'9102': '2019有你想有的东西',
'9444732965739290427429': '恭喜你找到了！\n'
}
#log.debug('hb201902 tips: %s' % tips)
tip = ''
max_try = 30
try:
try_cnt = get_hit_count('#'.join(['201902', str(click_time)]))
log.debug('try_cnt: %d' % try_cnt)
if try_cnt > max_try: # 30次
node_id = '1'
tip = failed_tip
except Exception as e:
log.error('try_cnt: %s' % str(e))
node_id = '1'
tip = 'redis stopped'
cpt = None
try:
if node_id not in ['-1', '0', '1']:
old_secret_key = ('%10.06f' %
(click_time + 20190101 + try_cnt - 1)).replace(
'.', '')
log.debug('Crypt old_secret_key: %s' % old_secret_key)
cpt = Crypt(old_secret_key)
try:
node_id = cpt.decrypt(node_id)
except Exception as e:
node_id = '0'
if tip in [failed_tip, 'redis stopped']:
try_cnt = 0
click_time = float('%10.06f' % time.time())
secret_key = ('%10.06f' % (click_time + 20190101 + try_cnt)).replace(
'.', '')
log.debug('Crypt secret_key: %s' % secret_key)
cpt = Crypt(secret_key)
except Exception as e:
log.error('Crypt: %s' % str(e))
node_id = '1'
tip = failed_tip
log.debug("create_hb201902 node_id: %s" % node_id)
if tip not in [failed_tip, 'redis stopped']:
tip = tips.get(str(node_id))
create_tips(chart_data, '剩余步数%s' % (max_try - try_cnt), '-1')
name = '0'
if tip:
tip_secret_key = ('%10.06f' % click_time).replace('.', '')
tip_cpt = Crypt(tip_secret_key)
if node_id == '2019':
name = tip_cpt.encrypt('the old place')
elif node_id == '9444732965739290427429': # 2**73+37
name = tip_cpt.encrypt(base64.b64encode(answer).decode('utf-8'))
create_tips(chart_data, tip, name)
log.debug('create_hb201902 tip:%s' % tip)
if node_id not in ['0', '-1']:
create_nodes(chart_data, [int(node_id)], 3, cpt)
if tip in [failed_tip, 'redis stopped']:
return False, click_time
return True, click_time
def assignJob():
""" Admin submitting a job -- add job to the specified agent's queue """
log.debug(f"<{request.remote_addr}> {get_jwt_identity()} assigning a job")
if missingParams := missing(request, data=["agentID", "filename", "argv"]):
log.warning(f"<{request.remote_addr}> {missingParams}")
return {"error": missingParams}, 400
def sendAgentInstaller():
""" Fetch or generate an agent installer for the given operating system """
log.debug(f"<{request.remote_addr}> requesting an agent installer")
if missingParams := missing(request, data=["os"]):
log.warning(f"<{request.remote_addr}> {missingParams}")
return {"error": missingParams}, 400
def getJobResults():
""" Get all jobs that have executed in the last 7 days, or optionally specify a different amount of time """
log.debug(f"<{request.remote_addr}> getting job results")
if missingParams := missing(request, data=["agentID"]):
log.warning(f"<{request.remote_addr}> {missingParams}")
return {"error": missingParams}, 400
def postJobResults():
""" Job has been executed -- save output and return code """
log.debug(f"<{request.remote_addr}> saving job results")
if missingParams := missing(request, headers=["Agent-ID"], data=["job"]):
log.warning(f"<{request.remote_addr}> {missingParams}")
return {"error": missingParams}, 400
def addNewJob():
""" Add a new executable to the Commander library """
log.debug(f"<{request.remote_addr}> adding new job")
if missingParams := missingJobForm(request, data=["job", "file"]):
log.warning(f"<{request.remote_addr}> {missingParams}")
return {"error": missingParams}, 400
def updateJob():
""" Update the file or description of an existing entry in the commander library """
log.debug(f"<{request.remote_addr}> updating job")
if missingParams := missingJobForm(request, data=["filename"]):
log.warning(f"<{request.remote_addr}> {missingParams}")
return {"error": missingParams}, 400
def deleteJob():
""" Delete an entry and its corresponding file from the commander library """
log.debug(f"<{request.remote_addr}> deleting job")
if missingParams := missing(request, data=["filename"]):
log.warning(f"<{request.remote_addr}> {missingParams}")
return {"error": missingParams}, 400
