def newNotebook(notebook):
existing_notebook = Notebook.query.filter(
Notebook.name == notebook.get("name")).one_or_none()
if existing_notebook is None:
newNotebook = Notebook(id=notebook.get("id"),
name=notebook.get("name"),
status="pending")
db.session.add(newNotebook)
db.session.commit()
request_rpc = RpcClient()
print(" [x] Requesting creating notebook")
message = {
'id': newNotebook.id,
'name': newNotebook.name,
'type': 'Notebook',
'action': 'Create',
}
response = request_rpc.call(message)
print(" [.] Successfully")
return jsonify(newNotebook.serialize()), 201
else:
abort(409, "Notebook name exists already")
def node_status():
# a = request.args.get('a', 0, type=int)
#j = jsonify([ dict(addr=k) for k in app.nodes ])
global client
if client is None:
client = RpcClient(
dict([('' + str(i), k) for i, k in enumerate(app.nodes)]))
# Make RPC requests for list_chunks and get_statistics asynchronously
timeout = 5.
ltokens = client.list_chunks(wait=False)
stats = client.get_statistics(timeout=timeout)
ch = client.wait_for_tokens(ltokens, timeout=timeout)
ch = [msgpack.unpackb(p[0]) if p is not None else None for p in ch]
# print('Chunks:')
# for bch in ch:
# if bch is None:
# continue
# for b,f0,f1,w in bch:
# Nchunk = 1024 * 400
# print(' beam', b, 'chunk', f0/Nchunk, '+', (f1-f0)/Nchunk, 'from', w)
#print('Stats[0]:', stats[0])
stat = [
dict(addr=k, status='ok', chunks=chi, stats=st)
for k, chi, st in zip(app.nodes, ch, stats)
]
j = json.dumps(stat)
# print('JSON:', j)
return j
def write_to_db():
request_data = request.get_json(force=True)
rpc_client = RpcClient(routing_key='writeQ')
res = rpc_client.call(json_msg=request_data)
res = json.loads(res)
print("Received response: " + str(res), file=sys.stdout)
if res == "Response(status=400)":
return Response(status=400)
return Response(status=200)
def clear_db():
query = {"clear": 1, "collections": ["rides", "users"]}
rpc_client = RpcClient(routing_key='writeQ')
res = rpc_client.call(json_msg=query)
res = json.loads(res)
print("Received response: " + str(res), file=sys.stdout)
if res == "Response(status=400)":
return Response(status=500)
f = open("seq.txt", "w")
f.write("0")
f.close()
return Response(status=200)
def compute(type, task, process):
global create_json
data = create_json(task, process, request.json)
print (json.dumps(data))
# create queue, send and receive response
# TODO: validate type
rpc = RpcClient(type)
response = rpc.call(json.dumps(data))
return response
def read_from_db():
increment_requests_count()
global c
c += 1
if c == 1:
subprocess.Popen("python3 scaling.py", shell=True)
request_data = request.get_json(force=True)
rpc_client = RpcClient(routing_key='readQ')
res = rpc_client.call(json_msg=request_data)
res = json.loads(res)
print("Received response: " + str(res), file=sys.stdout)
if res == "Response(status=400)":
return Response(status=400)
return jsonify(res)
def get_rpc_client():
global _rpc_client
if _rpc_client is None:
from rpc_client import RpcClient
from collections import OrderedDict
servers = OrderedDict([('' + str(i), k)
for i, k in enumerate(app.nodes)])
_rpc_client = RpcClient(servers)
return _rpc_client
def deleteNotebook(id):
notebook = Notebook.query.filter(Notebook.id == id).one_or_none()
if notebook is not None:
notebook_request_rpc = RpcClient()
print(" [x] Requesting deleting notebook")
message = {'id': notebook.id, 'type': 'Notebook', 'action': 'Delete'}
response = notebook_request_rpc.call(message)
print(" [.] Successfully")
db.session.delete(notebook)
db.session.commit()
return make_response(
"Notebook with id:{id} successfully deleted".format(id=id), 200)
else:
abort(404, "Notebook with this id: {id} not found".format(id=id))
def newTraining(name):
uploaded_file = request.files['file']
newTraining = Training(name=str(name), status="pending")
if uploaded_file.filename != '':
file_path = os.path.join(app.config['UPLOAD_FOLDER'],
uploaded_file.filename)
newTraining.file_path = file_path
newTraining.file_name = uploaded_file.filename
uploaded_file.save(file_path)
db.session.add(newTraining)
db.session.commit()
request_rpc = RpcClient()
print(" [x] Requesting creating training")
message = {
'id': newTraining.id,
'name': newTraining.name,
'type': 'Training',
'action': 'Create',
'file_path': file_path,
'file_name': uploaded_file.filename
}
response = request_rpc.call(message)
print(" [.] Successfully")
return jsonify(newTraining.serialize()), 201
else:
abort(409, "This is not valid training")
def autoplay():
global clock, screen
screen = pygame.display.set_mode((1000, 600))
rpc = RpcClient()
rpc_response = {}
for i in range(1, 31):
rpc_response = rpc.room_register(f'room-{i}')
if rpc_response.get('status', 200) == 200: break
else: print(rpc_response)
room, name = rpc_response['roomId'], rpc_response['tankId']
room_state = RoomEvents(room)
room_state.start()
ai = AI(name)
counter = 0
seconds = 0
winner = ''
lost = False
kicked = False
game_over = False
mainloop = True
while mainloop:
millis = clock.tick(FPS)
seconds += millis / 1000
for event in pygame.event.get():
if event.type == pygame.QUIT:
mainloop = False
if event.type == pygame.KEYDOWN:
if event.key == pygame.K_ESCAPE:
mainloop = False
game_over = True
if not room_state.ready:
wait_text = 'Loading...'
wait_text = big_font.render(wait_text, True, (50, 50, 50))
screen.fill((255, 255, 255))
text_rect = wait_text.get_rect(center=(screen.get_size()[0] // 2,
screen.get_size()[1] // 2))
screen.blit(wait_text, text_rect)
elif room_state.response:
screen.fill((201, 175, 135))
cur_state = room_state.response
tanks = cur_state['gameField']['tanks']
bullets = cur_state['gameField']['bullets']
ai.start(tanks, bullets)
if ai.fire:
rpc_response = rpc.fire()
ai.fire = False
ai.last_action = seconds
if ai.turn_direction:
print(ai.turn_direction)
rpc_response = rpc.turn_tank(ai.turn_direction)
ai.turn_direction = ''
ai.last_action = seconds
if ai.last_action + 10 < seconds:
ai.turn_direction = random.choice(ai.directions)
remaining_time = cur_state.get('remainingTime', 0)
text = font.render(f'Remaining time: {remaining_time}', True,
(0, 0, 0))
text_rect = text.get_rect(center=(400, 50))
screen.blit(text, text_rect)
drawScoreboard(name, tanks, room)
for tank in tanks:
draw_tank(seconds, name, **tank)
for bullet in bullets:
draw_bullet(name, **bullet)
if len(bullets) > counter: shoot_sound.play(maxtime=1600)
counter = len(bullets)
if room_state.new and cur_state['hits']:
room_state.new = False
explosion_sound.play()
if next((x for x in cur_state['losers'] if x['tankId'] == name),
None):
lost = True
mainloop = False
game_over = True
elif next((x for x in cur_state['kicked'] if x['tankId'] == name),
None):
kicked = True
mainloop = False
game_over = True
elif cur_state['winners']:
mainloop = False
game_over = True
win_text = 'Congrats! Score: {}. Winner(-s): '.format(
cur_state["winners"][0]["score"])
winner = win_text + ', '.join(
map(
lambda i: i['tankId']
if i['tankId'] != name else 'You',
cur_state['winners']))
elif next((x for x in room_state.winners if x['tankId'] == name),
None):
mainloop = False
game_over = True
win_text = 'Congrats! Score: {}. Winner(-s): '.format(
room_state.winners[0]["score"])
winner = win_text + ', '.join(
map(
lambda i: i['tankId']
if i['tankId'] != name else 'You', room_state.winners))
elif not next((x for x in tanks if x['id'] == name), None):
lost = True
mainloop = False
game_over = True
pygame.display.flip()
room_state.kill = True
room_state.join()
screen = pygame.display.set_mode((800, 640))
return game_over, winner, lost, kicked
# -*- coding: utf-8 -*-
import datetime
from events_spider.utils.tools import LOGGER, APP_CONF, SCHEDULER
from rpc_client import RpcClient
from SimpleXMLRPCServer import SimpleXMLRPCServer
CLIENT = RpcClient()
def callback():
LOGGER.info("New Master Confirmed.")
CLIENT.call()
# SCHEDULER.add_job(CLIENT.call, 'interval', id='call', minutes=APP_CONF['config']['crawl_frequency'], next_run_time=datetime.datetime.now())
LOGGER.info(SCHEDULER.get_jobs())
server = SimpleXMLRPCServer((APP_CONF['config']['localhost'], 8888))
server.register_function(callback, "call")
LOGGER.info("Awaiting Being Eelcted.")
server.serve_forever()
from __future__ import print_function
import yaml
from rpc_client import RpcClient
if __name__ == '__main__':
y = yaml.load(open('l1_configs/l1_production_8beam_webapp.yaml'))
#nodes = y['rpc_address']
#print('Nodes:', nodes)
#nodes = nodes[:3]
nodes = ['tcp://10.6.201.10:5555']
servers = dict([(n,n) for n in nodes])
client = RpcClient(servers)
fpgas = client.get_max_fpga_counts(timeout=5.)
print('Got:', fpgas)
for i,fpga in enumerate(fpgas):
print()
#print(i, fpga)
for where,beam,f in fpga:
if where == 'bonsai':
print('FPGA', f)
print((f / 384) % 4096)
def main():
# Expect to be run with a YAML config file as argument.
# (eg, l1_configs/l1_production_8beam_webapp.yaml)
import sys
import yaml
if len(sys.argv) != 2:
print('Use: %s <config.yaml>' % sys.argv[0])
return -1
config_fn = sys.argv[1]
try:
y = yaml.load(open(config_fn))
except:
print("ch-logger: couldn't parse yaml config file '%s'" % config_fn)
import traceback
traceback.print_exc()
return -1
if not isinstance(y, dict) or not 'rpc_address' in y:
print("ch-logger: no 'rpc_address' field found in yaml file '%s'" %
config_fn)
return -1
nodes = y['rpc_address']
# allow a single string (not a list)
if isinstance(nodes, basestring):
nodes = [nodes]
if not isinstance(nodes, list) or not all(
isinstance(x, basestring) for x in nodes):
print(
"ch_logger: expected 'rpc_address' field to be a list of strings in config file %s"
% config_fn)
return -1
database = y.get('log_database', 'sqlite:///log.sqlite3')
print('Using log database:', database)
servers = dict([(i, k) for i, k in enumerate(nodes)])
client = RpcClient(servers, identity='ch-logger')
logaddr = y.get('logger_address', None)
if logaddr is None:
logaddr = 'tcp://eno1:*'
# Parse IP address part of name
hostname = logaddr.replace('tcp://', '')
hostname, port = hostname.split(':')
try:
import socket
ip = socket.gethostbyname(hostname)
except:
# Is it a network interface name?
import netifaces
try:
ip = netifaces.ifaddresses(hostname)[netifaces.AF_INET][0]['addr']
print('Converting network interface name', hostname, 'to', ip)
hostname = ip
except:
# Not a network interface name. Continue but expect to fail...?
pass
logaddr = 'tcp://' + hostname + ':' + port
print('Logging to address:', logaddr)
logger = ChLogToDatabase(addr=logaddr, daemon=False, database=database)
addr = logger.address
print('Telling L1 processes to log to', addr)
#logger.handle_message(['slinky.local L1-RPC-server INFO l1-rpc.cpp:412 [int L1RpcServer::_handle_request(zmq::message_t *, zmq::message_t *)] 2018-01-09-20:44:27.827 chlog remote logging request received. Hello!'])
client.start_logging(addr)
engine = sqlalchemy.create_engine(database)
session_maker = sessionmaker(bind=engine)
session = session_maker()
while True:
import time
# time.sleep(10.)
#
# # Send RPCs
# client.get_statistics(timeout=3.)
#
# time.sleep(3.)
# Query DB.
#engine = logger.engine #sqlalchemy.create_engine(database, echo=True)
#session_maker = sessionmaker(bind=engine)
#session = session_maker()
#session = logger.session_maker()
print()
print('(Most recent) Log messages:')
for logmsg in session.query(LogMessage).order_by(desc(
LogMessage.date)).limit(10):
print(' ', logmsg)
time.sleep(15.)
import simulate_l0
import numpy as np
from rpc_client import read_msgpack_file, RpcClient
from time import sleep
import subprocess
import os
from glob import glob
print('Deleting existing msgpack files...')
fns = glob('chunk-test-rfi-mask-*.msgpack')
for fn in fns:
print(' ', fn)
os.remove(fn)
l0 = simulate_l0.l0sim('l0_configs/l0_test_rfi.yml', 1.0)
client = RpcClient({'a': 'tcp://127.0.0.1:5555'})
if True:
l1cmd = './ch-frb-l1 -fv l1_configs/l1_test_rfi.yml rfi_configs/rfi_testing.json bonsai_production_noups_nbeta1_v2.hdf5 xxx'
need_rfi = True
else:
l1cmd = './ch-frb-l1 -fv l1_configs/l1_test_norfi.yml rfi_configs/rfi_testing_no.json bonsai_production_noups_nbeta1_v2.hdf5 xxx'
need_rfi = False
l1 = subprocess.Popen(l1cmd, shell=True)
# wait until L1 is ready to receive
sleep(5)
beam_id = 0
fpga_counts_per_sample = 384
nt = 1024
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
'--config',
default='l1_configs/l1_production_8beam_webapp.yaml',
help='YAML config file listing L1 server RPC addresses')
parser.add_argument('--period',
default=300,
type=float,
help='Time to wait between samples, in seconds')
parser.add_argument('--output',
default='packet-rate-%04i.json',
help='Packet rate output filename pattern')
args = parser.parse_args()
config_fn = args.config
try:
y = yaml.load(open(config_fn))
except:
print("Failed to parse yaml config file '%s'" % config_fn)
raise
if not isinstance(y, dict) or not 'rpc_address' in y:
print("No 'rpc_address' field found in yaml file '%s'" % config_fn)
sys.exit(1)
nodes = y['rpc_address']
# allow a single string (not a list)
if isinstance(nodes, basestring):
nodes = [nodes]
if not isinstance(nodes, list) or not all(
isinstance(x, basestring) for x in nodes):
print("%s: expected 'rpc_address' field to be a list of strings" %
config_fn)
sys.exit(1)
client = RpcClient(dict([('' + str(i), k) for i, k in enumerate(nodes)]))
step = 0
t0 = time.time()
while True:
start_time = t0 - args.period
end_time = t0
rates = client.get_packet_rate(start=start_time,
period=args.period,
timeout=args.period)
#print('Packet rates:', rates)
# Find the set of all L0 senders.
senders = set()
for node, rate in zip(nodes, rates):
if rate is None:
continue
senders.update(rate.packets.keys())
senders = list(senders)
# Decide on the order of senders in the matrix
senders.sort()
senderindex = dict([(s, i) for i, s in enumerate(senders)])
# Build the matrix
matrix = [[0 for s in senders] for n in nodes]
for i, rate in enumerate(rates):
if rate is None:
continue
period = rate.period
for sender, packets in rate.packets.items():
#matrix[i][senderindex[sender]] = float(packets) / float(period)
# Save about 3 digits
matrix[i][senderindex[sender]] = 0.001 * int(
(1000. * packets) / float(period))
#print('Matrix:', matrix)
output = (datetime.now().isoformat(), nodes, senders, matrix)
outfn = args.output % step
print('Writing to file', outfn)
json.dump(output, open(outfn, 'w'))
step += 1
tnow = time.time()
tsleep = t0 + args.period - tnow
print('Sleeping', tsleep)
time.sleep(tsleep)
t0 += args.period