def handle_request_main(req, filename=None):
acl.try_reload()
init_conf("src")
status.push("handling email request")
ret = handle_request(req, filename=filename)
status.pop()
return ret
def main():
if lock.lock("sending-files", non_block=1) == None:
return
init_conf()
maybe_flush_queue(path.notify_queue_dir)
maybe_flush_queue(path.buildlogs_queue_dir)
maybe_flush_queue(path.ftp_queue_dir)
def main(srv_ssl=False):
write_css()
write_js()
socket.setdefaulttimeout(30)
try:
init_conf()
host = ""
port = config.request_handler_server_port
if srv_ssl:
port = config.request_handler_server_ssl_port
try:
server = HTTPServer((host, port), MyHandler)
if srv_ssl:
server.socket = ssl.wrap_socket(
server.socket,
keyfile=config.request_handler_server_ssl_key,
certfile=config.request_handler_server_ssl_cert,
ca_certs="/etc/certs/ca-certificates.crt",
server_side=True)
except Exception, e:
log.notice(
"request_handler_server: can't start server on [%s:%d], ssl=%s: %s"
% (host, port, str(srv_ssl), e))
print >> sys.stderr, "ERROR: Can't start server on [%s:%d], ssl=%s: %s" % (
host, port, str(srv_ssl), e)
sys.exit(1)
log.notice('request_handler_server: started on [%s:%d], ssl=%s...' %
(host, port, str(srv_ssl)))
server.serve_forever()
def main():
if lock.lock("sending-files", non_block = 1) == None:
return
init_conf()
maybe_flush_queue(path.notify_queue_dir)
maybe_flush_queue(path.buildlogs_queue_dir)
maybe_flush_queue(path.ftp_queue_dir)
def handle_request_main(req, filename = None):
acl.try_reload()
init_conf("src")
status.push("handling email request")
ret = handle_request(req, filename = filename)
status.pop()
return ret
def send_stc_pkt(f):
"""send ipv4 packets according configuration file.
:param f: configuration file .
:returns: None
:raises: None
"""
init_conf(f)
traffic_config = get_conf()
dst_ip_list = []
global traffic_results_ret
p2 = build_stc_eth()
if "l3_protocol" in traffic_config:
if traffic_config["l3_protocol"] == "ipv4":
p3 = build_stc_ipv4()
elif traffic_config["l3_protocol"] == "ipv6":
p3 = build_stc_ipv6()
else:
logging.error("layer 3 version must be 4 or 6")
traffic_results_ret['status'] = '0'
if "ip_protocol" in traffic_config:
if traffic_config["ip_protocol"] == '6':
p4 = build_stc_tcp()
elif traffic_config["ip_protocol"] == '17':
p4 = build_stc_udp()
else:
logging.error("layer 4 version must be 6 or 17")
traffic_results_ret['status'] = '0'
#Layer Beyond the TCP, StcPacket() should only use defaut input parameters
#since the StcPacket layer is created according to the *traffic_config.xml
p5 = StcPacket()
#TODO:should fetch burst_loop_count from *_p1_tx.py
burst_loop_count = 1000
dst_ip = traffic_config["ip_dst_addr"]
dst_ip_list.append(dst_ip)
for i in range(burst_loop_count - 1):
dst_ip = get_next_valid_ip(dst_ip)
dst_ip_list.append(dst_ip)
p3.dst = dst_ip_list
p = p2 / p3 / p4 / p5
packetList = sendp(p, return_packets=True)
#print packetList.summary()
traffic_results_ret['status'] = '1'
def main():
init_conf()
for b in builders_order():
run_rpm_builder(b)
# give builder some time to aquire lock
time.sleep(1)
# wait for children to die out
try:
while 1: os.wait()
except:
pass
def main():
write_css();
write_js();
socket.setdefaulttimeout(30)
try:
init_conf()
host = ""
port = config.request_handler_server_port
try:
server = HTTPServer((host, port), MyHandler)
except Exception, e:
log.notice("request_handler_server: can't start server on [%s:%d]: %s" % (host, port, e))
print >> sys.stderr, "ERROR: Can't start server on [%s:%d]: %s" % (host, port, e)
sys.exit(1)
log.notice('request_handler_server: started on [%s:%d]...' % (host, port))
server.serve_forever()
def main():
init_conf("src")
if lock("building-srpm", non_block=1) == None:
return
while True:
status.push("srpm: processing queue")
q = B_Queue(path.queue_file)
if not q.lock(1):
status.pop()
return
q.read()
if q.requests == []:
q.unlock()
status.pop()
return
r = pick_request(q)
q.write()
q.unlock()
status.pop()
status.push("srpm: handling request from %s" % r.requester)
handle_request(r)
status.pop()
def main():
init_conf("src")
if lock("building-srpm", non_block = 1) == None:
return
while True:
status.push("srpm: processing queue")
q = B_Queue(path.queue_file)
if not q.lock(1):
status.pop()
return
q.read()
if q.requests == []:
q.unlock()
status.pop()
return
r = pick_request(q)
q.write()
q.unlock()
status.pop()
status.push("srpm: handling request from %s" % r.requester)
handle_request(r)
status.pop()
def main():
lck = lock.lock("request_fetcher", non_block = True)
if lck == None:
sys.exit(1)
init_conf()
acl.try_reload()
status.push("fetching requests")
if has_new(config.control_url):
q = fetch_queue(config.control_url)
max_no = 0
q_new = []
for r in q:
if r.no > max_no:
max_no = r.no
if r.no > last_count:
q_new.append(r)
for b in config.binary_builders:
handle_reqs(b, q_new)
f = open(path.last_req_no_file, "w")
f.write("%d\n" % max_no)
f.close()
status.pop()
lck.close()
def main():
lck = lock.lock("request_fetcher", non_block=True)
if lck == None:
sys.exit(1)
init_conf()
acl.try_reload()
status.push("fetching requests")
if has_new(config.control_url):
q = fetch_queue(config.control_url)
max_no = 0
q_new = []
for r in q:
if r.no > max_no:
max_no = r.no
if r.no > last_count:
q_new.append(r)
for b in config.binary_builders:
handle_reqs(b, q_new)
f = open(path.last_req_no_file, "w")
f.write("%d\n" % max_no)
f.close()
status.pop()
lck.close()
continue
tmp=i.split()
mtime=int(tmp[5])
pkgname=tmp[6]
if curtime - mtime > config.max_keep_time:
rmpkgs.append(pkgname)
i=0
while rmpkgs[i:i+1000]:
chroot.run("cd /spools/ready; rm -f %s" % ' '.join(rmpkgs[i:i+1000]), 'root')
i=i+1000
f.close()
chroot.run("poldek --mo=nodiff --mkidxz -s /spools/ready")
if __name__ == '__main__':
init_conf()
bb=config.binary_builders[:]
clean_dir(path.spool_dir+'/builds', 2592000) # a month
if config.src_builder:
try:
init_conf(config.src_builder)
except:
pass
else:
handle_src()
sys.exit(0)
for b in bb:
try:
init_conf(b)
except:
continue
def simple_burst(port_a, port_b, burst_size, rate, f):
"""send ipv4 packets according configuration file.
:param f: configuration file .
:returns: None
:raises: None
"""
# create client
c = STLClient()
passed = True
init_conf(f)
traffic_config = get_conf()
dst_ip_list = []
global traffic_results_ret
#p2 = build_stc_eth()
p2 = Ether()
if "l3_protocol" in traffic_config:
if traffic_config["l3_protocol"] == "ipv4":
p3 = build_stc_ipv4()
elif traffic_config["l3_protocol"] == "ipv6":
p3 = build_stc_ipv6()
else:
logging.error("layer 3 version must be 4 or 6")
traffic_results_ret['status'] = '0'
if "ip_protocol" in traffic_config:
if traffic_config["ip_protocol"] == '6':
p4 = build_stc_tcp()
elif traffic_config["ip_protocol"] == '17':
p4 = build_stc_udp()
else:
logging.error("layer 4 version must be 6 or 17")
traffic_results_ret['status'] = '0'
#Layer Beyond the TCP, StcPacket() should only use defaut input parameters
#since the StcPacket layer is created according to the *traffic_config.xml
p5 = StcPacket()
p = p2 / p3 / p4 / p5
print ls(p)
try:
pkt = STLPktBuilder(pkt=p)
print pkt
#pkt.dump_pkt_to_pcap("test.pcap")
# create a single bursts and link them
s1 = STLStream(
name='A',
packet=pkt,
mode=STLTXSingleBurst(total_pkts=burst_size),
)
# connect to server
c.connect()
# prepare our ports
c.reset(ports=[port_a, port_b])
# add both streams to ports
stream_ids = c.add_streams([s1], ports=[port_a])
c.clear_stats()
c.start(ports=[port_a], mult=rate)
c.wait_on_traffic(ports=[port_a, port_b])
stats = c.get_stats()
ipackets = stats['total']['ipackets']
print("Packets Received: ", ipackets)
except STLError as e:
passed = False
print(e)
finally:
c.disconnect()
if c.get_warnings():
print("\n\n*** test had warnings ****\n\n")
for w in c.get_warnings():
print(w)
if passed and not c.get_warnings():
print("\nTest has passed :-)\n")
else:
print("\nTest has failed :-(\n")
b = get_next_valid_ip(b)
return b
def get_next_ip(s):
"""ip=ip+1, include x.x.x.0 and x.x.x.255,
:param s: string type of ip .
:returns: ip+1
:raises:
"""
return str(long2ip(ip2long(s) + 1))
if __name__ == '__main__':
init_conf("StcConf\case91_traffic_config.xml")
traffic_config = get_conf()
print traffic_config
#p = IP()/TCP()/StcPacket(StcSignature="1234567890"*2, StcPadding="0"*1264, CustomPattern="1"*176)
#L3 packet construction using Stc traffic parameters in traffic_config
p3 = IP()
if "ip_hdr_length" in traffic_config:
p3.ihl = int(traffic_config["ip_hdr_length"])
if "ip_tos_field" in traffic_config:
p3.tos = int(traffic_config["ip_tos_field"])
if "l3_length" in traffic_config:
p3.len = int(traffic_config["l3_length"])
if "ip_id" in traffic_config:
class Tester(object):
def __init__(self):
self.device = torch.device("cuda" if cfg.use_cuda else "cpu")
model_path = os.path.join(cfg.checkpoint, cfg.model_name)
self.net = Net()
self.net.load_state_dict(torch.load(model_path))
self.net.to(self.device)
self.data_set = DataSet()
def test(self):
self.net.eval()
correct = 0
with torch.no_grad():
for data, target in self.data_set.test_loader:
data = data.to(self.device)
target = target.to(self.device)
output = self.net(data)
pred = output.argmax(dim=1, keepdim=True)
correct += pred.eq(target.view_as(pred)).sum().item()
logging.info('\nTest set:Accuracy: {}/{} ({:.02f}%)\n'.format(
correct, len(self.data_set.test_loader.dataset),
100. * correct / len(self.data_set.test_loader.dataset)))
if __name__ == "__main__":
init_conf(cfg)
init_logger(file_name=cfg.log_name, directory=cfg.checkpoint)
tester = Tester()
tester.test()
def main_for(builder):
msg = ""
init_conf(builder)
q = B_Queue(path.queue_file + "-" + config.builder)
q.lock(0)
q.read()
if q.requests == []:
q.unlock()
return
req = pick_request(q)
q.unlock()
# high priority tasks have priority < 0, normal tasks >= 0
if req.priority >= 0:
# allow only one build in given builder at once
if not lock.lock("building-rpm-for-%s" % config.builder, non_block=1):
return
# don't kill server
check_load()
# not more then job_slots builds at once
locked = 0
for slot in range(config.job_slots):
if lock.lock("building-rpm-slot-%d" % slot, non_block=1):
locked = 1
break
if not locked:
return
# record fact that we got lock for this builder, load balancer
# will use it for fair-queuing
l = lock.lock("got-lock")
f = open(path.got_lock_file, "a")
f.write(config.builder + "\n")
f.close()
l.close()
else:
msg = "HIGH PRIORITY: "
msg += "handling request %s (%d) for %s from %s, priority %s" \
% (req.id, req.no, config.builder, req.requester, req.priority)
log.notice(msg)
status.push(msg)
handle_request(req)
status.pop()
def otherreqs(r):
if r.no == req.no:
return False
else:
return True
q = B_Queue(path.queue_file + "-" + config.builder)
q.lock(0)
q.read()
previouslen = len(q.requests)
q.requests = filter(otherreqs, q.requests)
if len(q.requests) < previouslen:
q.write()
q.unlock()
mtime = int(tmp[5])
pkgname = tmp[6]
if curtime - mtime > config.max_keep_time:
rmpkgs.append(pkgname)
i = 0
while rmpkgs[i:i + 1000]:
chroot.run("cd /spools/ready; rm -f %s" % ' '.join(rmpkgs[i:i + 1000]),
'root')
i = i + 1000
f.close()
chroot.run("poldek --mo=nodiff --mkidxz -s /spools/ready")
if __name__ == '__main__':
init_conf()
bb = config.binary_builders[:]
clean_dir(path.spool_dir + '/builds', 2592000) # a month
if config.src_builder:
try:
init_conf(config.src_builder)
except:
pass
else:
handle_src()
sys.exit(0)
for b in bb:
try:
init_conf(b)
except:
continue
def main_for(builder):
msg = ""
init_conf(builder)
q = B_Queue(path.queue_file + "-" + config.builder)
q.lock(0)
q.read()
if q.requests == []:
q.unlock()
return
req = pick_request(q)
q.unlock()
# high priority tasks have priority < 0, normal tasks >= 0
if req.priority >= 0:
# allow only one build in given builder at once
if not lock.lock("building-rpm-for-%s" % config.builder, non_block = 1):
return
# don't kill server
check_load()
# not more then job_slots builds at once
locked = 0
for slot in range(config.job_slots):
if lock.lock("building-rpm-slot-%d" % slot, non_block = 1):
locked = 1
break
if not locked:
return
# record fact that we got lock for this builder, load balancer
# will use it for fair-queuing
l = lock.lock("got-lock")
f = open(path.got_lock_file, "a")
f.write(config.builder + "\n")
f.close()
l.close()
else:
msg = "HIGH PRIORITY: "
msg += "handling request %s (%d) for %s from %s, priority %s" \
% (req.id, req.no, config.builder, req.requester, req.priority)
log.notice(msg)
status.push(msg)
handle_request(req)
status.pop()
def otherreqs(r):
if r.no==req.no:
return False
else:
return True
q = B_Queue(path.queue_file + "-" + config.builder)
q.lock(0)
q.read()
previouslen=len(q.requests)
q.requests=filter(otherreqs, q.requests)
if len(q.requests)<previouslen:
q.write()
q.unlock()
