def __init__(self, builder: Optional["StencilBuilder"] = None):
self._builder = builder
self.args_data = {}
with open(self.TEMPLATE_PATH, "r") as f:
self.template = jinja2.Template(f.read())
self.gw6_1 = next(self.hosts6)
self.gw6_2 = next(self.hosts6)
self.prefixlen6 = self.net6.prefixlen
with open(args.config, 'r') as f:
config = yaml.load(f)
conf = config['conf']
helpers = conf['helpers']
dhcpv6 = conf['dhcpv6']
vlans = []
for name, vconf in config['vlans'].items():
vid = vconf['vid']
vlans.append(Vlan(vid, name, vconf, helpers, dhcpv6))
vlans.sort(key=lambda x: x.vid)
outfiles = [('templates/vlan.j2', 'output/{}-vlan.txt'),
('templates/gw.j2', 'output/{}-gw.txt'),
('templates/gw1.j2', 'output/{}-gw1.txt'),
('templates/gw2.j2', 'output/{}-gw2.txt')]
for tmplfile, outfile in outfiles:
with open(tmplfile, 'r') as f:
tmpl = jinja2.Template(f.read())
with open(outfile.format(event), 'w') as f:
f.write(tmpl.render(vlans=vlans))
print('Written {}'.format(outfile).format(event))
using std::cout;
using std::endl;
using namespace uTensor;
{{ test }}
"""
const_container_str = """
#ifndef {{ constants_header | replace(".", "_") }}
#define {{ constants_header | replace(".", "_") }}
{{ constant_snippet }}
#endif
"""
test_Template = jinja2.Template(test_str)
const_Template = jinja2.Template(const_str)
container_Template = jinja2.Template(container_str)
const_container_Template = jinja2.Template(const_container_str)
if __name__ == "__main__":
tests=[]
constants=[]
reference_tensors = []
out_tensor = {}
ref_test_data = import_test_data()
name_map = get_name_map()
for key in name_map:
# -*- coding: utf-8 -*-
"""
Created on Sat Jan 11 21:08:14 2020
@author: Synophic
"""
import jinja2
import csv
config = 'Sub_interface_template.j2'
with open(config) as e:
output = e.read()
csv_file = 'Sub_interface_creation_variables.csv'
with open(csv_file) as f:
read_csv = csv.DictReader(f)
for var in read_csv:
var
t = jinja2.Template(output)
#print(var['circle'] + ' PEYTO_IP ' + var['Loopback0_IP1'])
print('#' * 80)
#print(t.render(var))
file = open(var['loopback_ip'] + '.txt', 'w')
file.write(t.render(var))
file.close()
def plot_d3(all_runs):
'''
Return html plots
'''
html_template = jinja2.Template('''
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<!--
Copyright (C) 2017 Sudeep Sahadevan
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>
-->
<script type="text/javascript" src="https://d3js.org/d3.v4.js" charset="utf-8"></script>
<link href='https://fonts.googleapis.com/css?family=Open+Sans' rel='stylesheet' type='text/css'>
<title>NextSeq 500 run stats</title>
<style type="text/css">
head, body {font-family: 'Open Sans', sans-serif;}
.plotDiv{float: left;}
#inputf3Label, #inputf5Label{
color: #cccccc;
}
#plotOptions{
padding-top: 50px;
padding-bottom: 5px;
}
#inputf7Label,#inputf8Label{
color:#cccccc;
}
#divButton1,#divButton2{
padding-top: 5px;
}
.axis{
font-size: 14px;
}
.axis path, .axis line {
fill: none;
stroke: grey;
stroke-width: 2;
shape-rendering: crispEdges;
}
.line {
fill: none;
stroke-width: 2;
opacity: 0.60
}
.title{
font-size: 20px;
font-weight: bold;
}
.label{
font-size: 16px;
}
.description1{
color: #666666;
}
.bar{fill: #0288d1}
.bar:hover{fill: #4fc3f7}
.bar2{fill: #388e3c}
.bar2:hover{fill: #81c784}
div.tooltip {
position: absolute;
text-align: center;
width: 30px;
height: 28px;
padding: 2px;
font-size: 14px;
font-weight: bold;
background-color: rgba(0, 0, 0, 0);
border: 0px;
border-radius: 8px;
pointer-events: none;
}
</style>
</head>
<body>
<div id="plotElem" class="plotDiv"></div>
<div id="plotOptions" class="plotDiv">
<div id="plotOpt1">
<label for="inputf1" id="inputf1Label"> Select data:<br>
<select id="inputf1" onchange="updater()" autocomplete="off">
<option name="inputOpts" value="count" selected="selected">Runs per month</option>
<option name="inputOpts" value="cd">Avg. Cluster density</option>
<option name="inputOpts" value="cpf">Avg. Clusters passing filter</option>
<option name="inputOpts" value="ey">Avg. Estimated yield</option>
</select>
</div>
<div d="plotOpt2">
<form id="inputf2" autocomplete="off"><br>
<input value="line" name="plotChbx1" id="pC1" type="checkbox" onclick="inputActivator()">Plot per month data
<label for="inputf3" id="inputf3Label"><br>Select year and month:<br>
<select id="inputf3" autocomplete="off" disabled="disabled">
<option name="Yopts" value="16">2016</option>
<option name="Yopts" value="17" selected="selected">2017</option>
<option name="Yopts" value="18">2018</option>
</select>
<select id="inputf4" autocomplete="off" disabled="disabled">
<option name="Mopts" value="01" selected="selected">1</option>
<option name="Mopts" value="02">2</option>
<option name="Mopts" value="03">3</option>
<option name="Mopts" value="04">4</option>
<option name="Mopts" value="05">5</option>
<option name="Mopts" value="06">6</option>
<option name="Mopts" value="07">7</option>
<option name="Mopts" value="08">8</option>
<option name="Mopts" value="09">9</option>
<option name="Mopts" value="10">10</option>
<option name="Mopts" value="11">11</option>
<option name="Mopts" value="12">12</option>
</select>
<label for="inputf5" id="inputf5Label" > <br>Select data:<br>
<select id="inputf5" autocomplete="off" disabled="disabled">
<option name="inputOpts5" value="cd" selected="selected">Cluster density</option>
<option name="inputOpts5" value="cpf">Clusters passing filter</option>
<option name="inputOpts5" value="ey">Estimated yield</option>
</select>
</div>
<div id="divButton1">
<button id="button1" type="button" onclick="updaterYM()" disabled="disabled">Plot</button>
</div>
<div id="plotOpt3">
<form id="inputf6" autocomplete="off"><br>
<input value="line" name="plotChbx2" id="pC2" type="checkbox" onclick="inputActivator()">Show scatter plots
<label for="inputf7" id="inputf7Label"><br>Select x axis:<br>
<select id="inputf7" autocomplete="off" disabled="disabled">
<option name="scXopts" value=2>Run number</option>
<option name="scXopts" value=3>Read1</option>
<option name="scXopts" value=4>Read2</option>
<option name="scXopts" value=10 selected="selected">Cluster density</option>
<option name="scXopts" value=11>Clusters passing filter</option>
<option name="scXopts" value=12>Estimated yield</option>
</select>
<label for="inputf8" id="inputf8Label"><br>Select y axis:<br>
<select id="inputf8" autocomplete="off" disabled="disabled">
<option name="scYopts" value=2>Run number</option>
<option name="scYopts" value=3>Read1</option>
<option name="scYopts" value=4>Read2</option>
<option name="scYopts" value=10>Cluster density</option>
<option name="scYopts" value=11>Clusters passing filter</option>
<option name="scYopts" value=12 selected="selected">Estimated yield</option>
</select>
</div>
<div id="divButton2">
<button id="button2" type="button" onclick="updaterScatter()" disabled="disabled">Plot</button>
</div>
<div id="error1"></div>
</div>
<script id="data1" type="text/javascript">
// column names: Date RunID RunNumber Read1 Read2 Index1Read Index2Read BaseSpaceRunId ExperimentName LibraryID ClusterDensity ClustersPassingFilter EstimatedYield CompletionStatus
var rundat = {{all_dat_json}};
</script>
<script id="functions1" type="text/javascript" >
//
// source: https://stackoverflow.com/questions/29544371/finding-the-average-of-an-array-using-js
//
const average = arr => arr.reduce((sume, el) => sume + el, 0) / arr.length;
//
// Plot renderer
//
function plotRender (svg,plotSelector,yLabel) {
var textArea = document.getElementById('error1');
textArea.innerHTML="";
var xScale = d3.scaleBand().rangeRound([0, width]).padding(0.1);
var yScale = d3.scaleLinear().range([height, 0]);
var xAxis = d3.axisBottom().scale(xScale);
xScale.domain(runarr.map(function(d) {return d.date; }));
var yAxis = d3.axisLeft().scale(yScale);
if (plotSelector=='count') {
yScale.domain([0, d3.max(runarr, function(d) {return d.ey.length; })]).nice();
}else if (plotSelector=='cd') {
yScale.domain([0, d3.max(runarr, function(d) {return average(d.cd); })]).nice();
}else if (plotSelector=='cpf') {
yScale.domain([0, d3.max(runarr, function(d) {return average(d.cpf); })]).nice();
}else if (plotSelector=='ey') {
yScale.domain([0, d3.max(runarr, function(d) {return average(d.ey); })]).nice();
}
// Define the div for the tooltip
var div = d3.select("#plotElem").append("div")
.attr("class", "tooltip")
.style("opacity", 0);
// X Axis
svg.append("g")
.attr("class", "x axis")
.attr("transform", "translate(0," + height + ")")
.call(xAxis)
.selectAll("text")
.attr("y", 0)
.attr("x", 9)
.attr("dy", ".35em")
.attr("transform", "rotate(90)")
.style("text-anchor", "start");
// Y Axis
svg.append("g")
.attr("class", "y axis")
.call(yAxis);
// Plot
svg.selectAll(".bar").data(runarr)
.enter().append("rect")
.attr("class", "bar")
.attr("x", function(d) {return xScale(d.date); })
.attr("width", xScale.bandwidth())
.attr("y", function(d) {
if (plotSelector=='count') {return yScale(d.ey.length);}
else if (plotSelector=='cd') {return yScale(average(d.cd));}
else if (plotSelector=='cpf') {return yScale(average(d.cpf));}
else if (plotSelector=='ey') {return yScale(average(d.ey));}
})
.attr("height", function(d) {
if (plotSelector=='count') {return height-yScale(d.ey.length);}
else if (plotSelector=='cd') {return height-yScale(average(d.cd));}
else if (plotSelector=='cpf') {return height-yScale(average(d.cpf));}
else if (plotSelector=='ey') {return height-yScale(average(d.ey));}
})
.on('mouseover', function (d) {
div.transition().duration(200).style("opacity", .9)
var hval=0;
if (plotSelector=='count') {hval = d.ey.length}
else if (plotSelector=='cd') {hval = average(d.cd).toFixed(2)}
else if (plotSelector=='cpf') {hval = average(d.cpf).toFixed(2)}
else if (plotSelector=='ey') {hval = average(d.ey).toFixed(2)}
div.html(hval).style("left", (d3.event.pageX-28) + "px").style("top", (d3.event.pageY-28) + "px");
})
.on("mouseout", function(d) {
div.transition().duration(500).style("opacity", 0);
});
// Add Title
svg.append("text")
.attr("class","title")
.attr("x", titlePosX)
.attr("y", titlePosY)
.attr("text-anchor", "middle")
.text("Nextseq 500 run stats");
// X Label
svg.append("text")
.attr("class","x label")
.attr("x", xlabX)
.attr("y", xlabY)
.attr("text-anchor", "middle")
.text('year-month');
// Y Label
svg.append("text")
.attr("class","y label")
.attr("transform", "rotate(-90)")
.attr("y", ylabY)
.attr("x",ylabX)
.style("text-anchor", "middle")
.text(yLabel);
}
// Per month-year plot renderer
function plotYMRender(svg,ymdata,plotdata,plotyear,plotmonth,yLabel) {
var xvals = [];
var xScale = d3.scaleBand().rangeRound([0, width]).padding(0.1);
var yScale = d3.scaleLinear().range([height, 0]);
for(var i=0;i<ymdata.run1.length;i++){
xvals.push(i);
}
var yvals = [];
if (plotdata=='cd') {
yvals = ymdata.cd;
yScale.domain([0,cdmax]).nice();
}else if (plotdata=='cpf') {
yvals = ymdata.cpf;
yScale.domain([0,cpfmax]).nice();
}else if (plotdata=='ey') {
yvals = ymdata.ey;
yScale.domain([0,eymax]).nice();
}
xScale.domain(xvals);
var xAxis = d3.axisBottom(xScale)
.tickFormat(function(d,i) {return ymdata.run1[i]; });
var yAxis = d3.axisLeft().scale(yScale);
// Define the div for the tooltip
var div = d3.select("#plotElem").append("div")
.attr("class", "tooltip")
.style("opacity", 0);
// X Axis
svg.append("g")
.attr("class", "x axis")
.attr("transform", "translate(0," + height + ")")
.call(xAxis)
.selectAll("text")
.attr("y", 0)
.attr("x", 9)
.attr("dy", ".35em")
.attr("transform", "rotate(90)")
.style("text-anchor", "start");
// Y Axis
svg.append("g")
.attr("class", "y axis")
.call(yAxis);
// Plot
svg.selectAll(".bar").data(yvals)
.enter().append("rect")
.attr("class", "bar2")
.attr("x", function (d,i) {return xScale(xvals[i])})
.attr("width", xScale.bandwidth())
.attr("y", function(d,i) {return yScale(d);})
.attr("height", function(d,i) {return height-yScale(d);})
.on('mouseover', function (d) {
div.transition().duration(200).style("opacity", .9)
div.html(d.toFixed(2)).style("left", (d3.event.pageX-28) + "px").style("top", (d3.event.pageY-28) + "px");
})
.on("mouseout", function(d) {
div.transition().duration(500).style("opacity", 0);
});
// Add Title
svg.append("text")
.attr("class","title")
.attr("x",titlePosX)
.attr("y",titlePosY)
.attr("text-anchor", "middle")
.text("Nextseq 500 run stats : 20"+plotyear+"."+plotmonth);
// X Label
svg.append("text")
.attr("class","x label")
.attr("x", xlabX)
.attr("y", xlabY)
.attr("text-anchor", "middle")
.text("Read length/run");
// Y Label
svg.append("text")
.attr("class","y label")
.attr("transform", "rotate(-90)")
.attr("y", ylabY)
.attr("x", ylabX)
.style("text-anchor", "middle")
.text(yLabel);
}
//
// scatter plotter
//
function scatterPlotter(svg,onx,ony,xLabel,yLabel) {
var onxvar = [];
var onyvar = [];
for (var i=0;i<rundat.length;i++) {
onxvar.push(rundat[i][onx]);
onyvar.push(rundat[i][ony]);
}
var xlimit = d3.extent(onxvar);
var ylimit = d3.extent(onyvar);
var xScale = d3.scaleLinear().domain(xlimit).nice().range([0, width]);
var yScale = d3.scaleLinear().domain(ylimit).nice().range([height, 0]);
var xAxis = d3.axisBottom(xScale);
var yAxis = d3.axisLeft().scale(yScale);
// push text element messge first
var textArea = document.getElementById('error1')
textArea.style.paddingTop="10px";
textArea.style.fontSize = "12px";
textArea.style.fontWeight = "bold";
textArea.style.color = "#000000";
textArea.innerHTML="Click on a dot for details";
svg.append("g")
.attr("class", "x axis")
.attr("transform", "translate(0," + height + ")")
.call(xAxis);
svg.append("g")
.attr("class", "y axis")
.call(yAxis);
svg.selectAll(".dots").data(onxvar)
.enter().append("circle")
.attr("class","dots")
.attr("r",5)
.attr("cx",function (d,i) {return xScale(onxvar[i]);})
.attr("cy",function (d,i) {return yScale(onyvar[i]);})
.style("fill","rgba(105, 50, 129,0.7)")
.on("click",clickEvent)
.on("mouseout",mouseOutEvent);
// Add Title
svg.append("text")
.attr("class","title")
.attr("x", titlePosX)
.attr("y", titlePosY)
.attr("text-anchor", "middle")
.text("Nextseq 500 run stats");
// X Label
svg.append("text")
.attr("class","x label")
.attr("x", xlabX)
.attr("y", xlabY)
.attr("text-anchor", "middle")
.text(xLabel);
// Y Label
svg.append("text")
.attr("class","y label")
.attr("transform", "rotate(-90)")
.attr("y", ylabY)
.attr("x", ylabX)
.style("text-anchor", "middle")
.text(yLabel);
// handle click event
function clickEvent(d,i) {
d3.select(this).transition()
.style("fill","#5b2c6f")
.attr("r", 8);
// textArea.style.fontWeight = "";
var dotInfo =rundat[i][8]+"<table id=infoTable><tr><td class=\\"description1\\">Run date</td><td>"+rundat[i][0]+"</td></tr>";
dotInfo+="<tr><td class=\\"description1\\">Read1</td><td>"+rundat[i][3]+"</td></tr><tr><td class=\\"description1\\">Read2</td><td>"+rundat[i][4]+"</td></tr>";
dotInfo+="<tr><td class=\\"description1\\">BaseSpaceRunId</td><td>"+rundat[i][7]+"</td>";
dotInfo+="<tr><td class=\\"description1\\">LibraryID</td><td>"+rundat[i][9]+"</td>";
textArea.innerHTML=dotInfo;
}
// handle mouse out event
function mouseOutEvent(d,i) {
d3.select(this).transition()
.style("fill","rgba(105, 50, 129,0.7)")
.attr("r", 5);
}
}
//
// data init and manipulation
//
var regx = /\d{2}$/i;
var runmap = {};
var cdmax = 0;
var cpfmax = 0;
var eymax = 0;
for (let rn of rundat) {
rn1 = rn[0].replace(/\d{2}$/i,'');
if ((rn1 in runmap) && (rn[12]>0)) {
runmap[rn1].run1.push(rn[3]);
runmap[rn1].run2.push(rn[4]);
runmap[rn1].cd.push(rn[10]);
runmap[rn1].cpf.push(rn[11]);
runmap[rn1].ey.push(rn[12]);
}else {
if (rn[12]>0) {
runmap[rn1] = {'date':rn1,'run1':[rn[3]],'run2':[rn[4]],'cd':[rn[10]],'cpf':[rn[11]],'ey':[rn[12]]};
}
}
if(rn[10]>cdmax){cdmax = rn[10];}
if(rn[11]>cpfmax){cpfmax = rn[11];}
if(rn[12]>eymax){eymax = rn[12];}
}
var runarr = [];
for (var key in runmap) {
runarr.push(runmap[key]);
}
// Plot stuff
var plotSelector = 'count'
W1 = 1050;
H1 = 675;
var margin = {top: 50, right: 20, bottom: 75, left: 75},
width = W1 - margin.left - margin.right,
height = H1 - margin.top - margin.bottom;
var svg = d3.select("#plotElem")
.append("svg")
.attr("height", H1)
.attr("width", W1)
.append("g")
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
// define positions for plot labels
var titlePosX, xlabX;
titlePosX = xlabX = width/2;
var titlePosY = 0-(margin.top)/2;
var xlabY = height+(margin.bottom*0.8);
var ylabX = 0 - (height / 2);
var ylabY = 0 - (margin.left*0.6);
//
// activate and deactivate options
//
var cchecker = 0; // variable to store previous state
function inputActivator() {
var checker = document.getElementById("pC1").checked;
var checker2 = document.getElementById("pC2").checked;
// first plot
var inputA = document.getElementById('inputf1');
var inputAlab = document.getElementById('inputf1Label');
// second plot
var inputY = document.getElementById('inputf3');
var inputM = document.getElementById('inputf4');
var inputYMlab = document.getElementById('inputf3Label');
var inputO = document.getElementById('inputf5');
var inputOlab = document.getElementById('inputf5Label');
var inputB = document.getElementById('button1');
// third plot
var inputXX = document.getElementById('inputf7');
var inputYY = document.getElementById('inputf8');
var inputXXlab = document.getElementById('inputf7Label');
var inputYYlab = document.getElementById('inputf8Label');
var inputB2 = document.getElementById('button2');
if (checker && checker2 && cchecker==1) { // checker was set before, so disable it
document.getElementById("pC1").checked = false;
checker = false;
}else if (checker && checker2 && cchecker==2) {// checker2 was set before, so disable it
document.getElementById("pC2").checked = false;
checker2 = false;
}
if(checker){
cchecker = 1;
inputY.disabled = false;
inputM.disabled = false;
inputO.disabled = false;
inputB.disabled = false;
inputY.focus();
inputM.focus();
inputO.focus();
inputB.focus();
inputYMlab.style.color = '#000000';
inputOlab.style.color = '#000000';
// disable first plot
inputA.disabled = true;
inputAlab.style.color = '#cccccc';
// disable third plot
inputXX.disabled = true;
inputYY.disabled = true;
inputB2.disabled = true;
inputXXlab.style.color = '#cccccc';
inputYYlab.style.color = '#cccccc';
}else if (checker2) {
cchecker = 2;
// enable third plot
inputXX.disabled = false;
inputYY.disabled = false;
inputB2.disabled = false;
inputXXlab.style.color = '#000000';
inputYYlab.style.color = '#000000';
// disable second plot
inputY.disabled = true;
inputM.disabled = true;
inputYMlab.style.color = '#cccccc';
inputO.disabled = true;
inputOlab.style.color = '#cccccc';
inputB.disabled = true;
// disable first plot
inputA.disabled = true;
inputAlab.style.color = '#cccccc';
}else{
// disable second plot
inputY.disabled = true;
inputM.disabled = true;
inputYMlab.style.color = '#cccccc';
inputO.disabled = true;
inputOlab.style.color = '#cccccc';
inputB.disabled = true;
// disable third plot
inputXX.disabled = true;
inputYY.disabled = true;
inputB2.disabled = true;
inputXXlab.style.color = '#cccccc';
inputYYlab.style.color = '#cccccc';
// enable first plot
inputA.disabled = false;
inputA.focus();
inputAlab.style.color = '#000000';
updater();
}
}
// call the first bar plot on init
// init y label
var yLabel = 'Runs per month';
plotRender(svg,plotSelector,yLabel);
//
// meta plot renderer
//
function updater() {
var plotvar = document.getElementById("inputf1");
var plotSelector = plotvar.options[document.getElementById("inputf1").selectedIndex].value;
yLabel = plotvar.options[document.getElementById("inputf1").selectedIndex].textContent;
svg.selectAll("g > *").remove();
plotRender(svg,plotSelector,yLabel);
}
//
// plot per month data
//
function updaterYM() {
var ploty = document.getElementById("inputf3");
var plotyear = ploty.options[document.getElementById("inputf3").selectedIndex].value;
var plotm = document.getElementById("inputf4");
var plotmonth = plotm.options[document.getElementById("inputf4").selectedIndex].value;
var plotd = document.getElementById("inputf5");
var plotdata = plotd.options[document.getElementById("inputf5").selectedIndex].value;
yLabel = plotd.options[document.getElementById("inputf5").selectedIndex].textContent;
plotym = plotyear+plotmonth;
var textArea = document.getElementById('error1');
textArea.contenteditable="true";
textArea.style.paddingTop="10px";
if (plotym in runmap) {
textArea.innerHTML="";
svg.selectAll("g > *").remove();
var ymdat = [];
ymdat.push(runmap[plotym])
plotYMRender(svg,runmap[plotym],plotdata,plotyear,plotmonth,yLabel);
}else {
textArea.style.fontSize = "16px";
textArea.style.fontWeight = "bold";
textArea.style.color = "#ff3300";
textArea.innerHTML="Error! Cannot find data for period: <font color=\\"#000000\\">20"+plotyear+"."+plotmonth+"</font>";
}
}
//
// plot scatter plots
//
function updaterScatter() {
var xx = document.getElementById("inputf7");
var xxvar = xx.options[document.getElementById("inputf7").selectedIndex].value;
var xLabel = xx.options[document.getElementById("inputf7").selectedIndex].textContent;
var yy = document.getElementById("inputf8");
var yyvar = xx.options[document.getElementById("inputf8").selectedIndex].value;
var yLabel = xx.options[document.getElementById("inputf8").selectedIndex].textContent;
svg.selectAll("g > *").remove();
scatterPlotter(svg,xxvar,yyvar,xLabel,yLabel);
}
</script>
</body>
</html>
''')
return html_template.render(all_dat_json=json.dumps(all_runs))
def build_config():
print('sys.argv:')
print(sys.argv)
parser = argparse.ArgumentParser()
parser.add_argument('--data_dir', default='')
parser.add_argument('--num_steps', default=1000)
parser.add_argument('--resize_min_dimension',
type=positive_int,
default=600)
parser.add_argument('--resize_max_dimension',
type=positive_int,
default=1024)
parser.add_argument('--resize_fixed_width', type=positive_int, default=0)
parser.add_argument('--resize_fixed_height', type=positive_int, default=0)
parser.add_argument('--grid_scales',
nargs='+',
default=[0.25, 0.5, 1.0, 2.0])
parser.add_argument('--grid_aspect_ratios',
nargs='+',
default=[0.5, 1.0, 2.0])
parser.add_argument('--tf_record_train_path', default='')
parser.add_argument('--tf_record_test_path', default='')
parser.add_argument('--label_map_path', default='')
parser.add_argument('--use_pretrained_checkpoint',
type=str_bool,
default=True)
parser.add_argument('--pretrained_checkpoint_path', default='')
args, _ = parser.parse_known_args()
targs = {
'data_dir': args.data_dir,
'num_steps': args.num_steps,
'resize_min_dimension': args.resize_min_dimension,
'resize_max_dimension': args.resize_max_dimension,
'resize_fixed_width': args.resize_fixed_width,
'resize_fixed_height': args.resize_fixed_height,
'grid_scales': args.grid_scales,
'grid_aspect_ratios': args.grid_aspect_ratios,
'tf_record_train_path': args.tf_record_train_path,
'tf_record_test_path': args.tf_record_test_path,
'label_map_path': args.label_map_path,
'use_pretrained_checkpoint': args.use_pretrained_checkpoint,
'pretrained_checkpoint_path': args.pretrained_checkpoint_path,
}
print('targs:')
print(targs)
if targs['resize_min_dimension'] == 0 or targs['resize_min_dimension'] == 0:
targs['resize_min_dimension'] = 0
targs['resize_max_dimension'] = 0
if targs['resize_fixed_width'] == 0 or targs['resize_fixed_height'] == 0:
targs['resize_fixed_width'] = 0
targs['resize_fixed_height'] = 0
if targs['resize_min_dimension'] == 0 and targs['resize_fixed_width'] == 0:
raise Exception(
'it should be set positive resize_min_dimension with resize_max_dimension or resize_fixed_width with resize_fixed_height'
)
if targs['num_steps'] == 0:
raise Exception('num_steps must be positive')
if targs['data_dir'] == '':
raise Exception('data_dir is not set')
if targs['tf_record_train_path'] == '':
targs[
'tf_record_train_path'] = '%s/pet_train_with_masks.record' % targs[
'data_dir']
if targs['tf_record_test_path'] == '':
targs['tf_record_test_path'] = '%s/pet_val_with_masks.record' % targs[
'data_dir']
if targs['label_map_path'] == '':
targs['label_map_path'] = '%s/pet_label_map.pbtxt' % targs['data_dir']
if targs['pretrained_checkpoint_path'] == '':
targs[
'pretrained_checkpoint_path'] = '%s/model.ckpt' % targs['data_dir']
t = open('faster_rcnn.config.template', 'r')
template = jinja2.Template(t.read())
t.close()
tw = open('faster_rcnn.config', 'w+')
cfg = str(template.render(args=targs))
tw.write(cfg)
tw.close()
pass
object = ObjectHandler()
saxparser = make_parser()
saxparser.setContentHandler(object)
toolssource = sys.argv[1]
datasource = open(sys.argv[2], "r")
destination = sys.argv[3]
saxparser.parse(datasource)
with open("%s/source.template" % toolssource, "r") as source:
template = jinja2.Template(source.read())
with open("%s/%sAttributes.cc" % (destination, object.name), "wt") as out:
out.write(template.render(object = object.name,
string_parameters = object.parameters["basic"],
factory_parameters = object.parameters["factory"],
include = object.include,
include_options = object.include_options,
date = object.generated,
tags = object.tags,
tag = object.tags[0],
prefix = object.prefix
)
)
with open("%s/header.template" % (toolssource), "r") as source:
template = jinja2.Template(source.read())
with open("%s/%sAttributes.h" % (destination, object.name), "wt") as out:
#!/usr/bin/python
import jinja2
import os
import socket
import glob
convert = lambda src, dst: open(dst, "w").write(
jinja2.Template(open(src).read()).render(**os.environ))
# Actual startup script
os.environ["FRONT_ADDRESS"] = socket.gethostbyname("front")
for dovecot_file in glob.glob("/conf/*"):
convert(dovecot_file,
os.path.join("/etc/dovecot", os.path.basename(dovecot_file)))
# Run postfix
os.system("chown -R mail:mail /mail /var/lib/dovecot")
os.execv("/usr/sbin/dovecot",
["dovecot", "-c", "/etc/dovecot/dovecot.conf", "-F"])
def _get_html_template(self):
returned = self._email_template
if returned is None:
source = resource_string('slash.plugins.builtin', 'email_template.j2').decode('utf-8')
returned = type(self)._email_template = jinja2.Template(source)
return returned
def runtemplate(template, row):
try:
renderer=jinja2.Template(template)
return renderer.render(row=row)
except Exception, e:
return e
def flatten(list_of_lists):
"""Flatten a list-of-lists into a single list."""
return list(itertools.chain.from_iterable(list_of_lists))
HIGHLIGHT_BUTTON_TMPL = jinja2.Template("""
<script>
colors_on = true;
function color_cells() {
var ffunc = function(i,e) {return e.innerText {{ filter_cond }}; }
var cells = $('table.dataframe').children('tbody')
.children('tr')
.children('td')
.filter(ffunc);
if (colors_on) {
cells.css('background', 'white');
} else {
cells.css('background', '{{ highlight_color }}');
}
colors_on = !colors_on;
}
$( document ).ready(color_cells);
</script>
<form action="javascript:color_cells()">
<input type="submit" value="Toggle highlighting (val {{ filter_cond }})"></form>
""")
RESIZE_CELLS_TMPL = jinja2.Template("""
<script>
var df = $('table.dataframe');
var cells = df.children('tbody').children('tr')
def format_template(template, **kw):
return jinja2.Template(template).render(**kw)
GALLERY_TEMPLATE = jinja2.Template(u"""
.. This document is auto-generated by the altair-gallery extension. Do not modify directly.
.. _{{ gallery_ref }}:
{{ title }}
{% for char in title %}-{% endfor %}
This gallery contains a selection of examples of the plots Altair can create.
Some may seem fairly complicated at first glance, but they are built by combining a simple set of declarative building blocks.
{% for grouper, group in examples %}
.. _gallery-category-{{ grouper }}:
{{ grouper }}
{% for char in grouper %}~{% endfor %}
.. raw:: html
<span class="gallery">
{% for example in group %}
<a class="imagegroup" href="{{ example.name }}.html">
<span class="image" alt="{{ example.title }}" style="background-image: url({{ image_dir }}/{{ example.name }}-thumb.png);"></span>
<span class="image-title">{{ example.title }}</span>
</a>
{% endfor %}
</span>
<div style='clear:both;'></div>
.. toctree::
:hidden:
{% for example in group %}
{{ example.name }}
{%- endfor %}
{% endfor %}
""")
def render(value, params):
"""Renders text, interpolating params.
"""
return str(jinja2.Template(value).render(params))
def _log_match_image_debug(imglog):
if not imglog.enabled:
return
from _stbt.core import _Annotation
for matched, position, _, level in imglog.data["pyramid_levels"]:
template = imglog.images["level%d-template" % level]
imglog.imwrite(
"level%d-source_with_match" % level,
imglog.images["level%d-source" % level],
Region(x=position.x,
y=position.y,
width=template.shape[1],
height=template.shape[0]),
_Annotation.MATCHED if matched else _Annotation.NO_MATCH)
for i, result in enumerate(imglog.data["matches"]):
imglog.imwrite(
"match%d-source_with_match" % i,
imglog.images["source"],
result.region,
_Annotation.MATCHED if result._first_pass_matched # pylint:disable=protected-access
else _Annotation.NO_MATCH)
try:
import jinja2
except ImportError:
warn("Not generating html view of the image-processing debug images,"
" because python 'jinja2' module is not installed.")
return
template = jinja2.Template("""
<!DOCTYPE html>
<html lang='en'>
<head>
<link href="http://netdna.bootstrapcdn.com/twitter-bootstrap/2.3.2/css/bootstrap-combined.min.css" rel="stylesheet">
<style>
h5 { margin-top: 40px; }
.table th { font-weight: normal; background-color: #eee; }
img {
vertical-align: middle; max-width: 150px; max-height: 36px;
padding: 1px; border: 1px solid #ccc; }
p { line-height: 40px; }
.table td { vertical-align: middle; }
</style>
</head>
<body>
<div class="container">
<h4>
<i>{{template_name}}</i>
{{"matched" if matched else "didn't match"}}
</h4>
<h5>First pass (find candidate matches):</h5>
<p>Searching for <b>template</b> {{link("template")}}
within <b>source</b> image {{link("source")}}
<table class="table">
<tr>
<th>Pyramid level</th>
<th>Match #</th>
<th>Searching for <b>template</b></th>
<th>within <b>source regions of interest</b></th>
<th>
OpenCV <b>matchTemplate heatmap</b>
with method {{match_parameters.match_method}}
({{"darkest" if match_parameters.match_method ==
"sqdiff-normed" else "lightest"}}
pixel indicates position of best match).
</th>
<th>
matchTemplate heatmap <b>above match_threshold</b>
of {{"%g"|format(match_parameters.match_threshold)}}
(white pixels indicate positions above the threshold).
</th>
<th><b>Matched?<b></th>
<th>Best match <b>position</b></th>
<th> </th>
<th><b>certainty</b></th>
</tr>
{% for matched, position, certainty, level in pyramid_levels %}
<tr>
<td><b>{{level}}</b></td>
<td><b>{{"0" if level == 0 else ""}}</b></td>
<td>{{link("template", level)}}</td>
<td>{{link("source_with_rois", level)}}</td>
<td>{{link("source_matchtemplate", level)}}</td>
<td>
{{link("source_matchtemplate_threshold", level) if matched else ""}}
</td>
<td>{{"Matched" if matched else "Didn't match"}}</td>
<td>{{position if level > 0 else matches[0].region}}</td>
<td>{{link("source_with_match", level)}}</td>
<td>{{"%.4f"|format(certainty)}}</td>
</tr>
{% endfor %}
{% for m in matches[1:] %}
{# note that loop.index is 1-based #}
<tr>
<td> </td>
<td><b>{{loop.index}}</b></td>
<td> </td>
<td> </td>
<td>{{link("heatmap", match=loop.index)}}</td>
<td></td>
<td>{{"Matched" if m._first_pass_matched else "Didn't match"}}</td>
<td>{{m.region}}</td>
<td>{{link("source_with_match", match=loop.index)}}</td>
<td>{{"%.4f"|format(m.first_pass_result)}}</td>
</tr>
{% endfor %}
</table>
{% if show_second_pass %}
<h5>Second pass (confirmation):</h5>
<p><b>Confirm method:</b> {{match_parameters.confirm_method}}</p>
{% if match_parameters.confirm_method != "none" %}
<table class="table">
<tr>
<th>Match #</th>
<th>Comparing <b>template</b></th>
<th>against <b>source image's region of interest</b></th>
{% if match_parameters.confirm_method == "normed-absdiff" %}
<th><b>Normalised template</b></th>
<th><b>Normalised source</b></th>
{% endif %}
<th><b>Absolute differences</b></th>
<th>
Differences <b>above confirm_threshold</b>
of {{"%.2f"|format(match_parameters.confirm_threshold)}}
</th>
<th>
After <b>eroding</b>
{{match_parameters.erode_passes}}
{{"time" if match_parameters.erode_passes == 1
else "times"}};
the template matches if no differences (white pixels) remain
</th>
</tr>
{% for m in matches %}
{% if m._first_pass_matched %}
<tr>
<td><b>{{loop.index0}}</b></td>
<td>{{link("confirm-template_gray", match=0)}}</td>
<td>{{link("confirm-source_roi_gray", match=loop.index0)}}</td>
{% if match_parameters.confirm_method == "normed-absdiff" %}
<td>{{link("confirm-template_gray_normalized", match=loop.index0)}}</td>
<td>{{link("confirm-source_roi_gray_normalized", match=loop.index0)}}</td>
{% endif %}
<td>{{link("confirm-absdiff", match=loop.index0)}}</td>
<td>{{link("confirm-absdiff_threshold", match=loop.index0)}}</td>
<td>{{link("confirm-absdiff_threshold_erode", match=loop.index0)}}</td>
</tr>
{% endif %}
{% endfor %}
</table>
{% endif %}
{% endif %}
<p>For further help please read
<a href="http://stb-tester.com/match-parameters.html">stb-tester
image matching parameters</a>.
</div>
</body>
</html>
""")
def link(name, level=None, match=None): # pylint: disable=redefined-outer-name
return ("<a href='{0}{1}{2}.png'><img src='{0}{1}{2}.png'></a>".format(
"" if level is None else "level%d-" % level,
"" if match is None else "match%d-" % match, name))
with open(os.path.join(imglog.outdir, "index.html"), "w") as f:
f.write(
template.render(
link=link,
match_parameters=imglog.data["match_parameters"],
matched=any(imglog.data["matches"]),
matches=imglog.data["matches"],
min=min,
pyramid_levels=imglog.data["pyramid_levels"],
show_second_pass=any(x._first_pass_matched
for x in imglog.data["matches"]), # pylint:disable=protected-access
template_name=imglog.data["template_name"],
))
from . import urls
__all__ = [
"prepared_data_to_html", "display", "show", "save_html", "save_json",
"enable_notebook", "disable_notebook"
]
# Simple HTML template. This works in standalone web pages for single visualizations,
# but will not work within the IPython notebook due to the presence of
# requirejs
SIMPLE_HTML = jinja2.Template("""
<script type="text/javascript" src="{{ d3_url }}"></script>
<script type="text/javascript" src="{{ ldavis_url }}"></script>
<link rel="stylesheet" type="text/css" href="{{ ldavis_css_url }}">
<div id={{ visid }}></div>
<script type="text/javascript">
!function(LDAvis){
new LDAvis("#" + {{ visid }}, {{ vis_json | safe}});
}(LDAvis);
</script>
""")
# RequireJS template. If requirejs and jquery are not defined, this will
# result in an error. This is suitable for use within the IPython notebook.
REQUIREJS_HTML = jinja2.Template("""
<link rel="stylesheet" type="text/css" href="{{ ldavis_css_url }}">
<div id={{ visid }}></div>
<script type="text/javascript">
class InteractiveContextTest(tf.test.TestCase):
def setUp(self):
super(InteractiveContextTest, self).setUp()
builtins.__dict__['__IPYTHON__'] = True
self._tmpdir = None
def tearDown(self):
if self._tmpdir:
shutil.rmtree(self._tmpdir, ignore_errors=True)
super(InteractiveContextTest, self).tearDown()
def _setupTestNotebook(self, notebook_name='test_notebook.ipynb'):
notebook = nbformat.v4.new_notebook(cells=[
nbformat.v4.new_markdown_cell(source='A markdown cell.'),
nbformat.v4.new_code_cell(source='foo = 1'),
nbformat.v4.new_markdown_cell(source='Another markdown cell.'),
nbformat.v4.new_code_cell(source=textwrap.dedent('''\
%%skip_for_export
!pip install something
!ls
x = 1
y = 2
print('this cell should not be exported')''')),
nbformat.v4.new_code_cell(source=textwrap.dedent('''\
def bar():
%some_line_magic print('this line should not be exported')
a = "hello"
b = "world"
return a + b''')),
nbformat.v4.new_code_cell(source=textwrap.dedent('''\
def baz():
c = "nyan"
d = "cat"
return c + d''')),
])
self._tmpdir = tempfile.mkdtemp()
self._exportdir = tempfile.mkdtemp()
self._notebook_fp = os.path.join(self._tmpdir, notebook_name)
nbformat.write(notebook, self._notebook_fp)
def testRequiresIPythonExecutes(self):
self.foo_called = False
def foo():
self.foo_called = True
interactive_context.requires_ipython(foo)()
self.assertTrue(self.foo_called)
def testRequiresIPythonNoOp(self):
del builtins.__dict__['__IPYTHON__']
self.foo_called = False
def foo():
self.foo_called = True
interactive_context.requires_ipython(foo)()
self.assertFalse(self.foo_called)
def testBasicRun(self):
class _FakeComponentSpec(types.ComponentSpec):
PARAMETERS = {}
INPUTS = {}
OUTPUTS = {}
class _FakeExecutor(base_executor.BaseExecutor):
CALLED = False
def Do(self, input_dict: Dict[Text, List[types.Artifact]],
output_dict: Dict[Text, List[types.Artifact]],
exec_properties: Dict[Text, Any]) -> None:
_FakeExecutor.CALLED = True
class _FakeComponent(base_component.BaseComponent):
SPEC_CLASS = _FakeComponentSpec
EXECUTOR_SPEC = executor_spec.ExecutorClassSpec(_FakeExecutor)
def __init__(self, spec: types.ComponentSpec):
super(_FakeComponent, self).__init__(spec=spec)
c = interactive_context.InteractiveContext()
component = _FakeComponent(_FakeComponentSpec())
c.run(component)
self.assertTrue(_FakeExecutor.CALLED)
def testRunMethodRequiresIPython(self):
del builtins.__dict__['__IPYTHON__']
c = interactive_context.InteractiveContext()
self.assertIsNone(c.run(None))
def testUnresolvedChannel(self):
class _FakeComponentSpec(types.ComponentSpec):
PARAMETERS = {}
INPUTS = {
'input': component_spec.ChannelParameter(type_name='Foo')
}
OUTPUTS = {}
class _FakeExecutor(base_executor.BaseExecutor):
CALLED = False
def Do(self, input_dict: Dict[Text, List[types.Artifact]],
output_dict: Dict[Text, List[types.Artifact]],
exec_properties: Dict[Text, Any]) -> None:
_FakeExecutor.CALLED = True
class _FakeComponent(base_component.BaseComponent):
SPEC_CLASS = _FakeComponentSpec
EXECUTOR_SPEC = executor_spec.ExecutorClassSpec(_FakeExecutor)
def __init__(self, spec: types.ComponentSpec):
super(_FakeComponent, self).__init__(spec=spec)
c = interactive_context.InteractiveContext()
foo = types.Channel(type_name='Foo', artifacts=[types.Artifact('Foo')])
component = _FakeComponent(_FakeComponentSpec(input=foo))
with self.assertRaisesRegexp(ValueError, 'Unresolved input channel'):
c.run(component)
@mock.patch.object(jinja2.Environment,
'get_template',
return_value=jinja2.Template('{{ notebook_content }}'))
def testExportToPipeline(self, mock_get_template):
self._setupTestNotebook()
c = interactive_context.InteractiveContext()
export_filepath = os.path.join(self._exportdir, 'exported_pipeline.py')
c.export_to_pipeline(notebook_filepath=self._notebook_fp,
export_filepath=export_filepath,
runner_type='beam')
with open(export_filepath, 'r') as exported_pipeline:
code = exported_pipeline.read()
self.assertEqual(
code,
textwrap.dedent('''\
foo = 1
def bar():
a = "hello"
b = "world"
return a + b
def baz():
c = "nyan"
d = "cat"
return c + d'''))
def testExportToPipelineRaisesErrorInvalidRunnerType(self):
self._setupTestNotebook()
c = interactive_context.InteractiveContext()
export_filepath = os.path.join(self._exportdir, 'exported_pipeline.py')
with self.assertRaisesRegexp(ValueError, 'runner_type'):
c.export_to_pipeline(notebook_filepath=self._notebook_fp,
export_filepath=export_filepath,
runner_type='foobar')
@mock.patch('tfx.orchestration.experimental.interactive.'
'standard_visualizations.ExampleAnomaliesVisualization.display'
)
def testShow(self, *unused_mocks):
context = interactive_context.InteractiveContext()
mock_object = mock.MagicMock()
standard_visualizations.ExampleAnomaliesVisualization.display = mock_object
mock_object.assert_not_called()
artifact = standard_artifacts.ExampleAnomalies()
context.show(
types.Channel(type=standard_artifacts.ExampleAnomalies,
artifacts=[artifact]))
mock_object.assert_called_with(artifact)
def load_docs(request):
URL = request.url_root
docs = htmldocs[request.endpoint]
docs = jinja2.Template(docs).render(URL=URL)
return markupsafe.Markup(docs)
def run(arguments, content, options, state_machine, state, lineno):
document = state_machine.document
config = document.settings.env.config
nofigs = 'nofigs' in options
formats = get_plot_formats(config)
default_fmt = formats[0][0]
options.setdefault('include-source', config.plot_include_source)
keep_context = 'context' in options
context_opt = None if not keep_context else options['context']
rst_file = document.attributes['source']
rst_dir = os.path.dirname(rst_file)
if len(arguments):
if not config.plot_basedir:
source_file_name = os.path.join(setup.app.builder.srcdir,
directives.uri(arguments[0]))
else:
source_file_name = os.path.join(setup.confdir, config.plot_basedir,
directives.uri(arguments[0]))
# If there is content, it will be passed as a caption.
caption = '\n'.join(content)
# If the optional function name is provided, use it
if len(arguments) == 2:
function_name = arguments[1]
else:
function_name = None
code = Path(source_file_name).read_text(encoding='utf-8')
output_base = os.path.basename(source_file_name)
else:
source_file_name = rst_file
code = textwrap.dedent("\n".join(map(str, content)))
counter = document.attributes.get('_plot_counter', 0) + 1
document.attributes['_plot_counter'] = counter
base, ext = os.path.splitext(os.path.basename(source_file_name))
output_base = '%s-%d.py' % (base, counter)
function_name = None
caption = ''
base, source_ext = os.path.splitext(output_base)
if source_ext in ('.py', '.rst', '.txt'):
output_base = base
else:
source_ext = ''
# ensure that LaTeX includegraphics doesn't choke in foo.bar.pdf filenames
output_base = output_base.replace('.', '-')
# is it in doctest format?
is_doctest = contains_doctest(code)
if 'format' in options:
if options['format'] == 'python':
is_doctest = False
else:
is_doctest = True
# determine output directory name fragment
source_rel_name = relpath(source_file_name, setup.confdir)
source_rel_dir = os.path.dirname(source_rel_name)
while source_rel_dir.startswith(os.path.sep):
source_rel_dir = source_rel_dir[1:]
# build_dir: where to place output files (temporarily)
build_dir = os.path.join(os.path.dirname(setup.app.doctreedir),
'plot_directive', source_rel_dir)
# get rid of .. in paths, also changes pathsep
# see note in Python docs for warning about symbolic links on Windows.
# need to compare source and dest paths at end
build_dir = os.path.normpath(build_dir)
if not os.path.exists(build_dir):
os.makedirs(build_dir)
# output_dir: final location in the builder's directory
dest_dir = os.path.abspath(
os.path.join(setup.app.builder.outdir, source_rel_dir))
if not os.path.exists(dest_dir):
os.makedirs(dest_dir) # no problem here for me, but just use built-ins
# how to link to files from the RST file
dest_dir_link = os.path.join(relpath(setup.confdir, rst_dir),
source_rel_dir).replace(os.path.sep, '/')
try:
build_dir_link = relpath(build_dir, rst_dir).replace(os.path.sep, '/')
except ValueError:
# on Windows, relpath raises ValueError when path and start are on
# different mounts/drives
build_dir_link = build_dir
source_link = dest_dir_link + '/' + output_base + source_ext
# make figures
try:
results = render_figures(code,
source_file_name,
build_dir,
output_base,
keep_context,
function_name,
config,
context_reset=context_opt == 'reset',
close_figs=context_opt == 'close-figs')
errors = []
except PlotError as err:
reporter = state.memo.reporter
sm = reporter.system_message(
2,
"Exception occurred in plotting {}\n from {}:\n{}".format(
output_base, source_file_name, err),
line=lineno)
results = [(code, [])]
errors = [sm]
# Properly indent the caption
caption = '\n'.join(' ' + line.strip()
for line in caption.split('\n'))
# generate output restructuredtext
total_lines = []
for j, (code_piece, images) in enumerate(results):
if options['include-source']:
if is_doctest:
lines = ['', *code_piece.splitlines()]
else:
lines = [
'.. code-block:: python', '',
*textwrap.indent(code_piece, ' ').splitlines()
]
source_code = "\n".join(lines)
else:
source_code = ""
if nofigs:
images = []
opts = [
':%s: %s' % (key, val) for key, val in options.items()
if key in ('alt', 'height', 'width', 'scale', 'align', 'class')
]
only_html = ".. only:: html"
only_latex = ".. only:: latex"
only_texinfo = ".. only:: texinfo"
# Not-None src_link signals the need for a source link in the generated
# html
if j == 0 and config.plot_html_show_source_link:
src_link = source_link
else:
src_link = None
result = jinja2.Template(config.plot_template or TEMPLATE).render(
default_fmt=default_fmt,
dest_dir=dest_dir_link,
build_dir=build_dir_link,
source_link=src_link,
multi_image=len(images) > 1,
only_html=only_html,
only_latex=only_latex,
only_texinfo=only_texinfo,
options=opts,
images=images,
source_code=source_code,
html_show_formats=config.plot_html_show_formats and len(images),
caption=caption)
total_lines.extend(result.split("\n"))
total_lines.extend("\n")
if total_lines:
state_machine.insert_input(total_lines, source=source_file_name)
# copy image files to builder's output directory, if necessary
Path(dest_dir).mkdir(parents=True, exist_ok=True)
for code_piece, images in results:
for img in images:
for fn in img.filenames():
destimg = os.path.join(dest_dir, os.path.basename(fn))
if fn != destimg:
shutil.copyfile(fn, destimg)
# copy script (if necessary)
Path(dest_dir, output_base + source_ext).write_text(
unescape_doctest(code) if source_file_name == rst_file else code,
encoding='utf-8')
return errors
def _render_jinja(u):
return jinja2.Template(u).render(**self.templates)
def load_template(self):
with open('event.html') as f:
self.body_template = jinja2.Template(f.read())
self.subject_template = jinja2.Template(
"[污染推播 {{ risk }}風險 #{{ uid }}] {{ area }} {{ start_time }} 時發生歷時 "
"{{ duration }}分鐘事件 - 最高 iot 數值: {{ max_value }}")
def main(sample=False,
dry_run=True,
limit=1,
no_limit=False,
database_filename=DATABASE_FILENAME_DEFAULT,
template_filename=TEMPLATE_FILENAME_DEFAULT,
config_filename=CONFIG_FILENAME_DEFAULT):
"""Python API for mailmerge.
mailmerge 0.1 by Andrew DeOrio <*****@*****.**>.
A simple, command line mail merge tool.
Render an email template for each line in a CSV database.
"""
# pylint: disable=too-many-arguments,too-many-locals,too-many-branches
# pylint: disable=too-many-statements
# NOTE: this function needs a refactor, then remove ^^^
# Create a sample email template and database if there isn't one already
if sample:
create_sample_input_files(
template_filename,
database_filename,
config_filename,
)
sys.exit(0)
if not os.path.exists(template_filename):
print("Error: can't find template email " + template_filename)
print("Create a sample (--sample) or specify a file (--template)")
sys.exit(1)
if not os.path.exists(database_filename):
print("Error: can't find database_filename " + database_filename)
print("Create a sample (--sample) or specify a file (--database)")
sys.exit(1)
try:
# Read template
with io.open(template_filename, "r") as template_file:
content = template_file.read() + u"\n"
template = jinja2.Template(content)
# Read CSV file database
database = []
with io.open(database_filename, "r") as database_file:
reader = csv.DictReader(database_file)
for row in reader:
database.append(row)
# Each row corresponds to one email message
for i, row in enumerate(database):
if not no_limit and i >= limit:
break
print(">>> message {}".format(i))
# Fill in template fields using fields from row of CSV file
raw_message = template.render(**row)
print(raw_message)
# Parse message headers and detect encoding
(text, sender, recipients) = parsemail(raw_message)
# Send message
if dry_run:
print(">>> sent message {} DRY RUN".format(i))
else:
# Send message
try:
sendmail(text, sender, recipients, config_filename)
except smtplib.SMTPException as err:
print(">>> failed to send message {}".format(i))
timestamp = '{:%Y-%m-%d %H:%M:%S}'.format(
datetime.datetime.now()
)
print(timestamp, i, err, sep=' ', file=sys.stderr)
else:
print(">>> sent message {}".format(i))
# Hints for user
if not no_limit:
print(">>> Limit was {} messages. ".format(limit) +
"To remove the limit, use the --no-limit option.")
if dry_run:
print((">>> This was a dry run. "
"To send messages, use the --no-dry-run option."))
except jinja2.exceptions.TemplateError as err:
print(">>> Error in Jinja2 template: {}".format(err))
sys.exit(1)
except csv.Error as err:
print(">>> Error reading CSV file: {}".format(err))
sys.exit(1)
except smtplib.SMTPAuthenticationError as err:
print(">>> Authentication error: {}".format(err))
sys.exit(1)
except configparser.Error as err:
print(">>> Error reading config file {}: {}".format(
config_filename, err))
sys.exit(1)
import generic.model.nonlinearity
import gpu.model.layer
import jinja2
_nonlinearity_module_template = jinja2.Template("""
__global__ void fprop_kernel(float* X, float* out, int N)
{
const int index = blockIdx.x * blockDim.x + threadIdx.x;
if (index < N) {
out[index] = {{ elementwise_fprop }};
}
}
__global__ void bprop_kernel(float* delta, float* Y, float* out, int N)
{
const int index = blockIdx.x * blockDim.x + threadIdx.x;
if (index < N) {
out[index] = {{ elementwise_bprop }};
}
}
""")
class ElementwiseNonlinearity(gpu.model.layer.Layer):
block_size = 512
def __init__(self, *args, **kwargs):
def main():
parser = argparse.ArgumentParser()
parser.add_argument('object',
help='The Swordfish/Redfish schema object to process.')
parser.add_argument('-t',
'--type',
default='swordfish',
const='swordfish',
nargs='?',
choices=['redfish', 'swordfish'],
help='Define the object type and go package')
parser.add_argument('-o',
'--output-file',
help='File to write results to. Default is to stdout.')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='Emit verbose output to help debug.')
args = parser.parse_args()
if args.type == 'redfish':
url = '%s%s.json' % (REDFISH_SCHEMA_BASE, args.object)
elif args.type == 'swordfish':
url = '%s%s.json' % (SWORDFISH_SCHEMA_BASE, args.object)
else:
raise NameError("Unknown schema type")
LOG.debug(url)
data = requests.get(url)
try:
base_data = data.json()
except Exception:
LOG.exception('Error with data:\n%s' % data)
return
for classdef in base_data.get('definitions', []):
if classdef == args.object:
refs = base_data['definitions'][classdef].get('anyOf', [])
for ref in refs:
reflink = ref.get('$ref', '')
if 'idRef' in reflink:
continue
refurl = reflink.split('#')[0]
if refurl > url:
url = refurl
break
object_data = requests.get(url).json()
params = {
'object_name': args.object,
'classes': [],
'enums': [],
'package': args.type
}
for name in object_data['definitions']:
if name == 'Actions':
continue
definition = object_data['definitions'][name]
if definition.get('type') == 'object':
properties = definition.get('properties', '')
if not ('target' in properties and 'title' in properties):
_add_object(params, name, definition)
elif definition.get('enum'):
_add_enum(params, name, definition)
else:
LOG.debug('Skipping %s', definition)
with io.open('source.tmpl', 'r', encoding='utf-8') as f:
template_body = f.read()
template = jinja2.Template(template_body)
print(template.render(**params))
def send_token_email(self,
email,
user=None,
template=None,
action_sender=None,
action_uri=None,
subject=None,
override_access_lifespan=None,
custom_token=None,
sender='no-reply@praetorian'):
"""
Sends an email to a user, containing a time expiring
token usable for several actions. This requires
your application is initiliazed with a `mail` extension,
which supports Flask-Mail's `Message()` object and
a `send()` method.
Returns a dict containing the information sent, along with the
`result` from mail send.
:param: email: The email address to use
(username, id, email, etc)
:param: user: The user object to tie claim to
(username, id, email, etc)
:param: template: HTML Template for confirmation email.
If not provided, a stock entry is
used
:param: action_sender: The sender that should be attached
to the confirmation email.
:param: action_uri: The uri that should be visited to
complete the token action.
:param: subject: The email subject.
:param: override_access_lifespan: Overrides the JWT_ACCESS_LIFESPAN
to set an access lifespan for the
registration token.
"""
notification = {
'result': None,
'message': None,
'user': str(user),
'email': email,
'token': custom_token,
'subject': subject,
'confirmation_uri': action_uri, # backwards compatibility
'action_uri': action_uri,
}
PraetorianError.require_condition(
action_sender,
"A sender is required to send confirmation email",
)
PraetorianError.require_condition(
custom_token,
"A custom_token is required to send notification email",
)
if template is None:
with open(self.confirmation_template) as fh:
template = fh.read()
with PraetorianError.handle_errors('fail sending email'):
flask.current_app.logger.debug(
"NOTIFICATION: {}".format(notification))
jinja_tmpl = jinja2.Template(template)
notification['message'] = jinja_tmpl.render(notification).strip()
msg = Message(html=notification['message'],
sender=action_sender,
subject=notification['subject'],
recipients=[notification['email']])
flask.current_app.logger.debug("Sending email to {}".format(email))
notification['result'] = flask.current_app.extensions['mail'].send(
msg)
return notification
__TEMPLATE__ = jinja2.Template(
"""# NOTE: Generated By HttpRunner v{{ version }}
# FROM: {{ testcase_path }}
{% if imports_list and diff_levels > 0 %}
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__){% for _ in range(diff_levels) %}.parent{% endfor %}))
{% endif %}
{% if parameters %}
import pytest
from rrtv_httprunner import Parameters
{% endif %}
from rrtv_httprunner import HttpRunner, Config, Step, RunRequest, RunTestCase
{% for import_str in imports_list %}
{{ import_str }}
{% endfor %}
class {{ class_name }}(HttpRunner):
{% if parameters %}
@pytest.mark.parametrize("param", Parameters({{parameters}}))
def test_start(self, param):
super().test_start(param)
{% endif %}
config = {{ config_chain_style }}
teststeps = [
{% for step_chain_style in teststeps_chain_style %}
{{ step_chain_style }},
{% endfor %}
]
if __name__ == "__main__":
{{ class_name }}().test_start()
""")
def read_template_html_file(filename):
content = jinja2.Template(pathlib.Path(filename).read_text())
return content
from .components import DashboardComponent, ProfileTimePlot
from .core import BokehServer
from .utils import transpose
from ..compatibility import WINDOWS
from ..diagnostics.progress_stream import color_of
from ..metrics import time
from ..utils import log_errors, key_split, format_bytes, format_time
logger = logging.getLogger(__name__)
import jinja2
with open(os.path.join(os.path.dirname(__file__), 'template.html')) as f:
template_source = f.read()
template = jinja2.Template(template_source)
template_variables = {
'pages': ['main', 'system', 'profile', 'crossfilter', 'counters']
}
class StateTable(DashboardComponent):
""" Currently running tasks """
def __init__(self, worker):
self.worker = worker
names = [
'Stored', 'Executing', 'Ready', 'Waiting', 'Connections', 'Serving'
]
self.source = ColumnDataSource({name: [] for name in names})
def Request(self):
"""Create the Approval object and notify the Approval Granter."""
approval_id = "approval:%X" % utils.PRNG.GetULong()
approval_urn = self.BuildApprovalUrn(approval_id)
subject_title = self.BuildSubjectTitle()
email_msg_id = email.utils.make_msgid()
with aff4.FACTORY.Create(approval_urn,
self.approval_type,
mode="w",
token=self.token) as approval_request:
approval_request.Set(
approval_request.Schema.SUBJECT(self.subject_urn))
approval_request.Set(
approval_request.Schema.REQUESTOR(self.token.username))
approval_request.Set(approval_request.Schema.REASON(self.reason))
approval_request.Set(
approval_request.Schema.EMAIL_MSG_ID(email_msg_id))
cc_addresses = (self.email_cc_address,
config.CONFIG.Get("Email.approval_cc_address"))
email_cc = ",".join(filter(None, cc_addresses))
# When we reply with the approval we want to cc all the people to whom the
# original approval was sent, to avoid people approving stuff that was
# already approved.
if email_cc:
reply_cc = ",".join((self.approver, email_cc))
else:
reply_cc = self.approver
approval_request.Set(approval_request.Schema.EMAIL_CC(reply_cc))
approval_request.Set(
approval_request.Schema.NOTIFIED_USERS(self.approver))
# We add ourselves as an approver as well (The requirement is that we have
# 2 approvers, so the requester is automatically an approver).
approval_request.AddAttribute(
approval_request.Schema.APPROVER(self.token.username))
approval_link_urns = self.BuildApprovalSymlinksUrns(approval_id)
for link_urn in approval_link_urns:
with aff4.FACTORY.Create(link_urn,
aff4.AFF4Symlink,
mode="w",
token=self.token) as link:
link.Set(link.Schema.SYMLINK_TARGET(approval_urn))
# Notify to the users.
for user in self.approver.split(","):
user = user.strip()
try:
fd = aff4.FACTORY.Open(aff4.ROOT_URN.Add("users").Add(user),
aff4_type=aff4_users.GRRUser,
mode="rw",
token=self.token)
except aff4.InstantiationError:
continue
fd.Notify("GrantAccess", approval_urn,
"Please grant access to %s" % subject_title, "")
fd.Close()
if not config.CONFIG.Get("Email.send_approval_emails"):
return approval_urn
subject_template = jinja2.Template(
"Approval for {{ user }} to access {{ client }}.", autoescape=True)
subject = subject_template.render(
user=utils.SmartUnicode(self.token.username),
client=utils.SmartUnicode(subject_title))
template = jinja2.Template("""
<html><body><h1>Approval to access
<a href='{{ admin_ui }}/#/{{ approval_url }}'>{{ subject_title }}</a>
requested.</h1>
The user "{{ username }}" has requested access to
<a href='{{ admin_ui }}/#/{{ approval_url }}'>{{ subject_title }}</a>
for the purpose of <em>{{ reason }}</em>.
Please click <a href='{{ admin_ui }}/#/{{ approval_url }}'>
here
</a> to review this request and then grant access.
<p>Thanks,</p>
<p>{{ signature }}</p>
<p>{{ image|safe }}</p>
</body></html>""",
autoescape=True)
body = template.render(
username=utils.SmartUnicode(self.token.username),
reason=utils.SmartUnicode(self.reason),
admin_ui=utils.SmartUnicode(config.CONFIG["AdminUI.url"]),
subject_title=utils.SmartUnicode(subject_title),
approval_url=utils.SmartUnicode(
self.BuildApprovalReviewUrlPath(approval_id)),
# If you feel like it, add a funny cat picture here :)
image=utils.SmartUnicode(
config.CONFIG["Email.approval_signature"]),
signature=utils.SmartUnicode(config.CONFIG["Email.signature"]))
email_alerts.EMAIL_ALERTER.SendEmail(self.approver,
utils.SmartStr(
self.token.username),
utils.SmartStr(subject),
utils.SmartStr(body),
is_html=True,
cc_addresses=email_cc,
message_id=email_msg_id)
return approval_urn
from __future__ import print_function, unicode_literals
import jinja2
crypto_vars = {
"crypto_encr": "aes",
"policy_group": "5",
"isakmp_enable": False,
}
crypto_template = '''
crypto isakmp policy 10
encr {{crypto_encr}}
authentication pre-share
group {{policy_group}}
{% if isakmp_enable -%}
crypto isakmp key my_key address 1.1.1.1 no-xauth
crypto isakmp keepalive 10 periodic
{% endif %}
'''
t = jinja2.Template(crypto_template)
print(t.render(crypto_vars))
