def setUp(self):
test_date_1 = datetime.datetime(2020, 12, 31)
test_date_2 = datetime.datetime(2015, 7, 6)
test_date_3 = datetime.datetime(2010, 5, 4)
self.ascii_data = [
[1, test_date_1, 1.0, "test ' abc"],
[74321, test_date_2, 3.0, "abc\n this is a really string"*100],
[4, test_date_3, 4.0, "asdf \""],
]
self.unicode_data = [
[1, test_date_1, 1.0, "\u265d"],
[74321, test_date_2, 3.0, "\u265c"],
[4, test_date_3, 4.0, "\u2704"],
[5, test_date_1, 5.0, "\u2704"],
]
self.unicode_with_none_data = [
[None, test_date_1, 1.0, "\u265d"],
[74321, None, 3.0, "\u265c"],
[4, test_date_3, None, "\u2704"],
[5, test_date_1, 5.0, None]
]
self.ascii_table = table3.ListTable(
columnTypes=[int, datetime.datetime, float, str],
colnames=["a1", "a2", "a3", "a4"],
data=self.ascii_data
)
self.unicode_table = table3.ListTable(
columnTypes=[int, datetime.datetime, float, str],
colnames=["a1\u26f1", "a2\u26fd", "a3", "a4"],
data=self.unicode_data
)
self.unicode_with_none_table = table3.ListTable(
columnTypes=[int, datetime.datetime, float, str],
colnames=["a1\u26f1", "a2\u26fd", "a3", "a4"],
data=self.unicode_with_none_data
)
def test_date_hack(self):
data = [[datetime.datetime(2020, 9, 8, 7, 6, 5).isoformat()]]
table = table3.ListTable(columnTypes=[str], colnames=["date"], data=data)
file = table2spss.table2sav(table)
pspp = subprocess.Popen(
["pspp", "-b"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
input = self.get_pspp_command(file)
stdout, stderr = pspp.communicate(input=input)
self.assertIn(b"07-SEP-2020 16:53:55", stdout)
def get_table(self):
probs = collections.defaultdict(lambda : collections.defaultdict(dict))
for (i, q, id, s) in self.get_probs():
probs[i][q.label][id] = s
assocTable = table3.ListTable(colnames=["Interval", "From", "To", "Association"])
for i in sorted(probs):
for q in probs[i]:
sumprob1 = float(sum(probs[i][q].values()))
if sumprob1 == 0: continue
for q2 in probs[i]:
if q == q2: continue
sumproduct = 0
for id, p1 in probs[i][q].iteritems():
p2 = probs[i][q2].get(id)
if not p2: continue
sumproduct += p1*p2
p = sumproduct / sumprob1
assocTable.addRow(i, q, q2, p)
return assocTable
def getTable(table, colnames=None):
if isinstance(table, (list, tuple)):
from amcat.tools.table import table3
table = table3.ListTable(table, colnames)
return table
def run(self):
es = amcates.ES()
filters = dict(keywordsearch.filters_from_form(self.data))
queries = list(keywordsearch.queries_from_form(self.data))
qargs = dict(filters=filters, score=True, fields=[])
probs = {}
p = lambda f: 1 - (.5 ** f)
probs = {q.label : {r.id : p(r.score) for r in
es.query_all(query=q.query, **qargs)}
for q in queries}
assocTable = table3.ListTable(colnames=["From", "To", "Association"])
for q in probs:
sumprob1 = float(sum(probs[q].values()))
if sumprob1 == 0: continue
for q2 in probs:
if q == q2: continue
sumproduct = 0
for id, p1 in probs[q].iteritems():
p2 = probs[q2].get(id)
if not p2: continue
sumproduct += p1*p2
p = sumproduct / sumprob1
assocTable.addRow(q, q2, p)
if self.options['network_output'] == 'ool':
self.output = 'json-html'
assocTable = table3.WrappedTable(assocTable, cellfunc = lambda a: self.format(a) if isinstance(a, float) else a)
return self.outputResponse(assocTable, AssociationsScript.output_type)
elif self.options['network_output'] == 'oo':
# convert list to dict and make into dict table
result = table3.DictTable()
result.rowNamesRequired=True
for x,y,a in assocTable:
result.addValue(x,y,self.format(a))
self.output = 'json-html'
return self.outputResponse(result, AssociationsScript.output_type)
elif self.options['network_output'] == 'oon':
g = dot.Graph()
threshold = self.options.get('graph_threshold')
if not threshold: threshold = 0
nodes = {}
def getnode(x):
if not x in nodes:
id = "node_%i_%s" % (len(nodes), re.sub("\W","",x))
nodes[x] = dot.Node(id, x)
return nodes[x]
for x,y,a in assocTable:
if threshold and a < threshold:
continue
opts = {}
if self.options['graph_label']: opts['label'] = self.format(a)
w = 1 + 10 * a
g.addEdge(getnode(x),getnode(y), weight=w, **opts)
html = g.getHTMLObject()
self.output = 'json-html'
return self.outputResponse(html, unicode)