def main():
graph = utils.load_graph()
position = utils.get_positions(graph)
utils.make_dir('images/louvain')
true_communities = utils.get_labels(graph, list(graph.nodes))
utils.plot_communities(graph, position, true_communities, labels=True, title='Butterfly Similarity Network - True Communities', path='images/louvain/communities_true.png')
communities = utils.group_communities(louvain_clustering(graph))
utils.plot_communities(graph, position, communities, labels=False, title='Butterfly Similarity Network - Louvain Communities', path='images/louvain/communities_louvain.png')
graph_nodes = sorted(list(graph.nodes))
predictions = utils.predict_majority_class(graph, communities)
preds = [predictions[n] for n in graph_nodes]
labels = [graph.nodes[n]['label'] for n in graph_nodes]
utils.accuracy(preds, labels)
utils.confusion_matrix(preds, labels, 'Confusion Matrix - Majority Label Predictions from Louvain Communities', 'images/louvain/cm_louvain.png')
def main():
graph = utils.load_graph()
position = utils.get_positions(graph)
utils.make_dir('images/spectral')
true_communities = utils.get_labels(graph, list(graph.nodes))
utils.plot_communities(graph, position, true_communities, labels=True, title='Butterfly Similarity Network - True Communities', path='images/spectral/communities_true.png')
node_assignments = spectral_clustering(graph)
nodes_to_communities = {k:v for (k,v) in zip(range(len(node_assignments)), node_assignments)}
communities = utils.group_communities(nodes_to_communities)
utils.plot_communities(graph, position, communities, labels=False, title='Butterfly Similarity Network - Spectral Communities', path='images/spectral/communities_spectral.png')
graph_nodes = sorted(list(graph.nodes))
predictions = utils.predict_majority_class(graph, communities)
preds = [predictions[n] for n in graph_nodes]
labels = [graph.nodes[n]['label'] for n in graph_nodes]
utils.accuracy(preds, labels)
utils.confusion_matrix(preds, labels, 'Confusion Matrix - Spectral Clustering', 'images/spectral/cm_spectral.png')
def predict_triggers(self, tokens_2d, entities_3d, postags_2d, seqlen_1d,
adjm):
tokens = torch.LongTensor(tokens_2d).to(self.device)
postags_2d = torch.LongTensor(postags_2d).to(self.device)
word_emb = self.word_embed(tokens)
pos_emb = self.postag_embed(postags_2d)
ent_emb = self.entity_embed(entities_3d)
BATCH_SIZE, SEQ_LEN = tokens.shape[:]
mask = torch.zeros(BATCH_SIZE, SEQ_LEN)
for i in range(BATCH_SIZE):
s_len = seqlen_1d[i]
mask[i, 0:s_len] = 1
mask = torch.FloatTensor(mask).to(self.device)
adj = torch.stack([
torch.sparse.FloatTensor(torch.LongTensor(adjmm[0]),
torch.FloatTensor(adjmm[1]),
torch.Size([3, SEQ_LEN,
SEQ_LEN])).to_dense()
for adjmm in adjm
])
adj = adj.to(self.device)
x_emb = torch.cat([word_emb, pos_emb, ent_emb], 2)
positional_sequences = get_positions(BATCH_SIZE, SEQ_LEN)
xx = []
for i in range(SEQ_LEN):
# encoding
posi_emb = self.position_embed(positional_sequences[i].to(
self.device))
lstm_input = torch.cat([x_emb, posi_emb], dim=2)
x, _ = self.rnn(lstm_input) # (batch_size, seq_len, d')
# gcns
gcn_o = self.mogcn(adj, x) #[batch_size, SEQ_LEN, d]
xx.append(gcn_o[:, i, :]) # (batch_size, d')
# output linear
xx = torch.stack(xx, dim=1) # (batch_size, seq_len, d')
xx_m = torch.mul(xx, mask.unsqueeze(2))
trigger_logits = self.fc_trigger(xx_m)
trigger_hat_2d = trigger_logits.argmax(-1)
return trigger_logits, trigger_hat_2d
import pandas as pd
data = load('./first_data_sample.csv')
starting_data = data.copy()
# MA for 100 days
data['100MA'] = get_ma(data)
# Bollinger's upper band
data['UpperBand'] = get_upper_band(data, 100, 2)
# Bollinger's lower band
data['LowerBand'] = get_lower_band(data, 100, 2)
data = get_positions(data)
# Plot start data
plot_data(data, 'Close', '100MA', 'UpperBand', 'LowerBand')
# Plot strategy data
plot_calculated_data(data, 'Close', '100MA', 'UpperBand',
'LowerBand', 'Without optimization')
# Plot profit without optimization
data['Trade'] = data['Close'] + data['TradeCost']
data['Return'] = data['Trade'].pct_change()
data['Strategy return'] = data['Return'] * data['Position']
plot_profit(data, 'Without optimization')
def find(data):
'''
takes {
query {
conditions [{}]
operator string
}
group string
keys [string]
sort [{}]
range [int, int]
}
'''
response = {}
q = query.parse(data)
if 'group' in q:
names = {}
groups = {}
key = 'group:' + hashlib.sha1(json.dumps(data).encode('utf-8')).hexdigest()
g = state.cache.get(key)
if g is None:
items = q['qs'].options(load_only('id'))
qs = models.Find.query.filter_by(key=q['group'])
if items.first():
qs = qs.filter(models.Find.item_id.in_(items))
for f in qs.values('value', 'findvalue'):
value = f[0]
findvalue = f[1]
if findvalue not in groups:
groups[findvalue] = 0
groups[findvalue] += 1
names[findvalue] = value
g = [{'name': names[k], 'items': groups[k]} for k in groups]
else:
g = []
if 'sort' in q:
sort_type = utils.get_by_id(settings.config['itemKeys'], q['group']).get('sortType')
def _sort_key(k):
if sort_type == 'person' and q['sort'][0]['key'] == 'name':
v = get_sort_name(k[q['sort'][0]['key']])
else:
v = k[q['sort'][0]['key']]
if isinstance(v, str):
v = unicodedata.normalize('NFKD', v).lower()
return v
g.sort(key=_sort_key, reverse=q['sort'][0]['operator'] == '-')
state.cache.set(key, g)
if 'positions' in data:
response['positions'] = {}
ids = [k['name'] for k in g]
response['positions'] = utils.get_positions(ids, data['positions'])
elif 'range' in data:
response['items'] = g[q['range'][0]:q['range'][1]]
else:
response['items'] = len(g)
elif 'position' in data:
ids = [i.id for i in q['qs'].options(load_only('id'))]
response['position'] = utils.get_positions(ids, [data['qs'][0].id])[0]
elif 'positions' in data:
ids = [i.id for i in q['qs'].options(load_only('id'))]
response['positions'] = utils.get_positions(ids, data['positions'])
elif 'keys' in data:
response['items'] = [
i.json(data['keys']) for i in q['qs'][q['range'][0]:q['range'][1]]
]
else:
size = [i.info.get('size', 0) for i in q['qs'].options(load_only('id', 'info'))]
response['items'] = len(size)
response['size'] = sum(size)
return response
def find(request):
'''
Example:
find({
query:{
conditions:[{ key: '*', value: 'paris', operator: '='}],
operator:'&'
},
keys: ['title', 'id'],
range: [0, 10],
sort: [{key: 'title', operator: '+'}]
})
takes {
'query': query,
'sort': array,
'range': array
clipsQuery: ...
}
query: query object, more on query syntax at
https://wiki.0x2620.org/wiki/pandora/QuerySyntax
sort: array of key, operator dics
[
{
key: "year",
operator: "-"
},
{
key: "director",
operator: ""
}
]
range: result range, array [from, to]
keys: array of keys to return
group: group elements by, country, genre, director...
with keys, items is list of dicts with requested properties:
returns {
items: [objects]
}
Groups
takes {
'query': query,
'key': string,
'group': string,
'range': array
clips: {}
}
query: query object, more on query syntax at
https://wiki.0x2620.org/wiki/pandora/QuerySyntax
range: result range, array [from, to]
keys: array of keys to return
group: group elements by, country, genre, director...
possible values for keys: name, items
with keys
items contains list of {'name': string, 'items': int}:
returns {
items: [objects]
}
without keys: return number of items in given query
returns {
items: int
}
Positions
takes {
'query': query,
'positions': [],
'sort': array
}
query: query object, more on query syntax at
https://wiki.0x2620.org/wiki/pandora/QuerySyntax
positions: ids of items for which positions are required
returns {
positions: {
id: position
}
}
'''
data = json.loads(request.POST['data'])
if settings.JSON_DEBUG:
print json.dumps(data, indent=2)
query = parse_query(data, request.user)
response = json_response({})
if 'group' in query:
response['data']['items'] = []
items = 'items'
item_qs = query['qs']
order_by = _order_by_group(query)
qs = models.Facet.objects.filter(key=query['group']).filter(
item__id__in=item_qs)
qs = qs.values('value').annotate(items=Count('id')).order_by(*order_by)
if 'positions' in query:
response['data']['positions'] = {}
ids = [j['value'] for j in qs]
response['data']['positions'] = utils.get_positions(
ids, query['positions'])
elif 'range' in data:
qs = qs[query['range'][0]:query['range'][1]]
response['data']['items'] = [{
'name': i['value'],
'items': i[items]
} for i in qs]
else:
response['data']['items'] = qs.count()
elif 'position' in query:
qs = _order_query(query['qs'], query['sort'])
ids = [j['itemId'] for j in qs.values('itemId')]
data['conditions'] = data['conditions'] + {
'value': query['position'],
'key': query['sort'][0]['key'],
'operator': '^'
}
query = parse_query(data, request.user)
qs = _order_query(query['qs'], query['sort'])
if qs.count() > 0:
response['data']['position'] = utils.get_positions(
ids, [qs[0].itemId])[0]
elif 'positions' in query:
qs = _order_query(query['qs'], query['sort'])
ids = [j['itemId'] for j in qs.values('itemId')]
response['data']['positions'] = utils.get_positions(
ids, query['positions'])
elif 'keys' in query:
response['data']['items'] = []
qs = _order_query(query['qs'], query['sort'])
_p = query['keys']
def get_clips(qs):
n = qs.count()
if n > query['clip_items']:
num = query['clip_items']
clips = []
step = int(n / (num + 1))
i = step
while i <= (n - step) and i < n and len(clips) < num:
clips.append(qs[i])
i += step
else:
clips = qs
return [
c.json(query['clip_keys'], query['clip_filter']) for c in clips
]
def only_p_sums(m):
r = {}
for p in _p:
if p == 'accessed':
r[p] = m.sort.accessed or ''
elif p == 'modified':
r[p] = m.sort.modified
elif p == 'timesaccessed':
r[p] = m.sort.timesaccessed
else:
r[p] = m.json.get(p, '')
if 'clip_qs' in query:
r['clips'] = get_clips(query['clip_qs'].filter(item=m))
return r
def only_p(m):
r = {}
if m:
m = json.loads(m, object_hook=ox.django.fields.from_json)
for p in _p:
r[p] = m.get(p, '')
if 'clip_qs' in query:
r['clips'] = get_clips(
query['clip_qs'].filter(item__itemId=m['id']))
return r
qs = qs[query['range'][0]:query['range'][1]]
#response['data']['items'] = [m.get_json(_p) for m in qs]
if filter(
lambda p: p in
('accessed', 'modified', 'timesaccessed', 'viewed'), _p):
qs = qs.select_related()
response['data']['items'] = [only_p_sums(m) for m in qs]
else:
response['data']['items'] = [
only_p(m['json']) for m in qs.values('json')
]
else: # otherwise stats
items = query['qs']
files = File.objects.filter(item__in=items).filter(size__gt=0)
r = files.aggregate(Sum('duration'), Sum('pixels'), Sum('size'))
totals = [i['id'] for i in settings.CONFIG['totals']]
if 'duration' in totals:
response['data']['duration'] = r['duration__sum']
if 'files' in totals:
response['data']['files'] = files.count()
if 'items' in totals:
response['data']['items'] = items.count()
if 'pixels' in totals:
response['data']['pixels'] = r['pixels__sum']
if 'runtime' in totals:
response['data']['runtime'] = items.aggregate(
Sum('sort__runtime'))['sort__runtime__sum'] or 0
if 'size' in totals:
response['data']['size'] = r['size__sum']
for key in ('runtime', 'duration', 'pixels', 'size'):
if key in totals and response['data'][key] == None:
response['data'][key] = 0
return render_to_json_response(response)
def find(request):
'''
Example:
find({
query:{
conditions:[{ key: '*', value: 'paris', operator: '='}],
operator:'&'
},
keys: ['title', 'id'],
range: [0, 10],
sort: [{key: 'title', operator: '+'}]
})
takes {
'query': query,
'sort': array,
'range': array
clipsQuery: ...
}
query: query object, more on query syntax at
https://wiki.0x2620.org/wiki/pandora/QuerySyntax
sort: array of key, operator dics
[
{
key: "year",
operator: "-"
},
{
key: "director",
operator: ""
}
]
range: result range, array [from, to]
keys: array of keys to return
group: group elements by, country, genre, director...
with keys, items is list of dicts with requested properties:
returns {
items: [objects]
}
Groups
takes {
'query': query,
'key': string,
'group': string,
'range': array
clips: {}
}
query: query object, more on query syntax at
https://wiki.0x2620.org/wiki/pandora/QuerySyntax
range: result range, array [from, to]
keys: array of keys to return
group: group elements by, country, genre, director...
possible values for keys: name, items
with keys
items contains list of {'name': string, 'items': int}:
returns {
items: [objects]
}
without keys: return number of items in given query
returns {
items: int
}
Positions
takes {
'query': query,
'positions': [],
'sort': array
}
query: query object, more on query syntax at
https://wiki.0x2620.org/wiki/pandora/QuerySyntax
positions: ids of items for which positions are required
returns {
positions: {
id: position
}
}
'''
data = json.loads(request.POST['data'])
if settings.JSON_DEBUG:
print json.dumps(data, indent=2)
query = parse_query(data, request.user)
response = json_response({})
if 'group' in query:
response['data']['items'] = []
items = 'items'
item_qs = query['qs']
order_by = _order_by_group(query)
qs = models.Facet.objects.filter(key=query['group']).filter(item__id__in=item_qs)
qs = qs.values('value').annotate(items=Count('id')).order_by(*order_by)
if 'positions' in query:
response['data']['positions'] = {}
ids = [j['value'] for j in qs]
response['data']['positions'] = utils.get_positions(ids, query['positions'])
elif 'range' in data:
qs = qs[query['range'][0]:query['range'][1]]
response['data']['items'] = [{'name': i['value'], 'items': i[items]} for i in qs]
else:
response['data']['items'] = qs.count()
elif 'position' in query:
qs = _order_query(query['qs'], query['sort'])
ids = [j['itemId'] for j in qs.values('itemId')]
data['conditions'] = data['conditions'] + {
'value': query['position'],
'key': query['sort'][0]['key'],
'operator': '^'
}
query = parse_query(data, request.user)
qs = _order_query(query['qs'], query['sort'])
if qs.count() > 0:
response['data']['position'] = utils.get_positions(ids, [qs[0].itemId])[0]
elif 'positions' in query:
qs = _order_query(query['qs'], query['sort'])
ids = [j['itemId'] for j in qs.values('itemId')]
response['data']['positions'] = utils.get_positions(ids, query['positions'])
elif 'keys' in query:
response['data']['items'] = []
qs = _order_query(query['qs'], query['sort'])
_p = query['keys']
def get_clips(qs):
n = qs.count()
if n > query['clip_items']:
num = query['clip_items']
clips = []
step = int(n / (num + 1))
i = step
while i <= (n - step) and i < n and len(clips) < num:
clips.append(qs[i])
i += step
else:
clips = qs
return [c.json(query['clip_keys'], query['clip_filter']) for c in clips]
def only_p_sums(m):
r = {}
for p in _p:
if p == 'accessed':
r[p] = m.sort.accessed or ''
elif p == 'modified':
r[p] = m.sort.modified
elif p == 'timesaccessed':
r[p] = m.sort.timesaccessed
else:
r[p] = m.json.get(p, '')
if 'clip_qs' in query:
r['clips'] = get_clips(query['clip_qs'].filter(item=m))
return r
def only_p(m):
r = {}
if m:
m = json.loads(m, object_hook=ox.django.fields.from_json)
for p in _p:
r[p] = m.get(p, '')
if 'clip_qs' in query:
r['clips'] = get_clips(query['clip_qs'].filter(item__itemId=m['id']))
return r
qs = qs[query['range'][0]:query['range'][1]]
#response['data']['items'] = [m.get_json(_p) for m in qs]
if filter(lambda p: p in (
'accessed', 'modified', 'timesaccessed', 'viewed'
), _p):
qs = qs.select_related()
response['data']['items'] = [only_p_sums(m) for m in qs]
else:
response['data']['items'] = [only_p(m['json']) for m in qs.values('json')]
else: # otherwise stats
items = query['qs']
files = File.objects.filter(item__in=items).filter(size__gt=0)
r = files.aggregate(
Sum('duration'),
Sum('pixels'),
Sum('size')
)
totals = [i['id'] for i in settings.CONFIG['totals']]
if 'duration' in totals:
response['data']['duration'] = r['duration__sum']
if 'files' in totals:
response['data']['files'] = files.count()
if 'items' in totals:
response['data']['items'] = items.count()
if 'pixels' in totals:
response['data']['pixels'] = r['pixels__sum']
if 'runtime' in totals:
response['data']['runtime'] = items.aggregate(Sum('sort__runtime'))['sort__runtime__sum'] or 0
if 'size' in totals:
response['data']['size'] = r['size__sum']
for key in ('runtime', 'duration', 'pixels', 'size'):
if key in totals and response['data'][key] == None:
response['data'][key] = 0
return render_to_json_response(response)
def find(data):
'''
takes {
query {
conditions [{}]
operator string
}
group string
keys [string]
sort [{}]
range [int, int]
}
'''
response = {}
q = query.parse(data)
if 'group' in q:
names = {}
groups = {}
key = 'group:' + hashlib.sha1(
json.dumps(data).encode('utf-8')).hexdigest()
g = state.cache.get(key)
if g is None:
items = q['qs'].options(load_only('id'))
qs = models.Find.query.filter_by(key=q['group'])
if items.first():
qs = qs.filter(models.Find.item_id.in_(items))
for f in qs.values('value', 'findvalue'):
value = f[0]
findvalue = f[1]
if findvalue not in groups:
groups[findvalue] = 0
groups[findvalue] += 1
names[findvalue] = value
g = [{'name': names[k], 'items': groups[k]} for k in groups]
else:
g = []
if 'sort' in q:
sort_type = utils.get_by_id(settings.config['itemKeys'],
q['group']).get('sortType')
def _sort_key(k):
if sort_type == 'person' and q['sort'][0]['key'] == 'name':
v = get_sort_name(k[q['sort'][0]['key']])
else:
v = k[q['sort'][0]['key']]
if isinstance(v, str):
v = unicodedata.normalize('NFKD', v).lower()
return v
g.sort(key=_sort_key, reverse=q['sort'][0]['operator'] == '-')
state.cache.set(key, g)
if 'positions' in data:
response['positions'] = {}
ids = [k['name'] for k in g]
response['positions'] = utils.get_positions(ids, data['positions'])
elif 'range' in data:
response['items'] = g[q['range'][0]:q['range'][1]]
else:
response['items'] = len(g)
elif 'position' in data:
ids = [i.id for i in q['qs'].options(load_only('id'))]
response['position'] = utils.get_positions(ids, [data['qs'][0].id])[0]
elif 'positions' in data:
ids = [i.id for i in q['qs'].options(load_only('id'))]
response['positions'] = utils.get_positions(ids, data['positions'])
elif 'keys' in data:
response['items'] = [
i.json(data['keys']) for i in q['qs'][q['range'][0]:q['range'][1]]
]
else:
size = [
i.info.get('size', 0)
for i in q['qs'].options(load_only('id', 'info'))
]
response['items'] = len(size)
response['size'] = sum(size)
return response