def get_latest_run_stats(self):
self.last_run_stats_refresh = datetime.datetime.now()
if not self.last_wf_run_id:
log(self.name + " > Getting latest run stats [full].")
fields, sql = sql_oracle.log_session_run_full
result = self.db.execute(
sql,
d(limit=999)
)
else:
log(self.name + " > Getting latest run stats [recent >= {s}].".format(
s=self.last_wf_run_id
)
)
fields, sql = sql_oracle.log_session_run_recent
result = self.db.execute(
sql,
d(last_wf_run_id=self.last_wf_run_id)
)
running_wf_run_id = None
folder_ids = set()
for row in result:
rec = get_rec(row, fields)
rec['start'] = rec['start'].strftime('%Y-%m-%d %H:%M:%S')
rec['success'] = 'No' if rec['error'] else 'Yes'
row_stats_str = ' S:{}/{} | T:{}/{} | E:{}'.format(
rec['src_success_rows'],
rec['src_failed_rows'],
rec['targ_success_rows'],
rec['targ_failed_rows'],
rec['total_trans_errs'],
)
rec['error'] = rec['error'] + ' --> ' + row_stats_str if rec['error'] else row_stats_str
try:
rec['end'] = rec['end'].strftime('%Y-%m-%d %H:%M:%S')
self.last_wf_run_id = rec['workflow_run_id'] if not self.last_wf_run_id or rec['workflow_run_id'] > self.last_wf_run_id else self.last_wf_run_id
except ValueError: # still in progress
rec['end'] = ''
rec['success'] = 'Running'
running_wf_run_id = rec['workflow_run_id'] if not running_wf_run_id or rec['workflow_run_id'] < running_wf_run_id else running_wf_run_id
rec['duration'] = float(rec['duration'])
rec['combo'] = str(rec.workflow_run_id) + '-' + str(rec.session_id)
self.run_stats_data[rec['combo']] = rec
if not rec['folder_id'] in self.folder_details:
folder_ids.add(rec['folder_id'])
self.folder_details[rec['folder_id']] = 'updating'
if len(folder_ids) > 0:
self.get_folder_details(list(folder_ids))
if running_wf_run_id:
self.last_wf_run_id = running_wf_run_id
def forward(self, x, mask):
#batch size, sequence length, embedding dimension
b, t, e = x.size()
h = self.heads
#each head inspects a fraction of the embedded space
#head dimension
s = e // h
#start index of position embedding
embedding_start = self.max_length - t
x = x.view(b, t, h, s)
queries, keys, values = [
w(x).transpose(1, 2) for w, x in zip(self.linears, (x, x, x))
]
if self.relative_pos:
#apply same position embeddings across the batch
#Is it possible to apply positional self-attention over
#only half of all relative distances?
Er = self.Er[:, embedding_start:, :].unsqueeze(0)
QEr = torch.matmul(queries, Er.transpose(-1, -2))
QEr = self._mask_positions(QEr)
#Get relative position attention scores
#combine batch with head dimension
SRel = self._skew(QEr).contiguous().view(b * h, t, t)
else:
SRel = torch.zeros([b * h, t, t], device=d())
queries, keys, values = map(lambda x: x.contiguous()\
.view(b*h, t, s), (queries, keys, values))
#Compute scaled dot-product self-attention
#scale pre-matrix multiplication
queries = queries / (e**(1 / 4))
keys = keys / (e**(1 / 4))
scores = torch.bmm(queries, keys.transpose(1, 2))
scores = scores + SRel
#(b*h, t, t)
subsequent_mask = torch.triu(torch.ones(1, t, t, device=d()), 1)
scores = scores.masked_fill(subsequent_mask == 1, -1e9)
if mask is not None:
mask = mask.repeat_interleave(h, 0)
wtf = (mask == 0).nonzero().transpose(0, 1)
scores[wtf[0], wtf[1], :] = -1e9
#Convert scores to probabilities
attn_probs = F.softmax(scores, dim=2)
attn_probs = self.dropout(attn_probs)
#use attention to get a weighted average of values
out = torch.bmm(attn_probs, values).view(b, h, t, s)
#transpose and recombine attention heads
out = out.transpose(1, 2).contiguous().view(b, t, s * h)
#last linear layer of weights
return self.recombine_heads(out)
def _mask_positions(self, qe):
# QEr is a matrix of queries (absolute position) dot distance embeddings (relative pos).
# Mask out invalid relative positions: e.g. if sequence length is L, the query at
# L-1 can only attend to distance r = 0 (no looking backward).
L = qe.shape[-1]
mask = torch.triu(torch.ones(L, L, device=d()), 1).flip(1)
return qe.masked_fill((mask == 1), 0)
def __init__(self,
n_tokens,
seq_length=None,
d_model=64,
n_heads=4,
depth=2,
d_feedforward=512,
dropout=0.1,
positional_encoding=False,
relative_pos=True,
xavier_init=False):
"""
Args:
n_tokens: number of commands/states in encoded musical sequence
seq_length: length of (padded) input/target sequences
d_model: dimensionality of embedded sequences
n_heads: number of attention heads
depth: number of stacked transformer layers
d_feedforward: dimensionality of dense sublayer
dropout: probability of dropout in dropout sublayer
relative_pos: (bool) if True, use relative positional embeddings
"""
super().__init__()
#number of commands in an encoded musical sequence
self.n_tokens = n_tokens
#embedding layer
self.d_model = d_model
self.embed = SequenceEmbedding(n_tokens, d_model)
#positional encoding layer
self.positional_encoding = positional_encoding
if self.positional_encoding:
pos = torch.zeros(5000, d_model)
position = torch.arange(5000).unsqueeze(1)
#geometric progression of wave lengths
div_term = torch.exp(torch.arange(0.0, d_model, 2) * \
- (math.log(10000.0) / d_model))
#even positions
pos[0:, 0::2] = torch.sin(position * div_term)
#odd positions
pos[0:, 1::2] = torch.cos(position * div_term)
#batch dimension
pos = pos.unsqueeze(0)
#move to GPU if needed
pos = pos.to(d())
self.register_buffer('pos', pos)
else:
if seq_length == None:
raise MusicTransformerError(
"seq_length not provided for positional embeddings")
self.pos = nn.Embedding(seq_length, d_model)
#last layer, outputs logits of next token in sequence
self.to_scores = nn.Linear(d_model, n_tokens)
self.layers = clones(
DecoderLayer(d_model, n_heads, d_feedforward, dropout,
relative_pos), depth)
self.norm = nn.LayerNorm(d_model)
if xavier_init:
self.initialize_weights()
def get_list_workflows(self):
"""
Obtain the list of workflows, ids in a folder.
"""
log(self.repo + " > Getting workflows for {0}.".format(self.name))
fields, sql = sql_oracle.list_workflow
result = self.db.execute(sql, d(folder_id=self.id))
for row in result:
rec = get_rec(row, fields)
self.workflows_id[rec.workflow_id] = self.workflows[rec.workflow_name] = Workflow(**rec)
fields, sql = sql_oracle.list_workflow_sessions
result = self.db.execute(sql, d(folder_id=self.id))
for row in result:
session = get_rec(row, fields)
if session.workflow_id in self.workflows_id:
self.workflows_id[session.workflow_id].session_ids.append(session.session_id)
def get_list_mappings(self):
"""
Obtain the list of mappings, ids in a folder.
"""
log(self.repo + " > Getting mappings for {0}.".format(self.name))
fields, sql = sql_oracle.list_mapping
result = self.db.execute(sql, d(folder_id=self.id))
data = [get_rec(row, fields) for row in result]
for rec in data:
self.mappings_id[rec.mapping_id] = self.mappings[rec.mapping_name] = rec
def get_list_fields(self):
"""
Obtain the list of fields for each transformation/source/target in all mappings in folder.
"""
log(self.repo + " > Getting transformation fields for {0}.".format(self.name))
fields, sql = sql_oracle.list_tranformation_fields
result = self.db.execute(sql, d(folder_id=self.id))
data_trans = [get_rec(row, fields) for row in result]
log(self.repo + " > Getting source/target fields for {0}.".format(self.name))
fields, sql = sql_oracle.list_source_target_fields
result = self.db.execute(sql, d(folder_id=self.id))
data_src_tgt = [get_rec(row, fields) for row in result]
if len(self.mappings_id) == 0: self.get_list_mappings()
if len(self.sources_id) == 0: self.get_list_sources()
if len(self.targets_id) == 0: self.get_list_targets()
for rec in data_trans:
rec['TYPE_'] = ' | '.join([
rec['datatype'],
'S:' + str(rec['scale']) if rec['scale'] else 'S:' + 'null',
'P:' + str(rec['precision']) if rec['precision'] else 'P:' + 'null',
'E:' + str(rec['expression']) if rec['expression'] else 'E:' + 'null',
])
self.mappings_transf_fields[rec.widget_field_id] = rec
for rec in data_src_tgt:
rec['TYPE_'] = ' | '.join([
rec['field_datatype'].replace('nvarchar2','varchar2').replace('nchar','char'),
'S:' + str(rec['field_scale']) if rec['field_scale'] else 'S:' + 'null',
'P:' + str(rec['field_precision']) if rec['field_precision'] else 'P:' + 'null',
'K:' + str(rec['field_key_type']) if rec['field_key_type'] else 'K:' + 'null',
'N:' + str(rec['field_nulltype']) if rec['field_nulltype'] else 'N:' + 'null',
])
if rec.type == 'SOURCE':
self.sources_fields[rec.field_id] = rec
else:
self.targets_fields[rec.field_id] = rec
def forward(self, x, mask=None):
x = self.embed(x) #(1024, 64)
b, t, e = x.size() # 1 64 1024
if self.positional_encoding:
positions = self.pos[:, :t, :]
else:
positions = self.pos(torch.arange(t,
device=d()))[None, :, :].expand(
b, t, e)
x = x + positions
for layer in self.layers:
x = layer(x, mask) #decoder
z = self.norm(x)
return self.to_scores(z)
def forward(self, x, mask=None):
x = self.embed(x)
b,t,e = x.size()
if self.positional_encoding:
positions = self.pos[:, :t, :]
else:
positions = self.pos(torch.arange(t,
device=d()))[None, :, :].expand(b, t, e)
x = x + positions
#another dropout layer here?
#pass input batch and mask through layers
for layer in self.layers:
x = layer(x, mask)
#one last normalization for good measure
z = self.norm(x)
return self.to_scores(z)
def __init__(self, d_model, heads=8, dropout=0.1, relative_pos=True):
super().__init__()
if d_model % heads != 0:
raise AttentionError("Number of heads does not divide model dimension")
self.d_model = d_model
self.heads = heads
s = d_model // heads
self.linears = torch.nn.ModuleList([nn.Linear(s, s, bias=False) for i in range(3)])
self.recombine_heads = nn.Linear(heads * s, d_model)
self.dropout = nn.Dropout(p=dropout)
self.max_length = 1024
#relative positional embeddings
self.relative_pos = relative_pos
if relative_pos:
self.Er = torch.randn([heads, self.max_length, s],
device=d())
else:
self.Er = None
def update_actions(state):
""" updates the available actions """
new_state = deepcopy(state)
actions = ["quit", "game"]
# starting and adding players
if len(state["players"]) >= 1 and not state["has_started"]:
actions.append("start")
if not state['has_started']:
actions.append('add_player')
# first roll of the game
if state['has_started']:
board = new_state['board']
player = new_state['current']['player']
prop = board[player["pos"]]
if state['current']['roll']['double_count'] < 3 and not state[
'current']['roll']['has_rolled']:
actions.append("roll")
# print("prop: ",prop)
# print("player: ", player)
# if state['current']['property']['is_for_sale'] and not d(state, 'current.property.owner'):
# if state['board'][['current']['player']['pos']]:
if not prop.get('owner') and not prop.get('special'):
actions.append("buy")
# if board['']
# if prop['']
# print(state['board'])
# if state['board
# actions.append("buy")
#should always be at the end
if d(state, 'current.roll.has_rolled'):
actions.append('end_turn')
new_state['actions'] = actions
return new_state
def get_list_sessions(self):
"""
Obtain the list of sessions, ids in a folder.
"""
log(self.repo + " > Getting sessions for {0}.".format(self.name))
fields, sql = sql_oracle.list_session
result = self.db.execute(sql, d(folder_id=self.id))
for row in result:
rec = get_rec(row, fields)
self.sessions_id[rec.session_id] = self.sessions[rec.session_name] = Session(**rec)
# Obtain connections
for ids in split_list(self.sessions_id.keys(), 499):
fields, sql = sql_oracle.list_session_conns
# result = self.db.execute(sql, session_id=str(tuple(self.sessions_id.keys())))
result = self.db.execute(sql.format(session_id=str(tuple(ids))))
for row in result:
rec = get_rec(row, fields)
self.sessions_id[rec.session_id].add_connection(rec.connection_type, rec.connection_name)
def generate_workflow_report_1(self, output_path='/__/temp/wf_report.csv', append=False):
"Generate a workflow report."
headers = "FOLDER WORKFLOW_NAME SESSION_NAME MAPPING_NAME SOURCE_CONNECTION SOURCE TARGET_CONNECTION TARGET ".split()
WFRecord = namedtuple('WFRecord', headers)
if append:
out_file = open(output_path, 'a')
else:
out_file = open(output_path, 'w')
out_file.write(','.join(headers) + '\n')
for wf_name, workflow in self.workflows.items():
for session_id in workflow.session_ids:
session = self.sessions_id[session_id]
source_connections = '|'.join([
conn_type+'.'+conn_name for conn_type, conn_name in session.sources
])
target_connections = '|'.join([
conn_type+'.'+conn_name for conn_type, conn_name in session.targets
])
record = d(
FOLDER=self.name,
WORKFLOW_NAME=workflow.name,
SESSION_NAME= session.name,
MAPPING_NAME= self.mappings_id[session.mapping_id].name,
SOURCE_CONNECTION=source_connections,
SOURCE='',
TARGET_CONNECTION=target_connections,
TARGET='',
)
out_file.write(','.join(WFRecord(**record)) + '\n')
# print(','.join(WFRecord(**record)))
out_file.close()
def on_exit(self, event):
d("exit btn clicker")
self.myapp_frame.Close()
def send(cmd):
d("> %s" % cmd)
s.write((cmd + "\n").encode())
def parseLine(line):
if len(line) and line[0] == "@":
v = line[1:].split(": ")
set_default_value(v[0], int(v[1]))
d(line)
def forward(self, x, mask):
# batch size, sequence length, embedding dimension
b, t, e = x.size()
h = self.heads
# each head inspects a fraction of the embedded space
# head dimension
s = e // h
# start index of position embedding
embedding_start = self.max_length - t
x = x.view(b, t, h, s)
queries, keys, values = [w(x).transpose(1, 2)
for w, x in zip(self.linears, (x, x, x))]
if self.relative_pos:
# apply same position embeddings across the batch
# Is it possible to apply positional self-attention over
# only half of all relative distances?
Er = self.Er[:, embedding_start:, :].unsqueeze(0)
QEr = torch.matmul(queries, Er.transpose(-1, -2))
QEr = self._mask_positions(QEr)
# Get relative position attention scores
# combine batch with head dimension
SRel = self._skew(QEr).contiguous().view(b * h, t, t)
else:
queries /= math.sqrt(s)
queries = queries.reshape(t, b, self.heads, s).transpose(0, 1)
keys = keys.reshape(t, b, self.heads, s).transpose(0, 1)
attn_scores = self._sliding_chunks_query_key_matmul(
queries, keys, self.one_sided_attn_window_size
)
# values to pad for attention probs
remove_from_windowed_attention_mask = (mask != 0)[:, :, None, None]
# cast to fp32/fp16 then replace 1's with -inf
float_mask = remove_from_windowed_attention_mask.type_as(queries).masked_fill(
remove_from_windowed_attention_mask, -10000.0
)
# diagonal mask with zeros everywhere and -inf inplace of padding
diagonal_mask = self._sliding_chunks_query_key_matmul(
float_mask.new_ones(size=float_mask.size()), float_mask, self.one_sided_attn_window_size
)
# pad local attention probs
attn_scores += diagonal_mask
attn_probs = F.softmax(attn_scores, dim=-1, dtype=torch.float32) # use fp32 for numerical stability
attn_probs = attn_probs.type_as(attn_scores)
del attn_scores
attn_probs = self.dropout(attn_probs)
values = values.reshape(t, b, self.heads, s).transpose(0, 1)
attn_output = self._sliding_chunks_matmul_attn_probs_value(
attn_probs, values, self.one_sided_attn_window_size
)
assert attn_output.size() == (b, t, self.heads, s), "Unexpected size"
attn_output = attn_output.transpose(0, 1).reshape(t, b, e).contiguous()
outputs = attn_output.transpose(0, 1)
return self.recombine_heads(outputs)
# Compute scaled dot-product self-attention
# scale pre-matrix multiplication
queries = queries / (e ** (1 / 4))
keys = keys / (e ** (1 / 4))
scores = torch.bmm(queries, keys.transpose(1, 2))
scores = scores + SRel
# (b*h, t, t)
subsequent_mask = torch.triu(torch.ones(1, t, t, device=d()),
1)
scores = scores.masked_fill(subsequent_mask == 1, -1e9)
if mask is not None:
mask = mask.repeat_interleave(h, 0)
wtf = (mask == 0).nonzero().transpose(0, 1)
scores[wtf[0], wtf[1], :] = -1e9
# Convert scores to probabilities
attn_probs = F.softmax(scores, dim=2)
attn_probs = self.dropout(attn_probs)
# use attention to get a weighted average of values
out = torch.bmm(attn_probs, values).view(b, h, t, s)
# transpose and recombine attention heads
out = out.transpose(1, 2).contiguous().view(b, t, s * h)
# last linear layer of weights
return self.recombine_heads(out)
def on_left_down(event):
d("Tray icon was left-clicked.")
def on_close(self, event):
d("destroy app")
stop()
self.myapp.RemoveIcon()
self.myapp.Destroy()
self.Destroy()
