def get_delta(self, scalper, lock):
_log(f"scraping delta for : {scalper['name']}")
self.createTable(scalper)
client = self.initConnector(scalper)
if scalper['init_calc'] == 0:
initial = (float(client.getBBO()['asks'][0][0]) +
float(client.getBBO()['bids'][0][0])) * 0.5
self.scalper.find_one_and_update({'name': scalper['name']},
{"$set": {
'init_calc': initial
}})
elif int(time.time()
) >= scalper['next_calc'] and scalper['next_calc'] != 0:
final = (float(client.getBBO()['asks'][0][0]) +
float(client.getBBO()['bids'][0][0])) * 0.5
delta = final - scalper['init_calc']
self.updateTable(delta=delta, key="delta", scalper=scalper)
self.scalper.find_one_and_update({'name': scalper['name']}, {
'$set': {
'next_calc':
int(int(time.time()) + scalper['time_interval']),
'init_calc': 0
}
})
elif scalper['next_calc'] == 0:
self.scalper.find_one_and_update({'name': scalper['name']}, {
'$set': {
'next_calc':
int(int(time.time()) + scalper['time_interval'])
}
})
lock.release()
return
def _jump_to_situation(self):
if self.jump_to.endswith(".csv"):
self.jump_to = "MainSituation_%s" % self.jump_to.split(".csv")[0]
sit = globals()[self.jump_to](self)
utils._log("JUMPING TO SITUATION: %s (%s)" % (self.jump_to, sit.__class__.__name__))
return sit
def cost(self, input, targets, mask=None, v_part=None, v_true=None):
if self.drop_rate > 0.:
input = _dropout_from_layer(input, self.drop_rate)
logit = self.apply_score(input, v_part)
self.softmax(logit)
golden_idx = targets.flatten() # must same with logit.shape[0]
# oh, the error may because the targets is (y_maxlen, batch),
# so v_true should also be (y_maxlen, batch) !!!
if self.use_mv:
_log(" building y flat idx with batch-level voc. ")
golden_flat_idx = T.arange(
golden_idx.shape[0]) * v_part.shape[0] + v_true.flatten()
else:
golden_flat_idx = T.arange(
golden_idx.shape[0]) * self.lr_out + golden_idx
gold_cost_flat = self.ce_p_y_give_x.flatten()[golden_flat_idx]
log_norm_flat = self.log_norm.flatten()
shape = [targets.shape[0], targets.shape[1]]
if mask is not None:
gold_cost_shape = gold_cost_flat.reshape(shape) * mask
log_norm_shape = log_norm_flat.reshape(shape) * mask
norm_cost_shape = gold_cost_shape + self.alpha * (log_norm_shape**2)
# to observe how much we compressed log |Z(x)|
return T.mean(norm_cost_shape), T.mean(T.abs_(log_norm_shape))
def multi_process(self, x_iter, n_process=5):
queue = Queue()
rqueue = Queue()
processes = [None] * n_process
for pidx in xrange(n_process):
processes[pidx] = Process(target=self.translate, args=(queue, rqueue, pidx))
processes[pidx].start()
def _send_jobs(x_iter):
for idx, line in enumerate(x_iter):
# _log(idx, line)
queue.put((idx, line))
return idx + 1
def _finish_processes():
for pidx in xrange(n_process):
queue.put(None)
def _retrieve_jobs(n_samples):
trans = [None] * n_samples
for idx in xrange(n_samples):
resp = rqueue.get()
trans[resp[0]] = resp[1]
if numpy.mod(idx + 1, 1) == 0:
_log('Sample {}/{} Done'.format((idx + 1), n_samples))
return trans
_log('Translating ...')
n_samples = _send_jobs(x_iter) # sentence number in source file
trans_res = _retrieve_jobs(n_samples)
_finish_processes()
_log('Done ...')
return '\n'.join(trans_res)
def _retrieve_jobs(n_samples):
trans = [None] * n_samples
for idx in xrange(n_samples):
resp = rqueue.get()
trans[resp[0]] = resp[1]
if numpy.mod(idx + 1, 1) == 0:
_log('Sample {}/{} Done'.format((idx + 1), n_samples))
return trans
def VariabilityIndex(self):
LcOutPath = self.LCfolder + self.config['target']['name']
utils._log('Computing Variability index ')
self.config['Spectrum']['FitsGeneration'] = 'no'
# ValueDC = self.GetDCValue()
ResultDicDC = utils.ReadResult(self.config)
LogL1 = []
LogL0 = []
Time = []
for i in xrange(self.Nbin):
CurConfig = get_config(self.configfile[i])
#Read the result. If it fails, it means that the bins has not bin computed. A warning message is printed
try :
ResultDic = utils.ReadResult(CurConfig)
except :
print "WARNING : fail reading the config file : ",CurConfig
print "Job Number : ",i
print "Please have a look at this job log file"
continue
# LogL1.append(ResultDic.get("log_like"))
#Update the time and time error array
Time.append((ResultDic.get("tmax")+ResultDic.get("tmin"))/2.)
##############################################################
# Compute the loglike value using the DC flux or prefactor
##############################################################
# Create one obs instance
CurConfig['Spectrum']['FitsGeneration'] = 'no'
_,Fit = GenAnalysisObject(CurConfig,verbose=0)#be quiet
Fit.ftol = float(self.config['fitting']['ftol'])
#Spectral index management!
utils.FreezeParams(Fit, self.srcname, 'Index', -self.config['LightCurve']['SpectralIndex'])
LogL1.append(-Fit.fit(0,optimizer=CurConfig['fitting']['optimizer']))
Model_type = Fit.model.srcs[self.srcname].spectrum().genericName()
if (Model_type == 'PowerLaw') :
utils.FreezeParams(Fit, self.srcname, 'Prefactor', utils.fluxNorm(ResultDicDC['Prefactor']))
if (Model_type == 'PowerLaw2') :
utils.FreezeParams(Fit, self.srcname, 'Integral', utils.fluxNorm(ResultDicDC['Integral']))
LogL0.append(-Fit.fit(0,optimizer=CurConfig['fitting']['optimizer']))
Can = _GetCanvas()
TgrDC = ROOT.TGraph(len(Time),np.array(Time),np.array(LogL0))
TgrDC.Draw("ALP*")
TgrDC = ROOT.TGraph(len(Time),np.array(Time),np.array(LogL0))
TgrDC.SetMarkerColor(2)
TgrDC.Draw("PL*")
#Save the canvas in the LightCurve subfolder
Can.Print(LcOutPath+'_VarIndex.eps')
Can.Print(LcOutPath+'_VarIndex.C')
print
print "TSvar = ",2*(sum(LogL1)-sum(LogL0))
print "NDF = ",len(LogL0)-1
print "Chi2 prob = ",ROOT.TMath.Prob(2*(sum(LogL1)-sum(LogL0)),len(LogL0)-1)
def Analysis(folder, config, tag="", convtyp='-1', verbose = 1):
""" run an analysis"""
Obs = Observation(folder, config, convtyp, tag=tag)
if verbose:
utils._log('SUMMARY: ' + tag)
Obs.printSum()
FitRunner = FitMaker(Obs, config)##Class
if config['Spectrum']['FitsGeneration'] == 'yes':
FitRunner.GenerateFits() #Generates fits files
return FitRunner
def single_trans_valid(self, x_iter):
total_trans = []
total_avg_merge_rate, total_sent_num = 0., 0
for idx, line in enumerate(x_iter):
s_filter = filter(lambda x: x != 0, line)
avg_merges, trans = self.trans_onesent(s_filter)
total_avg_merge_rate += avg_merges
total_sent_num += 1
total_trans.append(trans)
if numpy.mod(idx + 1, 10) == 0:
_log('Sample {} Done'.format((idx + 1)))
_log('Done ...')
return total_avg_merge_rate / total_sent_num, '\n'.join(total_trans)
def calc_size(self, scalper):
leverage = self.updateLeverage(scalper)
cost = (scalper['marginBatch'] / 100) * scalper['margin'] * leverage
client = self.initConnector(scalper)
if scalper['side'] == "BUY":
size = round(cost / float(client.getBBO()['asks'][0][0]), 3)
elif scalper['side'] == "SELL":
size = round(cost / float(client.getBBO()['bids'][0][0]), 3)
# unit test
_log(
f"size: {size},leverage: {leverage}, marginBatch: {scalper['marginBatch']}, cost: {cost}, scalper: {scalper['name']}"
)
return size, cost
def translate(self, queue, rqueue, pid):
while True:
req = queue.get()
if req == None:
break
idx, src = req[0], req[1]
_log('{}-{}'.format(pid, idx))
s_filter = filter(lambda x: x != 0, src)
_, trans = self.trans_onesent(s_filter)
rqueue.put((idx, trans))
return
def apply_score(self, input, v_part=None, drop=False):
# input: (trg_sent_len, batch_size, n_out) for training
# (1 or beamsize, n_out) for decoding
# W0: (n_out, voc_size)
if drop is True and self.drop_rate > 0.:
input = input * (1 - self.drop_rate)
if self.use_mv:
_log(" using batch-level voc. logit[v]")
W_tran = T.transpose(self.W0)
# self.b need to broadcasting
logit = theano.dot(input, T.transpose(
W_tran[v_part])) + self.b[v_part]
# logit.shape: array([1, 3]) or array([80, 3]) if input.shape[0]==80
else:
_log(" using full output voc.")
logit = theano.dot(input, self.W0) + self.b
if logit.ndim == 3:
logit = logit.reshape(
[logit.shape[0] * logit.shape[1], logit.shape[2]])
# when training, the logit.ndim is 3 (trg_sent_len, batch_size, voc_size)
# when decoding, the logit.ndim is 2 (1 or beam_size, voc_size)
return logit
def updateOrder(self, **kwargs):
scalper = kwargs['scalper']
if scalper['status'] == "LIVE":
docs = {
"name": scalper['name'],
"orderId": kwargs['response']['clientOrderId'],
"timestamp": utc_timestamp(),
"side": kwargs['response']['side'],
"symbol": kwargs['response']['symbol'],
"entryPrice": kwargs['entryPrice'],
"exit": kwargs['exit'],
"qty": kwargs['response']['origQty'],
"cost": kwargs['cost'],
"pnl": kwargs['pnl'],
"paper_pnl": kwargs['paper_pnl']
}
updateOrder = self.liveId.insert_one(dict(docs))
_log(f"newOrder: {docs['orderId']} inserted into liveId collecton",
"info")
elif scalper['status'] == "DEMO":
docs = {
"name": scalper['name'],
"orderId": kwargs['orderId'],
"timestamp": utc_timestamp(),
"side": scalper['side'],
"symbol": scalper['symbol'],
"entryPrice": kwargs['entryPrice'],
"exit": kwargs['exit'],
"qty": kwargs['size'],
"cost": kwargs['cost'],
"pnl": kwargs['pnl'],
"paper_pnl": kwargs['paper_pnl']
}
updateOrder = self.demoId.insert_one(dict(docs))
_log(f"newOrder: {docs['orderId']} inserted into demoId collecton",
"info")
return
def trans_samples(self, srcs, trgs):
for index in range(len(srcs)):
# print 'before filter: '
# print s[index]
# [ 37 785 600 44 160 4074 152 3737 2 399 1096 170 4 8 29999 0 0 0 0 0 0 0]
s_filter = filter(lambda x: x != 0, srcs[index])
_log('\n[{:3}] {}'.format('src', _index2sentence(s_filter, self.svcb_i2w)))
# ndarray -> list
# s_filter: [ 37 785 600 44 160 4074 152 3737 2 399
# 1096 170 4 8 29999]
t_filter = filter(lambda x: x != 0, trgs[index])
_log('[{:3}] {}'.format('ref', _index2sentence(t_filter, self.tvcb_i2w)))
_, trans = self.trans_onesent(s_filter)
_log('[{:3}] {}\n'.format('out', trans))
def cube_prune_trans(self, src_sent):
self.translations = []
src_sent = src_sent[0] if self.ifvalid else src_sent # numpy ndarray
self.ptv = numpy.asarray(src_sent[1],
dtype='int32') if self.ifvalid else None
np_src_sent = numpy.asarray(src_sent, dtype='int64')
if np_src_sent.ndim == 1: # x (5,)
# x(5, 1), (src_sent_len, batch_size)
np_src_sent = np_src_sent[:, None]
src_sent_len = np_src_sent.shape[0]
self.maxlen = 2 * src_sent_len # x(src_sent_len, batch_size)
s_im1, self.context, self.uh = self.fn_init(
np_src_sent) # np_src_sent (sl, 1), beam==1
# (1, trg_nhids), (src_len, 1, src_nhids*2)
init_beam(self.beam,
cnt=self.maxlen,
init_state=s_im1,
detail=False,
cp=True)
best_trans, best_loss = self.cube_pruning()
_log('@source[{}], translation(without eos)[{}], maxlen[{}], loss[{}]'.
format(src_sent_len, len(best_trans), self.maxlen, best_loss))
_log(
'init[{}] nh[{}] na[{}] ns[{}] mo[{}] ws[{}] ps[{}] ce[{}]'.format(
*self.lqc[0:8]))
avg_merges = format(self.lqc[9] / self.lqc[8], '0.3f')
_log('average merge count[{}/{}={}]'.format(self.lqc[9], self.lqc[8],
avg_merges))
return _filter_reidx(self.bos_id, self.eos_id, best_trans,
self.tvcb_i2w, self.ifmv, self.ptv)
def test_log_raise_on_unknown_level():
with raises(utils.NoSuchLogLevel):
utils._log('fake_service', 'fake_line', 'build', 'BOGUS_LEVEL')
def create_cube(self, bidx, eq_classes):
# eq_classes: (score_im1, y_im1, hi, ai, loc_in_prevb) NEW
cube = []
cnt_transed = len(self.translations)
for whichsubcub, leq_class in eq_classes.iteritems(): # sub cube
each_subcube_rowsz = len(leq_class)
self.prev_beam_ptrs += each_subcube_rowsz
#print self.prev_beam_ptrs
#if bidx >= 2 and self.prev_beam_ptrs > self.avg_bp_by_cur_step + 5:
# return cube
score_im1_r0, s_im1_r0, y_im1, y_im2, y_im3, _ = leq_class[0]
subcube = []
subcube_line_cache = []
_avg_si, _avg_hi, _avg_ai, _avg_scores_i = None, None, None, None
_cube_lm_krank_ces_i, _cube_krank_scores_i = None, None
if each_subcube_rowsz == 1:
_avg_sim1 = s_im1_r0
self.onerow_subcube_cnt += 1
else:
merged_score_im1 = [tup[0] for tup in leq_class]
merged_sim1 = [tup[1] for tup in leq_class[0:1]]
np_merged_score_im1 = numpy.array(merged_score_im1,
dtype='float32')
np_merged_sim1 = numpy.array(merged_sim1)
# arithmetic mean
_avg_sim1 = numpy.mean(np_merged_sim1, axis=0)
# geometric mean , not work
#_avg_sim1 = numpy.power(numpy.prod(np_merged_sim1, axis=0), 1.0 /
# np_merged_sim1.shape[0])
# harmonic mean
#_avg_sim1 = np_merged_sim1.shape[0] / numpy.sum(1.0 / np_merged_sim1, axis=0)
# weighted harmonic mean
#assert(np_merged_sim1.shape[0] == np_merged_score_im1.shape[0])
#_avg_sim1 = numpy.sum(np_merged_score_im1, axis=0) / numpy.sum(
# np_merged_score_im1[:,None,None] / np_merged_sim1, axis=0)
# weighted mean
#exp_score_im1 = numpy.exp(np_merged_score_im1 -
# numpy.max(np_merged_score_im1, axis=0))
#softmax_score_im1 = exp_score_im1 / exp_score_im1.sum()
#_avg_sim1 = numpy.sum(softmax_score_im1[:,None,None] * np_merged_sim1, axis=0)
# quadratic mean, not work
#_avg_sim1 = numpy.power(numpy.mean(numpy.power(np_merged_sim1, 2), axis=0),
# 1.0 / np_merged_sim1.shape[0])
#
# for tup in leq_class: watch the attention prob pi dist here ....
if self.lm is not None and bidx >= 4:
# TODO sort the row dimension by language model words distribution
debug('sort by lm: -3 -2 -1 => {} {} {}'.format(
y_im3, y_im2, y_im1))
if self.ngram == 2:
gram = [y_im1]
elif self.ngram == 3:
gram = [y_im1] if y_im2 == -1 else [y_im2, y_im1]
elif self.ngram == 4:
gram = [y_im1] if y_im3 == -1 and y_im2 == -1 else (
[y_im2, y_im1]
if y_im3 == -1 else [y_im3, y_im2, y_im1])
else:
raise NotImplementedError
lm_next_logps, next_ids = vocab_prob_given_ngram(
self.lm, gram, self.tvcb, self.tvcb_i2w)
np_lm_next_neg_logps = -numpy.asarray(lm_next_logps)
np_next_ids = numpy.asarray(next_ids)
_next_krank_ids = part_sort(np_lm_next_neg_logps,
self.k - cnt_transed)
_cube_lm_krank_ces_i = np_lm_next_neg_logps[_next_krank_ids]
_next_krank_wids = np_next_ids[_next_krank_ids]
for idx in gram:
_log(idx if idx == -1 else self.tvcb_i2w[idx] + ' ',
nl=False)
_log('=> ', nl=False)
for wid in _next_krank_wids:
_log('{}({}) '.format(self.tvcb_i2w[wid],
np_lm_next_neg_logps[wid]),
nl=False)
_log('')
self.pop_subcube_approx_cache.append(None)
else:
# TODO sort the row dimension by average scores
debug('sort by averge scores')
_y_emb_im1, _avg_hi = self.fn_nh(y_im1, _avg_sim1)
_, _avg_ai = self.fn_na(self.context, self.uh, _avg_hi)
_avg_si = self.fn_ns(_avg_hi, _avg_ai)
_avg_moi = self.fn_mo(_y_emb_im1, _avg_ai, _avg_si)
_avg_scores_i = self.fn_pws(_avg_moi,
self.ptv) # the larger the better
_avg_ces_i = self.fn_ce(_avg_scores_i).flatten()
_next_krank_wids = part_sort(_avg_ces_i, self.k - cnt_transed)
_cube_krank_scores_i = _cube_krank_ces_ith = _avg_ces_i[
_next_krank_wids]
self.pop_subcube_approx_cache.append(
(_y_emb_im1, _avg_hi, _avg_ai, _avg_si, _avg_moi,
_avg_scores_i, _next_krank_wids, _cube_krank_ces_ith))
self.push_subcube_approx_cache.append(None)
# add cnt for error The truth value of an array with more than one element is ambiguous
for i, tup in enumerate(leq_class):
subcube.append([
tup +
(_avg_sim1, None if _cube_lm_krank_ces_i is None else
_cube_lm_krank_ces_i[j], None if
_cube_krank_scores_i is None else _cube_krank_scores_i[j],
wid, i, j, whichsubcub, each_subcube_rowsz)
for j, wid in enumerate(_next_krank_wids)
])
subcube_line_cache.append(None)
cube.append(subcube)
self.subcube_lines_cache.append(subcube_line_cache)
self.printCube(cube)
return cube
def test_log_raise_on_unknown_level():
with raises(utils.NoSuchLogLevel):
utils._log('fake_service', 'fake_line', 'build', 'BOGUS_LEVEL')
def cube_pruning(self):
for bidx in range(1, self.maxlen + 1):
eq_classes = OrderedDict()
self.approx_items, self.cube_lines_mergeout = [], []
self.merge(bidx, eq_classes)
# create cube and generate next beam from cube
cube = self.create_cube(bidx, eq_classes)
if self.cube_prune(bidx, cube):
_log('early stop! see {} samples ending with EOS.'.format(
self.k))
avg_bp = format(self.locrt[0] / self.locrt[1], '0.3f')
_log('average location of back pointers [{}/{}={}]'.format(
self.locrt[0], self.locrt[1], avg_bp))
sorted_samples = sorted(self.translations,
key=lambda tup: tup[0])
best_sample = sorted_samples[0]
_log('translation length(with EOS) [{}]'.format(
best_sample[-1]))
for sample in sorted_samples: # tuples
_log('{}'.format(sample))
return back_tracking(self.beam, best_sample, False)
self.beam[bidx] = sorted(self.beam[bidx], key=lambda tup: tup[0])
debug('beam {} ----------------------------'.format(bidx))
for b in self.beam[bidx]:
debug('{}'.format(b))
# debug('{}'.format(b[0:1] + b[2:]))
# because of the the estimation of P(f|abcd) as P(f|cd), so the generated beam by
# cube pruning may out of order by loss, so we need to sort it again here
# losss from low to high
# no early stop, back tracking
avg_bp = format(self.locrt[0] / self.locrt[1], '0.3f')
_log('average location of back pointers [{}/{}={}]'.format(
self.locrt[0], self.locrt[1], avg_bp))
if len(self.translations) == 0:
_log('no early stop, no candidates ends with EOS, selecting from '
'len {} candidates, may not end with EOS.'.format(
self.maxlen))
best_sample = (self.beam[self.maxlen][0][0],) + \
self.beam[self.maxlen][0][2:] + (self.maxlen, )
_log('translation length(with EOS) [{}]'.format(best_sample[-1]))
return back_tracking(self.beam, best_sample, False)
else:
_log(
'no early stop, not enough {} candidates end with EOS, selecting the best '
'sample ending with EOS from {} samples.'.format(
self.k, len(self.translations)))
sorted_samples = sorted(self.translations, key=lambda tup: tup[0])
best_sample = sorted_samples[0]
_log('translation length(with EOS) [{}]'.format(best_sample[-1]))
for sample in sorted_samples: # tuples
_log('{}'.format(sample))
return back_tracking(self.beam, best_sample, False)
def create_cube(self, bidx, eq_classes):
# eq_classes: (score_im1, y_im1, hi, ai, loc_in_prevb) NEW
cube = []
cnt_transed = len(self.translations)
for whichsubcub, leq_class in eq_classes.iteritems(): # sub cube
each_subcube_rowsz = len(leq_class)
score_im1_r0, s_im1_r0, y_im1, y_im2, y_im3, _ = leq_class[0]
subcube = []
subcube_line_mergeout = []
_avg_si, _avg_hi, _avg_ai, _avg_scores_i = None, None, None, None
_cube_lm_krank_ces_i, _cube_krank_scores_i = None, None
if each_subcube_rowsz == 1:
_avg_sim1 = s_im1_r0
else:
merged_sim1 = [tup[1] for tup in leq_class]
_avg_sim1 = numpy.mean(numpy.array(merged_sim1), axis=0)
# for tup in leq_class: watch the attention prob pi dist here ....
if self.lm is not None and bidx >= 4:
# TODO sort the row dimension by language model words distribution
debug('sort by lm: -3 -2 -1 => {} {} {}'.format(
y_im3, y_im2, y_im1))
if self.ngram == 2:
gram = [y_im1]
elif self.ngram == 3:
gram = [y_im1] if y_im2 == -1 else [y_im2, y_im1]
elif self.ngram == 4:
gram = [y_im1] if y_im3 == -1 and y_im2 == -1 else (
[y_im2, y_im1]
if y_im3 == -1 else [y_im3, y_im2, y_im1])
else:
raise NotImplementedError
lm_next_logps, next_wids = vocab_prob_given_ngram(
self.lm, gram, self.tvcb, self.tvcb_i2w)
np_lm_next_logps = numpy.asarray(lm_next_logps)
np_next_wids = numpy.asarray(next_wids)
np_lm_next_neg_logps = -np_lm_next_logps
_next_krank_ids = part_sort(np_lm_next_neg_logps,
self.k - cnt_transed)
_cube_lm_krank_ces_i = np_lm_next_neg_logps[_next_krank_ids]
_next_krank_wids = np_next_wids[_next_krank_ids]
for idx in gram:
_log(idx if idx == -1 else self.tvcb_i2w[idx] + ' ',
nl=False)
_log('=> ', nl=False)
for wid in _next_krank_wids:
_log('{}({}) '.format(self.tvcb_i2w[wid],
np_lm_next_neg_logps[wid]),
nl=False)
_log('')
self.approx_items.append(None)
else:
# TODO sort the row dimension by average scores
debug('sort by averge scores')
_y_emb_im1, _avg_hi = self.fn_nh(y_im1, _avg_sim1)
_, _avg_ai = self.fn_na(self.context, self.uh, _avg_hi)
_avg_si = self.fn_ns(_avg_hi, _avg_ai)
_avg_moi = self.fn_mo(_y_emb_im1, _avg_ai, _avg_si)
_avg_scores_i = self.fn_pws(_avg_moi,
self.ptv) # the larger the better
_avg_scores_i_flat = _avg_scores_i.flatten()
_next_krank_ids = part_sort(-_avg_scores_i_flat,
self.k - cnt_transed)
_next_krank_wids = _next_krank_ids
_cube_krank_scores_i = _avg_scores_i_flat[_next_krank_wids]
#_avg_ces_i = self.fn_ce(_avg_scores_i).flatten()
#_cube_krank_scores_i = _avg_ces_i[_next_krank_wids]
self.approx_items.append(
(_y_emb_im1, _avg_hi, _avg_ai, _avg_si, _avg_moi,
_avg_scores_i, _next_krank_wids))
# add cnt for error The truth value of an array with more than one element is ambiguous
for i, tup in enumerate(leq_class):
subcube.append([
tup +
(_avg_sim1, None if _cube_lm_krank_ces_i is None else
_cube_lm_krank_ces_i[j], None if
_cube_krank_scores_i is None else _cube_krank_scores_i[j],
wid, i, j, whichsubcub, each_subcube_rowsz)
for j, wid in enumerate(_next_krank_wids)
])
subcube_line_mergeout.append(None)
cube.append(subcube)
self.cube_lines_mergeout.append(subcube_line_mergeout)
# print created cube before generating current beam for debug ...
debug(
'\n################################ CUBE ################################'
)
nsubcube = len(cube)
debug('MERGE => ', nl=False)
for subcube_id in xrange(nsubcube):
nmergings = len(cube[subcube_id])
debug('{} '.format(nmergings), nl=False)
debug('')
for subcube_id in xrange(nsubcube):
subcube = cube[subcube_id]
nmergings = len(subcube)
debug('Group: {} contains {} mergings:'.format(
subcube_id, nmergings))
for mergeid in xrange(nmergings):
line_in_subcube = subcube[mergeid]
first_item = line_in_subcube[0]
score_im1, y_im1 = first_item[0], first_item[2]
y_im1_w = None if y_im1 == -1 else self.tvcb_i2w[y_im1]
debug('{}={}({: >7}) => '.format(y_im1, y_im1_w,
format(score_im1, '0.2f')),
nl=False)
for cubetup in line_in_subcube:
wid = cubetup[-5]
lm_score = cubetup[-7]
model_score = cubetup[-6]
debug('{}={}({: >5}&+{: >5}={: >5}) | '.format(
wid, self.tvcb_i2w[wid],
None if lm_score is None else format(lm_score, '0.2f'),
None if model_score is None else format(
model_score, '0.2f'),
None if model_score is None else format(
score_im1 + model_score, '0.2f')),
nl=False)
debug('')
debug(
'######################################################################'
)
return cube
def initMongo(self) -> MongoClient:
mclient = MongoClient("uri")["collection"]
_log(f"mongoDB connection succesfull", "info")
return mclient
def exit_trigger(self, scalper, lock):
client = self.initConnector(scalper)
if scalper['status'] == "LIVE":
orders = list(
self.liveId.find(
{"$and": [{
'name': scalper['name']
}, {
'exit': False
}]}))
if scalper['status'] == "DEMO":
orders = list(
self.demoId.find(
{"$and": [{
'name': scalper['name']
}, {
'exit': False
}]}))
positionSide = scalper['positionSide']
orderType = "MARKET"
cumulative_margin = round(
len(orders) * (scalper['marginBatch'] / 100) * scalper['margin'],
3)
cumulative_price = 0
cumulative_size = 0
cumulative_pnl = 0
if len(orders) > 0:
for order in orders:
cumulative_price += float(order['entryPrice'])
cumulative_size += float(order['qty'])
average_price = cumulative_price / len(orders)
if scalper['side'] == "BUY":
cumulative_pnl = round(
cumulative_size *
(float(client.getBBO()['bids'][0][0]) - average_price) -
cumulative_size * (self.fee / 100), 2)
if scalper['side'] == "SELL":
cumulative_pnl = round(
cumulative_size *
(float(client.getBBO()['asks'][0][0]) - average_price) -
cumulative_size * (self.fee / 100), 2) * -1
_log(
f"totalSize: {cumulative_size}, marginUsed: {cumulative_margin}, totalPnl: {cumulative_pnl} scalper: {scalper['name']}"
)
cumulative_size = round(cumulative_size, 3)
if (cumulative_pnl >= scalper['margin'] *
(scalper['take_profit'] / 100)) or (
cumulative_pnl <= -1 * (scalper['margin'] *
(scalper['stop_loss'] / 100))):
if scalper['side'] == "BUY":
side = "SELL"
if scalper['status'] == "LIVE":
response = client.post_order(scalper['symbol'],
cumulative_size, side,
positionSide, orderType)
try:
if response['status'] == "NEW":
for order in orders:
self.liveId.find_one_and_update(
{"orderId": order['orderId']},
{"$set": {
'exit': True
}})
self.updateTrades(scalper, cumulative_pnl,
time.ctime())
except:
_log(response, "error")
if scalper['status'] == "DEMO":
for order in orders:
self.demoId.find_one_and_update(
{"orderId": order['orderId']},
{"$set": {
'exit': True
}})
self.initMongo().scalper.find_one_and_update(
{"name": scalper['name']}, {"$set": {
"marginUsed": 0
}})
if scalper['side'] == "SELL":
side = "BUY"
if scalper['status'] == "LIVE":
response = client.post_order(scalper['symbol'],
cumulative_size, side,
positionSide, orderType)
try:
if response['status'] == "NEW":
for order in orders:
self.liveId.find_one_and_update(
{"orderId": order['orderId']},
{"$set": {
'exit': True
}})
self.updateTrades(scalper, cumulative_pnl,
time.ctime())
except:
_log(response, "error")
if scalper['status'] == "DEMO":
for order in orders:
self.demoId.find_one_and_update(
{"orderId": order['orderId']},
{"$set": {
'exit': True
}})
self.initMongo().scalper.find_one_and_update(
{"name": scalper['name']}, {"$set": {
"marginUsed": 0
}})
self.initMongo().scalper.find_one_and_update(
{"name": scalper['name']},
{"$inc": {
"performance": cumulative_pnl
}})
return
def entry_trigger(self, scalper, lock):
client = self.initConnector(scalper)
client.hedge_mode()
positionSide = scalper['positionSide']
orderType = "MARKET"
ub, lb, zs = self.calc_zscore(scalper)
_log(f"Status {scalper['name']} => lb: {lb} | zs: {zs} | ub: {ub}")
if (scalper['side']
== "BUY") and (scalper['marginUsed'] < scalper['margin']):
if zs > ub:
self.scalper.find_one_and_update({"name": scalper['name']},
{"$inc": {
"signal": 1
}})
_log(f"Buy Triggered for {scalper['name']}")
orderId = gen_uuid()
size, cost = self.calc_size(scalper)
marginBatch = (scalper['marginBatch'] /
100) * scalper['margin']
if scalper['status'] == "LIVE":
response = client.post_order(scalper['symbol'], size,
scalper['side'], positionSide,
orderType)
try:
if response['status'] == "NEW":
_log(f"Live order executed for Id: {orderId}",
'info')
entryPrice = client.getBBO()['asks'][0][0]
self.updateOrder(response=response,
entryPrice=entryPrice,
cost=cost,
exit=False,
pnl=0,
paper_pnl=0,
scalper=scalper)
self.initMongo().scalper.find_one_and_update(
{"name": scalper['name']},
{"$inc": {
"marginUsed": marginBatch
}})
except KeyError:
_log(response, 'error')
elif scalper['status'] == "DEMO":
entryPrice = client.getBBO()['asks'][0][0]
_log(f"Demo order executed for Id: {orderId}", 'info')
self.updateOrder(orderId=orderId,
cost=cost,
size=size,
entryPrice=entryPrice,
exit=False,
pnl=0,
paper_pnl=0,
scalper=scalper)
self.initMongo().scalper.find_one_and_update(
{"name": scalper['name']},
{"$inc": {
"marginUsed": marginBatch
}})
if (scalper['side']
== "SELL") and (scalper['marginUsed'] < scalper['margin']):
if zs < lb:
self.scalper.find_one_and_update({"name": scalper['name']},
{"$inc": {
"signal": 1
}})
_log(f"Sell Triggered for {scalper['name']}")
orderId = gen_uuid()
size, cost = self.calc_size(scalper)
marginBatch = (scalper['marginBatch'] /
100) * scalper['margin']
if scalper['status'] == "LIVE":
response = client.post_order(scalper['symbol'], size,
scalper['side'], positionSide,
orderType)
try:
if response['status'] == "NEW":
_log(f"Live order executed for Id: {orderId}",
'info')
entryPrice = client.getBBO()['bids'][0][0]
self.updateOrder(response=response,
entryPrice=entryPrice,
cost=cost,
exit=False,
pnl=0,
paper_pnl=0,
scalper=scalper)
self.initMongo().scalper.find_one_and_update(
{"name": scalper['name']},
{"$inc": {
"marginUsed": marginBatch
}})
except KeyError:
_log(response, "error")
elif scalper['status'] == "DEMO":
entryPrice = client.getBBO()['bids'][0][0]
_log(f"Demo order executed for Id: {orderId}", 'info')
self.updateOrder(orderId=orderId,
cost=cost,
size=size,
entryPrice=entryPrice,
exit=False,
pnl=0,
paper_pnl=0,
scalper=scalper)
self.initMongo().scalper.find_one_and_update(
{"name": scalper['name']},
{"$inc": {
"marginUsed": marginBatch
}})
lock.release()
return
def test_log_raise_on_unknown_level():
with raises(utils.NoSuchLogLevel):
utils._log("fake_service", "fake_line", "build", "BOGUS_LEVEL")
switchs = [args.use_valid, args.use_batch, args.use_score, args.use_norm, args.use_mv,
args.watch_adist, args.merge_way, args.ifapprox_dist, args.ifapprox_att,
args.add_lmscore, args.ifsplit]
valid_set = args.valid_set
kl = args.m_threshold
nprocess = args.n_process
lmpath = args.lm_path if args.lm_path is not None else None
ngram = args.ngram
alpha = args.length_norm
beta = args.cover_penalty
dec_conf(switchs, beam_size, search_mode, kl, nprocess, lmpath, ngram, alpha, beta, valid_set)
config = getattr(configurations, 'get_config_cs2en')()
_log('init decoder ... ')
trans = Translate(**config)
_log('done')
_log('build decode funcs: f_init f_nh f_na f_ns f_mo f_ws f_ps f_ce f_next f_emb ... ', nl=False)
fs = trans.build_sample()
_log('done')
y_im1 = [2]
npy = np.asarray(y_im1)
if npy.ndim == 1:
x = npy[None, :]
# context = np.random.sample((7, 1, 2048)).astype(np.float32)
# s_im1 = np.random.sample((1, 1024)).astype(np.float32)
debug('............. time testing ..............................')
s = time.time()
# _log(idx, line)
queue.put((idx, line))
return idx + 1
def _finish_processes():
for pidx in xrange(n_process):
queue.put(None)
def _retrieve_jobs(n_samples):
trans = [None] * n_samples
for idx in xrange(n_samples):
resp = rqueue.get()
trans[resp[0]] = resp[1]
if numpy.mod(idx + 1, 1) == 0:
_log('Sample {}/{} Done'.format((idx + 1), n_samples))
return trans
_log('Translating ...')
n_samples = _send_jobs(x_iter) # sentence number in source file
trans_res = _retrieve_jobs(n_samples)
_finish_processes()
_log('Done ...')
return '\n'.join(trans_res)
if __name__ == "__main__":
import sys
res = valid_bleu(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4])
_log(res)
def PrepareEbin(Fit, FitRunner):
""" Prepare the computation of spectral point in energy bins by
i) removing the weak sources (TS<1) # not true
ii) updating the config file (option and energy)
and save it in a new ascii file
iii) changing the spectral model and saving it in a new xml file.
A list of the ascii files is returned"""
NEbin = int(FitRunner.config['Ebin']['NumEnergyBins'])
config = FitRunner.config
config['verbose'] ='no' #Be quiet
#Replace the evt file with the fits file produced before
#in order to speed up the production of the fits files
config['file']['event'] = FitRunner.obs.eventfile
#update the config to allow the fit in energy bins
config['UpperLimit']['envelope'] = 'no'
config['Ebin']['NumEnergyBins'] = '0'#no new bin in energy!
config['out'] = FitRunner.config['out'] + '/Ebin' + str(NEbin)
config['Spectrum']['ResultPlots'] = 'no' #no SED plot/modelmap
#copy the chose of the user for the enery bin computing
config['Spectrum']['FitsGeneration'] = config['Ebin']['FitsGeneration']
config['UpperLimit']['TSlimit'] = config['Ebin']['TSEnergyBins']
tag = FitRunner.config['file']['tag']
lEmax = np.log10(float(FitRunner.config['energy']['emax']))
lEmin = np.log10(float(FitRunner.config['energy']['emin']))
utils._log("Preparing submission of fit into energy bins")
print("Emin = ", float(FitRunner.config['energy']['emin']),
" Emax = ", float(FitRunner.config['energy']['emax']),
" Nbins = ", NEbin)
ener = np.logspace(lEmin, lEmax, NEbin + 1)
os.system("mkdir -p " + config['out'])
paramsfile = []
srcname = FitRunner.config['target']['name']
if config['UpperLimit']['TSlimit']>Fit.Ts(srcname) :
_log('Re-optimize', False)
print "An upper limit has been computed. The fit need to be re-optmized"
Fit.optimize(0)
# utils.RemoveWeakSources(Fit,srcname)#remove source with TS<1 to be sure that MINUIT will converge
Pref = utils.ApproxPref(Fit, ener, srcname)
Gamma = utils.ApproxGamma(Fit, ener, srcname)
Model_type = Fit.model.srcs[srcname].spectrum().genericName()
# if the model is not PowerLaw : change the model
if not(Model_type == 'PowerLaw') :
Fit.logLike.getSource(srcname).setSpectrum("PowerLaw") #Change model
for ibin in xrange(NEbin):#Loop over the energy bins
E = utils.GetE0(ener[ibin + 1],ener[ibin])
print "Submition # ", ibin, " at energy ", E
#Update the model for the bin
NewFitObject = ChangeModel(Fit, ener[ibin], ener[ibin + 1], srcname, Pref[ibin] ,Gamma[ibin])
Xmlname = (config['out'] + "/" + srcname +
"_" + str(ibin) + ".xml")
NewFitObject.writeXml(Xmlname)# dump the corresponding xml file
config['file']['xml'] = Xmlname
#update the energy bounds
config['energy']['emin'] = str(ener[ibin])
config['energy']['emax'] = str(ener[ibin + 1])
config['file']['tag'] = tag + '_Ebin' + str(NEbin) + '_' + str(ibin)
filename = config['target']['name'] + "_" + str(ibin) + ".conf"
paramsfile.append(filename)
config.write(open(config['out'] + '/' +paramsfile[ibin], 'w')) #save the config file in a ascii file
return paramsfile
def original_trans(self, x):
x = x[0] if self.ifvalid else x # numpy ndarray
# subdict set [0,2,6,29999, 333]
self.ptv = numpy.asarray(
x[1], dtype='int32') if self.ifvalid and self.ifmv else None
# k is the beam size we have
x = numpy.asarray(x, dtype='int64')
if x.ndim == 1:
x = x[None, :]
src_sent_len = x.shape[1]
maxlen = src_sent_len * 2
x = x.T
sample = []
sample_score = []
live_k = 1
dead_k = 0
hyp_samples = [[]] * live_k
hyp_scores = numpy.zeros(live_k).astype('float32')
hyp_states = []
# get initial state of decoder rnn and encoder context
s_im1, ctx0, c_x0 = self.fn_init(x)
y_im1 = [-1] # indicator for the first target word (bos target)
for ii in xrange(maxlen):
# (src_sent_len, 1, 2*src_nhids) -> (src_sent_len, live_k, 2*src_nhids)
ctx = numpy.tile(ctx0, [live_k, 1])
debug('ctx')
debug(ctx)
c_x = numpy.tile(c_x0, [live_k, 1])
debug('y_im1.................................................')
debug(y_im1)
debug('s_im1.................................................')
debug(s_im1)
yemb_im1, hi = self.fn_nh(y_im1, s_im1)
debug('hi.................................................')
debug(hi)
pi, ai = self.fn_na(ctx, c_x, hi)
debug('pi.................................................')
debug(pi)
debug('ai.................................................')
debug(ai)
s_im1 = s_i = self.fn_ns(hi, ai) # note, s_im1 should be updated!
debug('si')
debug(s_i)
mo = self.fn_mo(yemb_im1, ai, s_i)
next_scores = self.fn_pws(mo, self.ptv) # the larger the better
next_ces = -next_scores if self.ifscore else self.fn_ce(
next_scores)
#cand_scores = hyp_scores[:, None] - numpy.log(next_scores)
cand_scores = hyp_scores[:, None] + next_ces
debug(str(ii) + ' ===============================================')
debug('ce... i')
debug(next_ces)
cand_flat = cand_scores.flatten()
# ranks_flat = cand_flat.argsort()[:(k-dead_k)]
# we do not need to generate k candidate here, because we just need to generate k-dead_k
# more candidates ending with eos, so for each previous candidate we just need to expand
# k-dead_k candidates
ranks_flat = part_sort(cand_flat, self.k - dead_k)
# print ranks_flat, cand_flat[ranks_flat[1]], cand_flat[ranks_flat[8]]
voc_size = next_scores.shape[1]
trans_indices = ranks_flat // voc_size
word_indices = ranks_flat % voc_size
costs = cand_flat[ranks_flat]
debug('ce... prev i')
debug(costs)
new_hyp_samples = []
new_hyp_scores = numpy.zeros(self.k - dead_k).astype('float32')
new_hyp_states = []
for idx, [ti, wi] in enumerate(zip(trans_indices, word_indices)):
new_hyp_samples.append(hyp_samples[ti] + [wi])
new_hyp_scores[idx] = copy.copy(costs[idx])
new_hyp_states.append(copy.copy(
s_i[ti])) # here should be s_i !!!
# check the finished samples
new_live_k = 0
hyp_samples = []
hyp_scores = []
hyp_states = []
# current beam, if the hyposise ends with eos, we do not
for idx in xrange(len(new_hyp_samples)):
if new_hyp_samples[idx][-1] == self.eos_id:
sample.append(new_hyp_samples[idx])
sample_score.append(new_hyp_scores[idx])
# print new_hyp_scores[idx], new_hyp_samples[idx]
dead_k += 1
else:
new_live_k += 1
hyp_samples.append(new_hyp_samples[idx])
hyp_scores.append(new_hyp_scores[idx])
hyp_states.append(new_hyp_states[idx])
hyp_scores = numpy.array(hyp_scores)
live_k = new_live_k
debug('hyp_scores... prev i')
debug(hyp_scores)
debug('hyp_samples... prev i')
for hyp_sample in hyp_samples:
debug(hyp_sample)
if new_live_k < 1:
break
if dead_k >= self.k:
break
y_im1 = numpy.array([w[-1] for w in hyp_samples])
s_im1 = numpy.array(hyp_states)
if live_k > 0:
for idx in xrange(live_k):
sample.append(hyp_samples[idx])
sample_score.append(hyp_scores[idx])
if self.ifnorm:
lengths = numpy.array([len(s) for s in sample])
avg_sample_score = sample_score / lengths
else:
avg_sample_score = sample_score
sidx = numpy.argmin(avg_sample_score)
best_sum_loss = sample_score[sidx]
best_avg_loss = avg_sample_score[sidx]
best_trans = sample[sidx]
_log(
'@source length[{}], translation length(with eos)[{}], maxlen[{}], avg loss'
'[{}]={}/{}'.format(src_sent_len, len(best_trans), maxlen,
avg_sample_score[sidx], sample_score[sidx],
lengths[sidx]))
_log('init[{}] nh[{}] na[{}] ns[{}] mo[{}] ws[{}] ps[{}] p[{}]'.format(
*self.lqc))
return _filter_reidx(self.bos_id, self.eos_id, best_trans,
self.tvcb_i2w, self.ifmv, self.ptv)
