def test_slot_creation(self):
session = m.get_session()
s1 = m.Slot(sid='a',
name='test_slot',
keys='k1|k2',
date=dt.datetime(2019, 6, 9).date(),
priority=1,
dbid='.')
s2 = m.Slot(sid='b',
name='implied_volatilty',
keys='k1|k2',
date=dt.datetime(2019, 6, 9).date(),
priority=1,
dbid='.',
state=m.SlotStatesEnum.current)
p1 = m.Producer(state=m.ProducerStatesEnum.completed,
pname='trs_fitter',
keys='k1|k2',
date=dt.datetime(2019, 6, 9).date(),
args='{}')
s1.listeners.append(p1)
p1.outputs.append(s2)
session.add(s1)
session.add(s2)
session.add(p1)
session.commit()
session.query(m.Slot).filter_by(sid='a').one()
session.query(m.Slot).filter_by(sid='b').one()
session.query(m.Producer).one()
def start_producers():
session_ = get_session()
pnames = [
x[0] for x in session_.query(Producer.pname).filter(
Producer.pending()).distinct()
]
for pname in pnames:
pobj = net.producers[pname].obj
batch_size = pobj.batch_size
min_delay = dt.timedelta(seconds=pobj.delay)
threshold = dt.datetime.utcnow() - min_delay
logger.debug("starting Producer '%s' (batch_size=%s, delay=%s" %
(pname, batch_size, min_delay))
producers = (session_.query(Producer).filter(
Producer.pending()).filter(Producer.pname == pname).filter(
Producer.last_updated <= threshold)).all()
logger.debug("%s task ready for execution" % len(producers))
for idx in range(0, len(producers), batch_size):
execute(producers[idx:(idx + batch_size)])
# Need to set status on input slots to pending
for producer in producers[idx:(idx + batch_size)]:
for slot in producer.inputs:
logger.debug(
"slot '%s' with sid='%s' set to PENDING due to running producer"
% (slot.name, slot.sid))
slot.pending.set()
session_.commit()
def test_slot_insertion(self):
session = m.get_session()
s1 = m.Slot(name='european_totem',
keys='k1|k2',
date=dt.datetime(2019, 6, 9).date(),
priority=1,
dbid='.')
m.insert_slot(s1)
s2 = m.Slot(name='varswap_totem',
keys='k1|k2',
date=dt.datetime(2019, 6, 9).date(),
priority=1,
dbid='.')
m.insert_slot(s2, None)
for item in session.query(m.InputSlot):
print item
def test_start_producer(self):
session = m.get_session()
keys = '1|EUR|euro|equity|index'
p1 = m.Producer(state=m.ProducerStatesEnum.completed,
pname='totem_daily',
keys=keys,
date=dt.datetime(2019, 6, 9).date(),
args='{}')
s1 = m.Slot(name='european_totem',
keys=keys,
date=dt.datetime(2019, 6, 9).date(),
priority=1,
dbid='.')
s2 = m.Slot(name='varswap_totem',
keys=keys,
date=dt.datetime(2019, 6, 9).date(),
priority=1,
dbid='.')
s3 = m.Slot(name='implied_vol',
keys=keys,
date=dt.datetime(2019, 6, 9).date(),
priority=1,
dbid='.')
s4 = m.Slot(name='dividends',
keys=keys,
date=dt.datetime(2019, 6, 9).date(),
priority=1,
dbid='.')
s5 = m.Slot(name='product_rate',
keys=keys,
date=dt.datetime(2019, 6, 9).date(),
priority=1,
dbid='.')
m.insert_slot(s1, p1)
m.insert_slot(s2, p1)
m.insert_slot(s3, p1)
m.insert_slot(s4, p1)
m.insert_slot(s5, p1)
p.schedule_producers()
for row in session.query(m.Producer):
print row
p.start_producers()
time.sleep(5)
p.start_producers()
time.sleep(5)
p.start_producers()
def get_producer(pid):
session = m.get_session()
return session.query(m.Producer).filter_by(pid=pid).one()
def get_producers():
session = m.get_session()
return session.query(m.Producer).all()
def get_inputslot(id_):
session = m.get_session()
return session.query(m.InputSlot).filter_by(id_=id_).one()
def get_inputslots():
session = m.get_session()
return session.query(m.InputSlot).all()
def get_slot(sid):
session = m.get_session()
return session.query(m.Slot).filter_by(sid=sid).one()
def get_slots():
session = m.get_session()
return session.query(m.Slot).all()
def run(self):
logger.debug("autofit.app.Loop STARTED!")
session_ = get_session()
while not self._stop:
if self.queue.empty():
time.sleep(2)
else:
parcel = self.queue.get()
if parcel['state'] != 'ok':
continue
pname = parcel['pname']
pid = parcel['pid']
logger.debug("Receiver data from producer with pid '%s'" % pid)
if pid:
query = session_.query(Producer).filter_by(pid=pid)
try:
p = query.one()
if p.halted():
logger.debug(
"Producer with pid='%s' was halted, update skipped"
% pid)
continue
if not p.running():
logger.error(
"Producer with pid='%s' in an inconsistent state '%s'"
% (pid, p.state))
continue
except NoResultFound:
logger.exception(
"producer with pid='%s' does not exist" % pid)
continue
except MultipleResultsFound:
logger.exception("multiple producer with pid='%s'" %
pid)
continue
except Exception as e:
logger.exception(str(e))
elif not isinstance(net.producers[pname].obj, Daemon):
logger.exception("non-daemon producer missing pid")
else: # the producer is a daemon, set p to None
p = None
logger.debug("valid data received from pid='%s'" % pid)
date = parcel['date']
priority = parcel['priority']
if p:
p.completed.set()
for slot in p.inputs:
is_completed = True
for producer in slot.listeners:
if not producer.completed():
is_completed = False
break
if is_completed:
slot.current.set()
for slot in parcel['slots']:
oslot = Slot(name=slot['sname'],
keys=slot['keys'],
date=date,
priority=priority,
state=SlotStatesEnum.pending,
dbid=slot['dbid'])
insert_slot(oslot, p)
schedule_producers()
start_producers()
logger.debug('mail loop halted')
def schedule_producers():
ids = set()
session_ = get_session()
potprods = session_.query(InputSlot.pname).filter(
InputSlot.updated()).distinct()
for producer in potprods:
pname = producer.pname
logger.debug("joining producer %s" % pname)
oproducer = net.producers[pname].obj # type: ptypes._ProducerBase
skeys = net.slots[oproducer.provides[0]].obj.keys.split('|')
spkeys = skeys + ['pname', 'date']
fkeys = [InputSlot.__dict__[key]
for key in spkeys] # keys mapped to the ORM attributes
ukeys = session_.query(*fkeys).filter(
and_(InputSlot.updated(), InputSlot.pname == pname))
for fkey in fkeys:
ukeys = ukeys.filter(
fkey.isnot(None)) # TODO: do I really need this?
ukeys = ukeys.group_by(*fkeys).distinct()
for ukey in ukeys:
date = ukey.date
keys_dict = ukey._asdict()
seed = alias(
select([
cast(literal(val), String).label(key)
for key, val in keys_dict.items()
]), 'seed')
joins = []
columns = []
ons = []
counter = {}
sids = set()
for jslot in oproducer.requires: # type: ptypes.JoinedSlot
name = jslot.alias or jslot.name
if name in counter:
name_ = jslot.alias or "%s_%s" % (name, counter[name])
counter[name] += 1
else:
name_ = name
counter[name] = 1
table = session_.query(aliased(
InputSlot, name=name_)).filter_by(sname=name,
pname=pname,
date=date).subquery()
joins.append(table)
this_column = table.c.sid.label(name_)
columns.append(this_column)
join_on = True
slot_keys = jslot.slot_keys or skeys
producer_keys = jslot.producer_keys or skeys
for s, p in zip(slot_keys, producer_keys):
join_on = and_(join_on, seed.c[p] == table.c[s])
for row in session_.query(table.c.sid).select_from(
seed.join(table, join_on)):
# need to keep track of this to link the producer to the slots it processes
#
sids.add(row.sid)
for row in (session_.query(table.c.id_).select_from(
seed.join(table, join_on)).filter(
table.c.state == InputSlotStatesEnum.updated)):
ids.add(
row.id_
) # need to keep track of this to set the 'processed' state to used InputSlot
logger.debug('added id_ %s' % row.id_)
ons.append(join_on)
seed = session_.query(seed, *columns)
for table, on, jslot in zip(joins, ons, oproducer.requires):
if jslot.exclusive:
seed = seed.outerjoin(table,
on).filter(table.c.id_.is_(None))
else:
seed = seed.join(table, on)
if seed.count():
args = [row._asdict() for row in seed]
args = json.dumps(args)
slots = session_.query(Slot).filter(Slot.sid.in_(sids)).all()
logger.debug("producer '%s' is READY" % pname)
this_keys = '|'.join([keys_dict[k] for k in skeys])
this_producer = Producer(state=ProducerStatesEnum.void,
pname=pname,
keys=this_keys,
date=date,
args=args)
try:
insert_producer(producer=this_producer, slots=slots)
except (InconsistentProducerState,
MultipleProducerException) as e:
logger.exception(str(e))
continue # some other action here
else:
logger.debug("producer '%s' is INCOMPLETE" % pname)
for input_slot in session_.query(InputSlot).filter(InputSlot.id_.in_(ids)):
logger.info("InputSlot.id_='%s', "
"pname='%s', sname='%s', sid='%s' was PROCESSED" %
(input_slot.id_, input_slot.pname, input_slot.sname,
input_slot.sid))
input_slot.processed.set()
session_.commit()
