def run(name, data):
swap = SWAP.load(name)
config = swap.config
parser = ClassificationParser(config)
with open(data, 'r') as file:
reader = csv.DictReader(file)
for i, row in enumerate(reader):
row = parser.parse(row)
if row is None:
logger.error(row)
continue
if i % 100 == 0:
sys.stdout.flush()
sys.stdout.write("%d records processed\r" % i)
swap.classify(**row)
# if i > 0 and i % 1e6 == 0:
# print()
# print('Applying records')
# swap()
# swap.truncate()
swap()
swap.retire(config.fpr, config.mdr)
swap.save()
code.interact(local={**globals(), **locals()})
def send(name):
swap = SWAP.load(name)
ce.Config.load(swap.config.online_name)
logger.info('Sending reductions to caesar')
Online.send(swap)
logger.debug('Done sending reductions to caesar')
code.interact(local={**globals(), **locals()})
def run(name):
swap = SWAP.load(name)
ce.Config.load(swap.config.online_name)
Online.receive(swap)
swap.save()
ce.Config.instance().save()
logger.debug('Saved swap status')
logger.info('Sending reductions to caesar')
Online.send(swap)
logger.debug('Done sending reductions to caesar')
code.interact(local={**globals(), **locals()})
def golds(name, path):
golds = []
with open(path, 'r') as file:
reader = csv.DictReader(file)
for i, row in enumerate(reader):
golds.append((int(row['subject']), int(row['gold'])))
if i % 100 == 0:
sys.stdout.flush()
sys.stdout.write("%d records processed\r" % i)
swap = SWAP.load(name)
swap.apply_golds(golds)
swap.save()
code.interact(local={**globals(), **locals()})
def config(name, online_name):
"""
Configure swap to use caesar external configuration
\b
Arguments
---------
name - Name of swap configuration
online-name - Name of caesar_external configuration
"""
swap = SWAP.load(name)
config = swap.config
config.online_name = online_name
code.interact(local={**globals(), **locals()})
swap.save()
def run_continuous(name):
swap = SWAP.load(name)
ce.Config.load(swap.config.online_name)
try:
logger.info('Starting SWAP (%s) in continuous online mode...' % name)
while True:
_, haveItems = Online.receive(swap)
if haveItems:
swap.save()
ce.Config.instance().save()
logger.debug('Saved swap status')
logger.info('Sending reductions to caesar')
Online.send(swap)
logger.debug('Done sending reductions to caesar')
except KeyboardInterrupt as e:
logger.debug('Received KeyboardInterrupt {}'.format(e))
logger.debug('Terminating SWAP instance ({}).'.format(name))
return
def load(name):
swap = SWAP.load(name)
code.interact(local={**globals(), **locals()})
def clear(name):
swap = SWAP.load(name)
swap = SWAP(name, swap.config)
swap.save()
def config(name, online_name):
swap = SWAP.load(name)
config = swap.config
config.online_name = online_name
code.interact(local={**globals(), **locals()})
swap.save()
