def main():
(optval, args) = getopt(sys.argv[1:], 'v')
options = dict(optval)
if len(args) < 2:
usage('specify database file and flow')
level = logging.DEBUG if '-v' in options else logging.INFO
logging.basicConfig(stream=sys.stderr, level=level)
logger = logging.getLogger('crow_dataflow_sh')
logger.info('top of script')
dbfile, flow = args[0:2]
if flow not in 'OI':
usage(f"flow must be O (output) or I (input) not {flow}")
primary = {'flow': flow, 'actor': None, 'slot': None}
meta = {}
for arg in args[2:]:
split = arg.split('=', 1)
if len(split) != 2:
usage(f'{arg}: arguments must be var=value')
(var, strvalue) = split
value = shell_to_python_type(strvalue)
if var in primary:
primary[var] = value
else:
meta[var] = value
logger.info(f'{dbfile}: open sqlite3 database')
db = Dataflow(dbfile)
if flow == 'O':
find = db.find_output_slot
message = 'find output slots'
else:
find = db.find_input_slot
message = 'find input slots'
if primary['actor']:
message += f' actor={primary["actor"]}'
else:
message += ' for all actors'
if primary['slot']: message += f' slot={primary["slot"]}'
if meta:
message += ' meta: '
for k, v in meta:
message += f' {k}={v}'
logger.info(message)
db.dump(sys.stderr)
for slot in find(primary['actor'], primary['slot'], meta):
localmeta = slot.get_meta()
sys.stderr.write(f'{slot} meta = {localmeta}\n')
if localmeta:
metas = [f'{k}={v}' for k, v in localmeta.items()]
print(f'{slot.flow} {slot.actor} {slot.slot} {" ".join(metas)}')
else:
print(f'{slot.flow} {slot.actor} {slot.slot}')
def check_cycle(d: Dataflow, cycle: datetime) -> None:
for islot in d.find_input_slot('fam.job3', 'islot', {'letter': 'A'}):
imessage = islot.at(cycle)
with imessage.open('rt') as fd:
print(f"{fd.readline().strip()}: {imessage}")
for islot in d.find_input_slot('fam.job3', 'islot', {'letter': 'B'}):
imessage = islot.at(cycle)
imessage.obtain('dummy_input')
with open('dummy_input', 'rt') as fd:
print(f"{fd.readline().strip()}: {imessage}")
def deliver_cycle(d: Dataflow, cycle: datetime) -> None:
text = f'dummy file for cycle {cycle:%Y%m%d%H}\n'
for oslot in d.find_output_slot('fam.job2', 'oslot', {'letter': 'A'}):
omessage = oslot.at(cycle)
with open('dummy_file', 'wt') as fd1:
fd1.write(str(omessage))
omessage.deliver('dummy_file')
for oslot in d.find_output_slot('fam.job2', 'oslot', {'letter': 'B'}):
omessage = oslot.at(cycle)
with omessage.open('wt') as fd2:
fd2.write(str(omessage))
if os.path.exists('dummy_file'): os.unlink('dummy_file')
def main():
(optval, args) = getopt(sys.argv[1:], 'o:i:vm')
options = dict(optval)
level = logging.DEBUG if '-v' in options else logging.INFO
logging.basicConfig(stream=sys.stderr, level=level)
logger = logging.getLogger('crow_dataflow_sh')
if len(args) != 3: usage("give exactly three non-option arguments")
dbfile, adddel, cyclestr = args[0:3]
if adddel not in ['add', 'del']: usage('Specify "add" or "del"')
cycle = None
for fmt in ALLOWED_DATE_FORMATS:
with suppress(ValueError):
cycle = datetime.strptime(cyclestr, fmt)
break
if cycle is None: usage(f'unknown cycle format: {cyclestr}')
db = Dataflow(dbfile)
logger.info(f'{dbfile}: {adddel} cycle {cycle:%Y-%m-%dt%H:%M:%S}')
if adddel == 'add': db.add_cycle(cycle)
else: db.del_cycle(cycle)
def main():
logging.basicConfig(stream=sys.stderr, level=logging.INFO)
if os.path.exists('test.db'):
os.unlink('test.db')
if os.path.exists('com'):
shutil.rmtree('com')
d = Dataflow('test.db')
PRE = 'com/{cycle:%Y%m%d%H}/{actor}/{slot}.t{cycle:%H}z'
d.add_output_slot('fam.job1', 'oslot', PRE + '.x')
d.add_input_slot('fam.job2', 'islot')
d.add_input_slot('fam.job2', 'tslot', {
'when': datetime.now(),
'why': True
})
for S in [1, 2, 3]:
for L in 'AB':
d.add_output_slot('fam.job2', 'oslot', PRE + '.{letter}{slotnum}',
{
'slotnum': S,
'letter': L
})
for S in [1, 2, 3]:
for L in 'AB':
d.add_input_slot('fam.job3', 'islot', {'plopnum': S, 'letter': L})
d.dump(sys.stdout)
six_hours = timedelta(seconds=3600 * 6)
for islot in d.find_input_slot('fam.job3', 'islot'):
meta = islot.get_meta()
print(meta)
found = False
for oslot in d.find_output_slot('fam.job2', 'oslot', {
'slotnum': meta['plopnum'],
'letter': meta['letter']
}):
islot.connect_to(oslot, rel_time=six_hours)
break
cycle1 = datetime.strptime('2017081500', '%Y%m%d%H')
cycle2 = datetime.strptime('2017081506', '%Y%m%d%H')
cycle3 = datetime.strptime('2017081512', '%Y%m%d%H')
d.add_cycle(cycle1)
d.add_cycle(cycle2)
deliver_cycle(d, cycle1)
check_cycle(d, cycle2)
d.add_cycle(cycle3)
d.del_cycle(cycle1)
deliver_cycle(d, cycle2)
check_cycle(d, cycle3)
def main():
logging.basicConfig(stream=sys.stderr,level=logging.DEBUG)
if os.path.exists('test.db'):
os.unlink('test.db')
if os.path.exists('com'):
shutil.rmtree('com')
d=Dataflow('test.db')
PRE='com/{cycle:%Y%m%d%H}/{actor}/{slot}.t{cycle:%H}z'
d.add_output_slot('fam.job1','oslot',PRE+'.x')
d.add_input_slot('fam.job2','islot')
d.add_input_slot('fam.job2','tslot',{
'when':datetime.now(), 'why':True })
for S in [1,2,3]:
for L in 'AB':
d.add_output_slot('fam.job2','oslot',PRE+'.{letter}{slotnum}',
{'slotnum':S, 'letter':L})
for S in [1,2,3]:
for L in 'AB':
d.add_input_slot('fam.job3','islot',{'plopnum':S, 'letter':L})
for islot in d.find_input_slot('fam.job2','tslot'):
for oslot in d.find_output_slot('fam.job1','oslot'):
islot.connect_to(oslot,rel_time=SIX_HOURS)
for islot in d.find_input_slot('fam.job3','islot'):
meta=islot.get_meta()
found=False
for oslot in d.find_output_slot('fam.job2','oslot',{
'slotnum':meta['plopnum'], 'letter':meta['letter'] }):
islot.connect_to(oslot,rel_time=SIX_HOURS)
def main():
(optval, args) = getopt(sys.argv[1:],'o:i:vmc')
options=dict(optval)
level=logging.DEBUG if '-v' in options else logging.INFO
logging.basicConfig(stream=sys.stderr,level=level)
logger=logging.getLogger('crow_dataflow_sh')
if ( '-i' in options ) == ( '-o' in options ):
usage('specify exactly one of -o and -i')
flow = 'O' if '-i' in options else 'I'
if len(args)<4:
usage('specify dataflow db file, cycle, actor, and at least one var=value')
( dbfile, cyclestr, actor ) = args[0:3]
cycle=None
for fmt in ALLOWED_DATE_FORMATS:
with suppress(ValueError):
cycle=datetime.strptime(cyclestr,fmt)
break
if cycle is None: usage(f'unknown cycle format: {cyclestr}')
slot=None
meta={}
for arg in args[3:]:
split=arg.split('=',1)
if len(split)!=2:
usage(f'{arg}: arguments must be var=value')
( var, strvalue ) = split
value=shell_to_python_type(strvalue)
if var=='slot':
slot=value
elif var=='flow':
usage(f'{arg}: cannot set flow; that is set automatically via -i or -o')
elif var=='actor':
usage(f'{arg}: cannot set actor; that is set via a positional argument')
else:
meta[var]=value
db=Dataflow(dbfile)
if flow=='I':
logger.info(f'{dbfile}: find input slot actor={actor} slot={slot} '
f'meta={meta}')
matches=iter(db.find_input_slot(actor,slot,meta))
local=options['-o']
else:
logger.info(f'{dbfile}: find output slot actor={actor} slot={slot} '
f'meta={meta}')
matches=iter(db.find_output_slot(actor,slot,meta))
local=options['-i']
slots = [ slot for slot in matches ]
any_have_meta_lists=False
for slot in slots:
logger.info(str(slot))
if has_meta_lists(slot):
any_have_meta_lists=True
logger.info('... has metadata lists')
#any_have_meta_lists = any([ has_meta_lists(slot) for slot in slots ])
multi = len(slots)>1 or any_have_meta_lists
slot1, slot2 = None, None
with suppress(StopIteration):
slot1=next(matches)
slot2=next(matches)
if not slots:
logger.error('No match for query. Such a slot does not exist.')
exit(1)
elif multi and '-m' not in options:
logger.error('Multiple matches, and -m not specified. Abort.')
exit(1)
elif not multi and '-m' in options:
logger.error('Single match but -m was specified. Abort.')
exit(1)
for slot in slots:
deliver_by_format(logger,flow,local,slot.at(cycle),'-c' in options)