def _perform(self):
'''
Expects action.args.name as fits file name
Returns HDUs or (later) data model
'''
name = self.action.args.name
self.logger.info(f"Reading {name}")
out_args = Arguments()
out_args.name = name
out_args.hdus = open_nowarning(name)
return out_args
def testArgumentValues(self):
a = Arguments("a b c")
self.assertEqual(a[1], 'b')
a = Arguments("\tapa 'b c d' epa \"f g\" l h i j")
self.assertEqual(a[0], 'apa')
self.assertEqual(a[1], 'b c d')
self.assertEqual(a[2], 'epa')
self.assertEqual(a[3], 'f g')
self.assertEqual(a[4], 'l')
self.assertEqual(a[5], 'h')
self.assertEqual(a.get_args_after(4), 'h i j')
self.assertEqual(a.get_args_after(2), '\"f g\" l h i j')
def process_flat(action, context):
args = simple_fits_reader(action, context)()
img = args.hdus[0].data
name = args.name
minV, maxV, std = img.min(), img.max(), img.std()
context.logger.info(f"{name}, min={minV}, max={maxV}, std={std}")
return Arguments(name="OK")
def action_planner(self, action, context):
if action.args.imtype == "BIAS":
bias_args = Arguments(
name="bias_args",
want_type="BIAS",
new_type="MASTER_BIAS",
min_files=context.config.instrument.bias_min_nframes,
new_file_name="master_bias.fits")
context.push_event("process_bias", bias_args)
elif "FLAT" in action.args.imtype:
context.push_event("process_flat", action.args)
elif "ARC" in action.args.imtype:
context.push_event("process_arc", action.args)
elif "OBJECT" in action.args.imtype:
context.push_event("process_object", action.args)
return Arguments(name="new_action")
def ingest_file(self, action, context):
context.data_set.append_item(action.args.name)
try:
v = context.data_set.getInfo(action.args.name, "IMTYPE")
except:
v = "Unknown"
fname = os.path.basename(action.args.name)
self.logger.warn(f"Unknown IMTYPE {fname}")
return Arguments(name=action.args.name, imtype=v)
def add_create_contact_sheet_event(self, req, qstr):
self._getParameters(qstr)
out_dir = self.DRPFramework.config.output_directory
args = Arguments(dir_name=out_dir,
pattern="*.png",
out_name="contact_sheet.html",
cnt=-1)
self.DRPFramework.append_event("contact_sheet", args)
return json.dumps("OK"), self.jsonText
def _perform(self):
"""
Returns an Argument() with the parameters that depends on this operation.
"""
args = self.action.args
df = self.context.data_set.data_table
files = df[df.IMTYPE == args.want_type]
stacked = self.average(list(files.index))
self.save_fits_like(args.new_file_name, stacked, files.index[0],
args.new_type)
return Arguments(name=args.new_file_name)
def _perform(self):
sigmas, sizes = self.sigmas, self.sizes
args = self.action.args
self.logger.info(f"noise removal args {args}")
hdus = args.hdus
img = hdus[0].data
for a, b in zip(sizes, sigmas):
img = self._denoise(img)
out_args = Arguments(name=args.name, img=img)
return out_args
def get_event(self):
try:
try:
return self.event_queue_hi.get_nowait()
except:
return self.event_queue.get(True, self.config.event_timeout)
except Exception as e:
ev = self.config.no_event_event
if ev is None:
return None
time.sleep(self.config.no_event_wait_time)
ev.args = Arguments(name=ev.name,
time=datetime.datetime.ctime(
datetime.datetime.now()))
return ev
def _perform(self):
output_dir = self.config.output_directory
os.makedirs(output_dir, exist_ok=True)
args = self.action.args
name = os.path.basename(args.name)
os.makedirs(output_dir, exist_ok=True)
out_name = output_dir + '/' + name.replace('.fits', '.png')
img = args.img
h, w = img.shape
img1 = np.stack((img, ) * 3, axis=-1)
plt.imsave(out_name, img1)
self.logger.info("Saved {}".format(out_name))
out_args = Arguments(name=out_name)
return out_args
def _perform(self):
'''
Expects action.args.name as fits file name
Returns HDUs or (later) data model
'''
name = self.action.args.name
self.logger.info(f"Reading {name}")
out_args = Arguments()
out_args.name = name
ccddata, table = self.kcwi_fits_reader(name)
out_args.ccddata = ccddata
out_args.table = table
out_args.imtype = out_args.hdus.header['IMTYPE']
return out_args
def _perform(self):
'''
'''
args = self.action.args
dir_name = args.dir_name
out_name = args.out_name
pattern = args.pattern
self.logger.info(
f"Creating contact sheet in {dir_name}, out_name={out_name}")
flist = sorted(glob.glob(dir_name + '/' + pattern))
out = []
for f in flist:
out.append(self.genEntry(f))
with open(dir_name + '/' + out_name, 'w') as fh:
print("<html><body>", file=fh)
print("\n".join(out), file=fh)
print("</body></html>", file=fh)
return Arguments()
def execute(self, action, context):
'''
Executes one action
The input for the action is in action.args.
The action returns action_output and it is passed to the next event if action is successful.
'''
pipeline = self.pipeline
action_name = action.name
try:
if pipeline.get_pre_action(action_name)(action, context):
if self.config.print_trace:
self.logger.info('Executing action ' + action.name)
action_output = pipeline.get_action(action_name)(action,
context)
if pipeline.get_post_action(action_name)(action, context):
if not action.new_event is None:
new_args = Arguments(
) if action_output is None else action_output
self._push_event(action.new_event, new_args)
if not action.next_state is None:
context.state = action.next_state
if self.config.print_trace:
self.logger.info('Action ' + action.name + ' done')
return
else:
# post-condition failed
context.state = 'stop'
else:
# Failed pre-condition
if self.config.pre_condition_failed_stop:
context.state = 'stop'
except:
self.logger.error(
"Exception while invoking {}. Execution stopped.".format(
action_name))
context.state = 'stop'
if self.config.print_trace:
traceback.print_exc()
def f(action, context, **kargs):
return Arguments(kargs=kargs, not_found=name)
import time
if __name__ == '__main__':
if len(sys.argv) >= 2:
path = sys.argv[1]
pipeline = Fits2png_pipeline_with_actions()
framework = Framework(pipeline, "config.cfg")
framework.logger.info("Framework initialized")
if os.path.isdir(path):
flist = glob.glob(path + '/*.fits')
for f in flist:
args = Arguments(name=f)
framework.append_event('next_file', args)
out_dir = self.config.output_directory
out_name = 'test_w_actions.html'
framework.append_event(
'contact_sheet',
Arguments(dir_name=out_dir,
pattern='*.png',
out_name=out_name,
cnt=len(flist)))
else:
args = Arguments(name=path)
framework.append_event('next_file', args)
'''
Created on Jul 8, 2019
@author: skwok
'''
from core.framework import Framework
from pipelines.test_pipeline import Test_pipeline
from models.arguments import Arguments
import time
if __name__ == '__main__':
pipeline = Test_pipeline()
framework = Framework(pipeline, "config.cfg")
framework.logger.info("Framwork initialized")
framework.start()
framework.append_event('event1', Arguments())
framework.waitForEver()
framework.logger.warning("Terminating")
def action3(self, action, context):
self.logger.info("Action 3 - sleep 2")
time.sleep(2)
self.logger.info("Action 3 - done")
return Arguments(name='a3')
import tweepy
from models.streamming import TweetStreamListener
import argparse
from models.kafka import KProducer
from models.arguments import Arguments
from models.api import KafkaApi
import sys
import time
if __name__ == '__main__':
players = Arguments(sys.argv[1:]).players
bootstrap_servers = Arguments(sys.argv[1:]).bootstrap_servers
api = KafkaApi('conf/config.cfg').getApi()
players_streammings = {}
print(bootstrap_servers)
producer = KProducer(bootstrap_servers)
for player in players:
print(f'{player}: ')
streamPlayerListener = TweetStreamListener(producer.submitJsonMsg,
'json_msg',
key=player)
players_streammings[player] = tweepy.Stream(
auth=api.auth, listener=streamPlayerListener)
players_streammings[player].filter(track=[player],
is_async=True,
languages=["pt"])
#time.sleep(5)
#for player in players:
def action2(self, action, contenxt):
self.logger.info("Action 2 - sleep 10")
time.sleep(10)
self.logger.info("Action 2 - done")
return Arguments(name='a2')
def action1(self, action, contenxt):
self.logger.info("Action 1 - sleep 3")
time.sleep(3)
self.logger.info("Action 1 - done")
return Arguments(name='a1')
from pipelines.ccd_basic_pipeline import ccd_basic_pipeline
from models.arguments import Arguments
#
# Local functions
#
if __name__ == '__main__':
if len(sys.argv) >= 2:
path = sys.argv[1]
pipeline = ccd_basic_pipeline()
framework = Framework(pipeline, "config.cfg")
framework.config.instrument = ConfigClass("instr.cfg")
framework.logger.info("Framework initialized")
if os.path.isdir(path):
flist = glob.glob(path + '/*.fits')
for f in flist:
args = Arguments(name=f)
framework.append_event('next_file', args)
else:
framework.logger.info("Exepected directory")
framework.start()
framework.waitForEver()
else:
print("Usage {} dir_or_file".format(sys.argv[0]))
def _perform(self):
self.context.data_set.append_item(self.action.args.name)
self.name = self.action.args.name
out_args = Arguments()
ccddata, table = kcwi_fits_reader(self.name)
out_args.ccddata = ccddata
out_args.table = table
try:
type = self.get_keyword("IMTYPE")
groupid = self.get_keyword("GROUPID")
except KeyError:
type = "Unknown"
groupid = "Unknown"
fname = os.path.basename(self.action.args.name)
self.logger.warn(f"Unknown IMTYPE {fname}")
out_args.name = self.action.args.name
out_args.imtype = type
out_args.groupid = groupid
# CAMERA
out_args.camera = self.camera()
# CAMANGLE
out_args.camangle = self.camang()
# FILTER
out_args.filter = self.filter()
# GRANGLE
out_args.grangle = self.grangle()
# GRATING
out_args.grating = self.grating()
# ADJANGLE
out_args.adjang = self.adjang()
# RHO
out_args.rho = self.rho()
# CWAVE
out_args.cwave = self.cwave()
# ATRES
out_args.atres = self.atres()
# NAMPS
out_args.namps = int(self.get_keyword('NVIDINP'))
# NASMASK
out_args.nasmak = self.nasmask()
# BINNING
out_args.xbinsize, out_args.ybinsize = map(
int,
self.get_keyword('BINNING').split(','))
# IFUNUM
out_args.ifunum = self.get_keyword('IFUNUM')
# IFUNAM
out_args.ifuname = self.get_keyword('IFUNAM')
# ILUM
out_args.illum = self.illum()
# MAPCCD
out_args.map_ccd = self.map_ccd()
# for arg in str(out_args).split(','):
# print(arg)
return out_args
class Rmodel(colander.MappingSchema):
arguments = Arguments(missing=colander.drop)
options = Options()
properties = Properties(missing=colander.drop)
databases = colander.SchemaNode(colander.List(), missing=colander.drop)
dbfarm = colander.SchemaNode(colander.String(), missing=colander.drop)
def add_next_file_event(self, req, qstr):
self._getParameters(qstr)
args = Arguments(name=self._http_file_name)
self.DRPFramework.append_event("next_file", args)
return json.dumps("OK"), self.jsonText
#subprocess.Popen('bokeh serve', shell=True)
#time.sleep(2)
pipeline = Kcwi_pipeline()
framework = Framework(pipeline, 'config.cfg')
framework.config.instrument = ConfigClass("instr.cfg")
framework.logger.info("Framework initialized")
framework.logger.info("Checking path for files")
if os.path.isdir(path):
#flist = glob.glob(path + '/*.fits')
flist = [
'/Users/lrizzi/KCWI_DATA_1/kb181012_00014.fits',
'/Users/lrizzi/KCWI_DATA_1/kb181012_00016.fits'
]
flist = ['/Users/lrizzi/KCWI_DATA_1/kb181012_00016.fits']
for f in flist:
args = Arguments(name=f)
framework.append_event('next_file', args)
else:
args = Arguments(name=path)
framework.append_event('next_file', args)
framework.start()
framework.waitForEver()
else:
print("Usage {} dir_or_file".format(sys.argv[0]))