def _perform(self): ''' Expects action.args.name as fits file name Returns HDUs or (later) data model ''' name = self.action.args.name self.logger.info(f"Reading {name}") out_args = Arguments() out_args.name = name out_args.hdus = open_nowarning(name) return out_args
def testArgumentValues(self): a = Arguments("a b c") self.assertEqual(a[1], 'b') a = Arguments("\tapa 'b c d' epa \"f g\" l h i j") self.assertEqual(a[0], 'apa') self.assertEqual(a[1], 'b c d') self.assertEqual(a[2], 'epa') self.assertEqual(a[3], 'f g') self.assertEqual(a[4], 'l') self.assertEqual(a[5], 'h') self.assertEqual(a.get_args_after(4), 'h i j') self.assertEqual(a.get_args_after(2), '\"f g\" l h i j')
def process_flat(action, context): args = simple_fits_reader(action, context)() img = args.hdus[0].data name = args.name minV, maxV, std = img.min(), img.max(), img.std() context.logger.info(f"{name}, min={minV}, max={maxV}, std={std}") return Arguments(name="OK")
def action_planner(self, action, context): if action.args.imtype == "BIAS": bias_args = Arguments( name="bias_args", want_type="BIAS", new_type="MASTER_BIAS", min_files=context.config.instrument.bias_min_nframes, new_file_name="master_bias.fits") context.push_event("process_bias", bias_args) elif "FLAT" in action.args.imtype: context.push_event("process_flat", action.args) elif "ARC" in action.args.imtype: context.push_event("process_arc", action.args) elif "OBJECT" in action.args.imtype: context.push_event("process_object", action.args) return Arguments(name="new_action")
def ingest_file(self, action, context): context.data_set.append_item(action.args.name) try: v = context.data_set.getInfo(action.args.name, "IMTYPE") except: v = "Unknown" fname = os.path.basename(action.args.name) self.logger.warn(f"Unknown IMTYPE {fname}") return Arguments(name=action.args.name, imtype=v)
def add_create_contact_sheet_event(self, req, qstr): self._getParameters(qstr) out_dir = self.DRPFramework.config.output_directory args = Arguments(dir_name=out_dir, pattern="*.png", out_name="contact_sheet.html", cnt=-1) self.DRPFramework.append_event("contact_sheet", args) return json.dumps("OK"), self.jsonText
def _perform(self): """ Returns an Argument() with the parameters that depends on this operation. """ args = self.action.args df = self.context.data_set.data_table files = df[df.IMTYPE == args.want_type] stacked = self.average(list(files.index)) self.save_fits_like(args.new_file_name, stacked, files.index[0], args.new_type) return Arguments(name=args.new_file_name)
def _perform(self): sigmas, sizes = self.sigmas, self.sizes args = self.action.args self.logger.info(f"noise removal args {args}") hdus = args.hdus img = hdus[0].data for a, b in zip(sizes, sigmas): img = self._denoise(img) out_args = Arguments(name=args.name, img=img) return out_args
def get_event(self): try: try: return self.event_queue_hi.get_nowait() except: return self.event_queue.get(True, self.config.event_timeout) except Exception as e: ev = self.config.no_event_event if ev is None: return None time.sleep(self.config.no_event_wait_time) ev.args = Arguments(name=ev.name, time=datetime.datetime.ctime( datetime.datetime.now())) return ev
def _perform(self): output_dir = self.config.output_directory os.makedirs(output_dir, exist_ok=True) args = self.action.args name = os.path.basename(args.name) os.makedirs(output_dir, exist_ok=True) out_name = output_dir + '/' + name.replace('.fits', '.png') img = args.img h, w = img.shape img1 = np.stack((img, ) * 3, axis=-1) plt.imsave(out_name, img1) self.logger.info("Saved {}".format(out_name)) out_args = Arguments(name=out_name) return out_args
def _perform(self): ''' Expects action.args.name as fits file name Returns HDUs or (later) data model ''' name = self.action.args.name self.logger.info(f"Reading {name}") out_args = Arguments() out_args.name = name ccddata, table = self.kcwi_fits_reader(name) out_args.ccddata = ccddata out_args.table = table out_args.imtype = out_args.hdus.header['IMTYPE'] return out_args
def _perform(self): ''' ''' args = self.action.args dir_name = args.dir_name out_name = args.out_name pattern = args.pattern self.logger.info( f"Creating contact sheet in {dir_name}, out_name={out_name}") flist = sorted(glob.glob(dir_name + '/' + pattern)) out = [] for f in flist: out.append(self.genEntry(f)) with open(dir_name + '/' + out_name, 'w') as fh: print("<html><body>", file=fh) print("\n".join(out), file=fh) print("</body></html>", file=fh) return Arguments()
def execute(self, action, context): ''' Executes one action The input for the action is in action.args. The action returns action_output and it is passed to the next event if action is successful. ''' pipeline = self.pipeline action_name = action.name try: if pipeline.get_pre_action(action_name)(action, context): if self.config.print_trace: self.logger.info('Executing action ' + action.name) action_output = pipeline.get_action(action_name)(action, context) if pipeline.get_post_action(action_name)(action, context): if not action.new_event is None: new_args = Arguments( ) if action_output is None else action_output self._push_event(action.new_event, new_args) if not action.next_state is None: context.state = action.next_state if self.config.print_trace: self.logger.info('Action ' + action.name + ' done') return else: # post-condition failed context.state = 'stop' else: # Failed pre-condition if self.config.pre_condition_failed_stop: context.state = 'stop' except: self.logger.error( "Exception while invoking {}. Execution stopped.".format( action_name)) context.state = 'stop' if self.config.print_trace: traceback.print_exc()
def f(action, context, **kargs): return Arguments(kargs=kargs, not_found=name)
import time if __name__ == '__main__': if len(sys.argv) >= 2: path = sys.argv[1] pipeline = Fits2png_pipeline_with_actions() framework = Framework(pipeline, "config.cfg") framework.logger.info("Framework initialized") if os.path.isdir(path): flist = glob.glob(path + '/*.fits') for f in flist: args = Arguments(name=f) framework.append_event('next_file', args) out_dir = self.config.output_directory out_name = 'test_w_actions.html' framework.append_event( 'contact_sheet', Arguments(dir_name=out_dir, pattern='*.png', out_name=out_name, cnt=len(flist))) else: args = Arguments(name=path) framework.append_event('next_file', args)
''' Created on Jul 8, 2019 @author: skwok ''' from core.framework import Framework from pipelines.test_pipeline import Test_pipeline from models.arguments import Arguments import time if __name__ == '__main__': pipeline = Test_pipeline() framework = Framework(pipeline, "config.cfg") framework.logger.info("Framwork initialized") framework.start() framework.append_event('event1', Arguments()) framework.waitForEver() framework.logger.warning("Terminating")
def action3(self, action, context): self.logger.info("Action 3 - sleep 2") time.sleep(2) self.logger.info("Action 3 - done") return Arguments(name='a3')
import tweepy from models.streamming import TweetStreamListener import argparse from models.kafka import KProducer from models.arguments import Arguments from models.api import KafkaApi import sys import time if __name__ == '__main__': players = Arguments(sys.argv[1:]).players bootstrap_servers = Arguments(sys.argv[1:]).bootstrap_servers api = KafkaApi('conf/config.cfg').getApi() players_streammings = {} print(bootstrap_servers) producer = KProducer(bootstrap_servers) for player in players: print(f'{player}: ') streamPlayerListener = TweetStreamListener(producer.submitJsonMsg, 'json_msg', key=player) players_streammings[player] = tweepy.Stream( auth=api.auth, listener=streamPlayerListener) players_streammings[player].filter(track=[player], is_async=True, languages=["pt"]) #time.sleep(5) #for player in players:
def action2(self, action, contenxt): self.logger.info("Action 2 - sleep 10") time.sleep(10) self.logger.info("Action 2 - done") return Arguments(name='a2')
def action1(self, action, contenxt): self.logger.info("Action 1 - sleep 3") time.sleep(3) self.logger.info("Action 1 - done") return Arguments(name='a1')
from pipelines.ccd_basic_pipeline import ccd_basic_pipeline from models.arguments import Arguments # # Local functions # if __name__ == '__main__': if len(sys.argv) >= 2: path = sys.argv[1] pipeline = ccd_basic_pipeline() framework = Framework(pipeline, "config.cfg") framework.config.instrument = ConfigClass("instr.cfg") framework.logger.info("Framework initialized") if os.path.isdir(path): flist = glob.glob(path + '/*.fits') for f in flist: args = Arguments(name=f) framework.append_event('next_file', args) else: framework.logger.info("Exepected directory") framework.start() framework.waitForEver() else: print("Usage {} dir_or_file".format(sys.argv[0]))
def _perform(self): self.context.data_set.append_item(self.action.args.name) self.name = self.action.args.name out_args = Arguments() ccddata, table = kcwi_fits_reader(self.name) out_args.ccddata = ccddata out_args.table = table try: type = self.get_keyword("IMTYPE") groupid = self.get_keyword("GROUPID") except KeyError: type = "Unknown" groupid = "Unknown" fname = os.path.basename(self.action.args.name) self.logger.warn(f"Unknown IMTYPE {fname}") out_args.name = self.action.args.name out_args.imtype = type out_args.groupid = groupid # CAMERA out_args.camera = self.camera() # CAMANGLE out_args.camangle = self.camang() # FILTER out_args.filter = self.filter() # GRANGLE out_args.grangle = self.grangle() # GRATING out_args.grating = self.grating() # ADJANGLE out_args.adjang = self.adjang() # RHO out_args.rho = self.rho() # CWAVE out_args.cwave = self.cwave() # ATRES out_args.atres = self.atres() # NAMPS out_args.namps = int(self.get_keyword('NVIDINP')) # NASMASK out_args.nasmak = self.nasmask() # BINNING out_args.xbinsize, out_args.ybinsize = map( int, self.get_keyword('BINNING').split(',')) # IFUNUM out_args.ifunum = self.get_keyword('IFUNUM') # IFUNAM out_args.ifuname = self.get_keyword('IFUNAM') # ILUM out_args.illum = self.illum() # MAPCCD out_args.map_ccd = self.map_ccd() # for arg in str(out_args).split(','): # print(arg) return out_args
class Rmodel(colander.MappingSchema): arguments = Arguments(missing=colander.drop) options = Options() properties = Properties(missing=colander.drop) databases = colander.SchemaNode(colander.List(), missing=colander.drop) dbfarm = colander.SchemaNode(colander.String(), missing=colander.drop)
def add_next_file_event(self, req, qstr): self._getParameters(qstr) args = Arguments(name=self._http_file_name) self.DRPFramework.append_event("next_file", args) return json.dumps("OK"), self.jsonText
#subprocess.Popen('bokeh serve', shell=True) #time.sleep(2) pipeline = Kcwi_pipeline() framework = Framework(pipeline, 'config.cfg') framework.config.instrument = ConfigClass("instr.cfg") framework.logger.info("Framework initialized") framework.logger.info("Checking path for files") if os.path.isdir(path): #flist = glob.glob(path + '/*.fits') flist = [ '/Users/lrizzi/KCWI_DATA_1/kb181012_00014.fits', '/Users/lrizzi/KCWI_DATA_1/kb181012_00016.fits' ] flist = ['/Users/lrizzi/KCWI_DATA_1/kb181012_00016.fits'] for f in flist: args = Arguments(name=f) framework.append_event('next_file', args) else: args = Arguments(name=path) framework.append_event('next_file', args) framework.start() framework.waitForEver() else: print("Usage {} dir_or_file".format(sys.argv[0]))