def start_run(config): ALLOWED_OUTPUTS = { 'output:mqtt': OutMQTT, 'output:pushbullet': Pushbullet} cfg_parse = configparser.ConfigParser() cfg_parse.read(config) cfg = {key: value for key, value in cfg_parse.items()} settings_mqtt = {} if cfg.get('settings:mqtt'): settings_mqtt = { key: value for key, value in cfg['settings:mqtt'].items()} settings_rules = {} if cfg.get('settings:rules'): settings_rules = { key: value for key, value in cfg['settings:rules'].items()} output = None for output_alias, output_cls in ALLOWED_OUTPUTS.items(): if cfg.get(output_alias): settings_output = { key: value for key, value in cfg[output_alias].items()} output = output_cls(**settings_output) break sentinel = Sentinel(**settings_mqtt) sentinel.set_db(**settings_rules) sentinel.set_output(output) sentinel.start()
def list_rule(config): cfg_parse = configparser.ConfigParser() cfg_parse.read(config) cfg = {key: value for key, value in cfg_parse.items()} settings_rules = {} if cfg.get('settings:rules'): settings_rules = { key: value for key, value in cfg['settings:rules'].items()} sentinel = Sentinel() sentinel.set_db(**settings_rules) rules = sentinel.list_rules() for rule in rules: click.echo(click.style(f'> {rule}', fg='green'))
def add_rule(config, topic, operator, equated): cfg_parse = configparser.ConfigParser() cfg_parse.read(config) cfg = {key: value for key, value in cfg_parse.items()} settings_rules = {} if cfg.get('settings:rules'): settings_rules = { key: value for key, value in cfg['settings:rules'].items()} rule = Rule(topic=topic, operator=operator, equated=equated) sentinel = Sentinel() sentinel.set_db(**settings_rules) sentinel.add_rule(rule)
def from_url(url, db=None, master_for_args={}, **kwargs): """ Returns an active Redis client generated from the given database URL. Will attempt to extract the database id from the path url fragment, if none is provided. When url is starting with sentinels://, a sentinel object will be returned if no query parameter service_name in url, otherwise the master redis client will be returned. master_for_args will be applied to Sentinel.master_for. >>> import redis >>> redis.from_url('sentinels://node1:17700,node2:17700') Sentinel<sentinels=[node1:17700,node2:17700]> >>> redis.from_url( 'sentinels://node1:17700,node2:17700?service_name=mymaster', db=1) StrictRedis<SentinelConnectionPool<service=mymaster(master)> >>> redis.from_url( 'sentinels://node1:17700,node2:17700?service_name=mymaster&db=3', db=1) StrictRedis<SentinelConnectionPool<service=mymaster(master)> >>> redis.from_url( 'sentinels://node1:17700,node2:17700?service_name=mymaster', db=1, master_for_args={'redis_class':redis.Redis}) Redis<SentinelConnectionPool<service=mymaster(master)> """ parse_result = urlparse(url) if parse_result.scheme == 'sentinels': from sentinel import Sentinel sentinel, db_from_url, service_name = Sentinel.from_url(url, **kwargs) if not service_name: return sentinel else: return sentinel.master_for(service_name, db=(db_from_url or db or 0), **master_for_args) from redis.client import Redis return Redis.from_url(url, db, **kwargs)
def start_irun(): custom_style = style_from_dict({ Token.Separator: '#cc5454', Token.QuestionMark: '#673ab7 bold', Token.Selected: '#cc5454', # default Token.Pointer: '#673ab7 bold', Token.Instruction: '', # default Token.Answer: '#f44336 bold', Token.Question: '', }) def call_single_question(question_type, msg, choices=None, default=None): question = [ { 'type': question_type, 'name': 'question', 'message': msg, } ] if choices: question[0]['choices'] = choices if default is not None: question[0]['default'] = default answer = prompt(question, style=custom_style) return answer['question'] sentinel = Sentinel() # Q-Choose a database for the rules db_choices = ['SQLite3 [Default]', ] db_answer = call_single_question( question_type='list', msg='Choose a database for the rules', choices=db_choices ) if db_answer == 'SQLite3 [Default]': sentinel.set_db('sqlite://sentinel.db') # Q-Choose an output service output_choices = ['MQTT', ] output_answer = call_single_question( question_type='list', msg='Choose an output service', choices=output_choices ) if output_answer == 'MQTT': sentinel.set_output(OutMQTT()) # Q-Do you want add rules? add_rule_answer = call_single_question( question_type='confirm', msg='Do you want to add rules?', default=True ) quit_signal = not add_rule_answer while not quit_signal: # Get topic and ask about the operation type rule_topic_question = [ { 'type': 'input', 'name': 'topic', 'message': 'Topic', }, { 'type': 'list', 'name': 'operation', 'message': 'Choose a operation:', 'choices': [ 'Custom parameters', 'Data relay', ] }, ] rule_topic_answer = prompt(rule_topic_question, style=custom_style) if rule_topic_answer['operation'] == 'Data relay': rule = Rule(rule_topic_answer['topic']) sentinel.add_rule(rule) else: # Q-If the received value is: rule_operator_choices = [ '==', '!=', '>=', '<=', '<', '>', ] rule_operator_answer = call_single_question( question_type='list', msg='If the received value is:', choices=rule_operator_choices ) # Q-Value {rule_operator_answer}:' rule_equated_msg = ( f'Value {str(rule_operator_answer)}') rule_equated_answer = call_single_question( question_type='input', msg=rule_equated_msg ) rule = Rule( topic=rule_topic_answer['topic'], operator=rule_operator_answer, equated=rule_equated_answer ) sentinel.add_rule(rule) # Q-Do you want to add more rules?' more_rule_answer = call_single_question( question_type='confirm', msg='Do you want to add more rules?', default=False ) quit_signal = not more_rule_answer sentinel.start()
import json from hashlib import md5 from subprocess import Popen, PIPE from os import listdir, getenv from os.path import isfile, join, exists from pathlib import Path # your Auth key generated from app.sentinelengine.ai/authkeys AUTH_KEY = getenv("SENTINEL_AUTH_KEY") # your Device ID DEVICE_ID = getenv("SENTINEL_DEVICE_ID") # The sync folder SYNC_DIR = getenv("SENTINEL_ENGINE_FILE_FOLDER") # Initialise a Sentinel API client SentinelClient = Sentinel(AUTH_KEY) def loadFile(path): with open(join(SYNC_DIR, path), 'rb') as file: return file.read() if exists(SYNC_DIR) is False: print("SYNC_DIR: %s missing, creating..." % SYNC_DIR) Path(SYNC_DIR).mkdir(parents=True, exist_ok=True) while True: # 1. Generates md5 hashes for existing files md5s = [{ "filename": fname,
def main (ifile = 'data/brdf_WW_1_A_1.kernelFiltered.dat', \ confFile='config_files/sentinel_Def.conf', \ solve = ['xlai','xkab','scen','xkw','xkm','xleafn','xs1']) : ''' Show that we can use this same setup to solve for the field data (MODIS) by using a different config file ''' ifile = 'data/brdf_WW_1_A_1.kernelFiltered.dat' ofileSingle = 'output/brdf_WW_1_A_1.kernelFilteredSingle.dat' ofile = 'output/brdf_WW_1_A_1.kernelFiltered.dat' s = Sentinel(solve=solve,confFile=confFile) # as above, solve for initial estimate s.solveSingle(ifile,ofileSingle) s.paramPlot(s.loadData(ofileSingle),s.loadData(ofileSingle),\ filename='plots/testFieldDataSingle.png') s.smooth(ofileSingle,ofile=ofileSingle+'_smooth') s.paramPlot(s.loadData('input/truth.dat'),\ s.loadData(ofileSingle+'_smooth'),\ filename='plots/%s.png'%(ofileSingle+'_smooth')) gamma = np.median(np.sqrt(s.gammaSolve.values()) * \ np.array(s.wScale.values())) gamma = int((np.sqrt(s.gammaSolve[solve[0]]) * \ np.array(s.wScale[solve[0]]))+0.5) s.solveRegular(ifile,ofile,modelOrder=2,gamma=gamma, \ initial=ofileSingle+'_smooth') s.paramPlot(s.loadData(ofileSingle+'_smooth'),\ s.loadData(ofile + '_result.dat'),\ filename='plots/%s_Gamma%08d.png'%(ofile,gamma))
import json import os import time from subprocess import Popen, PIPE, TimeoutExpired from sentinel import Sentinel # your Auth key generated from app.sentinelengine.ai/authkeys AUTH_KEY = os.getenv("SENTINEL_AUTH_KEY") # your Device ID DEVICE_ID = os.getenv("SENTINEL_DEVICE_ID") # Initialise a Sentinel API client SentinelClient = Sentinel(AUTH_KEY) while True: # 1. Get latest commands. commands = SentinelClient.pollCommands(DEVICE_ID) # 2. Execute commands. for command in commands: print("Executing command: [" + command['command'] + "]") process = Popen(command['command'], stdout=PIPE, stderr=PIPE) try: # will raise error and kill any process that runs longer than 60 seconds stdout, stderr = process.communicate(timeout=10) except TimeoutExpired as e: process.kill() stdout, stderr = process.communicate() # 3. Upload command output. print("Uploading output...") SentinelClient.uploadOutput(command['upload'], stdout, stderr)
class PR(object): def __init__(self, source, target, review, build, number, title): assert isinstance(target, FQSHA), target assert isinstance(source, FQSHA), source assert number is None or isinstance(number, str) assert title is None or isinstance(title, str) assert review in ['pending', 'approved', 'changes_requested'] self.source = source self.target = target self.review = review self.build = build self.number = number self.title = title keep = Sentinel() def copy(self, source=keep, target=keep, review=keep, build=keep, number=keep, title=keep): return PR(source=self.source if source is PR.keep else source, target=self.target if target is PR.keep else target, review=self.review if review is PR.keep else review, build=self.build if build is PR.keep else build, number=self.number if number is PR.keep else number, title=self.title if title is PR.keep else title) def _maybe_new_shas(self, new_source=None, new_target=None): assert new_source is not None or new_target is not None assert new_source is None or isinstance(new_source, FQSHA) assert new_target is None or isinstance(new_target, FQSHA) if new_source and self.source != new_source: assert not self.is_merged() if new_target and self.target != new_target: log.info( f'new source and target sha {new_target.short_str()} {new_source.short_str()} {self.short_str()}' ) return self._new_target_and_source(new_target, new_source) else: log.info( f'new source sha {new_source.short_str()} {self.short_str()}' ) return self._new_source(new_source) else: if new_target and self.target != new_target: if self.is_merged(): log.info( f'ignoring new target sha for merged PR {self.short_str()}' ) return self else: log.info( f'new target sha {new_target.short_str()} {self.short_str()}' ) return self._new_target(new_target) else: return self def _new_target_and_source(self, new_target, new_source): return self.copy(source=new_source, target=new_target, review='pending')._new_build( # FIXME: if I already have an image, just use it try_new_build(new_source, new_target)) def _new_target(self, new_target): return self.copy(target=new_target)._new_build( determine_buildability(self.source, new_target)) def _new_source(self, new_source): return self.copy(source=new_source, review='pending')._new_build( # FIXME: if I already have an image, just use it try_new_build(new_source, self.target)) def _new_build(self, new_build): if self.build != new_build: self.notify_github(new_build) return self.copy(build=self.build.transition(new_build)) else: return self def build_it(self): # FIXME: if I already have an image, just use it return self._new_build(try_new_build(self.source, self.target)) # FIXME: this should be a verb def merged(self): return self._new_build(Merged(self.target.sha)) def notify_github(self, build): log.info(f'notifying github of {build} for {self.short_str()}') json = { 'state': build.gh_state(), 'description': str(build), 'context': CONTEXT } if isinstance(build, Failure) or isinstance(build, Mergeable): json['target_url'] = \ f'https://storage.googleapis.com/{GCS_BUCKET}/ci/{self.source.sha}/{self.target.sha}/index.html' try: post_repo(self.target.ref.repo.qname, 'statuses/' + self.source.sha, json=json, status_code=201) except BadStatus as e: if e.status_code == 422: log.exception( f'Too many statuses applied to {self.source.sha}! This is a ' f'dangerous situation because I can no longer block merging ' f'of failing PRs.') else: raise e @staticmethod def fresh(source, target, number=None, title=None): return PR(source, target, 'pending', Unknown(), number, title) def __str__(self): return json.dumps(self.to_json()) def short_str(self): return ( f'[PR {self.number}]{self.target.short_str()}..{self.source.short_str()};' f'{self.review};{self.build};') @staticmethod def from_json(d): assert 'target' in d, d assert 'source' in d, d assert 'review' in d, d assert 'build' in d, d assert 'number' in d, d assert 'title' in d, d return PR( FQSHA.from_json(d['source']), FQSHA.from_json(d['target']), d['review'], build_state_from_json(d['build']), d['number'], d['title'], ) def to_json(self): return { 'target': self.target.to_json(), 'source': self.source.to_json(), 'review': self.review, 'build': self.build.to_json(), 'number': self.number, 'title': self.title } def is_mergeable(self): return (isinstance(self.build, Mergeable) and self.review == 'approved') def is_approved(self): return self.review == 'approved' def is_running(self): return isinstance(self.build, Building) def is_pending_build(self): return isinstance(self.build, Buildable) def is_merged(self): return isinstance(self.build, Merged) def update_from_github_push(self, push): assert isinstance(push, FQSHA) assert self.target.ref == push.ref, f'{push} {self.short_str()}' return self._maybe_new_shas(new_target=push) def update_from_github_pr(self, gh_pr): assert isinstance(gh_pr, GitHubPR) assert self.target.ref == gh_pr.target_ref assert self.source.ref == gh_pr.source.ref # this will build new PRs when the server restarts if gh_pr.target_sha: result = self._maybe_new_shas(new_source=gh_pr.source, new_target=FQSHA( gh_pr.target_ref, gh_pr.target_sha)) else: result = self._maybe_new_shas(new_source=gh_pr.source) if self.title != gh_pr.title: log.info( f'found new title from github {gh_pr.title} {self.short_str()}' ) result = result.copy(title=gh_pr.title) if self.number != gh_pr.number: log.info( f'found new PR number from github {gh_pr.title} {self.short_str()}' ) result = result.copy(number=gh_pr.number) return result def update_from_github_review_state(self, review): if self.review != review: log.info( f'review state changing from {self.review} to {review} {self.short_str()}' ) return self.copy(review=review) else: return self def update_from_github_status(self, build): if isinstance(self.build, Unknown): if self.target.sha == build.target_sha: log.info( f'recovering from unknown build state via github. {build} {self.short_str()}' ) return self.copy(build=build) else: log.info('ignoring github build state for wrong target. ' f'{build} {self.short_str()}') return self else: log.info( f'ignoring github build state. {build} {self.short_str()}') return self def refresh_from_batch_job(self, job): state = job.cached_status()['state'] if state == 'Complete': return self.update_from_completed_batch_job(job) elif state == 'Cancelled': log.error( f'a job for me was cancelled {short_str_build_job(job)} {self.short_str()}' ) job.delete() return self._new_build(try_new_build(self.source, self.target)) else: assert state == 'Created', f'{state} {job.id} {job.attributes} {self.short_str()}' assert 'target' in job.attributes, job.attributes assert 'image' in job.attributes, job.attributes target = FQSHA.from_json(json.loads(job.attributes['target'])) image = job.attributes['image'] if target == self.target: return self._new_build(Building(job, image, target.sha)) else: log.info( f'found deploy job {job.id} for wrong target {target}, should be {self.target}' ) job.delete() return self def update_from_completed_batch_job(self, job): assert isinstance(job, Job) job_status = job.cached_status() exit_code = job_status['exit_code'] job_source = FQSHA.from_json(json.loads(job.attributes['source'])) job_target = FQSHA.from_json(json.loads(job.attributes['target'])) assert job_source.ref == self.source.ref assert job_target.ref == self.target.ref if job_target.sha != self.target.sha: log.info(f'notified of job for old target {job.id}' # too noisy: f' {job.attributes} {self.short_str()}' ) x = self elif job_source.sha != self.source.sha: log.info(f'notified of job for old source {job.id}' # too noisy: f' {job.attributes} {self.short_str()}' ) x = self elif exit_code == 0: log.info( f'job finished success {short_str_build_job(job)} {self.short_str()}' ) x = self._new_build(Mergeable(self.target.sha)) else: log.info( f'job finished failure {short_str_build_job(job)} {self.short_str()}' ) x = self._new_build( Failure(exit_code, job.attributes['image'], self.target.sha)) job.delete() return x
def main (ifile = 'data/brdf_WW_1_A_1.kernelFiltered.dat', \ confFile='config_files/sentinel_Def.conf', \ solve = ['xlai','xkab','scen','xkw','xkm','xleafn','xs1']) : ''' Show that we can use this same setup to solve for the field data (MODIS) by using a different config file ''' ifile = 'data/brdf_WW_1_A_1.kernelFiltered.dat' ofileSingle = 'output/brdf_WW_1_A_1.kernelFilteredSingle.dat' ofile = 'output/brdf_WW_1_A_1.kernelFiltered.dat' s = Sentinel(solve=solve, confFile=confFile) # as above, solve for initial estimate s.solveSingle(ifile, ofileSingle) s.paramPlot(s.loadData(ofileSingle),s.loadData(ofileSingle),\ filename='plots/testFieldDataSingle.png') s.smooth(ofileSingle, ofile=ofileSingle + '_smooth') s.paramPlot(s.loadData('input/truth.dat'),\ s.loadData(ofileSingle+'_smooth'),\ filename='plots/%s.png'%(ofileSingle+'_smooth')) gamma = np.median(np.sqrt(s.gammaSolve.values()) * \ np.array(s.wScale.values())) gamma = int((np.sqrt(s.gammaSolve[solve[0]]) * \ np.array(s.wScale[solve[0]]))+0.5) s.solveRegular(ifile,ofile,modelOrder=2,gamma=gamma, \ initial=ofileSingle+'_smooth') s.paramPlot(s.loadData(ofileSingle+'_smooth'),\ s.loadData(ofile + '_result.dat'),\ filename='plots/%s_Gamma%08d.png'%(ofile,gamma))