def bind_pg_to_websocket(self, filter_id=None): self.db.execute('LISTEN todos_updates;') last_ping = utc.now() while True: # Handle sending keepalives on the socket. now = utc.now() elapsed = (now - last_ping).total_seconds() if elapsed > self.app.settings.websocket_ping_interval: self.ws.send_frame('', self.ws.OPCODE_PING) last_ping = now # Block on notifications from Postgres, with 5 sec timeout. if select.select([self.dbconn], [], [], 5) == ([], [], []): logger.debug("No messages for 5 seconds.") else: self.dbconn.poll() while self.dbconn.notifies: notify = self.dbconn.notifies.pop() payload = json.loads(notify.payload) if filter_id is None or payload['id'] == filter_id: # Handle payloads too big for a PG NOTIFY. if 'error' in payload and 'id' in payload: payload = self.get_todo(payload['id']) self.ws.send(json.dumps(payload))
def bind_queue_to_websocket(self, exchange, routing_keys): settings = self.app.settings self.rabbit_conn = pika.BlockingConnection(pika.URLParameters(settings.rabbit_url)) channel = self.rabbit_conn.channel() queue = channel.queue_declare(exclusive=True) queue_name = queue.method.queue logger.debug('Rabbit socket on %s/%s' % (exchange, routing_keys)) channel.exchange_declare(exchange=exchange, type='topic', durable=True) for rk in routing_keys: channel.queue_bind(exchange=exchange, queue=queue_name, routing_key=rk) channel.basic_consume(self.on_rabbit_message, no_ack=True, queue=queue_name) last_ping = utc.now() while True: now = utc.now() elapsed = (now - last_ping).total_seconds() if elapsed > settings.websocket_ping_interval: self.ws.send_frame('', self.ws.OPCODE_PING) last_ping = now self.rabbit_conn.process_data_events()
def do_scheduled_tasks(settings): start_time = utc.now() db = make_session_cls(settings.db_url)() # write to checkins db.merge(Checkin(proc_name='timer', time=start_time)) db.commit() # connect to RabbitMQ rabbit_conn = pika.BlockingConnection(pika.URLParameters(settings.rabbit_url)) rabbit = rabbit_conn.channel() # write message for dispatcher to be consumed rabbit.exchange_declare(exchange=settings.dispatcher_ping_exchange, type='topic', durable=True) rabbit.basic_publish( exchange=settings.dispatcher_ping_exchange, routing_key='timer', body='timer' ) declare_exchanges(rabbit) check_pipelines(settings, db, rabbit) db.commit() check_jobs(settings, db, rabbit) db.commit() cleanup_logs(settings, db) db.commit() run_time = utc.now() - start_time logger.info("Finished scheduled tasks. Took %s seconds" % run_time.total_seconds())
def test_jwtsession_expire_days(): class A(spa.Handler): def get(self): return spa.Response(self.request.environ['jwtsession']['foo']) routes = ( ('/', 'a', A), ) app = spa.App(routes) secret = 'foobar' app = JWTSessionMiddleware(app, secret_key=secret, expire_days=1) c = Client(app, spa.Response) tok = jwt.encode({ 'foo': 'bar', 'iat': utc.now() - datetime.timedelta(days=1) }, secret) c.set_cookie('localhost', 'session', tok) with pytest.raises(KeyError): # The handler should fail to read 'foo' from the session, because it # will have been dropped when the middleware saw that its 'iat' # timestamp was too old. c.get('/')
def read(): """Reads the most recent update result, filtering results that are too old. Args: None. Returns: An update.result.Result based on the most recent update result file within the time threshold or None if none exist. """ result_files = glob.glob( os.path.join(_RESULT_FILE_DIR, _UPDATE_RESULT_FILENAME_FORMAT % '*')) if not result_files: return None # Filenames start with a timestamp, so the last one lexicographically is the # most recently created file. most_recent_result_file = sorted(result_files)[-1] with open(most_recent_result_file) as result_file: most_recent_result = update.result.read(result_file) # Ignore the result if it's too old. delta = utc.now() - most_recent_result.timestamp if delta.total_seconds() > _RECENT_UPDATE_THRESHOLD_SECONDS: return None return most_recent_result
def unserialize(cls, string, secret_key, algorithm='HS256', expire_days=None): """Load the secure cookie from a serialized string. :param string: the cookie value to unserialize. :param secret_key: the secret key used to serialize the cookie. :return: a new :class:`JWTCookie`. """ if isinstance(string, text_type): string = string.encode('utf-8', 'replace') if isinstance(secret_key, text_type): secret_key = secret_key.encode('utf-8', 'replace') items = jwt.decode(string, secret_key, algorithms=[algorithm]) if expire_days: if 'iat' not in items: raise TokenTimestampError('No iat claim in token') issued_at = utc.fromtimestamp(items['iat']) time_passed = utc.now() - issued_at if time_passed > datetime.timedelta(days=expire_days): raise TokenTimestampError('Token is too old') return cls(items, secret_key, algorithm)
def __call__(self, environ, start_response): qs_params = {k: v[0] for k, v in parse_qs(environ['QUERY_STRING']).items()} if self.qs_name not in qs_params: return self.app(environ, start_response) try: session_vals = jwt.decode(qs_params[self.qs_name], key=self.secret_key) except jwt.DecodeError: # silently drop malformed tokens return self.app(environ, start_response) if self.expire_days: if 'iat' not in session_vals: # We can't enforce token expiration if the token has no issued # at claim. So ignore the token. return self.app(environ, start_response) issued_at = utc.fromtimestamp(session_vals['iat']) if (utc.now() - issued_at).days > self.expire_days: # Token has an issued at claim, but it's too old. Ignore the # token. return self.app(environ, start_response) environ[self.wsgi_name].update(session_vals) return self.app(environ, start_response)
def test_qs_middleware(): class A(spa.Handler): def get(self): session = self.request.environ['jwtsession'] assert 'foo' in session return spa.Response(session['foo']) routes = ( ('/', 'a', A), ) app = spa.App(routes) secret = 'foobar' app = JWTSessionParamMiddleware(app, secret_key=secret) app = JWTSessionMiddleware(app, secret_key=secret) c = Client(app, spa.Response) token = jwt.encode({'foo': 'bar', 'iat': utc.now()}, secret).decode('utf-8') url = '/?session_token=%s' % token resp = c.get(url) assert resp.data == b'bar' cookieparts = resp.headers['Set-Cookie'].split('; ')[0].split('=') assert cookieparts[0] == 'session' assert jwt.decode(cookieparts[1], secret)['foo'] == 'bar'
def get_targets(self, target_time): # The HawaiianPipeline is in no hurry. If you call get_targets with a # target_time that's too recent, it will nack and make you wait. now = utc.now() wait_until = target_time + timedelta(days=4) if now < wait_until: raise mp.PipelineNack("What's the rush, man?", wait_until) return { "flour": [], "water": [], "yeast": [], "sugar": [], "salt": [], "olive oil": [], "mix": ["flour", "water", "yeast", "sugar", "salt", "olive oil"], "raise": ["mix"], "roll": ["raise"], "sauce": ["roll"], "cheese": ["sauce"], "ham": ["cheese"], "pineapple": ["cheese"], "bake": ["ham", "pineapple"], "box": ["bake"], "deliver": ["box"], "eat": ["deliver"] }
def parse_date(request): if 'date' in request.args: try: date_string = request.args.get('date') # Deal with a URL that has a + sign replaced by a space dt = parser.parse(date_string.replace(' ', '+')) d = utc.to_utc(dt) except Exception as err: raise Exception("Invalid Date '{}' {}".format(date_string, err)) else: d = utc.now() current_date = utc.now() if ((d - current_date).total_seconds() > 86400.0): raise Exception("Date '{}' more than 24 hours in future.".format(date_string)) return d
def _claim_job(self, target_time, target): """ Make a an RPC call to the dispatcher to claim self.job_id. If claiming the job_id is successful, the dispatcher will return '1', and this function will return True. If claiming the job is unsuccessful, probably because some other worker has already claimed it, the dispatcher will return '0', and this function will return False. """ self.corr_id = str(uuid.uuid4()) logger.info('Claiming job %s.' % self.job_id) self.chan.queue_declare(queue=self.queue, exclusive=True) start_time = utc.now() expire_time = self.get_expire_time(target_time, target, start_time) consumer_tag = self.chan.basic_consume(self._on_claim_response, no_ack=True, queue=self.queue) try: mp.claim_job(self.chan, self.job_id, self.queue, start_time.isoformat(), expire_time.isoformat(), self.corr_id) # Block while we wait for a response, as in the RabbitMQ RPC example # doc. wait_start = utc.now() while self._claim_response == None: self.conn.process_data_events() elapsed = (utc.now() - wait_start).total_seconds() if elapsed > self.assignment_wait_secs: logger.warning('Timed out waiting for job grant %s.' % self.job_id) return False granted = self._claim_response == u'1' if granted: logger.info('Claimed job %s.' % self.job_id) else: logger.info('Failed to claim job %s.' % self.job_id) finally: self.chan.basic_cancel(consumer_tag) return granted
def parse_date(request): if 'date' in request.args: try: date_string = request.args.get('date') # Deal with a URL that has a + sign replaced by a space dt = parser.parse(date_string.replace(' ', '+')) d = utc.to_utc(dt) except Exception as err: raise Exception("Invalid Date '{}' {}".format(date_string, err)) else: d = utc.now() current_date = utc.now() if ((d - current_date).total_seconds() > 86400.0): raise Exception( "Date '{}' more than 24 hours in future.".format(date_string)) return d
def on_pipeline_run_nack(settings, rabbit, db, data): logger.info( "Pipeline run nack {service}:{pipeline}:{run_id}".format(**data)) run = db.query(PipelineRun).filter_by(id=data['run_id']).one() # create a new nack record. if data['reannounce_time'] is None: rtime = None # If reannounce_time is None, then give up on this pipeline run. run.ack_time = utc.now() run.end_time = utc.now() else: rtime = isodate.parse_datetime(data['reannounce_time']) db.add( PipelineRunNack( pipeline_run=run, message=data['message'], reannounce_time=rtime, ))
def on_pipeline_run_ack(settings, rabbit, db, data): logger.info( "Pipeline run ack {service}:{pipeline}:{run_id}".format(**data)) run = db.query(PipelineRun).filter_by(id=data['run_id']).one() if run.ack_time is None: run.ack_time = utc.now() run.targets = data['targets'] run.target_parameters = data.get('target_parameters', {}) if run.is_ended(db): if run.end_time is None: run.end_time = utc.now() if run.all_targets_succeeded(db): run.succeeded = True else: for target in run.get_ready_targets(db): job = run.make_job(db, target) if job: lock_and_announce_job(db, rabbit, job)
def main(): settings = get_settings() rabbit_conn = pika.BlockingConnection( pika.URLParameters(settings.rabbit_url)) rabbit = rabbit_conn.channel() mp.declare_exchanges(rabbit) queue_name = 'mettle_dispatcher' rabbit.queue_declare(queue=queue_name, exclusive=False, durable=True) rabbit.queue_bind(exchange=mp.ANNOUNCE_SERVICE_EXCHANGE, queue=queue_name, routing_key='#') rabbit.queue_bind(exchange=mp.ACK_PIPELINE_RUN_EXCHANGE, queue=queue_name, routing_key='#') rabbit.queue_bind(exchange=mp.NACK_PIPELINE_RUN_EXCHANGE, queue=queue_name, routing_key='#') rabbit.queue_bind(exchange=mp.CLAIM_JOB_EXCHANGE, queue=queue_name, routing_key='#') rabbit.queue_bind(exchange=mp.END_JOB_EXCHANGE, queue=queue_name, routing_key='#') rabbit.queue_bind(exchange=settings.dispatcher_ping_exchange, queue=queue_name, routing_key='timer') Session = make_session_cls(settings.db_url) for method, properties, body in rabbit.consume(queue=queue_name): db = Session() if method.exchange == mp.ANNOUNCE_SERVICE_EXCHANGE: on_announce_service(settings, db, json.loads(body)) elif method.exchange == mp.ACK_PIPELINE_RUN_EXCHANGE: on_pipeline_run_ack(settings, rabbit, db, json.loads(body)) elif method.exchange == mp.NACK_PIPELINE_RUN_EXCHANGE: on_pipeline_run_nack(settings, rabbit, db, json.loads(body)) elif method.exchange == mp.CLAIM_JOB_EXCHANGE: on_job_claim(settings, rabbit, db, json.loads(body), properties.correlation_id) elif method.exchange == mp.END_JOB_EXCHANGE: on_job_end(settings, rabbit, db, json.loads(body)) # get messages from process timer restart queue elif method.exchange == settings.dispatcher_ping_exchange: db.merge(Checkin(proc_name='dispatcher', time=utc.now())) db.commit() rabbit.basic_ack(method.delivery_tag)
def serialize(self, expires=None): """Serialize the secure cookie into a string. If expires is provided, the session will be automatically invalidated after expiration when you unseralize it. This provides better protection against session cookie theft. :param expires: an optional expiration date for the cookie (a :class:`datetime.datetime` object) """ if self.secret_key is None: raise RuntimeError('no secret key defined') if expires: self['exp'] = expires self['iat'] = utc.now() return jwt.encode(self, self.secret_key, self.algorithm)
def __init__(self, hash_val: str, keywords, file_path: str, create_date: datetime, edit_date: datetime, file_size: int, num_words: int, parse_date: datetime = utc.now()): self._hash = hash_val self._keywords = keywords self._file_path = file_path self._parse_date = parse_date self._create_date = create_date self._edit_date = edit_date self._file_size = file_size self._num_words = num_words
def _get_latest_update_result(): result_files = glob.glob( os.path.join(_RESULT_FILE_DIR, _UPDATE_RESULT_FILENAME_FORMAT % '*')) if not result_files: return None # Filenames start with a timestamp, so the last one lexicographically is the # most recently created file. most_recent_result_file = sorted(result_files)[-1] with open(most_recent_result_file) as result_file: most_recent_result = update_result.read(result_file) # Ignore the result if it's too old. delta = utc.now() - most_recent_result.timestamp if delta.total_seconds() > _RECENT_UPDATE_THRESHOLD_SECONDS: return None return most_recent_result
def setup_sentry(): instance = current_instance() instance.add_tag('Status', 'deploy') instance.add_tag('StatusDetail', 'setup_sentry') run('sudo apt-get install -y build-essential python python-dev python-pip ' 'python-virtualenv python-setuptools libevent-dev supervisor nginx ' 'postgresql postgresql-server-dev-all') run('sudo -u postgres createuser -D -S -R sentry') run('sudo -u postgres createdb -O sentry sentry') run('sudo -u postgres psql -c ' '"alter user sentry with encrypted password \'sentry\'"') put('deploy/sentry.conf.py', '/home/ubuntu/sentry.conf.py') run('echo -e "SENTRY_KEY = \'`openssl rand -hex 8`\'" >> ' '/home/ubuntu/sentry.conf.py') run('sudo -u ubuntu mkdir -p /home/ubuntu/sentry') run('virtualenv --distribute /home/ubuntu/sentry') with cd('/home/ubuntu/sentry'): ve = lambda cmd: virtualenv_run(cmd, env='/home/ubuntu/sentry') ve('pip install psycopg2 gevent sentry nydus') ve('sentry --config=/home/ubuntu/sentry.conf.py upgrade --noinput') ve('echo "from django.contrib.auth.models import User; ' 'User.objects.create_superuser(\'sentry\', ' '\'[email protected]\', \'sentry\')" | ' 'sentry --config=/home/ubuntu/sentry.conf.py shell') ve('sentry --config=/home/ubuntu/sentry.conf.py ' 'repair --owner=sentry') put('deploy/sentry.supervisord.conf', '/home/ubuntu/sentry.supervisord.conf') ve('sudo cp /home/ubuntu/sentry.supervisord.conf ' '/etc/supervisor/conf.d/sentry.conf') ve('sudo supervisorctl update') run('sudo touch /etc/nginx/sites-enabled/sentry') run('sudo unlink /etc/nginx/sites-enabled/sentry') put('deploy/sentry.nginx.conf', '/home/ubuntu/sentry.nginx.conf') run('sudo cp /home/ubuntu/sentry.nginx.conf ' '/etc/nginx/sites-available/sentry.conf') run('sudo ln -s /etc/nginx/sites-available/sentry.conf ' '/etc/nginx/sites-enabled/sentry.conf') run('sudo /etc/init.d/nginx restart') instance.add_tag('Status', 'work') instance.remove_tag('StatusDetail') instance.add_tag('DeployedAt', now().strftime('%Y-%m-%d %H:%M:%S'))
def test_read_session_cookie(): class A(spa.Handler): def get(self): return spa.Response(self.request.environ['jwtsession']['foo']) routes = ( ('/', 'a', A), ) app = spa.App(routes) secret = 'foobar' app = JWTSessionMiddleware(app, secret_key=secret) c = Client(app, spa.Response) c.set_cookie('localhost', 'session', jwt.encode({'foo': 'bar', 'iat': utc.now()}, secret)) resp = c.get('/') assert resp.data == b'bar'
def post(self, notification_id): n = self.db.query(Notification).filter_by(id=notification_id).one() if n.acknowledged_by is None: # TODO: check data length first so we can't be DOSed with a huge # payload. data = json.loads(self.request.get_data()) if data.get('acknowledged') == True: user = self.request.session['username'] n.acknowledged_by = user n.acknowledged_time = utc.now() print n self.db.commit() return redirect('/api/notifications/{id}/'.format( id=notification_id), code=303) else: return BadRequest('Must include acknowledged: true to ' 'acknowledge a notification.') else: return BadRequest('Notification already acknowledged.')
def setup_sentry(): instance = current_instance() instance.add_tag("Status", "deploy") instance.add_tag("StatusDetail", "setup_sentry") run( "sudo apt-get install -y build-essential python python-dev python-pip " "python-virtualenv python-setuptools libevent-dev supervisor nginx " "postgresql postgresql-server-dev-all" ) run("sudo -u postgres createuser -D -S -R sentry") run("sudo -u postgres createdb -O sentry sentry") run("sudo -u postgres psql -c " "\"alter user sentry with encrypted password 'sentry'\"") put("deploy/sentry.conf.py", "/home/ubuntu/sentry.conf.py") run("echo -e \"SENTRY_KEY = '`openssl rand -hex 8`'\" >> " "/home/ubuntu/sentry.conf.py") run("sudo -u ubuntu mkdir -p /home/ubuntu/sentry") run("virtualenv --distribute /home/ubuntu/sentry") with cd("/home/ubuntu/sentry"): ve = lambda cmd: virtualenv_run(cmd, env="/home/ubuntu/sentry") ve("pip install psycopg2 gevent sentry nydus") ve("sentry --config=/home/ubuntu/sentry.conf.py upgrade --noinput") ve( 'echo "from django.contrib.auth.models import User; ' "User.objects.create_superuser('sentry', " "'*****@*****.**', 'sentry')\" | " "sentry --config=/home/ubuntu/sentry.conf.py shell" ) ve("sentry --config=/home/ubuntu/sentry.conf.py " "repair --owner=sentry") put("deploy/sentry.supervisord.conf", "/home/ubuntu/sentry.supervisord.conf") ve("sudo cp /home/ubuntu/sentry.supervisord.conf " "/etc/supervisor/conf.d/sentry.conf") ve("sudo supervisorctl update") run("sudo touch /etc/nginx/sites-enabled/sentry") run("sudo unlink /etc/nginx/sites-enabled/sentry") put("deploy/sentry.nginx.conf", "/home/ubuntu/sentry.nginx.conf") run("sudo cp /home/ubuntu/sentry.nginx.conf " "/etc/nginx/sites-available/sentry.conf") run("sudo ln -s /etc/nginx/sites-available/sentry.conf " "/etc/nginx/sites-enabled/sentry.conf") run("sudo /etc/init.d/nginx restart") instance.add_tag("Status", "work") instance.remove_tag("StatusDetail") instance.add_tag("DeployedAt", now().strftime("%Y-%m-%d %H:%M:%S"))
def tar(self, appname, appversion): """ Given an app name and version to be used in the tarball name, create a tar.bz2 file with all of this folder's contents inside. Return a Build object with attributes for appname, appversion, time, and path. """ name_tmpl = '%(app)s-%(version)s-%(time)s.tar.bz2' time = utc.now() name = name_tmpl % { 'app': appname, 'version': appversion, 'time': time.strftime('%Y-%m-%dT%H-%M') } if not os.path.exists(TARBALL_HOME): os.mkdir(TARBALL_HOME) tarball = os.path.join(TARBALL_HOME, name) tar_params = {'filename': tarball, 'folder': self.folder} tar_result = run('tar -C %(folder)s -cjf %(filename)s .' % tar_params) tar_result.raise_for_status() return Build(appname, appversion, time, tarball)
def create_object_from_file(self, local_path): return Sp4Ephemerides(local_path, 1.5e6, name_list=["QZS-4", "QZS-3"]) class GPSCache(EphemerisFileCache): def __init__(self): EphemerisFileCache.__init__(self, "norad_gps") def get_url(self, utc_date): return "http://www.celestrak.com/NORAD/elements/gps-ops.txt" def create_object_from_file(self, local_path): return Sp4Ephemerides(local_path, 1.5e6) class GalileoCache(EphemerisFileCache): def __init__(self): EphemerisFileCache.__init__(self, "norad_galileo") def get_url(self, utc_date): return "http://www.celestrak.com/NORAD/elements/galileo.txt" def create_object_from_file(self, local_path): return Sp4Ephemerides(local_path, 1.5e6) if __name__=="__main__": cache = NORADCache() print(cache.get_positions(utc.now()))
def cleanup_logs(settings, db): logger.info("Cleaning up old logs.") cutoff_time = utc.now() - timedelta(days=settings.max_log_days) db.query(JobLogLine).filter( JobLogLine.received_time<cutoff_time ).delete(synchronize_session=False)
def next_run_time(self): if self.chained_from: return self.chained_from.next_run_time() schedule = croniter(self.crontab, utc.now()) return schedule.get_next(datetime.datetime)
def run_pipelines(service_name, rabbit_url, pipelines, queue_name=None): while True: try: # Expects 'pipelines' to be a dict of pipeline names (as keys) and # classes (as values), queue_name = queue_name or mp.service_queue_name(service_name) rabbit = RabbitChannel(rabbit_url, service_name, pipelines, queue_name) for method, properties, body in rabbit.consume(queue=queue_name): data = json.loads(body.decode('utf-8')) pipeline_name = data['pipeline'] pipeline_cls = pipelines[pipeline_name] target_time = isodate.parse_datetime(data['target_time']) run_id = data['run_id'] if method.exchange == mp.ANNOUNCE_PIPELINE_RUN_EXCHANGE: pipeline = pipeline_cls(rabbit.conn, rabbit, service_name, pipeline_name, run_id) # If it's a pipeline run announcement, then call get_targets # and publish result. try: targets = pipeline.get_targets(target_time) target_params = pipeline.get_target_parameters( target_time) logger.info( "Acking pipeline run %s:%s:%s" % (service_name, data['pipeline'], data['run_id'])) mp.ack_pipeline_run(rabbit, service_name, data['pipeline'], data['target_time'], run_id, targets, target_params) except PipelineNack as pn: logger.info( "Nacking pipeline run %s:%s:%s" % (service_name, data['pipeline'], data['run_id'])) reannounce_time = None if pn.reannounce_time: reannounce_time = pn.reannounce_time.isoformat() mp.nack_pipeline_run(rabbit, service_name, data['pipeline'], run_id, reannounce_time, pn.message) elif method.exchange == '' and method.routing_key == queue_name: # Job message published directly to our queue, not going # through an exchange. job_id = data['job_id'] target = data['target'] pipeline = pipeline_cls(rabbit.conn, rabbit, service_name, pipeline_name, run_id, target, job_id) # If it's a job announcement, then publish ack, run job, # then publish completion. # publish ack claimed = pipeline._claim_job(target_time, data['target']) if claimed: # WOOO! Actually do some work here. succeeded = pipeline.make_target( target_time, target, data['target_parameters']) mp.end_job(rabbit, service_name, data['pipeline'], data['target_time'], data['target'], job_id, utc.now().isoformat(), succeeded) else: logger.info('Failed to claim job %s.' % job_id) rabbit.basic_ack(method.delivery_tag) except (pika.exceptions.AMQPError, AttributeError) as e: if isinstance(e, AttributeError) and ( "'NoneType' object has no attribute 'sendall'" in str(e) or "'NoneType' object has no attribute 'send'" in str(e)): logger.exception('Unexpected RabbitMQ exception: %s.' % str(e)) logger.info( 'Connection will be re-established in %s seconds!' % SLEEP_INTERVAL_ON_RABBITMQ_EXCEPTION) time.sleep(SLEEP_INTERVAL_ON_RABBITMQ_EXCEPTION) else: raise
def validate_crontab(self, key, cronspec): if cronspec is not None: # If the cronspec is not parseable, croniter will raise an exception # here. croniter(cronspec, utc.now()) return cronspec
def create_object_from_file(self, local_path): return Sp4Ephemerides(local_path, 1.5e6) class GalileoCache(EphemerisFileCache): def __init__(self): EphemerisFileCache.__init__(self, "norad_galileo") def get_url(self, utc_date): return "http://www.celestrak.com/NORAD/elements/galileo.txt" def create_object_from_file(self, local_path): return Sp4Ephemerides(local_path, 1.5e6) class BeidouCache(EphemerisFileCache): def __init__(self): EphemerisFileCache.__init__(self, "norad_beidou") def get_url(self, utc_date): return "https://www.celestrak.com/NORAD/elements/beidou.txt" def create_object_from_file(self, local_path): return Sp4Ephemerides(local_path, 1.5e6) if __name__ == "__main__": cache = NORADCache() print(cache.get_positions(utc.now()))
def check_jobs(settings, db, rabbit): logger.info("Checking jobs.") now = utc.now() expired_jobs = db.query(Job).filter( Job.end_time==None, Job.expires<now, ) log_cutoff_time = now - timedelta(minutes=settings.job_log_lookback_minutes) for job in expired_jobs: recent_log_lines_count = db.query(JobLogLine).filter( JobLogLine.job==job, JobLogLine.received_time>log_cutoff_time ).count() if recent_log_lines_count: pipeline = job.pipeline_run.pipeline subj = "Job %s running past expire time." % job.target msg = textwrap.dedent("""Job {target_time} {target}, from pipeline {pipeline} has passed its expire time, but is still emitting log output. Will let it continue. Please consider modifiying the service to provide a more accurate expiration time.""".format( target_time=job.pipeline_run.target_time.isoformat(), target=job.target, pipeline=pipeline.name, )) notify_failed_run(db, job.pipeline_run, subj, msg) else: # This expired job is no longer doing stuff. As far as we can tell. job.end_time = now # See if we have run out of retries pipeline = job.pipeline_run.pipeline attempts_count = db.query(Job).filter_by( pipeline_run=job.pipeline_run, target=job.target).count() if attempts_count < pipeline.retries: # Make a new job new_job = Job( pipeline_run=job.pipeline_run, target=job.target, target_parameters=job.target_parameters ) db.add(new_job) else: # No more retries. Send a failure notification subj = "Job %s out of retries" % job.target msg = """Job {target_time} {target}, from pipeline {pipeline} has passed its expire time, has not recently emitted log messages, and has no retries remaining. You should look into it.""".format( target_time=job.pipeline_run.target_time.isoformat(), target=job.target, pipeline=pipeline.name, ) notify_failed_run(db, job.pipeline_run, subj, msg) # Handle jobs that haven't been acked. They should be announced. Any # expired ones have already been cleaned up by the time we get here. new_jobs = db.query(Job).filter( Job.start_time==None, ) db.commit() for job in new_jobs: lock_and_announce_job(db, rabbit, job)
def check_pipelines(settings, db, rabbit): logger.info("Checking pipelines.") scheduled_pipelines = db.query(Pipeline).filter( Pipeline.active==True, Pipeline.crontab!=None, Pipeline.chained_from==None, ) # Create any needed pipeline runs for pipelines with a crontab now = utc.now() start = now - timedelta(days=settings.lookback_days) for pipeline in scheduled_pipelines: for target_time in crontimes(pipeline.crontab, start): if target_time < now: ensure_pipeline_run(db, pipeline, target_time) else: break # Create any needed pipeline runs for pipelines that are chained to other # pipelines. chained_pipelines = db.query(Pipeline).filter( Pipeline.active==True, ).join(Pipeline.chained_from, aliased=True).filter( Pipeline.active==True, ) for pipeline in chained_pipelines: # get successful parent runs within lookback_days parent_runs = db.query(PipelineRun).filter( PipelineRun.pipeline==pipeline.chained_from, PipelineRun.target_time>=now - timedelta(days=settings.lookback_days), PipelineRun.succeeded==True, ) for pr in parent_runs: chained_runs = db.query(PipelineRun).filter( PipelineRun.pipeline==pipeline, PipelineRun.chained_from_id==pr.id, ) if chained_runs.count() == 0: logger.info("Chaining run of pipeline %s from %s for " "target time %s" % (pipeline.name, pr.pipeline.name, pr.target_time.isoformat())) db.add(PipelineRun( pipeline=pipeline, target_time=pr.target_time, started_by='timer', chained_from_id=pr.id, )) db.commit() # Now announce all unacked pipeline runs, whether created by this timer or # by someone manually, or by the dispatcher (using a trigger off some other # run) unacked_runs = db.query(PipelineRun).filter( PipelineRun.created_time>start, PipelineRun.ack_time==None, PipelineRun.end_time==None, ) for run in unacked_runs: # If run has previously been nacked, and we haven't reached the # reannounce time, then don't announce yet. announce_time = run.get_announce_time() if announce_time is None or announce_time < now: lock_and_announce_run(db, rabbit, run) else: logger.info("Skipping announcement for run %s until %s" % (run.id, announce_time)) # Finally, check for any acked runs without an end_time, and see if they're # actually finished. unended_runs = db.query(PipelineRun).filter( PipelineRun.created_time>start, PipelineRun.ack_time!=None, PipelineRun.end_time==None ) for run in unended_runs: ready_targets = run.get_ready_targets(db) if ready_targets: for target in ready_targets: # This job will be announced later in the check_jobs function. run.make_job(db, target) elif run.is_ended(db): run.end_time=now if run.all_targets_succeeded(db): run.succeeded = True elif run.is_failed(db): # Job has ended with failure (attemps reach the maximum retries allowed) notify_failed_run(db, run)
import sun import location import sky_object import utc import angle import numpy as np class SunObject(sky_object.SkyObject): def __init__(self): sky_object.SkyObject.__init__(self, "Sun") def get_az_el(self, utc_date, lat, lon, alt): s = sun.Sun() loc = location.Location(lat, lon, alt=alt) ra, decl = s.radec(utc_date) _el, _az = loc.equatorial_to_horizontal(utc_date, ra,decl) el, az = np.round([_el.to_degrees(), _az.to_degrees()], decimals=6) ret = [] ret.append({'name': 'sun', 'r': 1e10, 'el':el, 'az':az, 'jy':10000.0}) return ret if __name__=="__main__": cache = SunObject() print(cache.get_az_el(utc.now(), lat=angle.from_dms(-45.86391200), lon=angle.from_dms(170.51348452), alt=46.5))