def take_picture(camera: gp.camera, rotation_pos: int, declination_pos: int, queue: beanstalk.Connection) -> str: """take a picture and save it to the USB drive or the google drive, if specified""" # take the picture file_path = gp.check_result(gp.gp_camera_capture( camera, gp.GP_CAPTURE_IMAGE)) file_name = 'P{dec:02d}{rot:02d}_'.\ format(dec=declination_pos, rot=rotation_pos) + file_path.name # read the photo from the camera camera_file = gp.check_result(gp.gp_camera_file_get(camera, file_path.folder, file_path.name, gp.GP_FILE_TYPE_NORMAL)) # if a google drive isn't specified, write to the local USB drive # read the image from the camera into memory # and upload it file_data = gp.check_result(gp.gp_file_get_data_and_size(camera_file)) # okay, upload to the Google drive via a thread... byte_stream = io.BytesIO(file_data) camera_bytes = byte_stream.read(-1) job = {'task': 'photo', 'filename': file_name, 'data': base64.encodebytes(camera_bytes).decode('ascii')} # now send the photo to the Google Drive process queue.use(google_drive.GDRIVE_QUEUE) job_str = json.dumps(job) print("photo job size is {0} bytes".format(len(job_str))) queue.put(job_str) return file_name
def post_status(queue: beanstalk.Connection, message: str) -> None: """post a simple message to whomever is listening""" print(message) # echo message to stdout if queue: queue.use(STATUS_QUEUE) status_json = json.dumps({'msg': message}) queue.put(status_json)
def send_cancel_request(queue: beanstalk.Connection) -> int: """send a cancel request to the rig controller""" clear_status_queue(queue) # so we see what home command triggers queue.use(CANCEL_QUEUE) task_body = json.dumps({'task': 'cancel'}) return queue.put(task_body)
def __init__(self, request): if hasattr(request, 'connection'): self.conn = Connection(request.connection[0], request.connection[1]) elif settings.BEANSTALK_SERVERS: server = settings.BEANSTALK_SERVERS[0] self.conn = Connection(server[0], server[1]) else: raise Exception("No servers defined.")
def send_home_command(queue: beanstalk.Connection) -> int: """send a home command to home the rig""" clear_status_queue(queue) # so we see what home command triggers queue.use(TASK_QUEUE) task_body = json.dumps({'task': 'home'}) return queue.put(task_body)
def test_basic(self): conn = Connection("database") q = conn.Queue("testing") self.assertRaises(Empty, q.get) q.put(serialize({"name": "George Constanza"})) self.assertEquals(deserialize(q.get()), {"name": "George Constanza"})
def _ensure_connect(self): if self.__bs is None: print 'beanstalkc client connecting ...' self.__bs = Connection(self.__host, self.__port) print 'beanstalkc client connected to %s:%s:[%s]' % ( self.__host, self.__port, self.__tube) if not self.__tube is None: self.__bs.use(self.__tube) print 'beanstalkc client using %s' % self.__bs.using()
def establish_connection(self): conninfo = self.connection conn = Connection(self.type, host=conninfo.hostname, user=conninfo.userid, password=conninfo.password, database=conninfo.virtual_host, port=conninfo.port) conn.drain_events = self.drain_events return conn
def send_token(queue: beanstalk.Connection, body_str: str) -> int: """send the google token so we can save photos to a google drive""" queue.use(TASK_QUEUE) job_body = json.dumps({'task':'token', 'value': body_str}) job_id = queue.put(job_body) # ********************** # test_write_file(queue) # ********************** return job_id
def connect(self): while True: try: self.__bs = Connection(self.__host, self.__port) print 'connected to %s:%s' % (self.__host, self.__port) self.__bs.watch(self.__tube) print 'watching %s' % self.__bs.watching() if not self.__bs is None: return except: time.sleep(1)
def clear_all_queues(queue: beanstalk.Connection) -> None: """clear out all the currently known tubes""" for tube in [ CANCEL_QUEUE, STATUS_QUEUE, TASK_QUEUE, google_drive.GDRIVE_QUEUE ]: queue.use(tube) while True: dummy = queue.reserve(timeout=0) if dummy: dummy.delete() else: break
def clear_tube(queue: beanstalk.Connection, tube: str): """ flush all messages tubes to the rig""" queue.use(tube) while True: try: job = queue.reserve(timeout=0) if job is None: return job.delete() except beanstalk.CommandFailed: pass except beanstalk.DeadlineSoon: pass
def send_scan_command(queue: beanstalk.Connection, declination_steps: int, rotation_steps: int, start: int, stop: int) -> int: """this is it - time to scan. send the # of steps for each axis and return""" queue.use(TASK_QUEUE) task_body = json.dumps({'task': 'scan', 'steps': {'declination': declination_steps, 'rotation': rotation_steps}, 'offsets': {'start': start, 'stop': stop} }) return queue.put(task_body)
def main(): parser = OptionParser() parser.add_option("--i", default=50) parser.add_option("--n", default=10) parser.add_option("--tube", default="default") options, args = parser.parse_args() conn = Connection(HOST, PORT) conn.use(options.tube) for i in range(int(options.n)): conn.put(str(i))
def get_status(queue: beanstalk.Connection) -> str: """return the status of the rig""" queue.watch(STATUS_QUEUE) try: job = queue.reserve(timeout=0) # don't wait if job is None: return None # we have status, let's get it and pull the # job out of the queue status_json = jsonify(job.body) job.delete() return status_json except beanstalk.DeadlineSoon: return None except beanstalk.CommandFailed: return None
def __init__(self, host, port=11300): """ Args: host: host1_ip:host2_ip:... port: 11300 """ self.host = host self.port = port self.current_use_server_index = 0 self.servers = [] hosts = host.split(":") serverlist = itertools.product(hosts, [port]) for s in serverlist: conn = Connection(host=s[0], port=s[1], connect_timeout=20) conn.connect() self.servers.append(conn)
def _ensure_connect(self): if self.__bs is None: print 'beanstalkc client connecting ...' self.__bs = Connection(self.__host, self.__port) print 'beanstalkc client connected to %s:%s:[%s]' % (self.__host, self.__port, self.__tube) if not self.__tube is None: self.__bs.use(self.__tube) print 'beanstalkc client using %s' % self.__bs.using()
def test_write_file(queue: beanstalk.Connection) -> None: """simple program to test out google drive file writing""" queue.use(TASK_QUEUE) queue.watch(TASK_QUEUE) job = queue.reserve(timeout=2) if job is None: return None job_json = json.loads(job.body) if job_json['task'] != 'token': return None # we have a token, read it out access_info = json.loads(job_json['value']) drive_obj = google_drive.GoogleDrive(access_info) drive_obj.find_root_folder('rpipg') return None
class HandlerBase(object): def __init__(self, args): self.__b = Connection(host=args.hostname, port=args.port) def __del__(self): self.__b.close() def get_job_string(self, job): if getattr(job, 'jid', None) is None: return ('<JOB (LOOKUP FAILED)>') else: return ('<JOB (%d)>' % (job.jid)) def write_human(self, message): stderr.write(message + "\n") def write_data(self, data, encode=True): if encode is True: data = dumps(data, indent=2, separators=(',', ': ')) stdout.write(data) stdout.write("\n") def build_job(self, job_id, body=None): # beanstalkc does a poor job of tracking (or, rather, guessing) whether # or not a job is properly reserved in order to do most things. Not # only does it just not do anything if not reserved, but this state- # tracking doesn't at all work for our use case (one-offs). We set # 'reserved' to True no matter what. return Job(self.__b, job_id, body, True) def check_job_valid(self, job): if job is None: raise KeyError("Job was not found (1).") elif issubclass(job.__class__, Job) is False: raise ValueError("Job is not valid.") elif getattr(job, 'jid', None) is None: raise KeyError("Job was not found (2).") @property def beanstalk(self): return self.__b
def wait_for_work(queue: beanstalk.Connection) -> str: """wait for work, return json""" while True: job = queue.reserve(timeout=0) if job: job_json = job.body job.delete() # remove from the queue return json.loads(job_json) time.sleep(0.01) # sleep for 10 ms to share the computer
class JobService(object): """ beanstalkd job service watching the appointed tube. """ def __init__(self, host, port, tube): self.__host = host self.__port = port self.__tube = tube def connect(self): while True: try: self.__bs = Connection(self.__host, self.__port) print 'connected to %s:%s' % (self.__host, self.__port) self.__bs.watch(self.__tube) print 'watching %s' % self.__bs.watching() if not self.__bs is None: return except: time.sleep(1) def process(self, job): """overwrite Realize your functions """ pass def finish(self, job): job.delete() def start(self): self.connect() while True: try: job = self.__bs.reserve() print '>>[new job]', job, job.body self.process(job) self.finish(job) except Exception as ex: logging.error('job process exception', exc_info=ex) self.connect()
def establish_connection(self): self.port = self.port or DEFAULT_PORT self.host = self.host or DEFAULT_HOST self.connection = Connection(host=self.host, port=self.port) dbname = self.database if not dbname or dbname == "/": dbname = "ghettoq" self.database = getattr(self.connection, dbname) col = self.database.messages col.ensure_index([("queue", 1)]) return col
def init_app(self, app): app.config.setdefault('BEANSTALK_HOST', '127.0.0.1') app.config.setdefault('BEANSTALK_PORT', 11300) if not hasattr(app, 'extensions'): app.extensions = {} app.extensions['beanstalk'] = _BeanstalkState(self, app) self._client = Connection( host=app.config.get('BEANSTALK_HOST'), port=app.config.get('BEANSTALK_PORT'))
def wait_for_work(queue: beanstalk.Connection, motor_controller: Raspi_MotorHAT) -> dict: """wait for work, return json""" idle_start = time.time() while True: queue.watch(TASK_QUEUE) job = queue.reserve(timeout=0) if job: job_json = job.body job.delete() # remove from the queue return json.loads(job_json) time.sleep(0.01) # sleep for 10 ms to share the computer # if we have been idle for too long # release the stepper motors so they # don't overheat if (time.time() - idle_start) > 600 and motor_controller.is_active: # 10 minutes post_status(queue, "long idle, motors disabled") motor_controller.release_motors()
def connect_beanstalkd(): """Connect to beanstalkd server(s) from settings file""" server = getattr(settings, 'BEANSTALK_SERVER', '127.0.0.1') port = 11300 if server.find(':') > -1: server, port = server.split(':', 1) try: port = int(port) return Connection(server, port) except (ValueError, SocketError), e: raise BeanstalkError(e)
def do_server_status(self): conn = Connection(self.host, self.port) srv_stats = conn.stats() for cmd in ('put', 'reserve-with-timeout', 'delete'): self.submit('counter', cmd, srv_stats['cmd-%s' % cmd]) self.submit('counter', 'total_jobs', srv_stats['total-jobs']) for tube in conn.tubes(): for prefix in self.tubes_prefix: if tube.startswith(prefix): tube_stats = conn.stats_tube(tube) self.submit('records', 'current_ready', tube_stats['current-jobs-ready'], tube) self.submit('counter', 'total_jobs', tube_stats['total-jobs'], tube) conn.close()
def _open(self): conninfo = self.connection.client port = conninfo.port or DEFAULT_PORT conn = Connection(host=conninfo.hostname, port=port) conn.connect() return conn
def __init__(self, args): self.__b = Connection(host=args.hostname, port=args.port)
def __init__(self, *args, **kwargs): Connection.__init__(self, host=settings.options.beanstalk.host, port=settings.options.beanstalk.port)
class JobWorkerClient(object): """ job client. """ def __init__(self, host, port, tube=None): self.__host = host self.__port = port self.__tube = tube self.__bs = None def _ensure_connect(self): if self.__bs is None: print 'beanstalkc client connecting ...' self.__bs = Connection(self.__host, self.__port) print 'beanstalkc client connected to %s:%s:[%s]' % (self.__host, self.__port, self.__tube) if not self.__tube is None: self.__bs.use(self.__tube) print 'beanstalkc client using %s' % self.__bs.using() def put(self,obj): self._ensure_connect() self.__bs.put(json.dumps(obj)) def use_put(self, tube, obj): self._ensure_connect() self.__bs.use(tube) self.__bs.put(json.dumps(obj)) def use(self, tube): self._ensure_connect() self.__bs.use(tube) self.__tube = tube def close(self): try: self.__bs.close() except: pass
def send_cancel(queue: beanstalk.Connection) -> int: """send a cancel to the rig software to stop whatever is happening""" queue.use(CANCEL_QUEUE) return queue.put('STOP!') # anything will do
def establish_connection(self): self.host = self.host or DEFAULT_HOST self.port = self.port or DEFAULT_PORT return Connection(host=self.host, port=self.port)
class Worker(object): log = logging.getLogger() def __init__(self, tube='default', host='localhost', port=11300): self.tube = tube self.host, self.port = host, port try: self._connect() except SocketError: self.log.error('start failed. unable to connect to queue.') self.reconnect() def _connect(self): self.conn = Connection(self.host, self.port) self._watch_only(self.tube) def _watch_only(self, tube): for t in self.conn.watching(): if t != tube: self.conn.ignore(t) self.conn.watch(tube) def reconnect(self): while True: self.log.info('trying to reconnect to work queue.') try: self._connect() self.log.info('reconnected to queue host=%s port=%d.' % (self.host, self.port)) break except SocketError: self.log.error('reconnect failed. waiting 10 seconds before retrying.') time.sleep(10) def run(self): """ Worker main loop. Reserve job and execute. """ try: while True: self.cycle() except KeyboardInterrupt: self.log.info('got exit request. bye bye!') pass def cycle(self): try: self.log.info('waiting for jobs.') job = self.conn.reserve() if job is not None: self.log.info('got job with id #%d' % job.jid) self.on_job(Job(job)) except SocketError: self.reconnect() except Exception, e: self.log.exception('got unexpected exception when running job') pass # nothing else to do. the worker should keep running
def queue(self, name): return Queue(Connection(self.host, self.port), name, self.max_size)
def _connect(self): self.conn = Connection(self.host, self.port) self._watch_only(self.tube)
def session_start(queue: beanstalk.Connection) -> None: """before scanning, we need to start a 'session', which basically means doing any pre-scanning work. So we will shoot the drive process a message to kick off this activity""" queue.use(google_drive.GDRIVE_QUEUE) queue.put(json.dumps({'task': 'session_start'}))
def forward_authorization(queue: beanstalk.Connection, job: dict): """Forward the Google Drive authentication credentials to our Google Drive process""" queue.use(google_drive.GDRIVE_QUEUE) queue.put(json.dumps(job))
def create_connection(database): return Connection("redis", host="localhost", database=database)
def create_bstalk_conn(): return Connection( host=BEANSTALKD['host'], port=BEANSTALKD['port'], connect_timeout=1, )