def take_picture(camera: gp.camera,
rotation_pos: int, declination_pos: int,
queue: beanstalk.Connection) -> str:
"""take a picture and save it to the USB drive
or the google drive, if specified"""
# take the picture
file_path = gp.check_result(gp.gp_camera_capture(
camera, gp.GP_CAPTURE_IMAGE))
file_name = 'P{dec:02d}{rot:02d}_'.\
format(dec=declination_pos, rot=rotation_pos) + file_path.name
# read the photo from the camera
camera_file = gp.check_result(gp.gp_camera_file_get(camera,
file_path.folder,
file_path.name,
gp.GP_FILE_TYPE_NORMAL))
# if a google drive isn't specified, write to the local USB drive
# read the image from the camera into memory
# and upload it
file_data = gp.check_result(gp.gp_file_get_data_and_size(camera_file))
# okay, upload to the Google drive via a thread...
byte_stream = io.BytesIO(file_data)
camera_bytes = byte_stream.read(-1)
job = {'task': 'photo',
'filename': file_name,
'data': base64.encodebytes(camera_bytes).decode('ascii')}
# now send the photo to the Google Drive process
queue.use(google_drive.GDRIVE_QUEUE)
job_str = json.dumps(job)
print("photo job size is {0} bytes".format(len(job_str)))
queue.put(job_str)
return file_name
def post_status(queue: beanstalk.Connection, message: str) -> None:
"""post a simple message to whomever is listening"""
print(message) # echo message to stdout
if queue:
queue.use(STATUS_QUEUE)
status_json = json.dumps({'msg': message})
queue.put(status_json)
def send_cancel_request(queue: beanstalk.Connection) -> int:
"""send a cancel request to the rig controller"""
clear_status_queue(queue) # so we see what home command triggers
queue.use(CANCEL_QUEUE)
task_body = json.dumps({'task': 'cancel'})
return queue.put(task_body)
def __init__(self, request):
if hasattr(request, 'connection'):
self.conn = Connection(request.connection[0], request.connection[1])
elif settings.BEANSTALK_SERVERS:
server = settings.BEANSTALK_SERVERS[0]
self.conn = Connection(server[0], server[1])
else:
raise Exception("No servers defined.")
def send_home_command(queue: beanstalk.Connection) -> int:
"""send a home command to home the rig"""
clear_status_queue(queue) # so we see what home command triggers
queue.use(TASK_QUEUE)
task_body = json.dumps({'task': 'home'})
return queue.put(task_body)
def test_basic(self):
conn = Connection("database")
q = conn.Queue("testing")
self.assertRaises(Empty, q.get)
q.put(serialize({"name": "George Constanza"}))
self.assertEquals(deserialize(q.get()), {"name": "George Constanza"})
def _ensure_connect(self):
if self.__bs is None:
print 'beanstalkc client connecting ...'
self.__bs = Connection(self.__host, self.__port)
print 'beanstalkc client connected to %s:%s:[%s]' % (
self.__host, self.__port, self.__tube)
if not self.__tube is None:
self.__bs.use(self.__tube)
print 'beanstalkc client using %s' % self.__bs.using()
def establish_connection(self):
conninfo = self.connection
conn = Connection(self.type, host=conninfo.hostname,
user=conninfo.userid,
password=conninfo.password,
database=conninfo.virtual_host,
port=conninfo.port)
conn.drain_events = self.drain_events
return conn
def establish_connection(self):
conninfo = self.connection
conn = Connection(self.type,
host=conninfo.hostname,
user=conninfo.userid,
password=conninfo.password,
database=conninfo.virtual_host,
port=conninfo.port)
conn.drain_events = self.drain_events
return conn
def send_token(queue: beanstalk.Connection, body_str: str) -> int:
"""send the google token so we can save photos to a google drive"""
queue.use(TASK_QUEUE)
job_body = json.dumps({'task':'token', 'value': body_str})
job_id = queue.put(job_body)
# **********************
# test_write_file(queue)
# **********************
return job_id
def connect(self):
while True:
try:
self.__bs = Connection(self.__host, self.__port)
print 'connected to %s:%s' % (self.__host, self.__port)
self.__bs.watch(self.__tube)
print 'watching %s' % self.__bs.watching()
if not self.__bs is None:
return
except:
time.sleep(1)
def clear_all_queues(queue: beanstalk.Connection) -> None:
"""clear out all the currently known tubes"""
for tube in [
CANCEL_QUEUE, STATUS_QUEUE, TASK_QUEUE, google_drive.GDRIVE_QUEUE
]:
queue.use(tube)
while True:
dummy = queue.reserve(timeout=0)
if dummy:
dummy.delete()
else:
break
def clear_tube(queue: beanstalk.Connection, tube: str):
""" flush all messages tubes to the rig"""
queue.use(tube)
while True:
try:
job = queue.reserve(timeout=0)
if job is None:
return
job.delete()
except beanstalk.CommandFailed:
pass
except beanstalk.DeadlineSoon:
pass
def send_scan_command(queue: beanstalk.Connection,
declination_steps: int,
rotation_steps: int,
start: int, stop: int) -> int:
"""this is it - time to scan. send the # of steps for each axis
and return"""
queue.use(TASK_QUEUE)
task_body = json.dumps({'task': 'scan',
'steps': {'declination': declination_steps,
'rotation': rotation_steps},
'offsets': {'start': start, 'stop': stop}
})
return queue.put(task_body)
def main():
parser = OptionParser()
parser.add_option("--i", default=50)
parser.add_option("--n", default=10)
parser.add_option("--tube", default="default")
options, args = parser.parse_args()
conn = Connection(HOST, PORT)
conn.use(options.tube)
for i in range(int(options.n)):
conn.put(str(i))
def get_status(queue: beanstalk.Connection) -> str:
"""return the status of the rig"""
queue.watch(STATUS_QUEUE)
try:
job = queue.reserve(timeout=0) # don't wait
if job is None:
return None
# we have status, let's get it and pull the
# job out of the queue
status_json = jsonify(job.body)
job.delete()
return status_json
except beanstalk.DeadlineSoon:
return None
except beanstalk.CommandFailed:
return None
def __init__(self, host, port=11300):
"""
Args:
host: host1_ip:host2_ip:...
port: 11300
"""
self.host = host
self.port = port
self.current_use_server_index = 0
self.servers = []
hosts = host.split(":")
serverlist = itertools.product(hosts, [port])
for s in serverlist:
conn = Connection(host=s[0], port=s[1], connect_timeout=20)
conn.connect()
self.servers.append(conn)
def _ensure_connect(self):
if self.__bs is None:
print 'beanstalkc client connecting ...'
self.__bs = Connection(self.__host, self.__port)
print 'beanstalkc client connected to %s:%s:[%s]' % (self.__host, self.__port, self.__tube)
if not self.__tube is None:
self.__bs.use(self.__tube)
print 'beanstalkc client using %s' % self.__bs.using()
def test_write_file(queue: beanstalk.Connection) -> None:
"""simple program to test out google drive file writing"""
queue.use(TASK_QUEUE)
queue.watch(TASK_QUEUE)
job = queue.reserve(timeout=2)
if job is None:
return None
job_json = json.loads(job.body)
if job_json['task'] != 'token':
return None
# we have a token, read it out
access_info = json.loads(job_json['value'])
drive_obj = google_drive.GoogleDrive(access_info)
drive_obj.find_root_folder('rpipg')
return None
class HandlerBase(object):
def __init__(self, args):
self.__b = Connection(host=args.hostname, port=args.port)
def __del__(self):
self.__b.close()
def get_job_string(self, job):
if getattr(job, 'jid', None) is None:
return ('<JOB (LOOKUP FAILED)>')
else:
return ('<JOB (%d)>' % (job.jid))
def write_human(self, message):
stderr.write(message + "\n")
def write_data(self, data, encode=True):
if encode is True:
data = dumps(data, indent=2, separators=(',', ': '))
stdout.write(data)
stdout.write("\n")
def build_job(self, job_id, body=None):
# beanstalkc does a poor job of tracking (or, rather, guessing) whether
# or not a job is properly reserved in order to do most things. Not
# only does it just not do anything if not reserved, but this state-
# tracking doesn't at all work for our use case (one-offs). We set
# 'reserved' to True no matter what.
return Job(self.__b, job_id, body, True)
def check_job_valid(self, job):
if job is None:
raise KeyError("Job was not found (1).")
elif issubclass(job.__class__, Job) is False:
raise ValueError("Job is not valid.")
elif getattr(job, 'jid', None) is None:
raise KeyError("Job was not found (2).")
@property
def beanstalk(self):
return self.__b
def wait_for_work(queue: beanstalk.Connection) -> str:
"""wait for work, return json"""
while True:
job = queue.reserve(timeout=0)
if job:
job_json = job.body
job.delete() # remove from the queue
return json.loads(job_json)
time.sleep(0.01) # sleep for 10 ms to share the computer
class JobService(object):
"""
beanstalkd job service
watching the appointed tube.
"""
def __init__(self, host, port, tube):
self.__host = host
self.__port = port
self.__tube = tube
def connect(self):
while True:
try:
self.__bs = Connection(self.__host, self.__port)
print 'connected to %s:%s' % (self.__host, self.__port)
self.__bs.watch(self.__tube)
print 'watching %s' % self.__bs.watching()
if not self.__bs is None:
return
except:
time.sleep(1)
def process(self, job):
"""overwrite
Realize your functions
"""
pass
def finish(self, job):
job.delete()
def start(self):
self.connect()
while True:
try:
job = self.__bs.reserve()
print '>>[new job]', job, job.body
self.process(job)
self.finish(job)
except Exception as ex:
logging.error('job process exception', exc_info=ex)
self.connect()
class JobService(object):
"""
beanstalkd job service
watching the appointed tube.
"""
def __init__(self, host, port, tube):
self.__host = host
self.__port = port
self.__tube = tube
def connect(self):
while True:
try:
self.__bs = Connection(self.__host, self.__port)
print 'connected to %s:%s' % (self.__host, self.__port)
self.__bs.watch(self.__tube)
print 'watching %s' % self.__bs.watching()
if not self.__bs is None:
return
except:
time.sleep(1)
def process(self, job):
"""overwrite
Realize your functions
"""
pass
def finish(self, job):
job.delete()
def start(self):
self.connect()
while True:
try:
job = self.__bs.reserve()
print '>>[new job]', job, job.body
self.process(job)
self.finish(job)
except Exception as ex:
logging.error('job process exception', exc_info=ex)
self.connect()
def connect(self):
while True:
try:
self.__bs = Connection(self.__host, self.__port)
print 'connected to %s:%s' % (self.__host, self.__port)
self.__bs.watch(self.__tube)
print 'watching %s' % self.__bs.watching()
if not self.__bs is None:
return
except:
time.sleep(1)
def establish_connection(self):
self.port = self.port or DEFAULT_PORT
self.host = self.host or DEFAULT_HOST
self.connection = Connection(host=self.host, port=self.port)
dbname = self.database
if not dbname or dbname == "/":
dbname = "ghettoq"
self.database = getattr(self.connection, dbname)
col = self.database.messages
col.ensure_index([("queue", 1)])
return col
def init_app(self, app):
app.config.setdefault('BEANSTALK_HOST', '127.0.0.1')
app.config.setdefault('BEANSTALK_PORT', 11300)
if not hasattr(app, 'extensions'):
app.extensions = {}
app.extensions['beanstalk'] = _BeanstalkState(self, app)
self._client = Connection(
host=app.config.get('BEANSTALK_HOST'),
port=app.config.get('BEANSTALK_PORT'))
def wait_for_work(queue: beanstalk.Connection,
motor_controller: Raspi_MotorHAT) -> dict:
"""wait for work, return json"""
idle_start = time.time()
while True:
queue.watch(TASK_QUEUE)
job = queue.reserve(timeout=0)
if job:
job_json = job.body
job.delete() # remove from the queue
return json.loads(job_json)
time.sleep(0.01) # sleep for 10 ms to share the computer
# if we have been idle for too long
# release the stepper motors so they
# don't overheat
if (time.time() -
idle_start) > 600 and motor_controller.is_active: # 10 minutes
post_status(queue, "long idle, motors disabled")
motor_controller.release_motors()
def connect_beanstalkd():
"""Connect to beanstalkd server(s) from settings file"""
server = getattr(settings, 'BEANSTALK_SERVER', '127.0.0.1')
port = 11300
if server.find(':') > -1:
server, port = server.split(':', 1)
try:
port = int(port)
return Connection(server, port)
except (ValueError, SocketError), e:
raise BeanstalkError(e)
def do_server_status(self):
conn = Connection(self.host, self.port)
srv_stats = conn.stats()
for cmd in ('put', 'reserve-with-timeout', 'delete'):
self.submit('counter', cmd, srv_stats['cmd-%s' % cmd])
self.submit('counter', 'total_jobs', srv_stats['total-jobs'])
for tube in conn.tubes():
for prefix in self.tubes_prefix:
if tube.startswith(prefix):
tube_stats = conn.stats_tube(tube)
self.submit('records', 'current_ready', tube_stats['current-jobs-ready'], tube)
self.submit('counter', 'total_jobs', tube_stats['total-jobs'], tube)
conn.close()
def _open(self):
conninfo = self.connection.client
port = conninfo.port or DEFAULT_PORT
conn = Connection(host=conninfo.hostname, port=port)
conn.connect()
return conn
def __init__(self, args):
self.__b = Connection(host=args.hostname, port=args.port)
def __init__(self, *args, **kwargs):
Connection.__init__(self, host=settings.options.beanstalk.host,
port=settings.options.beanstalk.port)
class JobWorkerClient(object):
"""
job client.
"""
def __init__(self, host, port, tube=None):
self.__host = host
self.__port = port
self.__tube = tube
self.__bs = None
def _ensure_connect(self):
if self.__bs is None:
print 'beanstalkc client connecting ...'
self.__bs = Connection(self.__host, self.__port)
print 'beanstalkc client connected to %s:%s:[%s]' % (self.__host, self.__port, self.__tube)
if not self.__tube is None:
self.__bs.use(self.__tube)
print 'beanstalkc client using %s' % self.__bs.using()
def put(self,obj):
self._ensure_connect()
self.__bs.put(json.dumps(obj))
def use_put(self, tube, obj):
self._ensure_connect()
self.__bs.use(tube)
self.__bs.put(json.dumps(obj))
def use(self, tube):
self._ensure_connect()
self.__bs.use(tube)
self.__tube = tube
def close(self):
try:
self.__bs.close()
except:
pass
def send_cancel(queue: beanstalk.Connection) -> int:
"""send a cancel to the rig software to stop whatever is
happening"""
queue.use(CANCEL_QUEUE)
return queue.put('STOP!') # anything will do
def establish_connection(self):
self.host = self.host or DEFAULT_HOST
self.port = self.port or DEFAULT_PORT
return Connection(host=self.host, port=self.port)
class Worker(object):
log = logging.getLogger()
def __init__(self, tube='default', host='localhost', port=11300):
self.tube = tube
self.host, self.port = host, port
try:
self._connect()
except SocketError:
self.log.error('start failed. unable to connect to queue.')
self.reconnect()
def _connect(self):
self.conn = Connection(self.host, self.port)
self._watch_only(self.tube)
def _watch_only(self, tube):
for t in self.conn.watching():
if t != tube: self.conn.ignore(t)
self.conn.watch(tube)
def reconnect(self):
while True:
self.log.info('trying to reconnect to work queue.')
try:
self._connect()
self.log.info('reconnected to queue host=%s port=%d.' % (self.host, self.port))
break
except SocketError:
self.log.error('reconnect failed. waiting 10 seconds before retrying.')
time.sleep(10)
def run(self):
""" Worker main loop. Reserve job and execute. """
try:
while True:
self.cycle()
except KeyboardInterrupt:
self.log.info('got exit request. bye bye!')
pass
def cycle(self):
try:
self.log.info('waiting for jobs.')
job = self.conn.reserve()
if job is not None:
self.log.info('got job with id #%d' % job.jid)
self.on_job(Job(job))
except SocketError:
self.reconnect()
except Exception, e:
self.log.exception('got unexpected exception when running job')
pass # nothing else to do. the worker should keep running
def queue(self, name):
return Queue(Connection(self.host, self.port), name, self.max_size)
def _connect(self):
self.conn = Connection(self.host, self.port)
self._watch_only(self.tube)
def _open(self):
conninfo = self.connection.client
port = conninfo.port or DEFAULT_PORT
conn = Connection(host=conninfo.hostname, port=port)
conn.connect()
return conn
def session_start(queue: beanstalk.Connection) -> None:
"""before scanning, we need to start a 'session', which
basically means doing any pre-scanning work. So we will
shoot the drive process a message to kick off this activity"""
queue.use(google_drive.GDRIVE_QUEUE)
queue.put(json.dumps({'task': 'session_start'}))
def forward_authorization(queue: beanstalk.Connection, job: dict):
"""Forward the Google Drive authentication credentials to our
Google Drive process"""
queue.use(google_drive.GDRIVE_QUEUE)
queue.put(json.dumps(job))
def create_connection(database):
return Connection("redis", host="localhost", database=database)
def create_bstalk_conn():
return Connection(
host=BEANSTALKD['host'],
port=BEANSTALKD['port'],
connect_timeout=1,
)
