def makeBQEnvVarRowList(self, valueDict, deviceId, rowsList):
# each received EnvVar type message must have these fields
if not utils.key_in_dict(valueDict, self.var_KEY ) or \
not utils.key_in_dict(valueDict, self.values_KEY ):
logging.error('makeBQEnvVarRowList: Missing key(s) in dict.')
return
idKey = 'Env'
varName = valueDict[self.var_KEY]
values = valueDict[self.values_KEY]
# clean / scrub / check the values.
deviceId = deviceId.replace('~', '')
varName = varName.replace('~', '')
# NEW ID format: <KEY>~<valName>~<created UTC TS>~<deviceId>
ID = idKey + '~{}~{}~' + deviceId
row = (
ID.format(varName, time.strftime('%FT%XZ',
time.gmtime())), # id column
values,
0,
0) # values column, with zero for X, Y
rowsList.append(row)
def save_data_to_Device(self, pydict, deviceId):
try:
if self.messageType_EnvVar != self.get_message_type(pydict) and \
self.messageType_CommandReply != self.get_message_type(pydict):
logging.debug(f"save_data_to_Device: invalid message type {self.get_message_type(pydict)}")
return
# each received EnvVar type message must have these fields
if not utils.key_in_dict(pydict, self.var_KEY ) or \
not utils.key_in_dict(pydict, self.values_KEY ):
logging.error('save_data_to_Device: Missing key(s) in dict.')
return
varName = pydict[ self.var_KEY ]
value = self.__string_to_value( pydict[ self.values_KEY ] )
name = self.__string_to_name( pydict[ self.values_KEY ] )
valueToSave = {
'timestamp': str( time.strftime( '%FT%XZ', time.gmtime())),
'name': str( name ),
'value': str( value ) }
datastore.save_device_data(deviceId, varName, valueToSave)
except Exception as e:
logging.critical(f"Exception in save_data_to_Device(): {e}")
def validate_message(self, message: Dict[str, str]) -> bool:
if not utils.key_in_dict(message, self.messageType_KEY):
return False
message_type = self.get_message_type(message)
if not (message_type == self.messageType_EnvVar or \
message_type == self.messageType_CommandReply or \
message_type == self.messageType_Image or \
message_type == self.messageType_ImageUpload or \
message_type == self.messageType_RecipeEvent):
return False
if message_type == self.messageType_EnvVar or \
message_type == self.messageType_CommandReply:
# mandatory keys for msg types 'EnvVar' and 'CommandReply'
if not (utils.key_in_dict(message, self.var_KEY) or \
utils.key_in_dict(message, self.values_KEY)):
return False
if message_type == self.messageType_Image or \
message_type == self.messageType_ImageUpload:
# mandatory keys for image messages
if not (utils.key_in_dict(message, self.varName_KEY) or \
utils.key_in_dict(message, self.imageType_KEY) or \
utils.key_in_dict(message, self.fileName_KEY)):
return False
if message_type == self.messageType_RecipeEvent:
# mandatory keys for recipe event messages
if not (utils.key_in_dict(message, self.recipeAction_KEY) or \
utils.key_in_dict(message, self.recipeName_KEY)):
return False
return True
def validate_message(self, message: Dict[str, str]) -> bool:
if not utils.key_in_dict(message, self.device_ID_key):
return False
if not utils.key_in_dict(message, self.message_type_key):
return False
if not utils.key_in_dict(message, self.message_key):
return False
message_type = message.get(self.message_type_key)
if not (message_type == self.recipe_start or \
message_type == self.recipe_stop or \
message_type == self.recipe_end or \
message_type == self.set_testing_hours):
return False
return True
def get_message_type(self, message):
if not utils.key_in_dict(message, self.messageType_KEY):
logging.error('Missing key %s' % self.messageType_KEY)
return None
if self.messageType_EnvVar == message.get(self.messageType_KEY):
return self.messageType_EnvVar
if self.messageType_CommandReply == message.get(self.messageType_KEY):
return self.messageType_CommandReply
# deprecated
if self.messageType_Image == message.get(self.messageType_KEY):
return self.messageType_Image
if self.messageType_ImageUpload == message.get(self.messageType_KEY):
return self.messageType_ImageUpload
if self.messageType_RecipeEvent == message.get(self.messageType_KEY):
return self.messageType_RecipeEvent
logging.error('get_message_type: Invalid value {} for key {}'.format(
message.get(self.messageType_KEY), self.messageType_KEY ))
return None
def save_uploaded_image(self, pydict, deviceId):
try:
if self.messageType_ImageUpload != self.get_message_type(pydict):
logging.error("save_uploaded_image: invalid message type")
return
# each received image message must have these fields
if not utils.key_in_dict(pydict, self.varName_KEY) or \
not utils.key_in_dict(pydict, self.fileName_KEY ):
logging.error('save_uploaded_image: missing key(s) in dict.')
return
var_name = pydict.get(self.varName_KEY)
file_name = pydict.get(self.fileName_KEY)
start = datetime.now()
# get a timedelta of the difference
delta = datetime.now() - start
# keep checking for image curl upload for 5 minutes
while delta.total_seconds() <= 5 * 60:
# Has this image already been handled?
# (this can happen since google pub-sub is "at least once"
# message delivery, the same message can get delivered again)
if storage.isUploadedImageInBucket(file_name,
env_vars.cs_bucket):
logging.info(f'save_uploaded_image: file {file_name} '
f'already handled.')
break
# Check if the file is in the upload bucket.
if not storage.isUploadedImageInBucket(file_name,
env_vars.cs_upload_bucket):
time.sleep(10)
delta = datetime.now() - start
logging.debug(f'save_uploaded_image: waited '
f'{delta.total_seconds()} secs for '
f'upload of {file_name}')
continue
# Move image from one gstorage bucket to another:
# openag-public-image-uploads > openag-v1-images
publicURL = storage.moveFileBetweenBuckets(
env_vars.cs_upload_bucket,
env_vars.cs_bucket, file_name)
if publicURL is None:
logging.warning(f'save_uploaded_image: '
f'image already moved: {file_name}')
break
#img_in_bucket = storage.isUploadedImageInBucket(file_name,
# env_vars.cs_bucket)
#logging.info(f'save_uploaded_image: {img_in_bucket} '
# f'file {file_name} is in {env_vars.cs_bucket}')
# use named temporary files to download and resize the image
f_split = os.path.splitext(file_name)
with tempfile.NamedTemporaryFile(suffix=f_split[1]) \
as downloaded_image_fp:
downloaded = storage.downloadFile(downloaded_image_fp,
env_vars.cs_bucket, file_name)
if not downloaded:
logging.error(f'save_uploaded_image: '
f'image not downloaded: {file_name}')
# save a medium sized version of the image
downloaded_image_fp.seek(0) # rewind to start of stream
with tempfile.NamedTemporaryFile(suffix=f_split[1]) \
as smaller_image_fp:
# halves each dimension by default
images.resize(downloaded_image_fp.name,
smaller_image_fp.name)
fn = f_split[0] + '_medium' + f_split[1]
storage.uploadFile(smaller_image_fp,
env_vars.cs_bucket, fn)
# save a small sized version of the image
downloaded_image_fp.seek(0) # rewind to start of stream
with tempfile.NamedTemporaryFile(suffix=f_split[1]) \
as smaller_image_fp:
images.resize(downloaded_image_fp.name,
smaller_image_fp.name,
(640, 480)) # good for ani gif
fn = f_split[0] + '_small' + f_split[1]
storage.uploadFile(smaller_image_fp,
env_vars.cs_bucket, fn)
# save a small sized version of the image
downloaded_image_fp.seek(0) # rewind to start of stream
with tempfile.NamedTemporaryFile(suffix=f_split[1]) \
as smaller_image_fp:
images.resize(downloaded_image_fp.name,
smaller_image_fp.name,
(128, 128)) # thumbnail size
fn = f_split[0] + '_thumbnail' + f_split[1]
storage.uploadFile(smaller_image_fp,
env_vars.cs_bucket, fn)
# Put the URL in the datastore for the UI to use.
datastore.saveImageURL(deviceId, publicURL, var_name)
# Put the URL as an env. var in BQ.
message_obj = {}
# keep old message type, UI code may depend on it
message_obj[ self.messageType_KEY ] = self.messageType_Image
message_obj[ self.var_KEY ] = var_name
valuesJson = "{'values':["
valuesJson += "{'name':'URL', 'type':'str', 'value':'%s'}" % \
(publicURL)
valuesJson += "]}"
message_obj[ self.values_KEY ] = valuesJson
# Generate the data that will be sent to BigQuery for insertion.
# Each value must be a row that matches the table schema.
rowsList = []
if self.makeBQRowList(message_obj, deviceId, rowsList):
bigquery.data_insert(rowsList)
delta = datetime.now() - start
logging.info(f"save_uploaded_image: Done with {file_name} "
f"in {delta.total_seconds()} secs")
break
# Remove any files in the uploads bucket that are over 2 hours old
storage.delete_files_over_two_hours_old(env_vars.cs_upload_bucket)
except Exception as e:
logging.critical(f"Exception in save_uploaded_image(): {e}")
def save_uploaded_image(self, pydict, deviceId):
try:
if self.messageType_ImageUpload != self.get_message_type(pydict):
logging.error("save_uploaded_image: invalid message type")
return
# each received image message must have these fields
if not utils.key_in_dict(pydict, self.varName_KEY) or \
not utils.key_in_dict(pydict, self.fileName_KEY ):
logging.error('save_uploaded_image: missing key(s) in dict.')
return
var_name = pydict.get(self.varName_KEY)
file_name = pydict.get(self.fileName_KEY)
start = datetime.now()
# get a timedelta of the difference
delta = datetime.now() - start
# keep checking for image curl upload for 5 minutes
while delta.total_seconds() <= 5 * 60:
# Has this image already been handled?
# (this can happen since google pub-sub is "at least once"
# message delivery, the same message can get delivered again)
if storage.isUploadedImageInBucket(file_name,
env_vars.cs_bucket):
logging.info(f'save_uploaded_image: file {file_name} '
f'already handled.')
break
# Check if the file is in the upload bucket.
if not storage.isUploadedImageInBucket(file_name,
env_vars.cs_upload_bucket):
time.sleep(10)
delta = datetime.now() - start
logging.debug(f'save_uploaded_image: waited '
f'{delta.total_seconds()} secs for '
f'upload of {file_name}')
continue
# Move image from one gstorage bucket to another:
# openag-public-image-uploads > openag-v1-images
publicURL = storage.moveFileBetweenBuckets(
env_vars.cs_upload_bucket,
env_vars.cs_bucket, file_name)
if publicURL is None:
logging.warning(f'save_uploaded_image: '
f'image already moved: {file_name}')
break
# Put the URL in the datastore for the UI to use.
datastore.saveImageURL(deviceId, publicURL, var_name)
# Put the URL as an env. var in BQ.
message_obj = {}
# keep old message type, UI code may depend on it
message_obj[ self.messageType_KEY ] = self.messageType_Image
message_obj[ self.var_KEY ] = var_name
valuesJson = "{'values':["
valuesJson += "{'name':'URL', 'type':'str', 'value':'%s'}" % \
(publicURL)
valuesJson += "]}"
message_obj[ self.values_KEY ] = valuesJson
# Generate the data that will be sent to BigQuery for insertion.
# Each value must be a row that matches the table schema.
rowsList = []
if self.makeBQRowList(message_obj, deviceId, rowsList):
bigquery.data_insert(rowsList)
delta = datetime.now() - start
logging.info(f"save_uploaded_image: Done with {file_name} "
f"in {delta.total_seconds()} secs")
break
# Remove any files in the uploads bucket that are over 2 hours old
storage.delete_files_over_two_hours_old(env_vars.cs_upload_bucket)
except Exception as e:
logging.critical(f"Exception in save_uploaded_image(): {e}")
def save_old_chunked_image(self, pydict, deviceId):
try:
# each received image message must have these fields
if not utils.key_in_dict( pydict, self.varName_KEY ) or \
not utils.key_in_dict( pydict, self.imageType_KEY ) or \
not utils.key_in_dict( pydict, self.chunk_KEY ) or \
not utils.key_in_dict( pydict, self.totalChunks_KEY ) or \
not utils.key_in_dict( pydict, self.imageChunk_KEY ) or \
not utils.key_in_dict( pydict, self.messageID_KEY ):
logging.error(
'save_old_chunked_image: Missing key(s) in dict.')
return
messageId = pydict[self.messageID_KEY]
varName = pydict[self.varName_KEY]
imageType = pydict[self.imageType_KEY]
chunkNum = pydict[self.chunk_KEY]
totalChunks = pydict[self.totalChunks_KEY]
imageChunk = pydict[self.imageChunk_KEY]
# Get rid of all chunks if we receive one bad chunk - so we don't
# make bad partial images.
if 0 == len(imageChunk):
logging.error(
"save_old_chunked_image: received empty imageChunk from {}, cleaning up turds"
.format(deviceId))
self.deleteImageChunksFromDatastore(deviceId, messageId)
self.saveTurd(deviceId, messageId)
return
# Clean up any smelly old turds from previous images (if they don't
# match the current messageId from this device).
turds = self.getTurds(deviceId)
for badImage in turds:
badMessageId = badImage['messageId']
if badMessageId != messageId:
self.deleteImageChunksFromDatastore(deviceId, badMessageId)
self.deleteTurd(deviceId, badMessageId)
# Save this chunk to the datastore cache.
self.saveImageChunkToDatastore(deviceId, messageId, varName,
imageType, chunkNum, totalChunks,
imageChunk)
# For every message received, check data store to see if we can
# assemble chunks. Messages will probably be received out of order.
# Start with a list of the number of chunks received:
listOfChunksReceived = []
for c in range(0, totalChunks):
listOfChunksReceived.append(False)
# What chunks have we already received?
oldChunks = self.getImageChunksFromDatastore(deviceId, messageId)
for oc in oldChunks:
listOfChunksReceived[oc['chunkNum']] = True
logging.debug('save_old_chunked_image: received {} of {} '
'for messageId={}'.format(
oc['chunkNum'], totalChunks, messageId))
# Do we have all chunks?
haveAllChunks = True
chunkCount = 0
for c in listOfChunksReceived:
logging.debug(
'save_old_chunked_image: listOfChunksReceived [{}]={}'.
format(chunkCount, c))
chunkCount += 1
if not c:
haveAllChunks = False
logging.debug('save_old_chunked_image: haveAllChunks={}'.format(
haveAllChunks))
# No, so just add this chunk to the datastore and return
if not haveAllChunks:
logging.debug(
'save_old_chunked_image: returning to wait for more chunks'
)
return
# YES! We have all our chunks, so reassemble the binary image.
# Delete the temporary datastore cache for the chunks
self.deleteImageChunksFromDatastore(deviceId, messageId)
self.deleteTurd(deviceId, messageId)
# Sort the chunks by chunkNum (we get messages out of order)
oldChunks = sorted(oldChunks, key=lambda k: k['chunkNum'])
# Reassemble the b64 chunks into one string (in order).
b64str = ''
for oc in oldChunks:
b64str += oc['imageChunk']
logging.debug(
'save_old_chunked_image: assemble {} of {}'.format(
oc['chunkNum'], oc['totalChunks']))
# Now covert our base64 string into binary image bytes
imageBytes = base64.b64decode(b64str)
# Put the image bytes in cloud storage as a file, and get an URL
publicURL = storage.saveFile(varName, imageType, imageBytes,
deviceId)
# Put the URL in the datastore for the UI to use.
datastore.saveImageURL(deviceId, publicURL, varName)
# Put the URL as an env. var in BQ.
message_obj = {}
message_obj[self.messageType_KEY] = self.messageType_Image
message_obj[self.var_KEY] = varName
valuesJson = "{'values':["
valuesJson += "{'name':'URL', 'type':'str', 'value':'%s'}" % \
( publicURL )
valuesJson += "]}"
message_obj[self.values_KEY] = valuesJson
rowsList = []
self.makeBQEnvVarRowList(message_obj, deviceId, rowsList)
bigquery.data_insert(rowsList)
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
logging.critical("Exception in save_old_chunked_image(): %s" % e)
traceback.print_tb(exc_traceback, file=sys.stdout)