def parse(self, device_ID: str, message: Dict[str, str]) -> None: if not self.validate_message(message): logging.error(f'{self.name}.parse: invalid message={message}') return if self.messageType_Image == self.get_message_type(message): #logging.warning(f'{self.name}.parse: ignoring old chunked images ' # 'from old clients.') deprecated = DeprecatedImageChunking() deprecated.save_old_chunked_image(message, device_ID) return # New way of handling (already) uploaded images. if self.messageType_ImageUpload == self.get_message_type(message): self.save_uploaded_image(message, device_ID) return # Device sent a recipe event (start or stop) and we must # republish a notification message to the notifications topic # using our NotificationMessaging class. if self.messageType_RecipeEvent == self.get_message_type(message): action = message.get(self.recipeAction_KEY) message_type = None name = message.get(self.recipeName_KEY) if action == 'start': message_type = NotificationMessaging.recipe_start self.runs.start(device_ID,name) elif action == 'stop': message_type = NotificationMessaging.recipe_stop self.runs.stop(device_ID) elif action == 'end': message_type = NotificationMessaging.recipe_end self.runs.stop(device_ID) if message_type is None: logging.error(f'{self.name}.parse: invalid recipe event ' f'action={action}') return # TODO: Re-enable this when notifications get turned back on (After removing saving of the # runs() data from the notification service. # self.notification_messaging.publish(device_ID, message_type, name) return # Save the most recent data as properties on the Device entity in the # datastore. self.save_data_to_Device(message, device_ID) # Also insert into BQ (Env vars and command replies) rowsList = [] if self.makeBQRowList(message, device_ID, rowsList): bigquery.data_insert(rowsList)
def save_uploaded_image(self, pydict, deviceId): try: if self.messageType_ImageUpload != self.get_message_type(pydict): logging.error("save_uploaded_image: invalid message type") return # each received image message must have these fields if not utils.key_in_dict(pydict, self.varName_KEY) or \ not utils.key_in_dict(pydict, self.fileName_KEY ): logging.error('save_uploaded_image: missing key(s) in dict.') return var_name = pydict.get(self.varName_KEY) file_name = pydict.get(self.fileName_KEY) start = datetime.now() # get a timedelta of the difference delta = datetime.now() - start # keep checking for image curl upload for 5 minutes while delta.total_seconds() <= 5 * 60: # Has this image already been handled? # (this can happen since google pub-sub is "at least once" # message delivery, the same message can get delivered again) if storage.isUploadedImageInBucket(file_name, env_vars.cs_bucket): logging.info(f'save_uploaded_image: file {file_name} ' f'already handled.') break # Check if the file is in the upload bucket. if not storage.isUploadedImageInBucket(file_name, env_vars.cs_upload_bucket): time.sleep(10) delta = datetime.now() - start logging.debug(f'save_uploaded_image: waited ' f'{delta.total_seconds()} secs for ' f'upload of {file_name}') continue # Move image from one gstorage bucket to another: # openag-public-image-uploads > openag-v1-images publicURL = storage.moveFileBetweenBuckets( env_vars.cs_upload_bucket, env_vars.cs_bucket, file_name) if publicURL is None: logging.warning(f'save_uploaded_image: ' f'image already moved: {file_name}') break # Put the URL in the datastore for the UI to use. datastore.saveImageURL(deviceId, publicURL, var_name) # Put the URL as an env. var in BQ. message_obj = {} # keep old message type, UI code may depend on it message_obj[ self.messageType_KEY ] = self.messageType_Image message_obj[ self.var_KEY ] = var_name valuesJson = "{'values':[" valuesJson += "{'name':'URL', 'type':'str', 'value':'%s'}" % \ (publicURL) valuesJson += "]}" message_obj[ self.values_KEY ] = valuesJson # Generate the data that will be sent to BigQuery for insertion. # Each value must be a row that matches the table schema. rowsList = [] if self.makeBQRowList(message_obj, deviceId, rowsList): bigquery.data_insert(rowsList) delta = datetime.now() - start logging.info(f"save_uploaded_image: Done with {file_name} " f"in {delta.total_seconds()} secs") break # Remove any files in the uploads bucket that are over 2 hours old storage.delete_files_over_two_hours_old(env_vars.cs_upload_bucket) except Exception as e: logging.critical(f"Exception in save_uploaded_image(): {e}")
def save_uploaded_image(self, pydict, deviceId): try: if self.messageType_ImageUpload != self.get_message_type(pydict): logging.error("save_uploaded_image: invalid message type") return # each received image message must have these fields if not utils.key_in_dict(pydict, self.varName_KEY) or \ not utils.key_in_dict(pydict, self.fileName_KEY ): logging.error('save_uploaded_image: missing key(s) in dict.') return var_name = pydict.get(self.varName_KEY) file_name = pydict.get(self.fileName_KEY) start = datetime.now() # get a timedelta of the difference delta = datetime.now() - start # keep checking for image curl upload for 5 minutes while delta.total_seconds() <= 5 * 60: # Has this image already been handled? # (this can happen since google pub-sub is "at least once" # message delivery, the same message can get delivered again) if storage.isUploadedImageInBucket(file_name, env_vars.cs_bucket): logging.info(f'save_uploaded_image: file {file_name} ' f'already handled.') break # Check if the file is in the upload bucket. if not storage.isUploadedImageInBucket(file_name, env_vars.cs_upload_bucket): time.sleep(10) delta = datetime.now() - start logging.debug(f'save_uploaded_image: waited ' f'{delta.total_seconds()} secs for ' f'upload of {file_name}') continue # Move image from one gstorage bucket to another: # openag-public-image-uploads > openag-v1-images publicURL = storage.moveFileBetweenBuckets( env_vars.cs_upload_bucket, env_vars.cs_bucket, file_name) if publicURL is None: logging.warning(f'save_uploaded_image: ' f'image already moved: {file_name}') break #img_in_bucket = storage.isUploadedImageInBucket(file_name, # env_vars.cs_bucket) #logging.info(f'save_uploaded_image: {img_in_bucket} ' # f'file {file_name} is in {env_vars.cs_bucket}') # use named temporary files to download and resize the image f_split = os.path.splitext(file_name) with tempfile.NamedTemporaryFile(suffix=f_split[1]) \ as downloaded_image_fp: downloaded = storage.downloadFile(downloaded_image_fp, env_vars.cs_bucket, file_name) if not downloaded: logging.error(f'save_uploaded_image: ' f'image not downloaded: {file_name}') # save a medium sized version of the image downloaded_image_fp.seek(0) # rewind to start of stream with tempfile.NamedTemporaryFile(suffix=f_split[1]) \ as smaller_image_fp: # halves each dimension by default images.resize(downloaded_image_fp.name, smaller_image_fp.name) fn = f_split[0] + '_medium' + f_split[1] storage.uploadFile(smaller_image_fp, env_vars.cs_bucket, fn) # save a small sized version of the image downloaded_image_fp.seek(0) # rewind to start of stream with tempfile.NamedTemporaryFile(suffix=f_split[1]) \ as smaller_image_fp: images.resize(downloaded_image_fp.name, smaller_image_fp.name, (640, 480)) # good for ani gif fn = f_split[0] + '_small' + f_split[1] storage.uploadFile(smaller_image_fp, env_vars.cs_bucket, fn) # save a small sized version of the image downloaded_image_fp.seek(0) # rewind to start of stream with tempfile.NamedTemporaryFile(suffix=f_split[1]) \ as smaller_image_fp: images.resize(downloaded_image_fp.name, smaller_image_fp.name, (128, 128)) # thumbnail size fn = f_split[0] + '_thumbnail' + f_split[1] storage.uploadFile(smaller_image_fp, env_vars.cs_bucket, fn) # Put the URL in the datastore for the UI to use. datastore.saveImageURL(deviceId, publicURL, var_name) # Put the URL as an env. var in BQ. message_obj = {} # keep old message type, UI code may depend on it message_obj[ self.messageType_KEY ] = self.messageType_Image message_obj[ self.var_KEY ] = var_name valuesJson = "{'values':[" valuesJson += "{'name':'URL', 'type':'str', 'value':'%s'}" % \ (publicURL) valuesJson += "]}" message_obj[ self.values_KEY ] = valuesJson # Generate the data that will be sent to BigQuery for insertion. # Each value must be a row that matches the table schema. rowsList = [] if self.makeBQRowList(message_obj, deviceId, rowsList): bigquery.data_insert(rowsList) delta = datetime.now() - start logging.info(f"save_uploaded_image: Done with {file_name} " f"in {delta.total_seconds()} secs") break # Remove any files in the uploads bucket that are over 2 hours old storage.delete_files_over_two_hours_old(env_vars.cs_upload_bucket) except Exception as e: logging.critical(f"Exception in save_uploaded_image(): {e}")
def save_old_chunked_image(self, pydict, deviceId): try: # each received image message must have these fields if not utils.key_in_dict( pydict, self.varName_KEY ) or \ not utils.key_in_dict( pydict, self.imageType_KEY ) or \ not utils.key_in_dict( pydict, self.chunk_KEY ) or \ not utils.key_in_dict( pydict, self.totalChunks_KEY ) or \ not utils.key_in_dict( pydict, self.imageChunk_KEY ) or \ not utils.key_in_dict( pydict, self.messageID_KEY ): logging.error( 'save_old_chunked_image: Missing key(s) in dict.') return messageId = pydict[self.messageID_KEY] varName = pydict[self.varName_KEY] imageType = pydict[self.imageType_KEY] chunkNum = pydict[self.chunk_KEY] totalChunks = pydict[self.totalChunks_KEY] imageChunk = pydict[self.imageChunk_KEY] # Get rid of all chunks if we receive one bad chunk - so we don't # make bad partial images. if 0 == len(imageChunk): logging.error( "save_old_chunked_image: received empty imageChunk from {}, cleaning up turds" .format(deviceId)) self.deleteImageChunksFromDatastore(deviceId, messageId) self.saveTurd(deviceId, messageId) return # Clean up any smelly old turds from previous images (if they don't # match the current messageId from this device). turds = self.getTurds(deviceId) for badImage in turds: badMessageId = badImage['messageId'] if badMessageId != messageId: self.deleteImageChunksFromDatastore(deviceId, badMessageId) self.deleteTurd(deviceId, badMessageId) # Save this chunk to the datastore cache. self.saveImageChunkToDatastore(deviceId, messageId, varName, imageType, chunkNum, totalChunks, imageChunk) # For every message received, check data store to see if we can # assemble chunks. Messages will probably be received out of order. # Start with a list of the number of chunks received: listOfChunksReceived = [] for c in range(0, totalChunks): listOfChunksReceived.append(False) # What chunks have we already received? oldChunks = self.getImageChunksFromDatastore(deviceId, messageId) for oc in oldChunks: listOfChunksReceived[oc['chunkNum']] = True logging.debug('save_old_chunked_image: received {} of {} ' 'for messageId={}'.format( oc['chunkNum'], totalChunks, messageId)) # Do we have all chunks? haveAllChunks = True chunkCount = 0 for c in listOfChunksReceived: logging.debug( 'save_old_chunked_image: listOfChunksReceived [{}]={}'. format(chunkCount, c)) chunkCount += 1 if not c: haveAllChunks = False logging.debug('save_old_chunked_image: haveAllChunks={}'.format( haveAllChunks)) # No, so just add this chunk to the datastore and return if not haveAllChunks: logging.debug( 'save_old_chunked_image: returning to wait for more chunks' ) return # YES! We have all our chunks, so reassemble the binary image. # Delete the temporary datastore cache for the chunks self.deleteImageChunksFromDatastore(deviceId, messageId) self.deleteTurd(deviceId, messageId) # Sort the chunks by chunkNum (we get messages out of order) oldChunks = sorted(oldChunks, key=lambda k: k['chunkNum']) # Reassemble the b64 chunks into one string (in order). b64str = '' for oc in oldChunks: b64str += oc['imageChunk'] logging.debug( 'save_old_chunked_image: assemble {} of {}'.format( oc['chunkNum'], oc['totalChunks'])) # Now covert our base64 string into binary image bytes imageBytes = base64.b64decode(b64str) # Put the image bytes in cloud storage as a file, and get an URL publicURL = storage.saveFile(varName, imageType, imageBytes, deviceId) # Put the URL in the datastore for the UI to use. datastore.saveImageURL(deviceId, publicURL, varName) # Put the URL as an env. var in BQ. message_obj = {} message_obj[self.messageType_KEY] = self.messageType_Image message_obj[self.var_KEY] = varName valuesJson = "{'values':[" valuesJson += "{'name':'URL', 'type':'str', 'value':'%s'}" % \ ( publicURL ) valuesJson += "]}" message_obj[self.values_KEY] = valuesJson rowsList = [] self.makeBQEnvVarRowList(message_obj, deviceId, rowsList) bigquery.data_insert(rowsList) except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() logging.critical("Exception in save_old_chunked_image(): %s" % e) traceback.print_tb(exc_traceback, file=sys.stdout)