def worker( self, channel, method, properties, body ): status = self.Status.unknown error = None response_queue = None if True: #try: self._debug( 'Have message, Worker STARTED: ' ) message = json.loads( body ) self._debug( 'worker message: ' + str(message) ) # Check message in right format try: silo = message['silo'] dataset = message['dataset'] files = message['files'] response_queue_name = message['response'] except KeyError: status, error = self._sendError( None, "message", "Message missing required key, silo, dataset, files, response : " + str( message ), message ) # Connect to queue we need to respond on if status != self.Status.error : response_queue = RandomExclusiveQueue( self.config.rabbitmq.host, int(self.config.rabbitmq.port), self.config.rabbitmq.virtual_host, self.config.rabbitmq.username, self.config.rabbitmq.password, response_queue_name ) if not self.queue.connected(): status, error = self._sendError( None, "message", "Unable to connect to message queue at: " + self.config.rabbitmq.host, message ) # Connect to databank if status != self.Status.error : try: databank = Databank( self.config.databank.host, self.config.databank.username, self.config.databank.password ) except: status, error = self._sendError( response_queue, "databank", "Unknown error when connecting to Databank: " + self.config.databank.host ) else: # Test connection connection_status, connection_error = self._checkDatabankConnection( databank ) if connection_status != self.Status.ok : status, error = self._sendError( response_queue, "databank", 'Problem connecting to databank: ' + connection_error ) # Attempt to Create silo if status != self.Status.error : silo_status = self._createSilo( databank, silo ) if silo_status != self.Status.ok : status, error = self._sendError( response_queue, "silo", 'Unable to create silo [' + silo + '] in Databank' ) if status != self.Status.error : self._debug( "We have silo : " + silo ) # Attempt to create dataset dataset_status = self._createDataset( databank, silo, dataset ) if dataset_status != self.Status.ok : status, error = self._sendError( response_queue, "dataset", "Unable to create dataset [" + dataset + "] in databank" ) else: self._debug( "Dataset created : " + dataset ) if status != self.Status.error : # COULD TODO: Maybe we should check the files aren't already there, but this is quite hard to know, Are the files there but the wrong size? Are the files earlier versions? May as well just upload again, databank will create new versions when necessary. # response = databank.getFiles( silo, dataset ) status = self.Status.ok for file in files: filename = None if isinstance( file, dict ) : # We check this so we can support older versions where "file" was a string if "filename" in file: filename = file['filename'] file = file["file"] self._debug( "Uploading : " + file ) if os.path.exists( file ): file_status = self._uploadFile( response_queue, databank, silo, dataset, file, filename ) if file_status == self.Status.ok : self._sendInfo( response_queue, file, "File uploaded" ) else: status, error = self._sendError( response_queue, "file", "Unable to upload file : [" + file + "]" ) # with mimetype : " + mime ) break # To continue or not to continue... Not to. else : self._sendWarning( response_queue, file, file + " - File doesn't exists!" ) # This file doesn't exists... is that Good or bad? Do we keep the message? We'll have to remove it as it'll never get processed and stay in the list. So we'll pretend it's OK... we have sent a warning. if status == self.Status.ok: channel.basic_ack( delivery_tag = method.delivery_tag ) # Say completed and remove from queue. self._debug( 'Worker FINISHED' ) else: self._debug( 'Worker ERRORED' ) #except : # # status, error = self._sendError( None, "exception", "There has been a serious unknown exception in " + __name__ ) # self._debug( 'Worker EXCEPTION - ' + error ) # self._debug( sys.exc_info()[0] ) # # #raise # Re throw the exception. # Responde and say if it succeeded on not. message['ok'] = status == self.Status.ok message['error'] = error self._sendMessage( response_queue, self.MessageType.final, None, message ) response_queue.close() # sys.exit() # Stop after one completed, for debugging! if status != self.Status.ok : # If something is broken, wait until ending, it might be a temporary thing (e.g. network down) time.sleep( 30 )