def aggregate(parameters, tableBuilder): # Get the ID of the call if it already exists uid = parameters.get("uid"); if uid is None or uid == "": # Create a unique id uid = str(uuid.uuid4()) # Add the table headers tableBuilder.addHeader("uid") tableBuilder.addHeader("completed") # Fill in relevant information row = tableBuilder.addRow() row.setCell("uid", uid) row.setCell("completed", False) # Launch the actual process in a separate thread thread = Thread(target=aggregateProcess, args=(parameters, tableBuilder, uid)) thread.start() # Return immediately return # The process is already running in a separate thread. We get current # results and return them resultToSend = LRCache.get(uid); if resultToSend is None: # This should not happen raise Exception("Could not retrieve results from result cache!") # Add the table headers tableBuilder.addHeader("uid") tableBuilder.addHeader("completed") tableBuilder.addHeader("success") tableBuilder.addHeader("message") tableBuilder.addHeader("nCopiedFiles") tableBuilder.addHeader("relativeExpFolder") tableBuilder.addHeader("zipArchiveFileName") tableBuilder.addHeader("mode") # Store current results in the table row = tableBuilder.addRow() row.setCell("uid", resultToSend["uid"]) row.setCell("completed", resultToSend["completed"]) row.setCell("success", resultToSend["success"]) row.setCell("message", resultToSend["message"]) row.setCell("nCopiedFiles", resultToSend["nCopiedFiles"]) row.setCell("relativeExpFolder", resultToSend["relativeExpFolder"]) row.setCell("zipArchiveFileName", resultToSend["zipArchiveFileName"]) row.setCell("mode", resultToSend["mode"])
def aggregate(parameters, tableBuilder): # Get the ID of the call if it already exists uid = parameters.get("uid"); if uid is None or uid == "": # Create a unique id uid = str(uuid.uuid4()) # Add the table headers tableBuilder.addHeader("uid") tableBuilder.addHeader("completed") # Fill in relevant information row = tableBuilder.addRow() row.setCell("uid", uid) row.setCell("completed", False) # Launch the actual process in a separate thread thread = Thread(target = aggregateProcess, args = (parameters, tableBuilder, uid)) thread.start() # Return immediately return # The process is already running in a separate thread. We get current # results and return them resultToSend = LRCache.get(uid); if resultToSend is None: # This should not happen raise Exception("Could not retrieve results from result cache!") # Add the table headers tableBuilder.addHeader("uid") tableBuilder.addHeader("completed") tableBuilder.addHeader("success") tableBuilder.addHeader("message") tableBuilder.addHeader("nCopiedFiles") tableBuilder.addHeader("relativeExpFolder") tableBuilder.addHeader("zipArchiveFileName") tableBuilder.addHeader("mode") # Store current results in the table row = tableBuilder.addRow() row.setCell("uid", resultToSend["uid"]) row.setCell("completed", resultToSend["completed"]) row.setCell("success", resultToSend["success"]) row.setCell("message", resultToSend["message"]) row.setCell("nCopiedFiles", resultToSend["nCopiedFiles"]) row.setCell("relativeExpFolder", resultToSend["relativeExpFolder"]) row.setCell("zipArchiveFileName", resultToSend["zipArchiveFileName"]) row.setCell("mode", resultToSend["mode"])
def aggregateProcess(parameters, tableBuilder, uid): # Make sure to initialize and store the results. We need to have them since # most likely the client will try to retrieve them again before the process # is finished. resultToStore = {} resultToStore["uid"] = uid resultToStore["success"] = True resultToStore["completed"] = False resultToStore["message"] = "" resultToStore["nCopiedFiles"] = "" resultToStore["relativeExpFolder"] = "" resultToStore["zipArchiveFileName"] = "" resultToStore["mode"] = "" LRCache.set(uid, resultToStore) # Get path to containing folder # __file__ does not work (reliably) in Jython dbPath = "../core-plugins/microscopy/4/dss/reporting-plugins/export_microscopy_datasets" # Path to the logs subfolder logPath = os.path.join(dbPath, "logs") # Make sure the logs subforder exist if not os.path.exists(logPath): os.makedirs(logPath) # Path for the log file logFile = os.path.join(logPath, "log.txt") # Set up logging logging.basicConfig(filename=logFile, level=logging.DEBUG, format='%(asctime)-15s %(levelname)s: %(message)s') logger = logging.getLogger() # Get parameters from plugin.properties properties = parsePropertiesFile() if properties is None: raise Exception("Could not process plugin.properties") # Get the COLLECTION experiment identifier experimentId = parameters.get("experimentId") # Get the MICROSCOPY_EXPERIMENT sample identifier expSamplePermId = parameters.get("expSamplePermId") # Get the MICROSCOPY_SAMPLE_TYPE sample identifier samplePermId = parameters.get("samplePermId") # Get the mode mode = parameters.get("mode") # Info logger.info("Aggregation plug-in called with following parameters:") logger.info("* COLLECTION experimentId = " + experimentId) logger.info("* MICROSCOPY_EXPERIMENT sample permId = " + expSamplePermId) logger.info("* MICROSCOPY_SAMPLE_TYPE sample permId = " + samplePermId) logger.info("* mode = " + mode) logger.info("* userId = " + userId) logger.info("Aggregation plugin properties:") logger.info(" * base_dir = " + properties['base_dir']) logger.info(" * export_dir = " + properties['export_dir']) logger.info(" * hrm_base_dir = " + properties['hrm_base_dir']) logger.info(" * hrm_src_subdir = " + properties['hrm_src_subdir']) # Instantiate the Mover object - userId is a global variable # made available to the aggregation plug-in mover = Mover(experimentId, expSamplePermId, samplePermId, mode, userId, properties, logger) # Process success = mover.process() # Compress if mode == "zip": mover.compressIfNeeded() # Get some results info nCopiedFiles = mover.getNumberOfCopiedFiles() errorMessage = mover.getErrorMessage() relativeExpFolder = mover.getRelativeRootExperimentPath() zipFileName = mover.getZipArchiveFileName() # Update results and store them resultToStore["uid"] = uid resultToStore["completed"] = True resultToStore["success"] = success resultToStore["message"] = errorMessage resultToStore["nCopiedFiles"] = nCopiedFiles resultToStore["relativeExpFolder"] = relativeExpFolder resultToStore["zipArchiveFileName"] = zipFileName resultToStore["mode"] = mode LRCache.set(uid, resultToStore) # Email result to the user if success == True: subject = "Microscopy: successfully processed requested data" if nCopiedFiles == 1: snip = "One file was " else: snip = str(nCopiedFiles) + " files were " if mode == "normal": body = snip + "successfully exported to {...}/" + relativeExpFolder + "." elif mode == "hrm": body = snip + "successfully exported to your HRM source folder." else: body = snip + "successfully packaged for download: " + zipFileName else: subject = "Microscopy: error processing request!" body = "Sorry, there was an error processing your request. " + \ "Please send your administrator the following report:\n\n" + \ "\"" + errorMessage + "\"\n" # Send try: mailService.createEmailSender().withSubject(subject).withBody(body).send() except: sys.stderr.write("export_microscopy_datasets: Failure sending email to user!")
def retrieveProcess(parameters, tableBuilder, uid): # Make sure to initialize and store the results. We need to have them since # most likely the client will try to retrieve them again before the process # is finished. resultToStore = {} resultToStore["uid"] = uid resultToStore["completed"] = False resultToStore["success"] = True resultToStore["message"] = "" resultToStore["data"] = "" # Get the parameters # Get the entity code code = parameters.get("code") resultToStore["code"] = code # Get the X-axis parameter paramX = parameters.get("paramX") resultToStore["paramX"] = paramX # Get the Y-axis parameter paramY = parameters.get("paramY") resultToStore["paramY"] = paramY # Get the X-axis scaling displayX = parameters.get("displayX") resultToStore["displayX"] = displayX # Get the Y-axis scaling displayY = parameters.get("displayY") resultToStore["displayY"] = displayY # Number of events known to be in the file numEvents = int(parameters.get("numEvents")) resultToStore["numEvents"] = numEvents # Maximum number of events to return maxNumEvents = int(parameters.get("maxNumEvents")) resultToStore["maxNumEvents"] = maxNumEvents # Sampling samplingMethod = parameters.get("samplingMethod") resultToStore["samplingMethod"] = samplingMethod # Node key nodeKey = parameters.get("nodeKey") resultToStore["nodeKey"] = nodeKey # Store them into the cache LRCache.set(uid, resultToStore) # Set up logging _logger = setUpLogging() # Log parameter info _logger.info("Requested events for dataset " + code + " and parameters (" + paramX + ", " + paramY + ")") _logger.info("Requested scaling for parameter " + paramX + ": " + displayX) _logger.info("Requested scaling for parameter " + paramY + ": " + displayY) _logger.info("Requested sampling method: " + samplingMethod) _logger.info("Number of events in file: " + str(numEvents) + "; maximum number of events to return: " + str(maxNumEvents)) # Get the FCS file to process dataSetFiles = getFileForCode(code) # Prepare the data dataJSON = "" if len(dataSetFiles) != 1: # Build the error message message = "Could not retrieve the FCS file to process!" # Log the error _logger.error(message) # Store the results and set the completed flag resultToStore["completed"] = True resultToStore["success"] = False resultToStore["message"] = message # Return here return else: # Get the FCS file path fcsFile = dataSetFiles[0] # Log _logger.info("Dataset code " + code + " corresponds to FCS file " + \ fcsFile) # Open the FCS file reader = FCSReader(java.io.File(fcsFile), True) # Parse the file with data if not reader.parse(): # Build the error message message = "Could not process file " + os.path.basename(fcsFile) # Log the error _logger.error(message) # Store the results and set the completed flag resultToStore["completed"] = True resultToStore["success"] = False resultToStore["message"] = message # Return here return # Preparation steps were successful parameterNames = reader.getParameterNames() # Find the indices of the requested parameters indxX = int(parameterNames.indexOf(paramX)) indxY = int(parameterNames.indexOf(paramY)) # Prepare the data arrays data = [] # Actual number of events to be extracted actualNumEvents = min(maxNumEvents, numEvents) # Data sampling method. # # Method 1: the get the requested number of events, we will sub- # sample the file by skipping a certain number of rows # ("step") in between the returned once. # Method 2: to get the requested number of events, we just return # the first N rows at the beginning of the file. This is # faster, and as far as the experts say, should still be # reasonably representative of the underlying population. if samplingMethod == "1": sample = True else: sample = False # Now collect the first maxNumEvents rows dataX = reader.getDataPerColumnIndex(indxX, actualNumEvents, sample) dataY = reader.getDataPerColumnIndex(indxY, actualNumEvents, sample) # Is the Hyperlog scaling requested? if displayX == "Hyperlog": params = Hyperlog.estimateParamHeuristic(dataX) Hx = Hyperlog(params[0], params[1], params[2], params[3]) dataX = Hx.transform(dataX) dataX = Hyperlog.arrayMult(dataX, params[0]) if displayY == "Hyperlog": params = Hyperlog.estimateParamHeuristic(dataY) Hy = Hyperlog(params[0], params[1], params[2], params[3]) dataY = Hy.transform(dataY) dataY = Hyperlog.arrayMult(dataY, params[0]) # Build array to JSONify and return to the client for i in range(actualNumEvents): data.append([float(dataX[i]), float(dataY[i])]) # JSON encode the data array dataJSON = json.dumps(data) # Success message message = "Successfully processed file " + fcsFile # Log _logger.info(message) # Success success = True # Store the results and set the completed flag resultToStore["completed"] = True resultToStore["success"] = True resultToStore["message"] = message resultToStore["data"] = dataJSON
def aggregate(parameters, tableBuilder): # Add the table headers tableBuilder.addHeader("uid") tableBuilder.addHeader("completed") tableBuilder.addHeader("success") tableBuilder.addHeader("message") tableBuilder.addHeader("data") tableBuilder.addHeader("code") tableBuilder.addHeader("paramX") tableBuilder.addHeader("paramY") tableBuilder.addHeader("displayX") tableBuilder.addHeader("displayY") tableBuilder.addHeader("numEvents") tableBuilder.addHeader("maxNumEvents") tableBuilder.addHeader("samplingMethod") tableBuilder.addHeader("nodeKey") # Get the ID of the call if it already exists uid = parameters.get("uid") if uid is None or uid == "": # Create a unique id uid = str(uuid.uuid4()) # Fill in relevant information row = tableBuilder.addRow() row.setCell("uid", uid) row.setCell("completed", False) row.setCell("success", True) row.setCell("message", "") row.setCell("data", "") row.setCell("code", "") row.setCell("paramX", "") row.setCell("paramY", "") row.setCell("displayX", "") row.setCell("displayY", "") row.setCell("numEvents", "") row.setCell("maxNumEvents", "") row.setCell("samplingMethod", "") row.setCell("nodeKey", "") # Launch the actual process in a separate thread thread = Thread(target=retrieveProcess, args=(parameters, tableBuilder, uid)) thread.start() # Return immediately return # The process is already running in a separate thread. We get current # results and return them resultToSend = LRCache.get(uid) if resultToSend is None: # This should not happen raise Exception("Could not retrieve results from result cache!") # Fill in relevant information row = tableBuilder.addRow() row.setCell("uid", resultToSend["uid"]) row.setCell("completed", resultToSend["completed"]) row.setCell("success", resultToSend["success"]) row.setCell("message", resultToSend["message"]) row.setCell("data", resultToSend["data"]) row.setCell("code", resultToSend["code"]) row.setCell("paramX", resultToSend["paramX"]) row.setCell("paramY", resultToSend["paramY"]) row.setCell("displayX", resultToSend["displayX"]) row.setCell("displayY", resultToSend["displayY"]) row.setCell("numEvents", resultToSend["numEvents"]) row.setCell("maxNumEvents", resultToSend["maxNumEvents"]) row.setCell("samplingMethod", resultToSend["samplingMethod"]) row.setCell("nodeKey", resultToSend["nodeKey"])
def aggregateProcess(parameters, tableBuilder, uid): # Make sure to initialize and store the results. We need to have them since # most likely the client will try to retrieve them again before the process # is finished. resultToStore = {} resultToStore["uid"] = uid resultToStore["success"] = True resultToStore["completed"] = False resultToStore["message"] = "" resultToStore["nCopiedFiles"] = "" resultToStore["relativeExpFolder"] = "" resultToStore["zipArchiveFileName"] = "" resultToStore["mode"] = "" LRCache.set(uid, resultToStore) # Get path to containing folder # __file__ does not work (reliably) in Jython dbPath = "../core-plugins/microscopy/1/dss/reporting-plugins/export_microscopy_datasets" # Path to the logs subfolder logPath = os.path.join(dbPath, "logs") # Make sure the logs subforder exist if not os.path.exists(logPath): os.makedirs(logPath) # Path for the log file logFile = os.path.join(logPath, "log.txt") # Set up logging logging.basicConfig(filename=logFile, level=logging.DEBUG, format='%(asctime)-15s %(levelname)s: %(message)s') logger = logging.getLogger() # Get parameters from plugin.properties properties = parsePropertiesFile() if properties is None: raise Exception("Could not process plugin.properties") # Get the experiment identifier experimentId = parameters.get("experimentId") # Get the sample identifier sampleId = parameters.get("sampleId") # Get the mode mode = parameters.get("mode") # Info logger.info("Aggregation plug-in called with following parameters:") logger.info("experimentId = " + experimentId) logger.info("sampleId = " + sampleId) logger.info("mode = " + mode) logger.info("userId = " + userId) logger.info("Aggregation plugin properties:") logger.info("properties = " + str(properties)) # Instantiate the Mover object - userId is a global variable # made available to the aggregation plug-in mover = Mover(experimentId, sampleId, mode, userId, properties, logger) # Process success = mover.process() # Compress if mode == "zip": mover.compressIfNeeded() # Get some results info nCopiedFiles = mover.getNumberOfCopiedFiles() errorMessage = mover.getErrorMessage(); relativeExpFolder = mover.getRelativeExperimentPath() zipFileName = mover.getZipArchiveFileName() # Update results and store them resultToStore["uid"] = uid resultToStore["completed"] = True resultToStore["success"] = success resultToStore["message"] = errorMessage resultToStore["nCopiedFiles"] = nCopiedFiles resultToStore["relativeExpFolder"] = relativeExpFolder resultToStore["zipArchiveFileName"] = zipFileName resultToStore["mode"] = mode LRCache.set(uid, resultToStore) # Email result to the user if success == True: subject = "Microscopy: successfully processed requested data" if nCopiedFiles == 1: snip = "One file was " else: snip = str(nCopiedFiles) + " files were " if mode == "normal": body = snip + "successfully exported to {...}/" + relativeExpFolder + "." elif mode == "hrm": body = snip + "successfully exported to your HRM source folder." else: body = snip + "successfully packaged for download: " + zipFileName else: subject = "Microscopy: error processing request!" body = "Sorry, there was an error processing your request. " + \ "Please send your administrator the following report:\n\n" + \ "\"" + errorMessage + "\"\n" # Send try: mailService.createEmailSender().withSubject(subject).withBody(body).send() except: sys.stderr.write("export_microscopy_datasets: Failure sending email to user!")
def retrieveProcess(parameters, tableBuilder, uid): # Make sure to initialize and store the results. We need to have them since # most likely the client will try to retrieve them again before the process # is finished. resultToStore = {} resultToStore["uid"] = uid resultToStore["completed"] = False resultToStore["success"] = True resultToStore["message"] = "" resultToStore["data"] = "" # Get the parameters # Get the entity code code = parameters.get("code") resultToStore["code"] = code # Get the X-axis parameter paramX = parameters.get("paramX") resultToStore["paramX"] = paramX # Get the Y-axis parameter paramY = parameters.get("paramY") resultToStore["paramY"] = paramY # Get the X-axis scaling displayX = parameters.get("displayX") resultToStore["displayX"] = displayX # Get the Y-axis scaling displayY = parameters.get("displayY") resultToStore["displayY"] = displayY # Number of events known to be in the file numEvents = int(parameters.get("numEvents")) resultToStore["numEvents"] = numEvents # Maximum number of events to return maxNumEvents = int(parameters.get("maxNumEvents")) resultToStore["maxNumEvents"] = maxNumEvents # Sampling samplingMethod = parameters.get("samplingMethod") resultToStore["samplingMethod"] = samplingMethod # Node key nodeKey = parameters.get("nodeKey") resultToStore["nodeKey"] = nodeKey # Store them into the cache LRCache.set(uid, resultToStore) # Set up logging _logger = setUpLogging() # Log parameter info _logger.info("Requested events for dataset " + code + " and parameters (" + paramX + ", " + paramY + ")") _logger.info("Requested scaling for parameter " + paramX + ": " + displayX) _logger.info("Requested scaling for parameter " + paramY + ": " + displayY) _logger.info("Requested sampling method: " + samplingMethod) _logger.info("Number of events in file: " + str(numEvents) + "; maximum number of events to return: " + str(maxNumEvents)) # Get the FCS file to process dataSetFiles = getFileForCode(code) # Prepare the data dataJSON = "" if len(dataSetFiles) != 1: # Build the error message message = "Could not retrieve the FCS file to process!" # Log the error _logger.error(message) # Store the results and set the completed flag resultToStore["completed"] = True resultToStore["success"] = False resultToStore["message"] = message # Return here return else: # Get the FCS file path fcsFile = dataSetFiles[0] # Log _logger.info("Dataset code " + code + " corresponds to FCS file " + \ fcsFile) # Open the FCS file reader = FCSReader(java.io.File(fcsFile), True); # Parse the file with data if not reader.parse(): # Build the error message message = "Could not process file " + os.path.basename(fcsFile) # Log the error _logger.error(message) # Store the results and set the completed flag resultToStore["completed"] = True resultToStore["success"] = False resultToStore["message"] = message # Return here return # Preparation steps were successful parameterNames = reader.getParameterNames() # Find the indices of the requested parameters indxX = int(parameterNames.indexOf(paramX)) indxY = int(parameterNames.indexOf(paramY)) # Prepare the data arrays data = [] # Actual number of events to be extracted actualNumEvents = min(maxNumEvents, numEvents) # Data sampling method. # # Method 1: the get the requested number of events, we will sub- # sample the file by skipping a certain number of rows # ("step") in between the returned once. # Method 2: to get the requested number of events, we just return # the first N rows at the beginning of the file. This is # faster, and as far as the experts say, should still be # reasonably representative of the underlying population. if samplingMethod == "1": sample = True else: sample = False # Now collect the first maxNumEvents rows dataX = reader.getDataPerColumnIndex(indxX, actualNumEvents, sample) dataY = reader.getDataPerColumnIndex(indxY, actualNumEvents, sample) # Is the Hyperlog scaling requested? if displayX == "Hyperlog": params = Hyperlog.estimateParamHeuristic(dataX) Hx = Hyperlog(params[0], params[1], params[2], params[3]) dataX = Hx.transform(dataX) dataX = Hyperlog.arrayMult(dataX, params[0]) if displayY == "Hyperlog": params = Hyperlog.estimateParamHeuristic(dataY) Hy = Hyperlog(params[0], params[1], params[2], params[3]) dataY = Hy.transform(dataY) dataY = Hyperlog.arrayMult(dataY, params[0]) # Build array to JSONify and return to the client for i in range (actualNumEvents): data.append([float(dataX[i]), float(dataY[i])]) # JSON encode the data array dataJSON = json.dumps(data) # Success message message = "Successfully processed file " + fcsFile # Log _logger.info(message) # Success success = True # Store the results and set the completed flag resultToStore["completed"] = True resultToStore["success"] = True resultToStore["message"] = message resultToStore["data"] = dataJSON
def aggregate(parameters, tableBuilder): # Add the table headers tableBuilder.addHeader("uid") tableBuilder.addHeader("completed") tableBuilder.addHeader("success") tableBuilder.addHeader("message") tableBuilder.addHeader("data") tableBuilder.addHeader("code") tableBuilder.addHeader("paramX") tableBuilder.addHeader("paramY") tableBuilder.addHeader("displayX") tableBuilder.addHeader("displayY") tableBuilder.addHeader("numEvents") tableBuilder.addHeader("maxNumEvents") tableBuilder.addHeader("samplingMethod") tableBuilder.addHeader("nodeKey") # Get the ID of the call if it already exists uid = parameters.get("uid"); if uid is None or uid == "": # Create a unique id uid = str(uuid.uuid4()) # Fill in relevant information row = tableBuilder.addRow() row.setCell("uid", uid) row.setCell("completed", False) row.setCell("success", True) row.setCell("message", "") row.setCell("data", "") row.setCell("code", "") row.setCell("paramX", "") row.setCell("paramY", "") row.setCell("displayX", "") row.setCell("displayY", "") row.setCell("numEvents", "") row.setCell("maxNumEvents", "") row.setCell("samplingMethod", "") row.setCell("nodeKey", "") # Launch the actual process in a separate thread thread = Thread(target = retrieveProcess, args = (parameters, tableBuilder, uid)) thread.start() # Return immediately return # The process is already running in a separate thread. We get current # results and return them resultToSend = LRCache.get(uid); if resultToSend is None: # This should not happen raise Exception("Could not retrieve results from result cache!") # Fill in relevant information row = tableBuilder.addRow() row.setCell("uid", resultToSend["uid"]) row.setCell("completed", resultToSend["completed"]) row.setCell("success", resultToSend["success"]) row.setCell("message", resultToSend["message"]) row.setCell("data", resultToSend["data"]) row.setCell("code", resultToSend["code"]) row.setCell("paramX", resultToSend["paramX"]) row.setCell("paramY", resultToSend["paramY"]) row.setCell("displayX", resultToSend["displayX"]) row.setCell("displayY", resultToSend["displayY"]) row.setCell("numEvents", resultToSend["numEvents"]) row.setCell("maxNumEvents", resultToSend["maxNumEvents"]) row.setCell("samplingMethod", resultToSend["samplingMethod"]) row.setCell("nodeKey", resultToSend["nodeKey"])
def aggregateProcess(parameters, tableBuilder, uid): # Make sure to initialize and store the results. We need to have them since # most likely the client will try to retrieve them again before the process # is finished. resultToStore = {} resultToStore["uid"] = uid resultToStore["success"] = True resultToStore["completed"] = False resultToStore["message"] = "" resultToStore["nCopiedFiles"] = "" resultToStore["relativeExpFolder"] = "" resultToStore["zipArchiveFileName"] = "" resultToStore["mode"] = "" LRCache.set(uid, resultToStore) # Get path to containing folder # __file__ does not work (reliably) in Jython dbPath = "../core-plugins/flow/3/dss/reporting-plugins/export_flow_datasets" # Path to the logs subfolder logPath = os.path.join(dbPath, "logs") # Make sure the logs subforder exist if not os.path.exists(logPath): os.makedirs(logPath) # Path for the log file logFile = os.path.join(logPath, "log.txt") # Set up logging logging.basicConfig(filename=logFile, level=logging.DEBUG, format='%(asctime)-15s %(levelname)s: %(message)s') logger = logging.getLogger() # Get parameters from plugin.properties properties = parsePropertiesFile() if properties is None: msg = "Could not process plugin.properties" logger.error(msg) raise Exception(msg) if properties['base_dir'] == "" or properties['export_dir'] == '': msg = "Please set valid value for 'base_dir' and 'export_dir' in plugin.properties" logger.error(msg) raise Exception(msg) # Dump the properties dictionary to log logger.info(str(parameters)) # Get the task task = parameters["task"] # Get the experiment identifier collectionId = parameters["collectionId"] # Get the experiment type collectionType = parameters["collectionType"] # Get the experiment sample identifier expSampleId = parameters.get("expSampleId") # Get the experiment sample perm identifier expSamplePermId = parameters.get("expSamplePermId") # Get the experiment sample type expSampleType = parameters.get("expSampleType") # Get the plate code platePermId = parameters.get("platePermId") # Get the plate type plateType = parameters.get("plateType") # Get the mode mode = parameters.get("mode") # Info logger.info("Aggregation plug-in called with following parameters:") logger.info("task = " + task) logger.info("collectionId = " + collectionId) logger.info("collectionType = " + collectionType) logger.info("expSampleId = " + expSampleId) logger.info("expSamplePermId = " + expSamplePermId) logger.info("expSampleType = " + expSampleType) logger.info("platePermId = " + platePermId) logger.info("plateType = " + plateType) logger.info("mode = " + mode) logger.info("userId = " + userId) logger.info("Aggregation plugin properties:") logger.info("properties = " + str(properties)) # Consistency check: task must be one of a known set if task != "EXPERIMENT_SAMPLE" and \ task != "ALL_PLATES" and \ task != "PLATE" and \ task != "TUBESET": msg = "The requested task " + task + " is not known!" logger.error(msg) raise Exception(msg) logger.info("Requested task: " + task) # Instantiate the Mover object - userId is a global variable # made available to the aggregation plug-in mover = Mover(task, collectionId, collectionType, expSampleId, expSamplePermId, expSampleType, platePermId, plateType, mode, userId, properties, logger) # Process success = mover.process() logger.info("Process ended successfully.") # Compress if mode == "zip": mover.compressIfNeeded() # Get some results info nCopiedFiles = mover.getNumberOfCopiedFiles() errorMessage = mover.getErrorMessage() relativeExpFolder = mover.getRelativeRootExperimentPath() zipFileName = mover.getZipArchiveFileName() # Update results and store them resultToStore["uid"] = uid resultToStore["completed"] = True resultToStore["success"] = success resultToStore["message"] = errorMessage resultToStore["nCopiedFiles"] = nCopiedFiles resultToStore["relativeExpFolder"] = relativeExpFolder resultToStore["zipArchiveFileName"] = zipFileName resultToStore["mode"] = mode LRCache.set(uid, resultToStore) # Email result to the user if success == True: subject = "Flow export: successfully processed requested data" if nCopiedFiles == 1: snip = "One file was " else: snip = str(nCopiedFiles) + " files were " if mode == "normal": body = snip + "successfully exported to {...}/" + relativeExpFolder + "." else: body = snip + "successfully packaged for download: " + zipFileName else: subject = "Flow export: error processing request!" body = "Sorry, there was an error processing your request. " + \ "Please send your administrator the following report:\n\n" + \ "\"" + errorMessage + "\"\n" # Send try: mailService.createEmailSender().withSubject(subject).withBody( body).send() except: sys.stderr.write( "export_flow_datasets: Failure sending email to user!")
def aggregateProcess(parameters, tableBuilder, uid): # Make sure to initialize and store the results. We need to have them since # most likely the client will try to retrieve them again before the process # is finished. resultToStore = {} resultToStore["uid"] = uid resultToStore["success"] = True resultToStore["completed"] = False resultToStore["message"] = "" resultToStore["nCopiedFiles"] = "" resultToStore["relativeExpFolder"] = "" resultToStore["zipArchiveFileName"] = "" resultToStore["mode"] = "" LRCache.set(uid, resultToStore) # Get parameters from plugin.properties properties = parsePropertiesFile() if properties is None: raise Exception("Could not process plugin.properties") # Get the experiment identifier experimentId = parameters.get("experimentId") # Get the experiment type experimentType = parameters.get("experimentType") # Get the entity type entityType = parameters.get("entityType") # Get the entity code entityId = parameters.get("entityId") # Get the specimen name specimen = parameters.get("specimen") # Get the mode mode = parameters.get("mode") # Instantiate the Mover object - userId is a global variable # made available to the aggregation plug-in mover = Mover(experimentId, experimentType, entityType, entityId, specimen, mode, userId, properties) # Process success = mover.process() # Compress if mode == "zip": mover.compressIfNeeded() # Get some results info nCopiedFiles = mover.getNumberOfCopiedFiles() errorMessage = mover.getErrorMessage(); relativeExpFolder = mover.getRelativeRootExperimentPath() zipFileName = mover.getZipArchiveFileName() # Update results and store them resultToStore["uid"] = uid resultToStore["completed"] = True resultToStore["success"] = success resultToStore["message"] = errorMessage resultToStore["nCopiedFiles"] = nCopiedFiles resultToStore["relativeExpFolder"] = relativeExpFolder resultToStore["zipArchiveFileName"] = zipFileName resultToStore["mode"] = mode LRCache.set(uid, resultToStore) # Email result to the user if success == True: subject = "BD FACS DIVA export: successfully processed requested data" if nCopiedFiles == 1: snip = "One file was " else: snip = str(nCopiedFiles) + " files were " if mode == "normal": body = snip + "successfully exported to {...}/" + relativeExpFolder + "." else: body = snip + "successfully packaged for download: " + zipFileName else: subject = "BD FACS DIVA export: error processing request!" body = "Sorry, there was an error processing your request. " + \ "Please send your administrator the following report:\n\n" + \ "\"" + errorMessage + "\"\n" # Send try: mailService.createEmailSender().withSubject(subject).withBody(body).send() except: sys.stderr.write("export_bdfacsdiva_datasets: Failure sending email to user!")