def receive_message(header, message): logging.getLogger().debug(message) print("Processing message", header['message-id']) try: mx_data_reduction_to_ispyb(message, mxprocessing=mxprocessing) except ISPyBKeyProblem as e: msg = getattr(e, 'message', repr(e)) logging.getLogger().error(msg) stomp.ack(header['message-id'], header['subscription'])
def test_mx_data_reduction_xml_to_ispyb(testdb, filename): mxprocessing = testdb.mx_processing xml_file = os.path.abspath( os.path.join(os.path.dirname(__file__), filename)) # Find the datacollection associated with this data reduction run xml_dir = os.path.split(xml_file)[0] try: with open(os.path.join(xml_dir, ".dc_id")) as fh: dc_id = int(fh.read()) print("Got DC ID %d from file system" % dc_id) except Exception: dc_id = None mx_data_reduction_dict = xml_file_to_dict(xml_file) (app_id, ap_id, scaling_id, integration_id) = mx_data_reduction_to_ispyb(mx_data_reduction_dict, dc_id, mxprocessing) # Output results xml xml = ('<?xml version="1.0" encoding="ISO-8859-1"?>' "<dbstatus><autoProcProgramId>%d</autoProcProgramId>" "<autoProcId>%d</autoProcId>" "<autoProcScalingId>%d</autoProcScalingId>" "<autoProcIntegrationId>%d</autoProcIntegrationId>" "<code>ok</code></dbstatus>" % (app_id, ap_id, scaling_id, integration_id)) print(xml)
with ispyb.open(conf_file) as conn: mxprocessing = conn.mx_processing xml_file = sys.argv[2] xml_dir = os.path.split(xml_file)[0] # Find the datacollection associated with this data reduction run try: dc_id = int(open(os.path.join(xml_dir, ".dc_id"), "r").read()) print("Got DC ID %d from file system" % dc_id) except: dc_id = None mx_data_reduction_dict = xml_file_to_dict(xml_file) (app_id, ap_id, scaling_id, integration_id) = mx_data_reduction_to_ispyb(mx_data_reduction_dict, dc_id, mxprocessing) # Write results to xml_out_file if len(sys.argv) > 3: xml = ('<?xml version="1.0" encoding="ISO-8859-1"?>' "<dbstatus><autoProcProgramId>%d</autoProcProgramId>" "<autoProcId>%d</autoProcId>" "<autoProcScalingId>%d</autoProcScalingId>" "<autoProcIntegrationId>%d</autoProcIntegrationId>" "<code>ok</code></dbstatus>" % (app_id, ap_id, scaling_id, integration_id)) f = open(sys.argv[3], "w") f.write(xml) f.close()
def insertResult(result, resultType, request, visitName, dc_id=None, xmlFileName=None): #xmlfilename for fastDP #keep in mind that request type can be standard and result type be fastDP - multiple results per req. cbfComm = db_lib.getBeamlineConfigParam(beamline, "cbfComm") try: sessionid = core.retrieve_visit_id(visitName) except ISPyBNoResultException as e: logger.error( "caught ISPyBNoResultException: %s. make sure visit name is in the format mx999999-1234" % e) propNum = visitName.split('-')[0] sessionid = createVisit(propNum) request_type = request['request_type'] if request_type in ('standard', 'vector'): sample = request[ 'sample'] # this needs to be created and linked to a DC group if (resultType == 'fastDP'): mx_data_reduction_dict = xml_file_to_dict(xmlFileName) (app_id, ap_id, scaling_id, integration_id) = mx_data_reduction_to_ispyb( mx_data_reduction_dict, dc_id, mxprocessing) mxprocessing.upsert_program_ex(program_id=app_id, status=1) elif resultType == 'mxExpParams': result_obj = result['result_obj'] request_obj = result_obj['requestObj'] directory = request_obj["directory"] filePrefix = request_obj['file_prefix'] basePath = request_obj["basePath"] visitName = daq_utils.getVisitName() jpegDirectory = visitName + "/jpegs/" + directory[ directory.find(visitName) + len(visitName):len(directory)] fullJpegDirectory = basePath + "/" + jpegDirectory jpegImagePrefix = fullJpegDirectory + "/" + filePrefix daq_utils.take_crystal_picture(filename=jpegImagePrefix) jpegImageFilename = jpegImagePrefix + ".jpg" jpegImageThumbFilename = jpegImagePrefix + "t.jpg" node = db_lib.getBeamlineConfigParam(beamline, "adxvNode") comm_s = "ssh -q " + node + " \"convert " + jpegImageFilename + " -resize 40% " + jpegImageThumbFilename + "\"&" logger.info('resizing image: %s' % comm_s) os.system(comm_s) seqNum = int(detSeqNumPV.get()) hdfSampleDataPattern = directory + "/" + filePrefix + "_" hdfRowFilepattern = hdfSampleDataPattern + str(int( float(seqNum))) + "_master.h5" # keep in mind I could do the jpeg conversion here, but maybe best to allow synchWeb on demand. cbfDir = directory CBF_conversion_pattern = cbfDir + "/" + filePrefix + "_" JPEG_conversion_pattern = fullJpegDirectory + "/" + filePrefix + "_" node = db_lib.getBeamlineConfigParam(beamline, "adxvNode") adxvComm = os.environ["PROJDIR"] + db_lib.getBeamlineConfigParam( daq_utils.beamline, "adxvComm") comm_s = "ssh -q " + node + " \"sleep 6;" + cbfComm + " " + hdfRowFilepattern + " 1:1 " + CBF_conversion_pattern + ";" + adxvComm + " -sa " + CBF_conversion_pattern + "000001.cbf " + JPEG_conversion_pattern + "0001.jpeg;convert " + JPEG_conversion_pattern + "0001.jpeg -resize 10% " + JPEG_conversion_pattern + "0001.thumb.jpeg\"&" logger.info('diffraction thumbnail image: %s' % comm_s) os.system(comm_s) # Create a new data collection group entry: params = mxacquisition.get_data_collection_group_params() params['parentid'] = sessionid if request_type == 'standard': params['experimenttype'] = 'OSC' elif request_type == 'vector': params['experimenttype'] = 'Helical' return createDataCollection(directory, filePrefix, jpegImageFilename, params, request_obj, sessionid)
def insertResult(result, resultType, request, visitName, dc_id=None, xmlFileName=None): #xmlfilename for fastDP #keep in mind that request type can be standard and result type be fastDP - multiple results per req. try: sessionid = core.retrieve_visit_id(visitName) except ISPyBNoResultException as e: logger.error( "insert result - caught ISPyBNoResultException: '%s'. make sure visit name is in the format mx999999-1234. NOT HAVING MX IN FRONT IS A SIGN OF PROBLEMS - try newVisit() in that case." % e) propNum = visitName.split('-')[0] sessionid = createVisit(propNum) request_type = request['request_type'] if request_type in ('standard', 'vector'): sample = request[ 'sample'] # this needs to be created and linked to a DC group if (resultType == 'fastDP'): mx_data_reduction_dict = xml_file_to_dict(xmlFileName) (app_id, ap_id, scaling_id, integration_id) = mx_data_reduction_to_ispyb( mx_data_reduction_dict, dc_id, mxprocessing) mxprocessing.upsert_program_ex(program_id=app_id, status=1) elif resultType == 'mxExpParams': result_obj = result['result_obj'] request_obj = result_obj['requestObj'] directory = request_obj["directory"] filePrefix = request_obj['file_prefix'] basePath = request_obj["basePath"] visitName = daq_utils.getVisitName() jpegDirectory = visitName + "/jpegs/" + directory[ directory.find(visitName) + len(visitName):len(directory)] fullJpegDirectory = basePath + "/" + jpegDirectory jpegImagePrefix = fullJpegDirectory + "/" + filePrefix daq_utils.take_crystal_picture(filename=jpegImagePrefix) jpegImageFilename = jpegImagePrefix + ".jpg" jpegImageThumbFilename = jpegImagePrefix + "t.jpg" node = db_lib.getBeamlineConfigParam(beamline, "adxvNode") comm_s = f"ssh -q {node} \"{os.environ['MXPROCESSINGSCRIPTSDIR']}resize.sh {jpegImageFilename} {jpegImageThumbFilename} 40% \"&" logger.info('resizing image: %s' % comm_s) os.system(comm_s) seqNum = int(detSeqNumPV.get()) node = db_lib.getBeamlineConfigParam(beamline, "adxvNode") request_id = result['request'] comm_s = f"ssh -q {node} \"{os.environ['MXPROCESSINGSCRIPTSDIR']}eiger2cbf.sh {request_id} 1 1 0 {seqNum}\"" logger.info(f'diffraction thumbnail conversion to cbf: {comm_s}') os.system(comm_s) comm_s = f"ssh -q {node} \"{os.environ['MXPROCESSINGSCRIPTSDIR']}cbf2jpeg.sh {request_id}\"" logger.info(f'diffraction thumbnail conversion to jpeg: {comm_s}') os.system(comm_s) # Create a new data collection group entry: params = mxacquisition.get_data_collection_group_params() params['parentid'] = sessionid if request_type == 'standard': params['experimenttype'] = 'OSC' elif request_type == 'vector': params['experimenttype'] = 'Helical' return createDataCollection(directory, filePrefix, jpegImageFilename, params, request_obj, sessionid)
def insertResult(result,resultType,request,visitName,dc_id=None,xmlFileName=None): #xmlfilename for fastDP #keep in mind that request type can be standard and result type be fastDP - multiple results per req. cbfComm = db_lib.getBeamlineConfigParam(daq_utils.beamline,"cbfComm") try: sessionid = core.retrieve_visit_id(visitName) except ISPyBNoResultException: print("caught ISPyBNoResultException") sessionid = createVisit(visitName) request_type = request['request_type'] if request_type in('standard', 'vector') : sample = request['sample'] # this needs to be created and linked to a DC group if (resultType == 'fastDP'): mx_data_reduction_dict = xml_file_to_dict(xmlFileName) (app_id, ap_id, scaling_id, integration_id) = mx_data_reduction_to_ispyb(mx_data_reduction_dict, dc_id, mxprocessing) elif resultType == 'mxExpParams': result_obj = result['result_obj'] request_obj = result_obj['requestObj'] directory = request_obj["directory"] filePrefix = request_obj['file_prefix'] basePath = request_obj["basePath"] visitName = daq_utils.getVisitName() jpegDirectory = visitName + "/jpegs/" + directory[directory.find(visitName)+len(visitName):len(directory)] # jpegDirectory = "jpegs" + directory[directory.find(basePath)+len(basePath):len(directory)] fullJpegDirectory = basePath + "/" + jpegDirectory jpegImagePrefix = fullJpegDirectory+"/"+filePrefix daq_utils.take_crystal_picture(filename=jpegImagePrefix) jpegImageFilename = jpegImagePrefix+".jpg" jpegImageThumbFilename = jpegImagePrefix+"t.jpg" node = db_lib.getBeamlineConfigParam(daq_utils.beamline,"adxvNode") # node = db_lib.getBeamlineConfigParam(daq_utils.beamline,"spotNode1") comm_s = "ssh -q " + node + " \"convert " + jpegImageFilename + " -resize 40% " + jpegImageThumbFilename + "\"&" print(comm_s) os.system(comm_s) # seqNum = int(det_lib.detector_get_seqnum()) seqNum = int(detSeqNumPV.get()) hdfSampleDataPattern = directory+"/"+filePrefix+"_" hdfRowFilepattern = hdfSampleDataPattern + str(int(float(seqNum))) + "_master.h5" #jpeg folder cbfDir = os.getcwd()+"/jpegs/"+directory[len(os.getcwd()):len(directory)] #good luck with that working # keep in mind I could do the jpeg conversion here, but maybe best to allow synchWeb on demand. cbfDir = directory CBF_conversion_pattern = cbfDir + "/" + filePrefix+"_" JPEG_conversion_pattern = fullJpegDirectory + "/" + filePrefix+"_" node = db_lib.getBeamlineConfigParam(daq_utils.beamline,"adxvNode") # node = db_lib.getBeamlineConfigParam(daq_utils.beamline,"spotNode1") adxvComm = os.environ["PROJDIR"] + db_lib.getBeamlineConfigParam(daq_utils.beamline,"adxvComm") comm_s = "ssh -q " + node + " \"sleep 6;" + cbfComm + " " + hdfRowFilepattern + " 1 " + CBF_conversion_pattern + "0001.cbf;" + adxvComm + " -sa " + CBF_conversion_pattern + "0001.cbf " + JPEG_conversion_pattern + "0001.jpeg;convert " + JPEG_conversion_pattern + "0001.jpeg -resize 10% " + JPEG_conversion_pattern + "0001.thumb.jpeg\"&" # comm_s = "ssh -q " + node + " \"sleep 6;" + cbfComm + " " + hdfRowFilepattern + " 1 " + CBF_conversion_pattern + "0001.cbf;" + adxvComm + " -sa " + CBF_conversion_pattern + "0001.cbf " + JPEG_conversion_pattern + "0001.jpeg;cp " + JPEG_conversion_pattern + "0001.jpeg " + JPEG_conversion_pattern + "0001.thumb.jpeg\"&" print(comm_s) os.system(comm_s) # Create a new data collection group entry: params = mxacquisition.get_data_collection_group_params() params['parentid'] = sessionid # params['sampleid'] = ? if request_type == 'standard': params['experimenttype'] = 'OSC' elif request_type == 'vector': params['experimenttype'] = 'Helical' params['starttime'] = datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S') params['endtime'] = datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S') # params['starttime'] = datetime.fromtimestamp(request['time']).strftime('%Y-%m-%d %H:%M:%S') # params['endtime'] = datetime.fromtimestamp(request['time']).strftime('%Y-%m-%d %H:%M:%S') dcg_id = mxacquisition.insert_data_collection_group(list(params.values())) print("dcg_id: %i" % dcg_id) params = mxacquisition.get_data_collection_params() params['parentid'] = dcg_id params['visitid'] = sessionid params['imgdir'] = directory params['imgprefix'] = filePrefix params['imgsuffix'] = 'cbf' # assume CBF ...? params['wavelength'] = request_obj['wavelength'] params['starttime'] = datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S') # params['starttime'] = datetime.fromtimestamp(request['time']).strftime('%Y-%m-%d %H:%M:%S') params['run_status'] = 'DataCollection Successful' # assume success / not aborted params['datacollection_number'] = request_obj['runNum'] params['n_images'] = int(round((request_obj['sweep_end'] - request_obj['sweep_start']) / request_obj['img_width'])) params['exp_time'] = request_obj['exposure_time'] params['start_image_number'] = request_obj['file_number_start'] params['axis_start'] = request_obj['sweep_start'] params['axis_end'] = request_obj['sweep_end'] params['axis_range'] = request_obj['img_width'] params['resolution'] = request_obj['resolution'] params['detector_distance'] = request_obj['detDist'] params['slitgap_horizontal'] = request_obj['slit_width'] params['slitgap_vertical'] = request_obj['slit_height'] params['transmission'] = request_obj['attenuation'] params['file_template'] = '%s_####.cbf' % (request_obj['file_prefix']) # assume cbf ... # params['file_template'] = '%s_%s_####.cbf' % (request_obj['file_prefix'], request_obj['runNum']) # assume cbf ... # params['flux'] = ? params['overlap'] = 0.0 params['rotation_axis'] = 'Omega' # assume Omega unless we know otherwise print("jpegimfilename = " + jpegImageFilename) params['xtal_snapshot1'] = jpegImageFilename # params['xtal_snapshot1'] = '/dls/i03/data/2016/cm14451-2/jpegs/20160413/test_xtal/xtal1_1_1_0.0.png' params['xtal_snapshot2'] = '/dls/i03/data/2016/cm14451-2/jpegs/20160413/test_xtal/xtal1_1_1_90.0.png' params['xtal_snapshot3'] = '/dls/i03/data/2016/cm14451-2/jpegs/20160413/test_xtal/xtal1_3_1_183.0.png' params['xtal_snapshot4'] = '/dls/i03/data/2016/cm14451-2/jpegs/20160413/test_xtal/xtal1_3_1_93.0.png' dc_id = mxacquisition.insert_data_collection(list(params.values())) print("dc_id: %i" % dc_id) return dc_id