def test_mx_data_reduction_xml_to_ispyb(testdb, filename):
mxprocessing = testdb.mx_processing
xml_file = os.path.abspath(
os.path.join(os.path.dirname(__file__), filename))
# Find the datacollection associated with this data reduction run
xml_dir = os.path.split(xml_file)[0]
try:
with open(os.path.join(xml_dir, ".dc_id")) as fh:
dc_id = int(fh.read())
print("Got DC ID %d from file system" % dc_id)
except Exception:
dc_id = None
mx_data_reduction_dict = xml_file_to_dict(xml_file)
(app_id, ap_id, scaling_id,
integration_id) = mx_data_reduction_to_ispyb(mx_data_reduction_dict,
dc_id, mxprocessing)
# Output results xml
xml = ('<?xml version="1.0" encoding="ISO-8859-1"?>'
"<dbstatus><autoProcProgramId>%d</autoProcProgramId>"
"<autoProcId>%d</autoProcId>"
"<autoProcScalingId>%d</autoProcScalingId>"
"<autoProcIntegrationId>%d</autoProcIntegrationId>"
"<code>ok</code></dbstatus>" %
(app_id, ap_id, scaling_id, integration_id))
print(xml)
conf_file = sys.argv[1]
with ispyb.open(conf_file) as conn:
mxprocessing = conn.mx_processing
xml_file = sys.argv[2]
xml_dir = os.path.split(xml_file)[0]
# Find the datacollection associated with this data reduction run
try:
dc_id = int(open(os.path.join(xml_dir, ".dc_id"), "r").read())
print("Got DC ID %d from file system" % dc_id)
except:
dc_id = None
mx_data_reduction_dict = xml_file_to_dict(xml_file)
(app_id, ap_id, scaling_id,
integration_id) = mx_data_reduction_to_ispyb(mx_data_reduction_dict,
dc_id, mxprocessing)
# Write results to xml_out_file
if len(sys.argv) > 3:
xml = ('<?xml version="1.0" encoding="ISO-8859-1"?>'
"<dbstatus><autoProcProgramId>%d</autoProcProgramId>"
"<autoProcId>%d</autoProcId>"
"<autoProcScalingId>%d</autoProcScalingId>"
"<autoProcIntegrationId>%d</autoProcIntegrationId>"
"<code>ok</code></dbstatus>" %
(app_id, ap_id, scaling_id, integration_id))
f = open(sys.argv[3], "w")
f.write(xml)
def insertResult(result,
resultType,
request,
visitName,
dc_id=None,
xmlFileName=None): #xmlfilename for fastDP
#keep in mind that request type can be standard and result type be fastDP - multiple results per req.
try:
sessionid = core.retrieve_visit_id(visitName)
except ISPyBNoResultException as e:
logger.error(
"insert result - caught ISPyBNoResultException: '%s'. make sure visit name is in the format mx999999-1234. NOT HAVING MX IN FRONT IS A SIGN OF PROBLEMS - try newVisit() in that case."
% e)
propNum = visitName.split('-')[0]
sessionid = createVisit(propNum)
request_type = request['request_type']
if request_type in ('standard', 'vector'):
sample = request[
'sample'] # this needs to be created and linked to a DC group
if (resultType == 'fastDP'):
mx_data_reduction_dict = xml_file_to_dict(xmlFileName)
(app_id, ap_id, scaling_id,
integration_id) = mx_data_reduction_to_ispyb(
mx_data_reduction_dict, dc_id, mxprocessing)
mxprocessing.upsert_program_ex(program_id=app_id, status=1)
elif resultType == 'mxExpParams':
result_obj = result['result_obj']
request_obj = result_obj['requestObj']
directory = request_obj["directory"]
filePrefix = request_obj['file_prefix']
basePath = request_obj["basePath"]
visitName = daq_utils.getVisitName()
jpegDirectory = visitName + "/jpegs/" + directory[
directory.find(visitName) + len(visitName):len(directory)]
fullJpegDirectory = basePath + "/" + jpegDirectory
jpegImagePrefix = fullJpegDirectory + "/" + filePrefix
daq_utils.take_crystal_picture(filename=jpegImagePrefix)
jpegImageFilename = jpegImagePrefix + ".jpg"
jpegImageThumbFilename = jpegImagePrefix + "t.jpg"
node = db_lib.getBeamlineConfigParam(beamline, "adxvNode")
comm_s = f"ssh -q {node} \"{os.environ['MXPROCESSINGSCRIPTSDIR']}resize.sh {jpegImageFilename} {jpegImageThumbFilename} 40% \"&"
logger.info('resizing image: %s' % comm_s)
os.system(comm_s)
seqNum = int(detSeqNumPV.get())
node = db_lib.getBeamlineConfigParam(beamline, "adxvNode")
request_id = result['request']
comm_s = f"ssh -q {node} \"{os.environ['MXPROCESSINGSCRIPTSDIR']}eiger2cbf.sh {request_id} 1 1 0 {seqNum}\""
logger.info(f'diffraction thumbnail conversion to cbf: {comm_s}')
os.system(comm_s)
comm_s = f"ssh -q {node} \"{os.environ['MXPROCESSINGSCRIPTSDIR']}cbf2jpeg.sh {request_id}\""
logger.info(f'diffraction thumbnail conversion to jpeg: {comm_s}')
os.system(comm_s)
# Create a new data collection group entry:
params = mxacquisition.get_data_collection_group_params()
params['parentid'] = sessionid
if request_type == 'standard':
params['experimenttype'] = 'OSC'
elif request_type == 'vector':
params['experimenttype'] = 'Helical'
return createDataCollection(directory, filePrefix,
jpegImageFilename, params, request_obj,
sessionid)
def insertResult(result,
resultType,
request,
visitName,
dc_id=None,
xmlFileName=None): #xmlfilename for fastDP
#keep in mind that request type can be standard and result type be fastDP - multiple results per req.
cbfComm = db_lib.getBeamlineConfigParam(beamline, "cbfComm")
try:
sessionid = core.retrieve_visit_id(visitName)
except ISPyBNoResultException as e:
logger.error(
"caught ISPyBNoResultException: %s. make sure visit name is in the format mx999999-1234"
% e)
propNum = visitName.split('-')[0]
sessionid = createVisit(propNum)
request_type = request['request_type']
if request_type in ('standard', 'vector'):
sample = request[
'sample'] # this needs to be created and linked to a DC group
if (resultType == 'fastDP'):
mx_data_reduction_dict = xml_file_to_dict(xmlFileName)
(app_id, ap_id, scaling_id,
integration_id) = mx_data_reduction_to_ispyb(
mx_data_reduction_dict, dc_id, mxprocessing)
mxprocessing.upsert_program_ex(program_id=app_id, status=1)
elif resultType == 'mxExpParams':
result_obj = result['result_obj']
request_obj = result_obj['requestObj']
directory = request_obj["directory"]
filePrefix = request_obj['file_prefix']
basePath = request_obj["basePath"]
visitName = daq_utils.getVisitName()
jpegDirectory = visitName + "/jpegs/" + directory[
directory.find(visitName) + len(visitName):len(directory)]
fullJpegDirectory = basePath + "/" + jpegDirectory
jpegImagePrefix = fullJpegDirectory + "/" + filePrefix
daq_utils.take_crystal_picture(filename=jpegImagePrefix)
jpegImageFilename = jpegImagePrefix + ".jpg"
jpegImageThumbFilename = jpegImagePrefix + "t.jpg"
node = db_lib.getBeamlineConfigParam(beamline, "adxvNode")
comm_s = "ssh -q " + node + " \"convert " + jpegImageFilename + " -resize 40% " + jpegImageThumbFilename + "\"&"
logger.info('resizing image: %s' % comm_s)
os.system(comm_s)
seqNum = int(detSeqNumPV.get())
hdfSampleDataPattern = directory + "/" + filePrefix + "_"
hdfRowFilepattern = hdfSampleDataPattern + str(int(
float(seqNum))) + "_master.h5"
# keep in mind I could do the jpeg conversion here, but maybe best to allow synchWeb on demand.
cbfDir = directory
CBF_conversion_pattern = cbfDir + "/" + filePrefix + "_"
JPEG_conversion_pattern = fullJpegDirectory + "/" + filePrefix + "_"
node = db_lib.getBeamlineConfigParam(beamline, "adxvNode")
adxvComm = os.environ["PROJDIR"] + db_lib.getBeamlineConfigParam(
daq_utils.beamline, "adxvComm")
comm_s = "ssh -q " + node + " \"sleep 6;" + cbfComm + " " + hdfRowFilepattern + " 1:1 " + CBF_conversion_pattern + ";" + adxvComm + " -sa " + CBF_conversion_pattern + "000001.cbf " + JPEG_conversion_pattern + "0001.jpeg;convert " + JPEG_conversion_pattern + "0001.jpeg -resize 10% " + JPEG_conversion_pattern + "0001.thumb.jpeg\"&"
logger.info('diffraction thumbnail image: %s' % comm_s)
os.system(comm_s)
# Create a new data collection group entry:
params = mxacquisition.get_data_collection_group_params()
params['parentid'] = sessionid
if request_type == 'standard':
params['experimenttype'] = 'OSC'
elif request_type == 'vector':
params['experimenttype'] = 'Helical'
return createDataCollection(directory, filePrefix,
jpegImageFilename, params, request_obj,
sessionid)
def insertResult(result,resultType,request,visitName,dc_id=None,xmlFileName=None): #xmlfilename for fastDP
#keep in mind that request type can be standard and result type be fastDP - multiple results per req.
cbfComm = db_lib.getBeamlineConfigParam(daq_utils.beamline,"cbfComm")
try:
sessionid = core.retrieve_visit_id(visitName)
except ISPyBNoResultException:
print("caught ISPyBNoResultException")
sessionid = createVisit(visitName)
request_type = request['request_type']
if request_type in('standard', 'vector') :
sample = request['sample'] # this needs to be created and linked to a DC group
if (resultType == 'fastDP'):
mx_data_reduction_dict = xml_file_to_dict(xmlFileName)
(app_id, ap_id, scaling_id, integration_id) = mx_data_reduction_to_ispyb(mx_data_reduction_dict, dc_id, mxprocessing)
elif resultType == 'mxExpParams':
result_obj = result['result_obj']
request_obj = result_obj['requestObj']
directory = request_obj["directory"]
filePrefix = request_obj['file_prefix']
basePath = request_obj["basePath"]
visitName = daq_utils.getVisitName()
jpegDirectory = visitName + "/jpegs/" + directory[directory.find(visitName)+len(visitName):len(directory)]
# jpegDirectory = "jpegs" + directory[directory.find(basePath)+len(basePath):len(directory)]
fullJpegDirectory = basePath + "/" + jpegDirectory
jpegImagePrefix = fullJpegDirectory+"/"+filePrefix
daq_utils.take_crystal_picture(filename=jpegImagePrefix)
jpegImageFilename = jpegImagePrefix+".jpg"
jpegImageThumbFilename = jpegImagePrefix+"t.jpg"
node = db_lib.getBeamlineConfigParam(daq_utils.beamline,"adxvNode")
# node = db_lib.getBeamlineConfigParam(daq_utils.beamline,"spotNode1")
comm_s = "ssh -q " + node + " \"convert " + jpegImageFilename + " -resize 40% " + jpegImageThumbFilename + "\"&"
print(comm_s)
os.system(comm_s)
# seqNum = int(det_lib.detector_get_seqnum())
seqNum = int(detSeqNumPV.get())
hdfSampleDataPattern = directory+"/"+filePrefix+"_"
hdfRowFilepattern = hdfSampleDataPattern + str(int(float(seqNum))) + "_master.h5"
#jpeg folder cbfDir = os.getcwd()+"/jpegs/"+directory[len(os.getcwd()):len(directory)] #good luck with that working
# keep in mind I could do the jpeg conversion here, but maybe best to allow synchWeb on demand.
cbfDir = directory
CBF_conversion_pattern = cbfDir + "/" + filePrefix+"_"
JPEG_conversion_pattern = fullJpegDirectory + "/" + filePrefix+"_"
node = db_lib.getBeamlineConfigParam(daq_utils.beamline,"adxvNode")
# node = db_lib.getBeamlineConfigParam(daq_utils.beamline,"spotNode1")
adxvComm = os.environ["PROJDIR"] + db_lib.getBeamlineConfigParam(daq_utils.beamline,"adxvComm")
comm_s = "ssh -q " + node + " \"sleep 6;" + cbfComm + " " + hdfRowFilepattern + " 1 " + CBF_conversion_pattern + "0001.cbf;" + adxvComm + " -sa " + CBF_conversion_pattern + "0001.cbf " + JPEG_conversion_pattern + "0001.jpeg;convert " + JPEG_conversion_pattern + "0001.jpeg -resize 10% " + JPEG_conversion_pattern + "0001.thumb.jpeg\"&"
# comm_s = "ssh -q " + node + " \"sleep 6;" + cbfComm + " " + hdfRowFilepattern + " 1 " + CBF_conversion_pattern + "0001.cbf;" + adxvComm + " -sa " + CBF_conversion_pattern + "0001.cbf " + JPEG_conversion_pattern + "0001.jpeg;cp " + JPEG_conversion_pattern + "0001.jpeg " + JPEG_conversion_pattern + "0001.thumb.jpeg\"&"
print(comm_s)
os.system(comm_s)
# Create a new data collection group entry:
params = mxacquisition.get_data_collection_group_params()
params['parentid'] = sessionid
# params['sampleid'] = ?
if request_type == 'standard':
params['experimenttype'] = 'OSC'
elif request_type == 'vector':
params['experimenttype'] = 'Helical'
params['starttime'] = datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')
params['endtime'] = datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')
# params['starttime'] = datetime.fromtimestamp(request['time']).strftime('%Y-%m-%d %H:%M:%S')
# params['endtime'] = datetime.fromtimestamp(request['time']).strftime('%Y-%m-%d %H:%M:%S')
dcg_id = mxacquisition.insert_data_collection_group(list(params.values()))
print("dcg_id: %i" % dcg_id)
params = mxacquisition.get_data_collection_params()
params['parentid'] = dcg_id
params['visitid'] = sessionid
params['imgdir'] = directory
params['imgprefix'] = filePrefix
params['imgsuffix'] = 'cbf' # assume CBF ...?
params['wavelength'] = request_obj['wavelength']
params['starttime'] = datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')
# params['starttime'] = datetime.fromtimestamp(request['time']).strftime('%Y-%m-%d %H:%M:%S')
params['run_status'] = 'DataCollection Successful' # assume success / not aborted
params['datacollection_number'] = request_obj['runNum']
params['n_images'] = int(round((request_obj['sweep_end'] - request_obj['sweep_start']) / request_obj['img_width']))
params['exp_time'] = request_obj['exposure_time']
params['start_image_number'] = request_obj['file_number_start']
params['axis_start'] = request_obj['sweep_start']
params['axis_end'] = request_obj['sweep_end']
params['axis_range'] = request_obj['img_width']
params['resolution'] = request_obj['resolution']
params['detector_distance'] = request_obj['detDist']
params['slitgap_horizontal'] = request_obj['slit_width']
params['slitgap_vertical'] = request_obj['slit_height']
params['transmission'] = request_obj['attenuation']
params['file_template'] = '%s_####.cbf' % (request_obj['file_prefix']) # assume cbf ...
# params['file_template'] = '%s_%s_####.cbf' % (request_obj['file_prefix'], request_obj['runNum']) # assume cbf ...
# params['flux'] = ?
params['overlap'] = 0.0
params['rotation_axis'] = 'Omega' # assume Omega unless we know otherwise
print("jpegimfilename = " + jpegImageFilename)
params['xtal_snapshot1'] = jpegImageFilename
# params['xtal_snapshot1'] = '/dls/i03/data/2016/cm14451-2/jpegs/20160413/test_xtal/xtal1_1_1_0.0.png'
params['xtal_snapshot2'] = '/dls/i03/data/2016/cm14451-2/jpegs/20160413/test_xtal/xtal1_1_1_90.0.png'
params['xtal_snapshot3'] = '/dls/i03/data/2016/cm14451-2/jpegs/20160413/test_xtal/xtal1_3_1_183.0.png'
params['xtal_snapshot4'] = '/dls/i03/data/2016/cm14451-2/jpegs/20160413/test_xtal/xtal1_3_1_93.0.png'
dc_id = mxacquisition.insert_data_collection(list(params.values()))
print("dc_id: %i" % dc_id)
return dc_id