def collectData(currentRequest):
global data_directory_name, currentIspybDCID, fastDPNodeCount, fastDPNodeCounter
if (daq_utils.beamline == "fmx"):
if (getPvDesc("detCoverRBV") == 0):
logger.info("opening det cover")
setPvDesc("detCoverOpen", 1)
logMe = 1
reqObj = currentRequest["request_obj"]
data_directory_name = str(reqObj["directory"])
exposure_period = reqObj["exposure_time"]
wavelength = reqObj["wavelength"]
resolution = reqObj["resolution"]
slit_height = reqObj["slit_height"]
slit_width = reqObj["slit_width"]
attenuation = reqObj["attenuation"]
img_width = reqObj["img_width"]
file_prefix = str(reqObj["file_prefix"])
logger.info(reqObj["protocol"])
prot = str(reqObj["protocol"])
sweep_start = reqObj["sweep_start"]
sweep_end = reqObj["sweep_end"]
range_degrees = abs(sweep_end - sweep_start)
sweep_start = reqObj["sweep_start"] % 360.0
file_number_start = reqObj["file_number_start"]
basePath = reqObj["basePath"]
visitName = daq_utils.getVisitName()
jpegDirectory = visitName + "/jpegs/" + data_directory_name[
data_directory_name.find(visitName) +
len(visitName):len(data_directory_name)]
colDist = reqObj["detDist"]
status = 1
if not (os.path.isdir(data_directory_name)):
logger.debug(f'creating {data_directory_name}')
comm_s = "mkdir -p " + data_directory_name
os.system(comm_s)
comm_s = "chmod 777 " + data_directory_name
os.system(comm_s)
comm_s = "mkdir -p " + jpegDirectory
os.system(comm_s)
comm_s = "chmod 777 " + jpegDirectory
os.system(comm_s)
logger.debug(
'starting initial motions - transmission and detector distance')
daq_macros.setTrans(attenuation)
if prot not in ["rasterScreen", "eScan"]:
beamline_lib.mvaDescriptor("detectorDist", colDist)
logger.debug(
'transmission and detector distance (if not fluorescence-related) done'
)
# now that the detector is in the correct position, get the beam center
currentRequest['request_obj']['xbeam'] = getPvDesc('beamCenterX')
currentRequest['request_obj']['ybeam'] = getPvDesc('beamCenterY')
db_lib.updateRequest(currentRequest)
if (prot == "raster"):
logger.info('entering raster')
status = daq_macros.snakeRaster(currentRequest["uid"])
logger.info('exiting raster')
elif (prot == "stepRaster"):
status = daq_macros.snakeStepRaster(currentRequest["uid"])
elif (prot == "specRaster"):
status = daq_macros.snakeStepRasterSpec(currentRequest["uid"])
elif (prot == "vector" or prot == "stepVector"):
imagesAttempted = collect_detector_seq_hw(sweep_start, range_degrees,
img_width, exposure_period,
file_prefix,
data_directory_name,
file_number_start,
currentRequest)
elif (prot == "multiCol"):
daq_macros.snakeRaster(currentRequest["uid"])
elif (prot == "rasterScreen"):
daq_macros.rasterScreen(currentRequest)
elif (prot == "multiColQ"):
daq_macros.multiCol(currentRequest)
elif (prot == "eScan"):
daq_macros.eScan(currentRequest)
else: #standard, screening, or edna - these may require autoalign, checking first
if (reqObj["pos_x"] != -999):
beamline_lib.mvaDescriptor("sampleX", reqObj["pos_x"])
beamline_lib.mvaDescriptor("sampleY", reqObj["pos_y"])
beamline_lib.mvaDescriptor("sampleZ", reqObj["pos_z"])
elif (reqObj["centeringOption"] == "Interactive"
): #robotic, pause and let user center
pause_data_collection()
check_pause()
else:
logger.info("autoRaster")
if not (daq_macros.autoRasterLoop(currentRequest)):
logger.info("could not center sample")
db_lib.updatePriority(currentRequest["uid"], -1)
refreshGuiTree()
return 0
else:
if (reqObj["centeringOption"] == "AutoLoop"):
reqObj[
"sweep_start"] = beamline_lib.motorPosFromDescriptor(
"omega") #%360.0?
sweep_start = reqObj["sweep_start"]
if (reqObj["centeringOption"] == "AutoRaster"):
reqObj[
"sweep_start"] = beamline_lib.motorPosFromDescriptor(
"omega") - 90.0 #%360.0?
sweep_start = reqObj["sweep_start"]
daq_macros.setTrans(attenuation)
if (reqObj["protocol"] == "screen"):
screenImages = 2
screenRange = 90
range_degrees = img_width
for i in range(0, screenImages):
sweep_start = reqObj["sweep_start"] + (i * screenRange)
sweep_end = sweep_start + screenRange
file_prefix = str(reqObj["file_prefix"] + "_" +
str(i * screenRange))
data_directory_name = str(reqObj["directory"]) # for now
file_number_start = reqObj["file_number_start"]
beamline_lib.mvaDescriptor("omega", sweep_start)
if (i == 0):
imagesAttempted = collect_detector_seq_hw(
sweep_start,
range_degrees,
img_width,
exposure_period,
file_prefix,
data_directory_name,
file_number_start,
currentRequest,
changeState=False)
else:
imagesAttempted = collect_detector_seq_hw(
sweep_start, range_degrees, img_width, exposure_period,
file_prefix, data_directory_name, file_number_start,
currentRequest)
seqNum = int(detector_get_seqnum())
node = getBlConfig("spotNode1")
comm_s = f'ssh -q {node} \"{os.environ["MXPROCESSINGSCRIPTSDIR"]}eiger2cbf.sh {currentRequest["uid"]} 1 1 sweep_start {seqNum}\"'
logger.info(comm_s)
os.system(comm_s)
elif (reqObj["protocol"] == "characterize"
or reqObj["protocol"] == "ednaCol"):
characterizationParams = reqObj["characterizationParams"]
index_success = daq_macros.dna_execute_collection3(
0.0, img_width, 2, exposure_period, data_directory_name + "/",
file_prefix, 1, -89.0, 1, currentRequest)
if (index_success):
resultsList = db_lib.getResultsforRequest(
currentRequest["uid"]
) # because for testing I keep running the same request. Probably not in usual use.
results = None
for i in range(0, len(resultsList)):
if (resultsList[i]['result_type'] ==
'characterizationStrategy'):
results = resultsList[i]
break
if (results != None):
strategyResults = results["result_obj"]["strategy"]
stratStart = strategyResults["start"]
stratEnd = strategyResults["end"]
stratWidth = strategyResults["width"]
stratExptime = strategyResults["exptime"]
stratTrans = strategyResults["transmission"]
stratDetDist = strategyResults["detDist"]
sampleID = currentRequest["sample"]
tempnewStratRequest = daq_utils.createDefaultRequest(
sampleID)
newReqObj = tempnewStratRequest["request_obj"]
newReqObj["sweep_start"] = stratStart
newReqObj["sweep_end"] = stratEnd
newReqObj["img_width"] = stratWidth
newReqObj["exposure_time"] = stratExptime
newReqObj["attenuation"] = stratTrans
newReqObj["detDist"] = stratDetDist
newReqObj["directory"] = data_directory_name
newReqObj["pos_x"] = beamline_lib.motorPosFromDescriptor(
"sampleX")
newReqObj["pos_y"] = beamline_lib.motorPosFromDescriptor(
"sampleY")
newReqObj["pos_z"] = beamline_lib.motorPosFromDescriptor(
"sampleZ")
newReqObj[
"fastDP"] = True # this is where you might want a "new from old" request to carry over stuff like this.
newReqObj["fastEP"] = reqObj["fastEP"]
newReqObj["dimple"] = reqObj["dimple"]
newReqObj["xia2"] = reqObj["xia2"]
runNum = db_lib.incrementSampleRequestCount(sampleID)
newReqObj["runNum"] = runNum
newStratRequest = db_lib.addRequesttoSample(
sampleID,
newReqObj["protocol"],
daq_utils.owner,
newReqObj,
priority=0,
proposalID=daq_utils.getProposalID())
if (reqObj["protocol"] == "ednaCol"):
logger.info("new strat req = ")
logger.info(newStratRequest)
db_lib.updatePriority(currentRequest["uid"], -1)
refreshGuiTree()
collectData(db_lib.getRequestByID(newStratRequest))
return 1
else: #standard
logger.info("moving omega to start " + str(time.time()))
beamline_lib.mvaDescriptor("omega", sweep_start)
imagesAttempted = collect_detector_seq_hw(
sweep_start, range_degrees, img_width, exposure_period,
file_prefix, data_directory_name, file_number_start,
currentRequest)
try:
if (logMe) and prot == 'raster':
logMxRequestParams(currentRequest, wait=False)
elif (logMe):
logMxRequestParams(currentRequest)
except TypeError:
logger.error("caught type error in logging")
except IndexError:
logger.error("caught index error in logging")
except KeyError as e:
logger.error('caught key error in logging: %s' % e)
# collection finished, start processing
if reqObj["protocol"] in ("standard", "vector", "raster"):
send_kafka_message(topic=f'{daq_utils.beamline}.lsdc.documents',
event='stop',
uuid=currentRequest['uid'],
protocol=reqObj["protocol"])
if (prot == "vector" or prot == "standard" or prot == "stepVector"):
seqNum = int(detector_get_seqnum())
comm_s = os.environ[
"LSDCHOME"] + "/runSpotFinder4syncW.py " + data_directory_name + " " + file_prefix + " " + str(
currentRequest["uid"]) + " " + str(seqNum) + " " + str(
currentIspybDCID) + "&"
logger.info(comm_s)
os.system(comm_s)
if img_width > 0: #no dataset processing in stills mode
if (reqObj["fastDP"]):
if (reqObj["fastEP"]):
fastEPFlag = 1
else:
fastEPFlag = 0
if (reqObj["dimple"]):
dimpleFlag = 1
else:
dimpleFlag = 0
nodeName = "fastDPNode" + str(
(fastDPNodeCounter % fastDPNodeCount) + 1)
fastDPNodeCounter += 1
node = getBlConfig(nodeName)
dimpleNode = getBlConfig("dimpleNode")
if (daq_utils.detector_id == "EIGER-16"):
seqNum = int(detector_get_seqnum())
comm_s = os.environ[
"LSDCHOME"] + "/runFastDPH5.py " + data_directory_name + " " + str(
seqNum) + " " + str(
currentRequest["uid"]
) + " " + str(fastEPFlag) + " " + node + " " + str(
dimpleFlag) + " " + dimpleNode + " " + str(
currentIspybDCID) + "&"
else:
comm_s = os.environ[
"LSDCHOME"] + "/runFastDP.py " + data_directory_name + " " + file_prefix + " " + str(
file_number_start) + " " + str(
int(round(range_degrees / img_width))
) + " " + str(currentRequest["uid"]) + " " + str(
fastEPFlag) + " " + node + " " + str(
dimpleFlag) + " " + dimpleNode + "&"
logger.info(f'Running fastdp command: {comm_s}')
visitName = daq_utils.getVisitName()
if (
not os.path.exists(visitName + "/fast_dp_dir")
) or subprocess.run(
['pgrep', '-f', 'loop-fdp-dple-populate'],
stdout=subprocess.PIPE
).returncode == 1: # for pgrep, return of 1 means string not found
os.system("killall -KILL loop-fdp-dple-populate.sh")
logger.info('starting fast dp result gathering script')
os.system("cd " + visitName +
";${LSDCHOME}/bin/loop-fdp-dple-populate.sh&")
os.system(comm_s)
if (reqObj["xia2"]):
comm_s = f"ssh -q xf17id2-srv1 \"{os.environ['MXPROCESSINGSCRIPTSDIR']}xia2.sh {currentRequest['uid']} \"&"
os.system(comm_s)
logger.info('processing should be triggered')
db_lib.updatePriority(currentRequest["uid"], -1)
refreshGuiTree()
logger.info('after refresh GUI tree')
return status
def collectData(currentRequest):
global data_directory_name, currentIspybDCID, fastDPNodeCount, fastDPNodeCounter
if (daq_utils.beamline == "fmx"):
if (getPvDesc("detCoverRBV") == 0):
logger.info("opening det cover")
setPvDesc("detCoverOpen", 1)
logMe = 1
reqObj = currentRequest["request_obj"]
data_directory_name = str(reqObj["directory"])
exposure_period = reqObj["exposure_time"]
wavelength = reqObj["wavelength"]
resolution = reqObj["resolution"]
slit_height = reqObj["slit_height"]
slit_width = reqObj["slit_width"]
attenuation = reqObj["attenuation"]
img_width = reqObj["img_width"]
file_prefix = str(reqObj["file_prefix"])
logger.info(reqObj["protocol"])
prot = str(reqObj["protocol"])
sweep_start = reqObj["sweep_start"]
sweep_end = reqObj["sweep_end"]
range_degrees = abs(sweep_end - sweep_start)
sweep_start = reqObj["sweep_start"] % 360.0
file_number_start = reqObj["file_number_start"]
basePath = reqObj["basePath"]
visitName = daq_utils.getVisitName()
jpegDirectory = visitName + "/jpegs/" + data_directory_name[
data_directory_name.find(visitName) +
len(visitName):len(data_directory_name)]
colDist = reqObj["detDist"]
status = 1
if not (os.path.isdir(data_directory_name)):
comm_s = "mkdir -p " + data_directory_name
os.system(comm_s)
comm_s = "chmod 777 " + data_directory_name
os.system(comm_s)
comm_s = "mkdir -p " + jpegDirectory
os.system(comm_s)
comm_s = "chmod 777 " + jpegDirectory
os.system(comm_s)
daq_macros.setTrans(attenuation)
beamline_lib.mvaDescriptor("detectorDist", colDist)
# now that the detector is in the correct position, get the beam center
currentRequest['request_obj']['xbeam'] = getPvDesc('beamCenterX')
currentRequest['request_obj']['ybeam'] = getPvDesc('beamCenterY')
db_lib.updateRequest(currentRequest)
if (prot == "raster"):
logger.info('entering raster')
status = daq_macros.snakeRaster(currentRequest["uid"])
logger.info('exiting raster')
elif (prot == "stepRaster"):
status = daq_macros.snakeStepRaster(currentRequest["uid"])
elif (prot == "specRaster"):
status = daq_macros.snakeStepRasterSpec(currentRequest["uid"])
elif (prot == "vector" or prot == "stepVector"):
imagesAttempted = collect_detector_seq_hw(sweep_start, range_degrees,
img_width, exposure_period,
file_prefix,
data_directory_name,
file_number_start,
currentRequest)
elif (prot == "multiCol"):
daq_macros.snakeRaster(currentRequest["uid"])
elif (prot == "rasterScreen"):
daq_macros.rasterScreen(currentRequest)
elif (prot == "multiColQ"):
daq_macros.multiCol(currentRequest)
elif (prot == "eScan"):
daq_macros.eScan(currentRequest)
else: #standard, screening, or edna - these may require autoalign, checking first
if (reqObj["pos_x"] != -999):
beamline_lib.mvaDescriptor("sampleX", reqObj["pos_x"])
beamline_lib.mvaDescriptor("sampleY", reqObj["pos_y"])
beamline_lib.mvaDescriptor("sampleZ", reqObj["pos_z"])
elif (reqObj["centeringOption"] == "Interactive"
): #robotic, pause and let user center
pause_data_collection()
check_pause()
else:
logger.info("autoRaster")
if not (daq_macros.autoRasterLoop(currentRequest)):
logger.info("could not center sample")
db_lib.updatePriority(currentRequest["uid"], -1)
refreshGuiTree()
return 0
else:
if (reqObj["centeringOption"] == "AutoLoop"):
reqObj[
"sweep_start"] = beamline_lib.motorPosFromDescriptor(
"omega") #%360.0?
sweep_start = reqObj["sweep_start"]
if (reqObj["centeringOption"] == "AutoRaster"):
reqObj[
"sweep_start"] = beamline_lib.motorPosFromDescriptor(
"omega") - 90.0 #%360.0?
sweep_start = reqObj["sweep_start"]
daq_macros.setTrans(attenuation)
if (reqObj["protocol"] == "screen"):
screenImages = 2
screenRange = 90
range_degrees = img_width
for i in range(0, screenImages):
sweep_start = reqObj["sweep_start"] + (i * screenRange)
sweep_end = sweep_start + screenRange
file_prefix = str(reqObj["file_prefix"] + "_" +
str(i * screenRange))
data_directory_name = str(reqObj["directory"]) # for now
file_number_start = reqObj["file_number_start"]
beamline_lib.mvaDescriptor("omega", sweep_start)
if (i == 0):
imagesAttempted = collect_detector_seq_hw(
sweep_start,
range_degrees,
img_width,
exposure_period,
file_prefix,
data_directory_name,
file_number_start,
currentRequest,
changeState=False)
else:
imagesAttempted = collect_detector_seq_hw(
sweep_start, range_degrees, img_width, exposure_period,
file_prefix, data_directory_name, file_number_start,
currentRequest)
seqNum = int(detector_get_seqnum())
cbfComm = getBlConfig("cbfComm")
cbfDir = data_directory_name + "/cbf"
comm_s = "mkdir -p " + cbfDir
os.system(comm_s)
hdfSampleDataPattern = data_directory_name + "/" + file_prefix + "_"
hdfRowFilepattern = hdfSampleDataPattern + str(
int(float(seqNum))) + "_master.h5"
CBF_conversion_pattern = cbfDir + "/" + file_prefix + "_" + str(
int(sweep_start)) + ".cbf"
comm_s = "eiger2cbf-linux " + hdfRowFilepattern
startIndex = 1
endIndex = 1
node = getBlConfig("spotNode1")
comm_s = "ssh -q " + node + " \"sleep 6;" + cbfComm + " " + hdfRowFilepattern + " " + str(
startIndex) + ":" + str(
endIndex) + " " + CBF_conversion_pattern + "\"&"
logger.info(comm_s)
os.system(comm_s)
elif (reqObj["protocol"] == "characterize"
or reqObj["protocol"] == "ednaCol"):
characterizationParams = reqObj["characterizationParams"]
index_success = daq_macros.dna_execute_collection3(
0.0, img_width, 2, exposure_period, data_directory_name + "/",
file_prefix, 1, -89.0, 1, currentRequest)
if (index_success):
resultsList = db_lib.getResultsforRequest(
currentRequest["uid"]
) # because for testing I keep running the same request. Probably not in usual use.
results = None
for i in range(0, len(resultsList)):
if (resultsList[i]['result_type'] ==
'characterizationStrategy'):
results = resultsList[i]
break
if (results != None):
strategyResults = results["result_obj"]["strategy"]
stratStart = strategyResults["start"]
stratEnd = strategyResults["end"]
stratWidth = strategyResults["width"]
stratExptime = strategyResults["exptime"]
stratTrans = strategyResults["transmission"]
stratDetDist = strategyResults["detDist"]
sampleID = currentRequest["sample"]
tempnewStratRequest = daq_utils.createDefaultRequest(
sampleID)
newReqObj = tempnewStratRequest["request_obj"]
newReqObj["sweep_start"] = stratStart
newReqObj["sweep_end"] = stratEnd
newReqObj["img_width"] = stratWidth
newReqObj["exposure_time"] = stratExptime
newReqObj["attenuation"] = stratTrans
newReqObj["detDist"] = stratDetDist
newReqObj["directory"] = data_directory_name
newReqObj["pos_x"] = beamline_lib.motorPosFromDescriptor(
"sampleX")
newReqObj["pos_y"] = beamline_lib.motorPosFromDescriptor(
"sampleY")
newReqObj["pos_z"] = beamline_lib.motorPosFromDescriptor(
"sampleZ")
newReqObj[
"fastDP"] = True # this is where you might want a "new from old" request to carry over stuff like this.
newReqObj["fastEP"] = reqObj["fastEP"]
newReqObj["dimple"] = reqObj["dimple"]
newReqObj["xia2"] = reqObj["xia2"]
runNum = db_lib.incrementSampleRequestCount(sampleID)
newReqObj["runNum"] = runNum
newStratRequest = db_lib.addRequesttoSample(
sampleID,
newReqObj["protocol"],
daq_utils.owner,
newReqObj,
priority=0,
proposalID=daq_utils.getProposalID())
if (reqObj["protocol"] == "ednaCol"):
logger.info("new strat req = ")
logger.info(newStratRequest)
db_lib.updatePriority(currentRequest["uid"], -1)
refreshGuiTree()
collectData(db_lib.getRequestByID(newStratRequest))
return 1
else: #standard
logger.info("moving omega to start " + str(time.time()))
beamline_lib.mvaDescriptor("omega", sweep_start)
imagesAttempted = collect_detector_seq_hw(
sweep_start, range_degrees, img_width, exposure_period,
file_prefix, data_directory_name, file_number_start,
currentRequest)
try:
if (logMe):
logMxRequestParams(currentRequest)
except TypeError:
logger.error("caught type error in logging")
except IndexError:
logger.error("caught index error in logging")
except KeyError as e:
logger.error('caught key error in logging: %s' % e)
if (prot == "vector" or prot == "standard" or prot == "stepVector"):
seqNum = int(detector_get_seqnum())
comm_s = os.environ[
"LSDCHOME"] + "/runSpotFinder4syncW.py " + data_directory_name + " " + file_prefix + " " + str(
currentRequest["uid"]) + " " + str(seqNum) + " " + str(
currentIspybDCID) + "&"
logger.info(comm_s)
os.system(comm_s)
if img_width > 0: #no dataset processing in stills mode
if (reqObj["fastDP"]):
if (reqObj["fastEP"]):
fastEPFlag = 1
else:
fastEPFlag = 0
if (reqObj["dimple"]):
dimpleFlag = 1
else:
dimpleFlag = 0
nodeName = "fastDPNode" + str(
(fastDPNodeCounter % fastDPNodeCount) + 1)
fastDPNodeCounter += 1
node = getBlConfig(nodeName)
dimpleNode = getBlConfig("dimpleNode")
if (daq_utils.detector_id == "EIGER-16"):
seqNum = int(detector_get_seqnum())
comm_s = os.environ[
"LSDCHOME"] + "/runFastDPH5.py " + data_directory_name + " " + file_prefix + " " + str(
seqNum) + " " + str(
int(round(range_degrees / img_width))
) + " " + str(currentRequest["uid"]) + " " + str(
fastEPFlag) + " " + node + " " + str(
dimpleFlag) + " " + dimpleNode + " " + str(
currentIspybDCID) + "&"
else:
comm_s = os.environ[
"LSDCHOME"] + "/runFastDP.py " + data_directory_name + " " + file_prefix + " " + str(
file_number_start) + " " + str(
int(round(range_degrees / img_width))
) + " " + str(currentRequest["uid"]) + " " + str(
fastEPFlag) + " " + node + " " + str(
dimpleFlag) + " " + dimpleNode + "&"
logger.info(comm_s)
if (daq_utils.beamline == "amx"):
visitName = daq_utils.getVisitName()
if (not os.path.exists(visitName + "/fast_dp_dir")):
os.system("killall -KILL loop-fdp-dple-populate")
os.system(
"cd " + visitName +
";/GPFS/CENTRAL/xf17id2/jjakoncic/Scripts/loop-fdp-dple-populate.sh&"
)
os.system(comm_s)
if (reqObj["xia2"]):
comm_s = "ssh -q xf17id1-srv1 \"" + os.environ[
"LSDCHOME"] + "/runXia2.py " + data_directory_name + " " + file_prefix + " " + str(
file_number_start) + " " + str(
int(round(range_degrees / img_width))) + " " + str(
currentRequest["uid"]) + "\"&"
os.system(comm_s)
logger.info('processing should be triggered')
db_lib.updatePriority(currentRequest["uid"], -1)
refreshGuiTree()
logger.info('after refresh GUI tree')
return status