def get_dc(id): conf_file = os.getenv('ISPYB_CONFIG_FILE') if conf_file is None: return with ispyb.open(conf_file) as i: mx_acquisition = i.mx_acquisition dc = mx_acquisition.get_data_collection(id) print dc.file_template_full_python print dc.file_directory return dc
def get_dc(id): conf_file = os.getenv('ISPYB_CONFIG_FILE') if conf_file is None: return with ispyb.open(conf_file) as i: mx_acquisition = i.mx_acquisition dc = mx_acquisition.get_data_collection(id) logging.getLogger('image-service').debug('DC File Template: {}'.format(dc.file_template_full_python)) logging.getLogger('image-service').debug('DC File Directory: {}'.format(dc.file_directory)) # print dc.file_template_full_python # print dc.file_directory return dc
# # 2014-09-24 # # Examples of how to use the modules in the ispyb package, # demonstrating how to store data from the whole # data acquisition and processing pipeline. # from __future__ import absolute_import, division, print_function import sys from datetime import datetime import ispyb with ispyb.open(sys.argv[1]) as conn: core = conn.core mxacquisition = conn.mx_acquisition mxprocessing = conn.mx_processing mxscreening = conn.mx_screening # Find the id for a given visit sessionid = core.retrieve_visit_id("cm14451-2") # Create a new data collection entry: params = mxacquisition.get_data_collection_group_params() params["parentid"] = sessionid # experimenttype must be one of the allowed values: None, 'SAD', 'SAD - Inverse Beam', 'OSC', 'Collect - Multiwedge', 'MAD', 'Helical', 'Multi-positional', 'Mesh', 'Burn', 'MAD - Inverse Beam', 'Screening' params["experimenttype"] = "OSC" params["starttime"] = datetime.strptime("2014-09-25 13:00:00", "%Y-%m-%d %H:%M:%S")
def test_ws_connection(testconfig_ws): with pytest.raises(NotImplementedError): with ispyb.open(testconfig_ws) as conn: assert conn
from __future__ import print_function import os import sys import ispyb from ispyb.xmltools import mx_data_reduction_to_ispyb, xml_file_to_dict if len(sys.argv) not in (3, 4): print("Usage:") print("%s conf_file xml_in_file" % sys.argv[0]) print("%s conf_file xml_in_file xml_out_file" % sys.argv[0]) sys.exit(1) conf_file = sys.argv[1] with ispyb.open(conf_file) as conn: mxprocessing = conn.mx_processing xml_file = sys.argv[2] xml_dir = os.path.split(xml_file)[0] # Find the datacollection associated with this data reduction run try: dc_id = int(open(os.path.join(xml_dir, ".dc_id"), "r").read()) print("Got DC ID %d from file system" % dc_id) except: dc_id = None mx_data_reduction_dict = xml_file_to_dict(xml_file) (app_id, ap_id, scaling_id, integration_id) = mx_data_reduction_to_ispyb(mx_data_reduction_dict, dc_id, mxprocessing)
def main(cmd_args=sys.argv[1:]): parser = OptionParser( usage="ispyb.job [options] JOBID", description= "Command line tool to manipulate ISPyB processing table entries.", ) recipe_dir = None if zocalo: zc = zocalo.configuration.from_file() zc.activate() if hasattr(zc, "storage"): recipe_dir = zc.storage.get("zocalo.recipe_directory") if recipe_dir and os.path.isdir(recipe_dir): available_recipes = sorted( r[6:-5] for r in os.listdir(recipe_dir) if r.startswith("ispyb-") and r.endswith(".json")) else: available_recipes = None parser.add_option("-?", action="help", help=SUPPRESS_HELP) parser.add_option( "-v", "--verbose", action="store_true", dest="verbose", default=False, help="show full job record", ) group = OptionGroup( parser, "Processing job options", "These options can be used to create or modify a processing job.", ) group.add_option( "--new", dest="new", action="store_true", default=False, help= "create a new processing job. If --new is specified you must not specify another JOBID", ) group.add_option( "--dcid", dest="dcid", action="store", type="string", default=None, help= "set the primary data collection ID for the processing job (default: DCID of first sweep)", ) group.add_option( "--display", dest="display", action="store", type="string", default=None, help="set the display name of the processing job", ) group.add_option( "--comment", dest="comment", action="store", type="string", default=None, help="set a comment string for the processing job", ) if available_recipes: group.add_option( "--recipe", dest="recipe", action="store", type="choice", default=None, choices=available_recipes, help= "set a recipe for the processing job. Recipe name must correspond to a filename " f"(plus ispyb- prefix and .json extension) in {recipe_dir}: {', '.join(available_recipes)}", ) else: group.add_option( "--recipe", dest="recipe", action="store", type="string", default=None, help="set a recipe for the processing job", ) group.add_option( "--source", dest="source", action="store", type="choice", default="user", choices=["user", "automatic"], help= "set whether the processing job was triggered by a 'user' (default) or by 'automatic' processing", ) group.add_option( "--add-param", dest="parameters", action="append", type="string", default=[], metavar="KEY:VALUE", help="add a 'KEY:VALUE' pair string parameter to a processing job", ) group.add_option( "--add-sweep", dest="sweeps", action="append", type="string", default=[], metavar="DCID:START:END", help= "add an image range from a sweep of any data collection ID to the processing job. " "If no sweep is defined all images from the primary data collection ID are used if the data collection ID can be inferred", ) if zocalo: group.add_option( "--trigger", dest="trigger", action="store_true", default=False, help="start the processing job immediately after creation", ) group.add_option( "--trigger-variable", dest="triggervariables", action="append", type="string", default=[], metavar="KEY:VALUE", help= "Set an additional variable for recipe evaluation when starting the processing job", ) parser.add_option_group(group) group = OptionGroup( parser, "Processing program options", "These options can be used to create or update " "processing program entries belonging to a processing job.", ) group.add_option( "-c", "--create", dest="create", action="store_true", default=False, help="create a new processing program entry for the JOBID", ) group.add_option( "-u", "--update", dest="update", action="store", type="int", default=None, help="update an existing processing program entry", ) parser.add_option_group(group) group = OptionGroup( parser, "Processing program attributes", "These options can be used when creating or updating " "processing program entries.", ) group.add_option( "-p", "--program", dest="program", action="store", type="string", default=None, help="set a program name for processing entry", ) group.add_option( "-l", "--cmdline", dest="cmdline", action="store", type="string", default=None, help="set full command line for processing entry", ) group.add_option( "-e", "--environment", dest="environment", action="store", type="string", default=None, help="set an environment string for processing entry", ) group.add_option( "-r", "--result", dest="result", action="store", type="choice", default=None, choices=["success", "failure"], help="set a job result: success, failure", ) group.add_option( "-s", "--status", dest="status", action="store", type="string", default=None, help="set program status information", ) group.add_option( "--start-time", dest="starttime", metavar="TIMESTAMP", action="store", type="string", default=None, help="set the program start time (default: now)", ) group.add_option( "--update-time", dest="updatetime", metavar="TIMESTAMP", action="store", type="string", default=None, help="date the updated information (default: now)", ) parser.add_option_group(group) (options, args) = parser.parse_args(cmd_args) if not args and not options.new: if cmd_args: print("No job ID specified\n") parser.print_help() sys.exit(0) if len(args) > 1: sys.exit("Only one job ID can be specified") if options.new and args: sys.exit("Can not create a new job ID when a job ID is specified") if options.new and options.update: sys.exit("Can not update a program when creating a new job ID") if zocalo and options.triggervariables and not options.trigger: sys.exit("--trigger-variable only makes sense with --trigger") i = ispyb.open() if options.new: jobid = create_processing_job(i, options) else: jobid = args[0] if options.create: i.mx_processing.upsert_program_ex( job_id=jobid, name=options.program, command=options.cmdline, environment=options.environment, time_start=options.starttime, time_update=options.updatetime, message=options.status, status={ "success": 1, "failure": 0 }.get(options.result), ) elif options.update: i.mx_processing.upsert_program_ex( program_id=options.update, status={ "success": 1, "failure": 0 }.get(options.result), time_start=options.updatetime, time_update=options.updatetime, message=options.status, ) rp = i.get_processing_job(jobid) try: rp.load() except ispyb.NoResult: print(f"Processing ID {jobid} not found") sys.exit(1) print(f"""Processing ID {rp.jobid}: Name: {rp.name} Recipe: {rp.recipe} Comments: {rp.comment} Primary DC: {rp.DCID} Defined: {rp.timestamp}""") if options.verbose: if rp.parameters: maxlen = max(max(map(len, dict(rp.parameters))), 11) print("\n Parameters:") print("\n".join("%%%ds: %%s" % maxlen % (p[0], p[1]) for p in sorted(rp.parameters))) if rp.sweeps: print("\n Sweeps: ", end="") print(("\n" + " " * 13).join( f"DCID {sweep.DCID:7} images{sweep.start:5} -{sweep.end:5}" for sweep in rp.sweeps)) if rp.programs: print_format = "\nProgram #{0.app_id}: {0.name}, {0.status_text}" if options.verbose: print_format += "\n Command: {0.command}" print_format += "\nEnvironment: {0.environment}" print_format += "\n Defined: {0.time_defined}" print_format += "\n Started: {0.time_start}" print_format += "\nLast Update: {0.time_update}" print_format += "\n Last Info: {0.message}" for program in rp.programs: print(print_format.format(program)) if options.verbose: try: attachments = (i.mx_processing. retrieve_program_attachments_for_program_id( program.app_id)) for filetype in sorted( {a["fileType"] for a in attachments}): for attachment in sorted( (a for a in attachments if a["fileType"] == filetype), key=lambda a: a["fileName"], ): print(" {att[fileType]:>10s}: {att[fileName]}". format(att=attachment)) except ispyb.NoResult: pass
def create_connection(conf_file): import warnings warnings.warn("deprecated, use ispyb.open()", DeprecationWarning) return ispyb.open(conf_file)
def test_session_from_envvar(testconfig, monkeypatch): monkeypatch.setenv("ISPYB_CREDENTIALS", testconfig) conn = ispyb.open() assert isinstance(conn, ISPyBMySQLSPConnector)
def testdb(testconfig): """Return an ISPyB connection object for the test database configuration.""" with ispyb.open(testconfig) as conn: yield conn