def test_check_config_file_behaviour(mockstomp):
"""Check that a specified configuration file is read, that command line
parameters have precedence and are passed on to the stomp layer."""
mockconn = mock.Mock()
mockstomp.Connection.return_value = mockconn
parser = optparse.OptionParser()
stomp = StompTransport()
stomp.add_command_line_options(parser)
# Temporarily create an example stomp configuration file
cfgfile = tempfile.NamedTemporaryFile(delete=False)
try:
cfgfile.write("""
# An example stomp configuration file
# Only lines in the [stomp] block will be interpreted
[stomp]
#host = 127.0.0.1
port = 1234
username = someuser
password = somesecret
prefix = namespace
""".encode("utf-8"))
cfgfile.close()
parser.parse_args(
["--stomp-conf", cfgfile.name, "--stomp-user", mock.sentinel.user])
# Command line parameters are shared for all instances
stomp = StompTransport()
stomp.connect()
# Reset configuration for subsequent tests by reloading StompTransport
importlib.reload(workflows.transport.stomp_transport)
globals(
)["StompTransport"] = workflows.transport.stomp_transport.StompTransport
mockstomp.Connection.assert_called_once_with([("localhost", 1234)])
mockconn.connect.assert_called_once_with(mock.sentinel.user,
"somesecret",
wait=False)
assert stomp.get_namespace() == "namespace"
finally:
os.remove(cfgfile.name)
# Loading a non-existing configuration file
with pytest.raises(workflows.Error):
parser.parse_args(["--stomp-conf", ""])
def test_anonymous_connection(mockstomp):
"""Check that a specified configuration file is read, that command line
parameters have precedence and are passed on to the stomp layer."""
mockconn = mock.Mock()
mockstomp.Connection.return_value = mockconn
parser = optparse.OptionParser()
stomp = StompTransport()
stomp.add_command_line_options(parser)
parser.parse_args(["--stomp-user="******"--stomp-pass="******"StompTransport"] = workflows.transport.stomp_transport.StompTransport
mockconn.connect.assert_called_once_with(wait=False)
def ensure_url(url: str) -> str:
"""Make sure a string URL has a schema, for urllib.parse.urlparse consumption"""
if "://" not in url:
return f"minio://{args.host}"
return url
logger = logging.getLogger()
parser = ArgumentParser(description="Submit an S3 bucket for PIA")
StompTransport.load_configuration_file(
"/dls_sw/apps/zocalo/secrets/credentials-testing.cfg")
StompTransport.add_command_line_options(parser)
parser.add_argument(
"s3_url",
metavar="S3_URL",
help="The access URL for the S3 bucket",
type=ensure_url,
)
# parser.add_argument("images", metavar="IMAGES", help="Image numbers to submit. Index or ranges '1,10' '1-10'. Defauts to all")
parser.add_argument("-v",
"--verbose",
help="increase output verbosity",
action="store_true")
args = parser.parse_args()
logging.basicConfig(format="%(message)s",
def run():
parser = OptionParser(
usage="zocalo.go [options] dcid",
description="Triggers processing of a standard "
"recipe, of an arbitrary recipe from a local file, or of an entry in "
"the ISPyB processing table.",
)
parser.add_option("-?", action="help", help=SUPPRESS_HELP)
parser.add_option(
"-r",
"--recipe",
dest="recipe",
metavar="RCP",
action="append",
default=[],
help="Name of a recipe to run. Can be used multiple times. Recipe names correspond to filenames (excluding .json) in /dls_sw/apps/zocalo/live/recipes",
)
parser.add_option(
"-a",
"--autoprocscalingid",
dest="autoprocscalingid",
metavar="APSID",
action="store",
type="string",
default=None,
help="An auto processing scaling ID for downstream processing recipes.",
)
parser.add_option(
"-f",
"--file",
dest="recipefile",
metavar="FILE",
action="store",
type="string",
default="",
help="Run recipe contained in this file.",
)
parser.add_option(
"-n",
"--no-dcid",
dest="nodcid",
action="store_true",
default=False,
help="Trigger recipe without specifying a data collection ID",
)
parser.add_option(
"--drop",
dest="dropfile",
action="store_true",
default=False,
help=SUPPRESS_HELP,
) # Write directly to file, do not attempt to send via stomp
parser.add_option(
"-p",
"--reprocessing",
dest="reprocess",
action="store_true",
default=False,
help="Means a reprocessing ID is given rather than a data collection ID",
)
parser.add_option(
"-s",
"--set",
dest="parameters",
action="append",
default=[],
metavar="KEY=VALUE",
help="Set an additional variable for recipe evaluation",
)
parser.add_option(
"-v",
"--verbose",
dest="verbose",
action="store_true",
default=False,
help="Show raw message before sending",
)
parser.add_option(
"--dry-run",
dest="dryrun",
action="store_true",
default=False,
help="Verify that everything is in place that the message could be sent, but don't actually send the message",
)
parser.add_option(
"--test",
action="store_true",
dest="test",
default=False,
help="Run in ActiveMQ testing (zocdev) namespace",
)
default_configuration = "/dls_sw/apps/zocalo/secrets/credentials-live.cfg"
allow_stomp_fallback = not any("stomp" in s.lower() for s in sys.argv)
if "--test" in sys.argv:
default_configuration = "/dls_sw/apps/zocalo/secrets/credentials-testing.cfg"
allow_stomp_fallback = False
# override default stomp host
try:
StompTransport.load_configuration_file(default_configuration)
except workflows.Error as e:
print("Error: %s\n" % str(e))
allow_stomp_fallback = False
StompTransport.add_command_line_options(parser)
(options, args) = parser.parse_args(sys.argv[1:])
def generate_headers():
return {
"zocalo.go.user": getpass.getuser(),
"zocalo.go.host": socket.gethostname(),
}
def write_message_to_dropfile(message, headers):
message_serialized = (
json.dumps({"headers": headers, "message": message}, indent=2) + "\n"
)
fallback = os.path.join("/dls_sw/apps/zocalo/dropfiles", str(uuid.uuid4()))
if options.dryrun:
print("Not storing message in %s (running with --dry-run)" % fallback)
return
with open(fallback, "w") as fh:
fh.write(message_serialized)
print("Message successfully stored in %s" % fallback)
def send_to_stomp_or_defer(message, headers=None):
if not headers:
headers = generate_headers()
if options.verbose:
pprint(message)
if allow_stomp_fallback and options.dropfile:
return write_message_to_dropfile(message, headers)
try:
stomp = StompTransport()
if options.dryrun:
print("Not sending message (running with --dry-run)")
return
stomp.connect()
stomp.send("processing_recipe", message, headers=headers)
except (
KeyboardInterrupt,
SyntaxError,
AssertionError,
AttributeError,
ImportError,
TypeError,
ValueError,
):
raise
except Exception:
if not allow_stomp_fallback:
raise
print("\n\n")
import traceback
traceback.print_exc()
print("\n\nAttempting to store message in fallback location")
write_message_to_dropfile(message, headers)
message = {"recipes": options.recipe, "parameters": {}}
for kv in options.parameters:
if "=" not in kv:
sys.exit(f"Invalid variable specification '{kv}'")
key, value = kv.split("=", 1)
message["parameters"][key] = value
if (
not options.recipe
and not options.recipefile
and not options.nodcid
and not options.reprocess
):
sys.exit("No recipes specified.")
if options.recipefile:
with open(options.recipefile) as fh:
custom_recipe = workflows.recipe.Recipe(json.load(fh))
custom_recipe.validate()
message["custom_recipe"] = custom_recipe.recipe
if options.nodcid:
if options.recipe:
print("Running recipes", options.recipe)
if options.recipefile:
print("Running recipe from file", options.recipefile)
print("without specified data collection.")
send_to_stomp_or_defer(message)
print("\nSubmitted.")
sys.exit(0)
if not args:
sys.exit("No data collection IDs specified.")
if len(args) > 1:
sys.exit("Only a single data collection ID can be specified.")
dcid = int(args[0])
assert dcid > 0, "Invalid data collection ID given."
if options.reprocess:
# Given ID is a reprocessing ID. Nothing else needs to be specified.
if options.recipe:
print("Running recipes", options.recipe)
message["parameters"]["ispyb_process"] = dcid
send_to_stomp_or_defer(message)
print("\nReprocessing task submitted for ID %d." % dcid)
sys.exit(0)
if message["recipes"]:
print("Running recipes", message["recipes"])
if options.recipefile:
print("Running recipe from file", options.recipefile)
if not message["recipes"] and not message.get("custom_recipe"):
sys.exit("No recipes specified.")
print("for data collection", dcid)
message["parameters"]["ispyb_dcid"] = dcid
if options.autoprocscalingid:
apsid = int(options.autoprocscalingid)
assert apsid > 0, "Invalid auto processing scaling ID given."
message["parameters"]["ispyb_autoprocscalingid"] = apsid
send_to_stomp_or_defer(message)
print("\nSubmitted.")
