def get_all_queues(rabbitmq_admin_url):
"""
List the queues available for job-running
Note: does not return celery internal queues
@param rabbitmq_admin_url: RabbitMQ admin URL
@return: list of queues
"""
try:
endpoint = os.path.join(rabbitmq_admin_url, "api/queues")
req = requests.get(endpoint)
req.raise_for_status()
data = req.json()
except requests.HTTPError as e:
if e.response.status_code == 401:
logger.error(
"Failed to authenticate {}. Ensure credentials are set in .netrc"
.format(rabbitmq_admin_url))
raise Exception(e)
return [
obj["name"] for obj in data if not obj["name"].startswith("celery")
and obj["name"] not in HYSDS_QUEUES
]
def get_all_queues(rabbitmq_admin_url, user=None, password=None):
"""
List the queues available for job-running (Note: does not return celery internal queues)
:param rabbitmq_admin_url: RabbitMQ admin URL
:param user:
:param password:
:return: list of queues
"""
endpoint = os.path.join(rabbitmq_admin_url, "api/queues")
try:
if user and password:
data = requests.get(endpoint, auth=HTTPBasicAuth(user, password), verify=False)
else:
data = requests.get(endpoint, verify=False)
except requests.HTTPError as e:
if e.response.status_code == 401:
logger.error("Failed to authenticate {}. Ensure credentials are set in .netrc.".format(rabbitmq_admin_url))
raise
results = []
for obj in data:
if not obj["name"].startswith("celery") and obj["name"] not in HYSDS_QUEUES \
and obj["name"] != "Recommended Queues" and obj["messages_ready"] > 0:
results.append(obj)
return results
def get_hostname():
"""Get hostname."""
try:
return socket.getfqdn()
except Exception as e:
logger.warning(e)
try: # get IP
return socket.gethostbyname(socket.gethostname())
except Exception as e:
logger.error(e)
raise RuntimeError("Failed to resolve hostname for full email address. Please check the system.")
def get_all_queues(rabbitmq_admin_url):
'''
List the queues available for job-running
Note: does not return celery internal queues
@param rabbitmq_admin_url: RabbitMQ admin URL
@return: list of queues
'''
try:
data = get_requests_json_response(os.path.join(rabbitmq_admin_url, "api/queues"))
#print(data)
except HTTPError, e:
if e.response.status_code == 401:
logger.error("Failed to authenticate to {}. Ensure credentials are set in .netrc.".format(rabbitmq_admin_url))
raise
def get_all_queues(rabbitmq_admin_url, user=None, password=None):
"""
List the queues available for job-running (Note: does not return celery internal queues)
:param rabbitmq_admin_url: RabbitMQ admin URL
:param user:
:param password:
:return: list of queues
"""
print("get_all_queues : {}/ {}".format(user, password))
try:
if user and password:
data = get_requests_json_response(
os.path.join(rabbitmq_admin_url, "api/queues"),
auth=HTTPBasicAuth(user, password),
verify=False)
else:
data = get_requests_json_response(os.path.join(
rabbitmq_admin_url, "api/queues"),
verify=False)
except HTTPError as e:
if e.response.status_code == 401:
logger.error(
"Failed to authenticate to {}. Ensure credentials are set in .netrc."
.format(rabbitmq_admin_url))
raise
for obj in data:
if not obj["name"].startswith(
"celery") and obj["name"] not in HYSDS_QUEUES:
if obj["name"] == 'Recommended Queues':
continue
if obj["name"] == "factotum-job_worker-scihub_throttled":
print(obj["name"])
print(obj)
print(json.dumps(obj, indent=2, sort_keys=True))
break
return [
obj for obj in data if not obj["name"].startswith("celery")
and obj["name"] not in HYSDS_QUEUES
and obj["name"] != 'Recommended Queues' and obj["messages_ready"] > 0
]
def get_all_queues(rabbitmq_admin_url):
'''
List the queues available for job-running
Note: does not return celery internal queues
@param rabbitmq_admin_url: RabbitMQ admin URL
@return: list of queues
'''
try:
data = get_requests_json_response(
os.path.join(rabbitmq_admin_url, "api/queues"))
# print(data)
except HTTPError as e:
if e.response.status_code == 401:
logger.error(
"Failed to authenticate to {}. Ensure credentials are set in .netrc."
.format(rabbitmq_admin_url))
raise
# '''
for obj in data:
if not obj["name"].startswith(
"celery") and obj["name"] not in HYSDS_QUEUES:
if obj["name"] == 'Recommended Queues':
continue
if obj["name"] == "factotum-job_worker-scihub_throttled":
print((obj["name"]))
print(obj)
print((json.dumps(obj, indent=2, sort_keys=True)))
break
# '''
return [
obj for obj in data if not obj["name"].startswith("celery")
and obj["name"] not in HYSDS_QUEUES
and obj["name"] != 'Recommended Queues' and obj["messages_ready"] > 0
]
def resolve_mozart_job(product,
rule,
hysdsio=None,
queue=None,
component=None):
"""
Resolve Mozart job JSON.
@param product - product result body
@param rule - rule specification body
@param hysdsio - (optional) hysds-io body
@param component - tosca/grq or mozart/figaro, retrieve hysds_io from ES index (hysds_ios-mozart vs hysds_ios-grq)
@param queue - (optional) job queue override
"""
logger.info("rule: %s" % json.dumps(rule, indent=2))
logger.info("hysdsio: %s" % json.dumps(hysdsio, indent=2))
logger.info("component: %s" % component)
logger.info("queue: %s" % queue)
queue = rule['queue'] if queue is None else queue # override queue
if rule['priority'] is not None: # ensure priority is int
rule["priority"] = int(rule['priority'])
# this is the common data for all jobs, and will be copied for each individual submission
if hysdsio is None and component is None:
message = "[ERROR] Must supply a hysds-io object or a ES-URL to request one"
logger.error(message)
raise RuntimeError(message)
elif hysdsio is None:
if component.lower() in ('tosca', 'grq'):
hysds_io_index = 'hysds_ios-grq'
else:
hysds_io_index = 'hysds_ios-mozart'
hysdsio = mozart_es.get_by_id(index=hysds_io_index,
id=rule["job_type"])
hysdsio = hysdsio['_source']
# initialize job JSON
job = {
"queue": queue,
"priority": rule["priority"],
"soft_time_limit": rule.get("soft_time_limit", None),
"time_limit": rule.get("time_limit", None),
"disk_usage": rule.get("disk_usage", None),
"type": hysdsio["job-specification"],
"tags": json.dumps([rule["rule_name"], hysdsio["id"]]),
"username": rule.get("username", get_username()),
"enable_dedup": rule.get('enable_dedup', True)
}
rule["name"] = rule["rule_name"]
# resolve parameters for job JSON
if not isinstance(product, dict):
logger.info("Products: %s" % product)
params = get_params_for_products_set(hysdsio,
json.loads(rule["kwargs"]), rule,
product)
else:
logger.info("Product: %s" % product)
params = get_params_for_submission(hysdsio, json.loads(rule["kwargs"]),
rule, product)
logger.info("params: %s" % json.dumps(params, indent=2))
job["params"] = json.dumps(params) # set params
return job