def __init__(self, name):
"""Constructor"""
super(Zone, self).__init__(name)
self._details = config("zone", name)
self.tiles = []
self.collision_tiles = []
self.active_sprites = []
self.sprites = None
self._raw_map = readMap(name).split("\n")
self._generateTileTable()
def deduplicate(self, title):
'''
Prevents the re-adding of sling downloads
@params title - name of granule to check if downloading
'''
key = config()['dedup_redis_key']
global POOL
if POOL is None:
POOL = ConnectionPool.from_url(REDIS_URL)
r = StrictRedis(connection_pool=POOL)
return r.sadd(key, title) == 0
def __init__(self, name):
self.name = name
self._details = config("spell", name)
self.image = pygame.image.load(spellPath(name))
self.image = self.image.convert()
self.rect = self.image.get_rect()
def __init__(self, name):
self.name = name
self._details = config("aether", name)
self.image = pygame.image.load(aetherPath(name))
self.image = self.image.convert_alpha()
self.rect = self.image.get_rect()
def parse_params(self, aoi, input_qtype, dns_list_str):
'''
parses the parameters from the aoi, determines proper start/end times and returns a dict object containing params
'''
params = {} #dict to be returned holding parsed query parameters
cfg = config() #load settings.json
#determine if end_time has passed, therefore query is expired
if "temporal" in aoi.keys() and "endquery" in aoi["temporal"].keys():
if determine_if_expired_query(aoi["temporal"]["endquery"]):
return None
if "endquery" in aoi.keys() and determine_if_expired_query(
aoi["endquery"]):
return None
#map the values outside the endpoint specific settings
#default to looking back window_size days from settings config file, unless window_size is in the aoi
if "window_size" in aoi.keys():
window = aoi["window_size"]
else:
window = cfg["window-size-days"]
end_time = datetime.datetime.utcnow().strftime(
"%Y-%m-%dT%H:%M:%S") #default end time is now
#default start time is now - window size
start_time = (datetime.datetime.utcnow() - datetime.timedelta(
days=float(window))).strftime("%Y-%m-%dT%H:%M:%S")
#but if the endpoint aoi is not in redis, default to retrieving everything
last_query_time = self.loadStamp(self.stampKeyname(aoi, input_qtype))
if last_query_time is None:
start_time = "1970-01-01T00:00:00"
else:
start_time = last_query_time
#determine if there is an event time
event_time = None
#check aoi
if "event" in aoi.keys() and "time" in aoi["event"].keys():
event_time = parse_datetime(aoi["event"]["time"])
#check aoi["metadata"]
if "metadata" in aoi.keys() and "event" in aoi[
"metadata"] and "time" in aoi["metadata"]["event"].keys():
event_time = parse_datetime(aoi["metadata"]["event"]["time"])
#now determine start/end times
#see if starttime and endtime are part of the aoi metadata
met_start, met_end = parse_start_end(aoi, event_time=event_time)
if met_start != None: start_time = met_start
if met_end != None: end_time = met_end
#set priority, the most nested being the one passed
priority = 0 #defaults to 0
if "priority" in aoi.keys():
priority = aoi["priority"]
if "metadata" in aoi.keys() and "priority" in aoi["metadata"].keys():
priority = aoi["metadata"]["priority"]
#for each query within the aoi
if not ("metadata" in aoi.keys()
and "query" in aoi["metadata"].keys()):
return None #exit if no query fields exist
for qtype in aoi["metadata"]["query"].keys():
#determine endpoint, skip if it does not match the qquery input endpoint
if qtype != input_qtype:
continue
#set the query to run
query = aoi["metadata"]["query"][qtype]
#determine priority
if "priority" in query.keys():
priority = query["priority"]
#parse event time for each query, if given
query_start, query_end = parse_start_end(query,
event_time=event_time)
if query_start != None:
#make sure the query start time is within the start/end window (if specified in aoi metadata)
if met_start != None and convert_to_dt(
query_start) < convert_to_dt(met_start):
#the start time in the query is before the metadata window
start_time = met_start
elif met_end != None and convert_to_dt(
query_start) > convert_to_dt(met_end):
#start of the query window is after the given metadata window
return None
else:
#start time is between the metdata start/end window (or the window doesn't exist)
start_time = query_start
if query_end != None:
#make sure the end time is within the start/end metadata window
if met_end != None and convert_to_dt(
query_end) < convert_to_dt(met_end) and convert_to_dt(
start_time) < convert_to_dt(query_end):
#the end time in the query is in the metadata window
end_time = query_end
elif met_end != None and convert_to_dt(
query_end) > convert_to_dt(met_end):
#the end time in the query is after the metadata window
end_time = met_end
else:
end_time = query_end
#determine products to query for within each endpoint
if "products" not in aoi["metadata"]["query"][qtype]: continue
products = aoi["metadata"]["query"][qtype]["products"]
tags = []
if "tag" in aoi["metadata"].keys():
tags.append(aoi["metadata"]["tag"])
if qtype in aoi["metadata"].keys(
) and "tag" in aoi["metadata"][qtype].keys():
for tag in tags:
tags.append(aoi["metadata"][qtype]["tag"])
# parses dns comma seperated string to array
dns_list = [x.strip() for x in dns_list_str.split(',')]
#fill parameters
params["starttime"] = start_time
params["endtime"] = end_time
params["priority"] = priority
params["products"] = products
params["tag"] = tags
params["dns_list"] = dns_list
return params
return None
def submit_sling_job(self,
aoi,
query_params,
qtype,
queue_grp,
title,
link,
rtag=None,
pds_queue=None):
#Query for all products, and return a list of (Title,URL)
yr, mo, dy = self.getDataDateFromTitle(title) #date
filename = title + "." + self.getFileType()
if not pds_queue:
# build payload items for job submission
tags = query_params["tag"]
md5 = hashlib.md5("{0}.{1}\n".format(
title, self.getFileType())).hexdigest()
cfg = config() # load settings.json
repo_url = "%s/%s/%s/%s/%s/%s.%s" % (
cfg["repository-base"], md5[0:8], md5[8:16], md5[16:24],
md5[24:32], title, self.getFileType())
location = {}
location['type'] = 'polygon'
location['aoi'] = aoi['id']
location['coordinates'] = aoi['location']['coordinates']
prod_met = {}
prod_met['source'] = qtype
prod_met['dataset_type'] = title[0:3]
prod_met['spatial_extent'] = location
prod_met['tag'] = tags
queue = "factotum-job_worker-%s_throttled" % (
qtype + str(queue_grp)) # job submission queue
job_header = 'job-sling:'
dedup_key = DEDUP_KEY
params = [{
"name": "download_url",
"from": "value",
"value": link,
}, {
"name": "repo_url",
"from": "value",
"value": repo_url,
}, {
"name": "prod_name",
"from": "value",
"value": title,
}, {
"name": "file_type",
"from": "value",
"value": self.getFileType(),
}, {
"name": "prod_date",
"from": "value",
"value": "{}".format("%s-%s-%s" % (yr, mo, dy)),
}, {
"name": "prod_met",
"from": "value",
"value": prod_met,
}, {
"name": "options",
"from": "value",
"value": "--force_extract"
}]
else:
# queue = "opds-%s-job_worker-small" % (qtype)
queue = pds_queue # job submission queue, no queue group for autoscalers
job_header = 'job-sling-extract-opds:'
dedup_key = DEDUP_KEY_PDS
params = [{
"name": "download_url",
"from": "value",
"value": link,
}, {
"name": "prod_name",
"from": "value",
"value": "%s-pds" % title,
}, {
"name": "file",
"from": "value",
"value": filename,
}, {
"name": "prod_date",
"from": "value",
"value": "{}".format("%s-%s-%s" % (yr, mo, dy)),
}]
#set sling job spec release/branch
if rtag is None:
try:
with open('_context.json') as json_data:
context = json.load(json_data)
job_spec = job_header + context['job_specification'][
'job-version']
except:
print('Failed on loading context.json')
else:
job_spec = job_header + rtag
rtime = datetime.datetime.utcnow()
job_name = "%s-%s-%s-%s-%s" % (job_spec, queue, title,
rtime.strftime("%d_%b_%Y_%H:%M:%S"),
aoi['id'])
job_name = job_name.lstrip('job-')
priority = query_params["priority"]
#Setup input arguments here
rule = {
"rule_name": job_spec,
"queue": queue,
"priority": priority,
"kwargs": '{}'
}
#check for dedup, if clear, submit job
if not self.deduplicate(filename, dedup_key):
submit_mozart_job({},
rule,
hysdsio={
"id": "internal-temporary-wiring",
"params": params,
"job-specification": job_spec
},
job_name=job_name)
else:
location = " to OpenDataset" if pds_queue else "to own bucket"
reason = "in OpenDataset" if pds_queue else "in OpenDataset or own bucket"
print(
"Will not submit sling job {0} to {1}, already processed {2}".
format(title, location, reason))
import hashlib
import os
import traceback
import re
import requests
import backoff
from redis import ConnectionPool, StrictRedis
from utilities import config, get_aois, get_redis_endpoint
from hysds_commons.job_utils import submit_mozart_job
from hysds.celery import app
REDIS_URL = get_redis_endpoint()
POOL = None
DEDUP_KEY = config()['dedup_redis_key']
DEDUP_KEY_PDS = config()['dedup_redis_key_pds']
class AbstractQuery(object):
'''
A class holding all generic query functions
'''
@classmethod
def getQueryHandler(clazz, qtype):
'''
Get a handler for this particular query type by searching registered subclasses
@param qtype - type of query
'''
try:
mod = importlib.import_module(qtype)
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("qtype", help="query endpoint, e.g. (asf|scihub|unavco)")
parser.add_argument("--dns_list", help="dns list for qtype to use from .netrc, comma separated", required=True)
parser.add_argument("--tag", help="PGE docker image tag (release, version, " +
"or branch) to propagate",
default="master", required=False)
parser.add_argument("--sling_tag", help="sling PGE docker image tag (release, version, " +
"or branch) to propagate", required=False)
args = parser.parse_args()
query_endpoint = args.qtype
dns_list = args.dns_list
qquery_rtag = args.tag
sling_rtag = qquery_rtag if args.sling_tag is None else args.sling_tag
cfg = config()
aoi = get_aois(cfg) #retrieves a list of aois that match the grq values in settings.cfg
aois = sorted(aoi,key=lambda aoi: 0 if (not "metadata" in aoi or not "priority" in aoi["metadata"]) else aoi["metadata"]["priority"],reverse=True)
for region in aois:
#for each aoi
user_tags = region.get('metadata', {}).get('user_tags', [])
if "inactive" in user_tags:
#if the region is inactive, skip
print("AOI {0} marked as inactive. Skipping".format(region["id"]))
continue
#skip regions without types_types map
if "query" not in region["metadata"].keys():
continue
#set query priority
def submit_sling_job(self,
aoi,
query_params,
qtype,
queue_grp,
title,
link,
rtag=None):
#Query for all products, and return a list of (Title,URL)
cfg = config() #load settings.json
priority = query_params["priority"]
products = query_params["products"]
tags = query_params["tag"]
#build payload items for job submission
yr, mo, dy = self.getDataDateFromTitle(title) #date
md5 = hashlib.md5("{0}.{1}\n".format(title,
self.getFileType())).hexdigest()
repo_url = "%s/%s/%s/%s/%s/%s.%s" % (cfg["repository-base"], md5[0:8],
md5[8:16], md5[16:24], md5[24:32],
title, self.getFileType())
location = {}
location['type'] = 'polygon'
location['aoi'] = aoi['id']
location['coordinates'] = aoi['location']['coordinates']
prod_met = {}
prod_met['source'] = qtype
prod_met['dataset_type'] = title[0:3]
prod_met['spatial_extent'] = location
prod_met['tag'] = tags
#required params for job submission
if hasattr(self, 'getOauthUrl'):
#sling via oauth
oauth_url = self.getOauthUrl()
job_type = "job:spyddder-sling-oauth_%s" % qtype
job_name = "spyddder-sling-oauth_%s-%s-%s.%s" % (
qtype, aoi['id'], title, self.getFileType())
else:
#normal sling
job_type = "job:spyddder-sling_%s" % qtype
job_name = "spyddder-sling_%s-%s-%s.%s" % (qtype, aoi['id'], title,
self.getFileType())
oauth_url = None
queue = "factotum-job_worker-%s_throttled" % (qtype + str(queue_grp)
) # job submission queue
#set sling job spec release/branch
if rtag is None:
try:
with open('_context.json') as json_data:
context = json.load(json_data)
job_spec = 'job-sling:' + context['job_specification'][
'job-version']
except:
print('Failed on loading context.json')
else:
job_spec = 'job-sling:' + rtag
rtime = datetime.datetime.utcnow()
job_name = "%s-%s-%s-%s-%s" % (job_spec, queue, title,
rtime.strftime("%d_%b_%Y_%H:%M:%S"),
aoi['id'])
job_name = job_name.lstrip('job-')
#Setup input arguments here
rule = {
"rule_name": job_spec,
"queue": queue,
"priority": priority,
"kwargs": '{}'
}
params = [{
"name": "download_url",
"from": "value",
"value": link,
}, {
"name": "repo_url",
"from": "value",
"value": repo_url,
}, {
"name": "prod_name",
"from": "value",
"value": title,
}, {
"name": "file_type",
"from": "value",
"value": self.getFileType(),
}, {
"name": "prod_date",
"from": "value",
"value": "{}".format("%s-%s-%s" % (yr, mo, dy)),
}, {
"name": "prod_met",
"from": "value",
"value": prod_met,
}, {
"name": "options",
"from": "value",
"value": "--force_extract"
}]
#check for dedup, if clear, submit job
if not self.deduplicate(title + "." + self.getFileType()):
submit_mozart_job({},
rule,
hysdsio={
"id": "internal-temporary-wiring",
"params": params,
"job-specification": job_spec
},
job_name=job_name)
else:
print(
"Will not submit sling job for {0}, already processed".format(
title))
