def fetch(endpoint, startTime, endTime, bbox, depth_min, depth_max, platforms=None, pageCallback=None):
results = []
startIndex = 0
mainBoundsConstrainer = geo.BoundsConstrainer(north=-90, south=90, west=180, east=-180)
# First isn't parellel so we can get the ttl results, forced items per page, etc...
pageResults, totalResults, pageStartIndex, itemsPerPageR, boundsConstrainer = __doQuery(endpoint, startTime,
endTime, bbox,
depth_min, depth_max,
endpoint["itemsPerPage"],
startIndex, platforms,
pageCallback)
results = results + pageResults
mainBoundsConstrainer.testOtherConstrainer(boundsConstrainer)
pool = ThreadPool(processes=endpoint["fetchThreads"])
mpResults = [pool.apply_async(__doQuery, args=(
endpoint, startTime, endTime, bbox, depth_min, depth_max, itemsPerPageR, x, platforms, pageCallback)) for x in
range(len(pageResults), totalResults, itemsPerPageR)]
pool.close()
pool.join()
'''
If pageCallback was supplied, we assume this call to be asynchronous. Otherwise combine all the results data and return it.
'''
if pageCallback is None:
mpResults = [p.get() for p in mpResults]
for mpResult in mpResults:
results = results + mpResult[0]
mainBoundsConstrainer.testOtherConstrainer(mpResult[4])
return results, mainBoundsConstrainer
def __doQuery(endpoint, startTime, endTime, bbox, depth_min=None, depth_max=None, itemsPerPage=10, startIndex=0, platforms=None,
pageCallback=None):
params = {"startTime": startTime, "endTime": endTime, "bbox": bbox, "itemsPerPage": itemsPerPage,
"startIndex": startIndex, "stats": "true"}
if depth_min is not None:
params['minDepth'] = depth_min
if depth_max is not None:
params['maxDepth'] = depth_max
if platforms is not None:
params["platform"] = platforms.split(",")
resultsRaw = __fetchJson(endpoint["url"], params)
boundsConstrainer = geo.BoundsConstrainer(north=-90, south=90, west=180, east=-180)
if resultsRaw["totalResults"] == 0 or len(resultsRaw["results"]) == 0: # Double-sanity check
return [], resultsRaw["totalResults"], startIndex, itemsPerPage, boundsConstrainer
try:
results = []
for resultdict in resultsRaw["results"]:
result = __resultRawToUsable(resultdict)
result["source"] = endpoint["name"]
boundsConstrainer.testCoords(north=result["y"], south=result["y"], west=result["x"], east=result["x"])
results.append(result)
if "stats_fields" in resultsRaw and len(resultsRaw["results"]) == 0:
stats = resultsRaw["stats_fields"]
if "lat" in stats and "lon" in stats:
boundsConstrainer.testCoords(north=stats['lat']['max'], south=stats['lat']['min'],
west=stats['lon']['min'], east=stats['lon']['max'])
if pageCallback is not None:
pageCallback(results)
'''
If pageCallback was supplied, we assume this call to be asynchronous. Otherwise combine all the results data and return it.
'''
if pageCallback is None:
return results, int(resultsRaw["totalResults"]), int(resultsRaw["startIndex"]), int(
resultsRaw["itemsPerPage"]), boundsConstrainer
else:
return [], int(resultsRaw["totalResults"]), int(resultsRaw["startIndex"]), int(
resultsRaw["itemsPerPage"]), boundsConstrainer
except:
print "Invalid or missing JSON in response."
traceback.print_exc()
raise NexusProcessingException(reason="Invalid or missing JSON in response.")
def fetchData(self, endpoints, startTime, endTime, bbox, depth_min, depth_max, platforms):
boundsConstrainer = geo.BoundsConstrainer(asString=bbox)
threads = []
for endpoint in endpoints:
thread = workerthread.WorkerThread(datafetch.fetchData,
params=(endpoint, startTime, endTime, bbox, depth_min, depth_max))
threads.append(thread)
workerthread.wait(threads, startFirst=True, poll=0.01)
data2 = []
for thread in threads:
data, bounds = thread.results
data2 += data
boundsConstrainer.testOtherConstrainer(bounds)
return data2, boundsConstrainer