def dqsegdbQueryTimes(protocol, server, ifo, name, version,
include_list_string, startTime, endTime):
"""
Issue query to server for ifo:name:version with start and end time
Returns the python loaded JSON response!
Parameters
----------
protocol : `string`
Ex: 'https'
server : `string`
Ex: 'dqsegdb5.phy.syr.edu'
ifo : `string`
Ex: 'L1'
name: `string`
Ex: 'DMT-SCIENCE'
version : `string` or `int`
Ex: '1'
include_list_string : `string`
Ex: "metadata,known,active"
startTime : `int`
Ex: 999999999
endTime : `int`
Ex: 999999999
"""
queryurl = urifunctions.constructSegmentQueryURLTimeWindow(
protocol, server, ifo, name, version, include_list_string, startTime,
endTime)
result = urifunctions.getDataUrllib2(queryurl)
result_json = json.loads(result)
return result_json, queryurl
def dqsegdbQueryTimes(protocol,server,ifo,name,version,include_list_string,startTime,endTime):
"""
Issue query to server for ifo:name:version with start and end time
Returns the python loaded JSON response!
Parameters
----------
protocol : `string`
Ex: 'https'
server : `string`
Ex: 'dqsegdb5.phy.syr.edu'
ifo : `string`
Ex: 'L1'
name: `string`
Ex: 'DMT-SCIENCE'
version : `string` or `int`
Ex: '1'
include_list_string : `string`
Ex: "metadata,known,active"
startTime : `int`
Ex: 999999999
endTime : `int`
Ex: 999999999
"""
queryurl=urifunctions.constructSegmentQueryURLTimeWindow(protocol,server,ifo,name,version,include_list_string,startTime,endTime)
result=urifunctions.getDataUrllib2(queryurl)
result_json=json.loads(result)
return result_json,queryurl
def dqsegdbQueryTimesCompatible(protocol, server, ifo, name, version,
include_list_string, startTime, endTime):
"""
Issue query to server for ifo:name:version with start and end time
This is the version that reproduces S6 style query results when the query is empty
Returns the python loaded JSON response!
Parameters
----------
protocol : `string`
Ex: 'https'
server : `string`
Ex: 'dqsegdb5.phy.syr.edu'
ifo : `string`
Ex: 'L1'
name: `string`
Ex: 'DMT-SCIENCE'
version : `string` or `int`
Ex: '1'
include_list_string : `string`
Ex: "metadata,known,active"
startTime : `int`
Ex: 999999999
endTime : `int`
Ex: 999999999
"""
queryurl = urifunctions.constructSegmentQueryURLTimeWindow(
protocol, server, ifo, name, version, include_list_string, startTime,
endTime)
try:
result = urifunctions.getDataUrllib2(queryurl)
result_json = json.loads(result)
except HTTPError as e:
if e.code == 404:
# For S6 executable compatibility, we need to return something anyway to make ligolw_segments_from_cats and segment_query work properly, in this case, we'll return a faked up dictionary with empty lists for keys 'known' and 'active', which the calling functions will correctly interperet (because it's the equivalent of asking for a flag outside known time for the S6 calls)
result_json = {"known": [], "active": []}
else:
raise
return result_json, queryurl
def dqsegdbQueryTimesCompatible(protocol,server,ifo,name,version,include_list_string,startTime,endTime,warnings=True):
"""
Issue query to server for ifo:name:version with start and end time
This is the version that reproduces S6 style query results when the query is empty
Returns the python loaded JSON response!
Parameters
----------
protocol : `string`
Ex: 'https'
server : `string`
Ex: 'dqsegdb5.phy.syr.edu'
ifo : `string`
Ex: 'L1'
name: `string`
Ex: 'DMT-SCIENCE'
version : `string` or `int`
Ex: '1'
include_list_string : `string`
Ex: "metadata,known,active"
startTime : `int`
Ex: 999999999
endTime : `int`
Ex: 999999999
warnings : `bool`
show warnings for `HTTPError` (as well as raising exception),
default: `True`
"""
queryurl=urifunctions.constructSegmentQueryURLTimeWindow(protocol,server,ifo,name,version,include_list_string,startTime,endTime)
try:
result=urifunctions.getDataUrllib2(queryurl, warnings=warnings)
result_json=json.loads(result)
except HTTPError as e:
if e.code==404:
# For S6 executable compatibility, we need to return something anyway to make ligolw_segments_from_cats and segment_query work properly, in this case, we'll return a faked up dictionary with empty lists for keys 'known' and 'active', which the calling functions will correctly interperet (because it's the equivalent of asking for a flag outside known time for the S6 calls)
result_json={"known":[],"active":[]}
else:
raise
return result_json,queryurl
def dqsegdbCascadedQuery(protocol,
server,
ifo,
name,
include_list_string,
startTime,
endTime,
debug=False):
"""
Queries server for needed flag_versions to generate the result of a
cascaded query (was called a versionless query in S6).
Returns a python dictionary representing the calculated result "versionless"
flag and also the python dictionaries (in a list) for the flag_versions
necessary to construct the result.
Parameters
----------
protocol : `string`
Ex: 'https'
server : `string`
Ex: 'dqsegdb5.phy.syr.edu'
ifo : `string`
Ex: 'L1'
name: `string`
Ex: 'DMT-SCIENCE'
version : `string` or `int`
Ex: '1'
include_list_string : `string`
Ex: "metadata,known,active"
startTime : `int`
Ex: 999999999
endTime : `int`
Ex: 999999999
debug : `bool`
Ex: False
"""
if debug == True:
verbose = True
else:
verbose = False
## Construct url and issue query to determine highest version from list
## of versions
versionQueryURL = urifunctions.constructVersionQueryURL(
protocol, server, ifo, name)
if verbose:
print versionQueryURL
try:
versionResult = urifunctions.getDataUrllib2(versionQueryURL)
except HTTPError as e:
if e.code == 404:
import warnings
warnings.warn(
"Provided IFO:FLAG: %s:%s not found in database, returning empty result"
% (ifo, name))
jsonResults = []
else:
# Parse the result
# Results should be a JSON object like this:
#{
# "meta": {
# query_uri" : "uri", // contains the URI specified with the GET HTTP method
# "query_time" : gpstime, // when the query was issued
# // optional query parameters
# "query_start" : t1,
# "query_end" : t2
# },
# "resource_type" : ["resource_uri_1", "resource_uri_2"]
#}
versionData = json.loads(versionResult) #JSON is nice... :)
## Construct urls and issue queries for the multiple versions and dump the results to disk locally for careful provenance
jsonResults = []
#urlList=versionData['resource_type']
version_list = versionData['version']
urlList = [
versionQueryURL + '/' + str(version) for version in version_list
]
# sort list by decreasing version number and call each URL:
sortedurlList = sorted(urlList,
key=lambda url: url.split('/')[-1],
reverse=True)
for versioned_url in sortedurlList:
# I am assuming I need to pull off the versions from the urls to use my existing library function.
# Alternatively, we could make a new library function that starts from the end of the version and takes the include_list_string and start and end times as inputs
version = versioned_url.split('/')[-1]
queryurl = urifunctions.constructSegmentQueryURLTimeWindow(
protocol, server, ifo, name, version, include_list_string,
startTime, endTime)
if verbose:
print queryurl
result = urifunctions.getDataUrllib2(queryurl)
result_parsed = json.loads(result)
jsonResults.append(result_parsed)
# Fix!!! Improvement: Executive Decision: Should we force these intermediate results to hit disk?
# For now, I say yes:
# Fix!!! Improvement: Choose a better location for files to go automatically, so this can be run from other directories
filename = queryurl.replace('/', '_').split(':')[-1] + '.json'
if debug:
try:
tmpfile = open(filename, 'w')
json.dump(result_parsed, tmpfile)
tmpfile.close()
print "Stored partial result for individual version to disk as %s" % filename
except:
print "Couldn't save partial results to disk.... continuing anyway."
## Construct output segments lists from multiple JSON objects
# The jsonResults are in order of decreasing versions,
# thanks to the sorting above
# This generates a results_flag object for dumping to JSON with the
# total_known_list across all versions, cascaded
# and we have the total active list across all versions, cascaded
# so we're done the math! :
result_flag, affected_results = clientutils.calculate_versionless_result(
jsonResults, startTime, endTime, ifo_input=ifo)
if verbose:
print "active segments:", result_flag['active']
print "known segments:", result_flag['known']
### Old before JSON spec change:
### Need to build the client_meta part of the JSON results
#meta={}
#meta['program_name']=os.path.basename(__file__)
#meta['options']=options
#meta['start_time']=startTime
#meta['end_time']=endTime
#meta['query_uris_called']=sortedurlList
## Note: Using ligolw/utils/process.py method of determining time:
#meta['query_time']=query_start
#meta['query_start']=query_start
#meta['query_end']=_UTCToGPS(time.gmtime())
### Now that we have the meta and the flags, which include the result, we need to build up the larger JSON
#json_result={}
#json_result['client_meta']=meta
#json_result['flags']=[]
#for flag in jsonResults:
# json_result['flags'].append(flag)
#json_result['flags'].append(result_flag)
# Now we need to return the reduced results and the intermediate JSON
# responses from the versioned queries
# Note: The result_flag will not have query_metadata
return result_flag, jsonResults, affected_results
def dqsegdbCascadedQuery(protocol, server, ifo, name, include_list_string, startTime, endTime):
"""
Queries server for needed flag_versions to generate the result of a
cascaded query (was called a versionless query in S6).
Returns a python dictionary representing the calculated result "versionless"
flag and also the python dictionaries (in a list) for the flag_versions
necessary to construct the result.
Parameters
----------
protocol : `string`
Ex: 'https'
server : `string`
Ex: 'dqsegdb5.phy.syr.edu'
ifo : `string`
Ex: 'L1'
name: `string`
Ex: 'DMT-SCIENCE'
version : `string` or `int`
Ex: '1'
include_list_string : `string`
Ex: "metadata,known,active"
startTime : `int`
Ex: 999999999
endTime : `int`
Ex: 999999999
"""
#verbose=True
verbose=False
## Construct url and issue query to determine highest version from list
## of versions
versionQueryURL=urifunctions.constructVersionQueryURL(protocol,server,ifo,name)
if verbose:
print versionQueryURL
try:
versionResult=urifunctions.getDataUrllib2(versionQueryURL)
except HTTPError as e:
if e.code==404:
import warnings
warnings.warn("Provided IFO:FLAG: %s:%s not found in database, returning empty result" % (ifo,name))
jsonResults=[]
else:
# Parse the result
# Results should be a JSON object like this:
#{
# "meta": {
# query_uri" : "uri", // contains the URI specified with the GET HTTP method
# "query_time" : gpstime, // when the query was issued
# // optional query parameters
# "query_start" : t1,
# "query_end" : t2
# },
# "resource_type" : ["resource_uri_1", "resource_uri_2"]
#}
versionData=json.loads(versionResult) #JSON is nice... :)
## Construct urls and issue queries for the multiple versions and dump the results to disk locally for careful provenance
jsonResults=[]
#urlList=versionData['resource_type']
version_list=versionData['version']
urlList=[versionQueryURL+'/'+str(version) for version in version_list]
# sort list by decreasing version number and call each URL:
sortedurlList=sorted(urlList, key=lambda url: url.split('/')[-1], reverse=True)
for versioned_url in sortedurlList:
# I am assuming I need to pull off the versions from the urls to use my existing library function.
# Alternatively, we could make a new library function that starts from the end of the version and takes the include_list_string and start and end times as inputs
version=versioned_url.split('/')[-1]
queryurl=urifunctions.constructSegmentQueryURLTimeWindow(protocol,server,ifo,name,version,include_list_string,startTime,endTime)
if verbose:
print queryurl
result=urifunctions.getDataUrllib2(queryurl)
result_parsed=json.loads(result)
jsonResults.append(result_parsed)
# Fix!!! Improvement: Executive Decision: Should we force these intermediate results to hit disk?
# For now, I say yes:
# Fix!!! Improvement: Choose a better location for files to go automatically, so this can be run from other directories
filename=queryurl.replace('/','_').split(':')[-1]+'.json'
try:
tmpfile=open(filename,'w')
json.dump(result_parsed,tmpfile)
tmpfile.close()
except:
print "Couldn't save partial results to disk.... continuing anyway."
## Construct output segments lists from multiple JSON objects
# The jsonResults are in order of decreasing versions,
# thanks to the sorting above
# This generates a results_flag object for dumping to JSON with the
# total_known_list across all versions, cascaded
# and we have the total active list across all versions, cascaded
# so we're done the math! :
result_flag,affected_results=clientutils.calculate_versionless_result(jsonResults,startTime,endTime,ifo_input=ifo)
if verbose:
print "active segments:", result_flag['active']
print "known segments:", result_flag['known']
### Old before JSON spec change:
### Need to build the client_meta part of the JSON results
#meta={}
#meta['program_name']=os.path.basename(__file__)
#meta['options']=options
#meta['start_time']=startTime
#meta['end_time']=endTime
#meta['query_uris_called']=sortedurlList
## Note: Using ligolw/utils/process.py method of determining time:
#meta['query_time']=query_start
#meta['query_start']=query_start
#meta['query_end']=_UTCToGPS(time.gmtime())
### Now that we have the meta and the flags, which include the result, we need to build up the larger JSON
#json_result={}
#json_result['client_meta']=meta
#json_result['flags']=[]
#for flag in jsonResults:
# json_result['flags'].append(flag)
#json_result['flags'].append(result_flag)
# Now we need to return the reduced results and the intermediate JSON
# responses from the versioned queries
# Note: The result_flag will not have query_metadata
return result_flag,jsonResults,affected_results