def reportKnown(protocol, server, includeSegments, verbose, gps_start_time, gps_end_time): """ Construct url and issue query to get the reported list of all known segments for all flags in the time window provided. From the API Doc: Get a JSON string resource containing the known segments for all flags between t1 and t2. Note that this returns exactly what /dq/IFO/FLAG/VERSION/known does, except for ALL flags over the query period instead of one flag. The clients must assume that they may get empty known lists for flags that are unknown between times t1 and t2. Parameters ---------- protocol : `string` Ex: 'https' server : `string` Ex: 'dqsegdb5.phy.syr.edu' includeSegments : `boolean` Ex: True verbose : `boolean` Ex: True gps_start_time: `int` Ex: 999999999 gps_end_time: `int` Ex: 999999999 """ if includeSegments: includeText = "" timeText = "?s=%d&e=%d" % (gps_start_time, gps_end_time) else: includeText = "?include=metadata" timeText = "&s=%d&e=%d" % (gps_start_time, gps_end_time) queryurl = protocol + "://" + server + "/report/known" + includeText + timeText if verbose: print queryurl result = urifunctions.getDataUrllib2(queryurl, timeout=1200) return result, queryurl
def dqsegdbCheckVersion(protocol,server,ifo,name,version): """ Checks for existence of a given version of a flag in the db. Returns true if version exists Parameters ---------- protocol : `string` Ex: 'https' server : `string` Ex: 'dqsegdb5.phy.syr.edu' ifo : `string` Ex: 'L1' name: `string` Ex: 'DMT-SCIENCE' version : `string` or `int` Ex: '1' """ ### Fix!!! This looks wrong: seems to check if the flag exists, not whether a version on the server matches what was passed to the function queryurl=urifunctions.constructVersionQueryURL(protocol,server,ifo,name) try: result=urifunctions.getDataUrllib2(queryurl) except HTTPError as e: if e.code==404: return False else: raise ult_json=json.loads(result) version_list=result_json['version'] if version in version_list: return True else: return False
def dqsegdbQueryTimeless(protocol, server, ifo, name, version, include_list_string): """ Issue query to server for ifo:name:version without start and end time Returns the python loaded JSON response converted into a dictionary and queryurl! Returns ---------- [dictionary,string(url)] Parameters ---------- protocol : `string` Ex: 'https' server : `string` Ex: 'dqsegdb5.phy.syr.edu' ifo : `string` Ex: 'L1' name: `string` Ex: 'DMT-SCIENCE' version : `string` or `int` Ex: '1' include_list_string : `string` Ex: "metadata,known,active" """ queryurl = urifunctions.constructSegmentQueryURL(protocol, server, ifo, name, version, include_list_string) result = urifunctions.getDataUrllib2(queryurl) result_json = json.loads(result) return result_json, queryurl
def get_known_flags(start, end, url='https://segments.ligo.org', ifo=None, badonly=None): """Return the list of all flags with known segments Parameters ---------- start : `int` the GPS start time of the query end : `int` the GPS end time of the query url : `str`, optional the FQDN of the target segment database ifo : `str`, optional the prefix for the IFO, if `None` all flags are returned Returns ------- flags : `list` of `str` a list of flag names (<ifo>:<name>:<version>) that are known by the database in the given [start, end) interval """ start = int(to_gps(start)) end = int(to_gps(end)) uri = '%s/report/known?s=%d&e=%d' % (url, start, end) out = decode_json(urifunctions.getDataUrllib2(uri)) def select_flag(f): if ifo is not None and f['ifo'] != ifo: return False if (badonly is not None and f['metadata']['active_indicates_ifo_badness'] != badonly): return False return True return sorted(['%s:%s:%d' % (f['ifo'], f['name'], f['version']) for f in out['results'] if select_flag(f)])
def dqsegdbQueryTimeless(protocol,server,ifo,name,version,include_list_string): """ Issue query to server for ifo:name:version without start and end time Returns the python loaded JSON response converted into a dictionary and queryurl! Returns ---------- [dictionary,string(url)] Parameters ---------- protocol : `string` Ex: 'https' server : `string` Ex: 'dqsegdb5.phy.syr.edu' ifo : `string` Ex: 'L1' name: `string` Ex: 'DMT-SCIENCE' version : `string` or `int` Ex: '1' include_list_string : `string` Ex: "metadata,known,active" """ queryurl=urifunctions.constructSegmentQueryURL(protocol,server,ifo,name,version,include_list_string) result=urifunctions.getDataUrllib2(queryurl) result_json=json.loads(result) return result_json,queryurl
def dqsegdbQueryTimes(protocol,server,ifo,name,version,include_list_string,startTime,endTime): """ Issue query to server for ifo:name:version with start and end time Returns the python loaded JSON response! Parameters ---------- protocol : `string` Ex: 'https' server : `string` Ex: 'dqsegdb5.phy.syr.edu' ifo : `string` Ex: 'L1' name: `string` Ex: 'DMT-SCIENCE' version : `string` or `int` Ex: '1' include_list_string : `string` Ex: "metadata,known,active" startTime : `int` Ex: 999999999 endTime : `int` Ex: 999999999 """ queryurl=urifunctions.constructSegmentQueryURLTimeWindow(protocol,server,ifo,name,version,include_list_string,startTime,endTime) result=urifunctions.getDataUrllib2(queryurl) result_json=json.loads(result) return result_json,queryurl
def reportKnown(protocol,server,includeSegments,verbose,gps_start_time,gps_end_time): """ Construct url and issue query to get the reported list of all known segments for all flags in the time window provided. From the API Doc: Get a JSON string resource containing the known segments for all flags between t1 and t2. Note that this returns exactly what /dq/IFO/FLAG/VERSION/known does, except for ALL flags over the query period instead of one flag. The clients must assume that they may get empty known lists for flags that are unknown between times t1 and t2. Parameters ---------- protocol : `string` Ex: 'https' server : `string` Ex: 'dqsegdb5.phy.syr.edu' includeSegments : `boolean` Ex: True verbose : `boolean` Ex: True gps_start_time: `int` Ex: 999999999 gps_end_time: `int` Ex: 999999999 """ if includeSegments: includeText="" timeText="?s=%d&e=%d"%(gps_start_time,gps_end_time) else: includeText="?include=metadata" timeText="&s=%d&e=%d"%(gps_start_time,gps_end_time) queryurl=protocol+"://"+server+"/report/known"+includeText+timeText if verbose: print queryurl result=urifunctions.getDataUrllib2(queryurl,timeout=1200) return result,queryurl
def dqsegdbCheckVersion(protocol, server, ifo, name, version): """ Checks for existence of a given version of a flag in the db. Returns true if version exists Parameters ---------- protocol : `string` Ex: 'https' server : `string` Ex: 'dqsegdb5.phy.syr.edu' ifo : `string` Ex: 'L1' name: `string` Ex: 'DMT-SCIENCE' version : `string` or `int` Ex: '1' """ ### Fix!!! This looks wrong: seems to check if the flag exists, not whether a version on the server matches what was passed to the function queryurl = urifunctions.constructVersionQueryURL(protocol, server, ifo, name) try: result = urifunctions.getDataUrllib2(queryurl) except HTTPError as e: if e.code == 404: return False else: raise ult_json = json.loads(result) version_list = result_json['version'] if version in version_list: return True else: return False
def dqsegdbQueryTimes(protocol, server, ifo, name, version, include_list_string, startTime, endTime): """ Issue query to server for ifo:name:version with start and end time Returns the python loaded JSON response! Parameters ---------- protocol : `string` Ex: 'https' server : `string` Ex: 'dqsegdb5.phy.syr.edu' ifo : `string` Ex: 'L1' name: `string` Ex: 'DMT-SCIENCE' version : `string` or `int` Ex: '1' include_list_string : `string` Ex: "metadata,known,active" startTime : `int` Ex: 999999999 endTime : `int` Ex: 999999999 """ queryurl = urifunctions.constructSegmentQueryURLTimeWindow( protocol, server, ifo, name, version, include_list_string, startTime, endTime) result = urifunctions.getDataUrllib2(queryurl) result_json = json.loads(result) return result_json, queryurl
def dqsegdbMaxVersion(protocol, server, ifo, name): """ Checks for existence of a flag in the db, returns maximum version if the flag exists exists, 0 if the flag does not exist. Parameters ---------- protocol : `string` Ex: 'https' server : `string` Ex: 'dqsegdb5.phy.syr.edu' ifo : `string` Ex: 'L1' name: `string` Ex: 'DMT-SCIENCE' """ queryurl = urifunctions.constructFlagQueryURL(protocol, server, ifo) try: result = urifunctions.getDataUrllib2(queryurl, warnings=False) except HTTPError as e: print "e.code: %s FIX!" % str(e.code) if int(e.code) == 404: return 0 else: # Print all the messages this time result = urifunctions.getDataUrllib2(queryurl, warnings=True) raise # Now parse result for max version: queryurl = urifunctions.constructVersionQueryURL(protocol, server, ifo, name) try: result = urifunctions.getDataUrllib2(queryurl, warnings=False) except HTTPError as e: if int(e.code) == 404: return 0 else: raise result_json = json.loads(result) version_list = result_json['version'] return max(version_list)
def dqsegdbMaxVersion(protocol,server,ifo,name): """ Checks for existence of a flag in the db, returns maximum version if the flag exists exists, 0 if the flag does not exist. Parameters ---------- protocol : `string` Ex: 'https' server : `string` Ex: 'dqsegdb5.phy.syr.edu' ifo : `string` Ex: 'L1' name: `string` Ex: 'DMT-SCIENCE' """ queryurl=urifunctions.constructFlagQueryURL(protocol,server,ifo) try: result=urifunctions.getDataUrllib2(queryurl,warnings=False) except HTTPError as e: print "e.code: %s FIX!" % str(e.code) if int(e.code)==404: return 0 else: # Print all the messages this time result=urifunctions.getDataUrllib2(queryurl,warnings=True) raise # Now parse result for max version: queryurl=urifunctions.constructVersionQueryURL(protocol,server,ifo,name) try: result=urifunctions.getDataUrllib2(queryurl) except HTTPError as e: if int(e.code)==404: return 0 else: raise result_json=json.loads(result) version_list=result_json['version'] return max(version_list)
def reportFlags(protocol, server, verbose): """ Construct url and issue query to get the reported list of all flags provided by dqsegdb. From the API Doc: Get a JSON formatted string resource describing all the flags in the database. This provides an optimization by returning all flag names and all associated versions in a single call. Parameters ---------- protocol : `string` Ex: 'https' server : `string` Ex: 'dqsegdb5.phy.syr.edu' """ queryurl = protocol + "://" + server + "/report/flags" if verbose: print queryurl result = urifunctions.getDataUrllib2(queryurl) return result
def queryAPIVersion(protocol, server, verbose): """ Construct url and issue query to get the reported list of all IFOs provided by dqsegd and the API version of the server. Parameters ---------- protocol : `string` Ex: 'https' server : `string` Ex: 'dqsegdb5.phy.syr.edu' """ queryurl = protocol + "://" + server + "/dq" if verbose: print queryurl result = urifunctions.getDataUrllib2(queryurl) dictResult = json.loads(result) apiVersion = str(dictResult['query_information']['api_version']) return apiVersion
def reportFlags(protocol,server,verbose): """ Construct url and issue query to get the reported list of all flags provided by dqsegdb. From the API Doc: Get a JSON formatted string resource describing all the flags in the database. This provides an optimization by returning all flag names and all associated versions in a single call. Parameters ---------- protocol : `string` Ex: 'https' server : `string` Ex: 'dqsegdb5.phy.syr.edu' """ queryurl=protocol+"://"+server+"/report/flags" if verbose: print queryurl result=urifunctions.getDataUrllib2(queryurl) return result
def queryAPIVersion(protocol,server,verbose): """ Construct url and issue query to get the reported list of all IFOs provided by dqsegd and the API version of the server. Parameters ---------- protocol : `string` Ex: 'https' server : `string` Ex: 'dqsegdb5.phy.syr.edu' """ queryurl=protocol+"://"+server+"/dq" if verbose: print queryurl result=urifunctions.getDataUrllib2(queryurl) dictResult=json.loads(result) apiVersion=str(dictResult['query_information']['api_version']) return apiVersion
def dqsegdbQueryTimesCompatible(protocol, server, ifo, name, version, include_list_string, startTime, endTime): """ Issue query to server for ifo:name:version with start and end time This is the version that reproduces S6 style query results when the query is empty Returns the python loaded JSON response! Parameters ---------- protocol : `string` Ex: 'https' server : `string` Ex: 'dqsegdb5.phy.syr.edu' ifo : `string` Ex: 'L1' name: `string` Ex: 'DMT-SCIENCE' version : `string` or `int` Ex: '1' include_list_string : `string` Ex: "metadata,known,active" startTime : `int` Ex: 999999999 endTime : `int` Ex: 999999999 """ queryurl = urifunctions.constructSegmentQueryURLTimeWindow( protocol, server, ifo, name, version, include_list_string, startTime, endTime) try: result = urifunctions.getDataUrllib2(queryurl) result_json = json.loads(result) except HTTPError as e: if e.code == 404: # For S6 executable compatibility, we need to return something anyway to make ligolw_segments_from_cats and segment_query work properly, in this case, we'll return a faked up dictionary with empty lists for keys 'known' and 'active', which the calling functions will correctly interperet (because it's the equivalent of asking for a flag outside known time for the S6 calls) result_json = {"known": [], "active": []} else: raise return result_json, queryurl
def get_known_flags(start, end, url='https://segments.ligo.org', ifo=None, badonly=None): """Return the list of all flags with known segments Parameters ---------- start : `int` the GPS start time of the query end : `int` the GPS end time of the query url : `str`, optional the FQDN of the target segment database ifo : `str`, optional the prefix for the IFO, if `None` all flags are returned Returns ------- flags : `list` of `str` a list of flag names (<ifo>:<name>:<version>) that are known by the database in the given [start, end) interval """ start = int(to_gps(start)) end = int(to_gps(end)) uri = '%s/report/known?s=%d&e=%d' % (url, start, end) out = decode_json(urifunctions.getDataUrllib2(uri)) def select_flag(f): if ifo is not None and f['ifo'] != ifo: return False if (badonly is not None and f['metadata']['active_indicates_ifo_badness'] != badonly): return False return True return sorted([ '%s:%s:%d' % (f['ifo'], f['name'], f['version']) for f in out['results'] if select_flag(f) ])
def dqsegdbQueryTimesCompatible(protocol,server,ifo,name,version,include_list_string,startTime,endTime,warnings=True): """ Issue query to server for ifo:name:version with start and end time This is the version that reproduces S6 style query results when the query is empty Returns the python loaded JSON response! Parameters ---------- protocol : `string` Ex: 'https' server : `string` Ex: 'dqsegdb5.phy.syr.edu' ifo : `string` Ex: 'L1' name: `string` Ex: 'DMT-SCIENCE' version : `string` or `int` Ex: '1' include_list_string : `string` Ex: "metadata,known,active" startTime : `int` Ex: 999999999 endTime : `int` Ex: 999999999 warnings : `bool` show warnings for `HTTPError` (as well as raising exception), default: `True` """ queryurl=urifunctions.constructSegmentQueryURLTimeWindow(protocol,server,ifo,name,version,include_list_string,startTime,endTime) try: result=urifunctions.getDataUrllib2(queryurl, warnings=warnings) result_json=json.loads(result) except HTTPError as e: if e.code==404: # For S6 executable compatibility, we need to return something anyway to make ligolw_segments_from_cats and segment_query work properly, in this case, we'll return a faked up dictionary with empty lists for keys 'known' and 'active', which the calling functions will correctly interperet (because it's the equivalent of asking for a flag outside known time for the S6 calls) result_json={"known":[],"active":[]} else: raise return result_json,queryurl
def queryAPIVersion(protocol,server,verbose,warnings=True): """ Construct url and issue query to get the reported list of all IFOs provided by dqsegd and the API version of the server. Parameters ---------- protocol : `string` Ex: 'https' server : `string` Ex: 'dqsegdb5.phy.syr.edu' warnings : `bool` show warnings for `HTTPError` (as well as raising exception), default: `True` """ queryurl=protocol+"://"+server+"/dq" if verbose: print(queryurl) result=urifunctions.getDataUrllib2(queryurl, warnings=warnings) dictResult=json.loads(result) apiVersion=str(dictResult['query_information']['api_version']) return apiVersion
def dqsegdbCascadedQuery(protocol, server, ifo, name, include_list_string, startTime, endTime): """ Queries server for needed flag_versions to generate the result of a cascaded query (was called a versionless query in S6). Returns a python dictionary representing the calculated result "versionless" flag and also the python dictionaries (in a list) for the flag_versions necessary to construct the result. Parameters ---------- protocol : `string` Ex: 'https' server : `string` Ex: 'dqsegdb5.phy.syr.edu' ifo : `string` Ex: 'L1' name: `string` Ex: 'DMT-SCIENCE' version : `string` or `int` Ex: '1' include_list_string : `string` Ex: "metadata,known,active" startTime : `int` Ex: 999999999 endTime : `int` Ex: 999999999 """ #verbose=True verbose=False ## Construct url and issue query to determine highest version from list ## of versions versionQueryURL=urifunctions.constructVersionQueryURL(protocol,server,ifo,name) if verbose: print versionQueryURL try: versionResult=urifunctions.getDataUrllib2(versionQueryURL) except HTTPError as e: if e.code==404: import warnings warnings.warn("Provided IFO:FLAG: %s:%s not found in database, returning empty result" % (ifo,name)) jsonResults=[] else: # Parse the result # Results should be a JSON object like this: #{ # "meta": { # query_uri" : "uri", // contains the URI specified with the GET HTTP method # "query_time" : gpstime, // when the query was issued # // optional query parameters # "query_start" : t1, # "query_end" : t2 # }, # "resource_type" : ["resource_uri_1", "resource_uri_2"] #} versionData=json.loads(versionResult) #JSON is nice... :) ## Construct urls and issue queries for the multiple versions and dump the results to disk locally for careful provenance jsonResults=[] #urlList=versionData['resource_type'] version_list=versionData['version'] urlList=[versionQueryURL+'/'+str(version) for version in version_list] # sort list by decreasing version number and call each URL: sortedurlList=sorted(urlList, key=lambda url: url.split('/')[-1], reverse=True) for versioned_url in sortedurlList: # I am assuming I need to pull off the versions from the urls to use my existing library function. # Alternatively, we could make a new library function that starts from the end of the version and takes the include_list_string and start and end times as inputs version=versioned_url.split('/')[-1] queryurl=urifunctions.constructSegmentQueryURLTimeWindow(protocol,server,ifo,name,version,include_list_string,startTime,endTime) if verbose: print queryurl result=urifunctions.getDataUrllib2(queryurl) result_parsed=json.loads(result) jsonResults.append(result_parsed) # Fix!!! Improvement: Executive Decision: Should we force these intermediate results to hit disk? # For now, I say yes: # Fix!!! Improvement: Choose a better location for files to go automatically, so this can be run from other directories filename=queryurl.replace('/','_').split(':')[-1]+'.json' try: tmpfile=open(filename,'w') json.dump(result_parsed,tmpfile) tmpfile.close() except: print "Couldn't save partial results to disk.... continuing anyway." ## Construct output segments lists from multiple JSON objects # The jsonResults are in order of decreasing versions, # thanks to the sorting above # This generates a results_flag object for dumping to JSON with the # total_known_list across all versions, cascaded # and we have the total active list across all versions, cascaded # so we're done the math! : result_flag,affected_results=clientutils.calculate_versionless_result(jsonResults,startTime,endTime,ifo_input=ifo) if verbose: print "active segments:", result_flag['active'] print "known segments:", result_flag['known'] ### Old before JSON spec change: ### Need to build the client_meta part of the JSON results #meta={} #meta['program_name']=os.path.basename(__file__) #meta['options']=options #meta['start_time']=startTime #meta['end_time']=endTime #meta['query_uris_called']=sortedurlList ## Note: Using ligolw/utils/process.py method of determining time: #meta['query_time']=query_start #meta['query_start']=query_start #meta['query_end']=_UTCToGPS(time.gmtime()) ### Now that we have the meta and the flags, which include the result, we need to build up the larger JSON #json_result={} #json_result['client_meta']=meta #json_result['flags']=[] #for flag in jsonResults: # json_result['flags'].append(flag) #json_result['flags'].append(result_flag) # Now we need to return the reduced results and the intermediate JSON # responses from the versioned queries # Note: The result_flag will not have query_metadata return result_flag,jsonResults,affected_results
def dqsegdbCascadedQuery(protocol, server, ifo, name, include_list_string, startTime, endTime, debug=False): """ Queries server for needed flag_versions to generate the result of a cascaded query (was called a versionless query in S6). Returns a python dictionary representing the calculated result "versionless" flag and also the python dictionaries (in a list) for the flag_versions necessary to construct the result. Parameters ---------- protocol : `string` Ex: 'https' server : `string` Ex: 'dqsegdb5.phy.syr.edu' ifo : `string` Ex: 'L1' name: `string` Ex: 'DMT-SCIENCE' version : `string` or `int` Ex: '1' include_list_string : `string` Ex: "metadata,known,active" startTime : `int` Ex: 999999999 endTime : `int` Ex: 999999999 debug : `bool` Ex: False """ if debug == True: verbose = True else: verbose = False ## Construct url and issue query to determine highest version from list ## of versions versionQueryURL = urifunctions.constructVersionQueryURL( protocol, server, ifo, name) if verbose: print versionQueryURL try: versionResult = urifunctions.getDataUrllib2(versionQueryURL) except HTTPError as e: if e.code == 404: import warnings warnings.warn( "Provided IFO:FLAG: %s:%s not found in database, returning empty result" % (ifo, name)) jsonResults = [] else: # Parse the result # Results should be a JSON object like this: #{ # "meta": { # query_uri" : "uri", // contains the URI specified with the GET HTTP method # "query_time" : gpstime, // when the query was issued # // optional query parameters # "query_start" : t1, # "query_end" : t2 # }, # "resource_type" : ["resource_uri_1", "resource_uri_2"] #} versionData = json.loads(versionResult) #JSON is nice... :) ## Construct urls and issue queries for the multiple versions and dump the results to disk locally for careful provenance jsonResults = [] #urlList=versionData['resource_type'] version_list = versionData['version'] urlList = [ versionQueryURL + '/' + str(version) for version in version_list ] # sort list by decreasing version number and call each URL: sortedurlList = sorted(urlList, key=lambda url: url.split('/')[-1], reverse=True) for versioned_url in sortedurlList: # I am assuming I need to pull off the versions from the urls to use my existing library function. # Alternatively, we could make a new library function that starts from the end of the version and takes the include_list_string and start and end times as inputs version = versioned_url.split('/')[-1] queryurl = urifunctions.constructSegmentQueryURLTimeWindow( protocol, server, ifo, name, version, include_list_string, startTime, endTime) if verbose: print queryurl result = urifunctions.getDataUrllib2(queryurl) result_parsed = json.loads(result) jsonResults.append(result_parsed) # Fix!!! Improvement: Executive Decision: Should we force these intermediate results to hit disk? # For now, I say yes: # Fix!!! Improvement: Choose a better location for files to go automatically, so this can be run from other directories filename = queryurl.replace('/', '_').split(':')[-1] + '.json' if debug: try: tmpfile = open(filename, 'w') json.dump(result_parsed, tmpfile) tmpfile.close() print "Stored partial result for individual version to disk as %s" % filename except: print "Couldn't save partial results to disk.... continuing anyway." ## Construct output segments lists from multiple JSON objects # The jsonResults are in order of decreasing versions, # thanks to the sorting above # This generates a results_flag object for dumping to JSON with the # total_known_list across all versions, cascaded # and we have the total active list across all versions, cascaded # so we're done the math! : result_flag, affected_results = clientutils.calculate_versionless_result( jsonResults, startTime, endTime, ifo_input=ifo) if verbose: print "active segments:", result_flag['active'] print "known segments:", result_flag['known'] ### Old before JSON spec change: ### Need to build the client_meta part of the JSON results #meta={} #meta['program_name']=os.path.basename(__file__) #meta['options']=options #meta['start_time']=startTime #meta['end_time']=endTime #meta['query_uris_called']=sortedurlList ## Note: Using ligolw/utils/process.py method of determining time: #meta['query_time']=query_start #meta['query_start']=query_start #meta['query_end']=_UTCToGPS(time.gmtime()) ### Now that we have the meta and the flags, which include the result, we need to build up the larger JSON #json_result={} #json_result['client_meta']=meta #json_result['flags']=[] #for flag in jsonResults: # json_result['flags'].append(flag) #json_result['flags'].append(result_flag) # Now we need to return the reduced results and the intermediate JSON # responses from the versioned queries # Note: The result_flag will not have query_metadata return result_flag, jsonResults, affected_results