def create_table_errors(conn): """ create the errors table if it does not exist """ # NB: the comments are kept by sqlite3 and can be accessed with command ".schema" query_create = """ CREATE TABLE IF NOT EXISTS errors ( id TEXT NOT NULL /* resource ivoid */ ,url TEXT NOT NULL /* access URL */ ,date TEXT /* validation date */ ,type TEXT /* error type ex: "error" "warning" "fatal" */ ,num INT /* error number (there can be several errors with the same name) */ ,name TEXT /* error name ex: "4.3.2" */ ,msg TEXT /* error msg - verbose - for taplint */ ,section TEXT /* error section - for taplint */ ) """ cur_create = db.execute_db(conn, query_create, [], True) #conn.commit() # no need, db.execute_db does it query_create_index = """ CREATE UNIQUE INDEX IF NOT EXISTS pk ON errors (id,url,date,type,num,name) """ cur_create_index = db.execute_db(conn, query_create_index, [], True)
def create_table_services(conn): """ create the services table if it does not exist """ # NB: the comments are kept by sqlite3 and can be accessed with command ".schema" query_create = """ CREATE TABLE IF NOT EXISTS services ( id TEXT NOT NULL /* resource ivoid */ ,url TEXT NOT NULL /* access URL */ ,title TEXT /* resource title */ ,short_name TEXT /* resource short name */ ,date_insert TEXT /* date row was inserted by query-rr.py = date when the service was first seen */ ,date_update TEXT /* date row was updated by query-rr.py = last date when the service was seen */ ,vor_status TEXT /* status of the resource, coming from the RR it will always be 'active' */ ,vor_created TEXT /* date of creation of the resource */ ,vor_updated TEXT /* date of update of the resource */ ,contact_name TEXT /* 1st contact name for curation */ ,contact_email TEXT /* 1st email for curation */ ,provenance TEXT /* ivoid of registry where resource comes from */ ,date TEXT /* validation date */ ,standard_id TEXT /* standard_id of the capability */ ,xsi_type TEXT /* type of the interface - should always be vs:paramhttp per the search() funtion implementation */ ,spec TEXT /* specification of the service in natural language as in spec_from_standardid */ ,specv TEXT /* version of the specification for the service */ ,params TEXT /* parameters for the validator */ ,val_mode TEXT /* validator mode ("not_run","normal","batch") */ ,result_vot TEXT /* validator result for VOTable ("yes","no","") */ ,result_spec TEXT /* validator result for spec ("yes","no","") */ ,nb_warn INT /* validator nb of warnings */ ,nb_err INT /* validator nb of errors */ ,nb_fail INT /* validator nb of failures (for TAP validator taplint) */ ,nb_fatal INT /* validator nb o f fatal errors */ ,days_same INT DEFAULT 0 /* nb of days the result have been the same */ ) """ cur_create = db.execute_db(conn, query_create, [], True) #conn.commit() # no need, db.execute_db does it query_create_index = """ CREATE UNIQUE INDEX IF NOT EXISTS pk ON services (id,url) """ cur_create_index = db.execute_db(conn, query_create_index, [], True)
def upsert_error(conn, ivoid, url, date, type, num, name, msg="", section=""): ''' insert or update an error in the errors table :param conn: sqlite3 DB connection object :param ivoid: service id :param url: service url :param date: date :param type: error type (warning, error, fatal, failure) :param num: sequence number of error :param name: name of error :param msg: error msg :param section: section for error ''' logging.info("Upserting error") # Check if the error already exists in the DB # NB: this query can take a long time if no index (id,url) has been defined on table errors ! # => create index errors_idx on errors(id,url); query = """ SELECT count(*) FROM errors WHERE id = ? and url = ? and date = ? and type = ? and num = ? and name = ? and msg = ? and section = ? """ cur = db.execute_db(conn, query, (ivoid, url, date, type, num, name, msg, section)) nb_errors = cur.fetchone()[0] #logging.info("nb_errors = %d",nb_errors) if (nb_errors == 0): # if the error does not already exist # insert the error query = """ INSERT INTO errors (id,url,date,type,num,name,msg,section) VALUES (?, ?, ?, ?, ?, ?, ?, ?) """ cur = db.execute_db(conn, query, (ivoid, url, date, type, num, name, msg, section)) conn.commit() # because INSERT return
def main(argv): ''' main program :param argv: parameters ''' program_version = "1.3" #global logger # Read program arguments service_type = None # no default db_file = None # no default log_file = None # no default try: opts, args = getopt.getopt(argv, "h", ["type=", "db=", "log="]) except getopt.GetoptError as err: print str(err) usage() sys.exit(2) for o, a in opts: if o in ("-h"): usage() sys.exit(0) elif o in ("--type"): service_type = a elif o in ("--db"): db_file = a elif o in ("--log"): log_file = a else: assert False, "unhandled option" if (service_type == None): print('ERROR: No service_type') usage() exit(2) if (db_file == None): print('ERROR: No db_file') usage() exit(2) # Setup logging # Try to use coloredlogs but does not work well # Create a logger object. #logger = logging.getLogger('val.py') # By default the install() function installs a handler on the root logger, # this means that log messages from your code and log messages from the # libraries that you use will all show up on the terminal. #coloredlogs.install(level='DEBUG',fmt='%(asctime)s %(filename)s %(levelname)s %(lineno)d %(processName)s %(funcName)s: %(message)s') # Try to use colorlog - does not work #colorlog.basicConfig(format='%(asctime)s %(filename)s %(levelname)s %(lineno)d %(processName)s %(funcName)s: %(message)s', level=logging.DEBUG) #colorlog.info("Starting argv=%s",argv) logging.basicConfig( format= '%(asctime)s %(filename)s %(levelname)s %(lineno)d %(processName)s %(funcName)s: %(message)s', level=logging.DEBUG, filename=log_file) # Add colors - from https://stackoverflow.com/questions/384076/how-can-i-color-python-logging-output logging.addLevelName( logging.WARNING, "\033[1;31m%s\033[1;0m" % logging.getLevelName(logging.WARNING)) logging.addLevelName( logging.ERROR, "\033[1;41m%s\033[1;0m" % logging.getLevelName(logging.ERROR)) logging.info("This is query-rr.py version %s. argv=%s", program_version, argv) # Try to open the DB file, conn = db.open_db(db_file) # create the tables if they don't exist create_table_services(conn) create_table_errors(conn) # URL of RR to use url_rr = "http://voparis-rr.obspm.fr/tap" # Look for services with type service_type in the RR logging.debug("Looking for services with type=%s in RR=%s", service_type, url_rr) services = search(baseurl=url_rr, servicetype=service_types[service_type]) # Get nb of services found nb_services = len(services) logging.debug("Nb of services found=%d", nb_services) s = 0 for service in services: s = s + 1 logging.info("Processing service %d/%d ivoid=%s url=%s standardid=%s", s, nb_services, service.ivoid, service.access_url, service.standard_id) if (False): # display all service attributes for a in service: logging.debug("%s: %s", a, service[a]) # get today's date in format 2017-12-21 date_today = datetime.datetime.today() date_today_s = date_today.strftime('%Y-%m-%d') # Check if the service exists in the DB query = """ SELECT count(*) FROM services WHERE id=? and url=? """ cur = db.execute_db(conn, query, (service.ivoid, service.access_url), True) nb_such_service = cur.fetchone()[0] #logging.debug("Nb of such service found in the DB=%d",nb_such_service) if (nb_such_service == 0 ): # service does not exist in the DB yet => insert it logging.debug("No such service found in the DB, inserting it") query_insert = """ INSERT INTO services (id,url,date_insert) VALUES (?, ?, ?) """ cur_insert = db.execute_db( conn, query_insert, (service.ivoid, service.access_url, date_today_s)) #conn.commit() # because INSERT # no need, db.execute_db does it # Update the service with data from registry if (True): # Extract the part before the # in the standardid and the fragment (#something at the end of the standard id) standardid = service.standard_id # in lowercase in the RR index_pound = standardid.find( '#') # index of char '#' in standardid #logging.debug("index_pound=%d",index_pound) fragment = None if (index_pound != -1): # pound found, copy only the part before the pound standardid_substring = standardid[:index_pound] fragment = standardid[index_pound:] else: # no pound found, copy the entire string standardid_substring = standardid logging.debug("Found standardid_substring=%s", standardid_substring) logging.debug("Found fragment=%s", fragment) spec = spec_from_standardid[standardid_substring] logging.debug("Found spec=%s", spec) # Try to extract version of std used by service # NB (per Markus after discussion in Santiago 2017-10) # => the correct way is to check the end of standardid then if no version found, std_version logging.debug("Determining spec version") specv = None if ( fragment != None ): # if a fragment was found, extract the spec from the fragment logging.debug("standardid's fragment found") specv_from_fragment = fragment[ 7:] # extract "2.0" from "#query-2.0" => skip "#query-" logging.debug("Found specv_from_fragment=%s", specv_from_fragment) if ( specv_from_fragment.replace(".", "", 1).isdigit() ): # https://stackoverflow.com/questions/4138202/using-isdigit-for-floats logging.debug( "specv_from_fragment can convert to float => using it") specv = specv_from_fragment if (specv == None): # version not found in fragment logging.debug( "No standardid's fragment found or version not found in fragment, checking std_version=%s", service['std_version']) if (service['std_version'] != ""): logging.debug("std_version not empty, using it") specv = service['std_version'] else: # if not found, use default version for this standard logging.debug( "std_version empty, using default specv from standardid" ) specv = default_specv_from_standardid[standardid_substring] logging.info("Found specv=%s", specv) # default params for validation are extracted from the array val.validatorParams (in vla.py) if ((spec == "Simple Image Access") and (specv == "2.0")): # for SIAv2 we need this case params = validatorParams[spec + " " + specv] else: params = validatorParams[spec] # if some attributes are void string "" then set them to N/A role_name = service['role_name'] if (role_name == ""): role_name = "N/A" email = service['email'] if (email == ""): email = "N/A" query_update = """ UPDATE services SET date_update = ? ,vor_created = ? ,vor_updated = ? ,vor_status = ? ,provenance = ? ,standard_id = ? ,title = ? ,short_name = ? ,contact_name = ? ,contact_email = ? ,xsi_type = ? ,spec = ? ,specv = ? ,params = ? WHERE id=? AND url=? """ logging.info("Updating table services for service") cur_update = db.execute_db( conn, query_update, [ date_today_s, service[ 'created'] # must be accessed like this and not by property because property not exposed in class RegistryResource , service['updated'] # idem etc. , 'active' # we assume that if a service was found in the RR, it is active. Reason: # See http://ivoa.net/documents/RegTAP/20171206/WD-RegTAP-1.1-20171206.html # "The status attribute of vr:Resource is considered an implementation detail of the XML serialization and is not kept here. # Neither inactive nor deleted records may be kept in the resource table. Since all other tables in the relational registry should keep # a foreign key on the ivoid column, this implies that only metadata on active records is being kept in the relational registry. # In other words, users can expect a resource to exist and work if they find it in a relational registry" , service[ 'harvested_from'] # the provenance registry's ivoid , standardid # the standardid , service.res_title, service.short_name, role_name # contact name , email # contact email , service[ 'intf_type'] # per the search() function's implementation this should always be 'vs:paramhttp' , spec, specv, params, service.ivoid, service.access_url ]) #conn.commit() # because UPDATE # no need, db.execute_db does it # if True # at the end, close the DB connection logging.info("Done. Closing connection") conn.close()
def main(argv): ''' main program :param argv: parameters ''' program_version = "1.9" #global logger # Read program arguments db_file = None # no default nb_ps = 1 # nb of processes to use timeout = 20 # timeout for validation of individual service, in secs log_file = None # no default try: opts, args = getopt.getopt(argv, "h", ["db=", "ps=", "timeout=", "log="]) except getopt.GetoptError as err: print str(err) usage() sys.exit(2) for o, a in opts: if o in ("-h"): usage() sys.exit(0) elif o in ("--db"): db_file = a elif o in ("--ps"): nb_ps = int(a) elif o in ("--timeout"): timeout = int(a) if (timeout < 1): logging.error("timeout must be greater or equal to 1") sys.exit(1) elif o in ("--log"): log_file = a else: assert False, "unhandled option" if (db_file == None): print('ERROR: No db_file') usage() exit(2) # Setup logging # Try to use coloredlogs but does not work well # Create a logger object. #logger = logging.getLogger('val.py') # By default the install() function installs a handler on the root logger, # this means that log messages from your code and log messages from the # libraries that you use will all show up on the terminal. #coloredlogs.install(level='DEBUG',fmt='%(asctime)s %(filename)s %(levelname)s %(lineno)d %(processName)s %(funcName)s: %(message)s') # Try to use colorlog - does not work #colorlog.basicConfig(format='%(asctime)s %(filename)s %(levelname)s %(lineno)d %(processName)s %(funcName)s: %(message)s', level=logging.DEBUG) #colorlog.info("Starting argv=%s",argv) logging.basicConfig( format= '%(asctime)s %(filename)s %(levelname)s %(lineno)d %(processName)s %(funcName)s: %(message)s', level=logging.DEBUG, filename=log_file) # Add colors - from https://stackoverflow.com/questions/384076/how-can-i-color-python-logging-output logging.addLevelName( logging.WARNING, "\033[1;31m%s\033[1;0m" % logging.getLevelName(logging.WARNING)) logging.addLevelName( logging.ERROR, "\033[1;41m%s\033[1;0m" % logging.getLevelName(logging.ERROR)) logging.info("This is val.py version %s. argv=%s", program_version, argv) # Configuration depending on hostname hostname = socket.gethostname() logging.info("Running on %s", hostname) conn = db.open_db(db_file) # Prepare where clause for extracting suitable services #min_update_date = datetime.date.today() - datetime.timedelta(2) # today - 2 days min_update_date = datetime.date.today( ) # 2018-04-18 changed to today => assume query-rr-*.py was run just before and the same day... min_update_date_s = min_update_date.strftime('%Y-%m-%d') logging.info("min_update_date_s is %s", min_update_date_s) where = "date_update >='" + min_update_date_s + "'" #where = where + " and id like '%vopdc%'" # debug: 3 services #where = where + " and url like '%.au%'" # debug: 7 services #where = where + " and id like '%irsa%'" # debug: 366 services #where = where + " and id='ivo://vopdc.obspm/imcce/skybot'" # debug: 1 service with 1 error #where = where + " and id='ivo://CDS.VizieR/J/AJ/127/1227'" # debug: 1 service with 2 errors #where = where + " and id='ivo://vopdc.obspm/imcce/dynastvo/epn' and url='http://voparis-tap-planeto.obspm.fr/__system__/tap/run/tap'" # debug: TAP query #where = where +" and url='http://camelot.star.le.ac.uk:8080/dsa-catalog/SubmitCone?DSACAT=ledas&DSATAB=a2rtraw&'" # debug: service for which the VO-Paris validator times out # Count nb of suitable services in the db query = "SELECT count(*) FROM services WHERE " + where cur = db.execute_db(conn, query, [], True) nb_services = cur.fetchone()[0] logging.info("nb_services = %d", nb_services) if (nb_services != 0): if (nb_ps <= nb_services): # Get suitable services order = "id asc, url asc" # query : get those columns only query = "SELECT id,url,spec,specv,params FROM services WHERE " + where + " ORDER BY " + order cur = db.execute_db(conn, query, [], True) services = cur.fetchall() conn.close() #sys.exit(0) #print(services[0]["id"]) # debug #print(services[0]) #sys.exit(0) #print(services[0]) # Slice the services array into nb_ps arrays of same size sservices = numpy.array_split(services, nb_ps) for i in range(len(sservices)): logging.info("Slice # %d len=%d first ivoid=%s", i, len(sservices[i]), (sservices[i][0][0])) #sys.exit(0) jobs = [] for i in range(nb_ps): p = multiprocessing.Process(target=validate_services, args=(sservices[i], timeout, db_file)) jobs.append(p) p.start() #p.join() else: logging.error("nb_ps>nb_services. Try with a lower nb_ps") sys.exit(1) else: logging.error("No suitable service found. Aborting.") sys.exit(10)
def validate_service(conn, service, timeout): ''' validate one service: calls validator and update the sqlite3 DB :param conn: sqlite3 connection object :param service: array containing attributes of the service per SQL query done :param timeout: timeout for calling the validator ''' # extract the service attributes, order is defined by SQL request done in main ivoid = service[0] url = service[1] spec = service[2] specv = service[3] params = service[4] logging.info("Processing service ivoid=%s url=%s spec=%s specv=%s", ivoid, url, spec, specv) # For TAP services, check if the url has already been validated today because # there are many TAP services which have a different IVOID but the same URL # get today's date in format 2017-05-18 date_today = datetime.datetime.today() date_today_s = date_today.strftime('%Y-%m-%d') # Check how many services with the same URL have been validated today logging.info("Checking how many services with same URL validated today") query = """ SELECT COUNT(*) FROM services WHERE url = ? AND date = ? """ cur = db.execute_db(conn, query, (url, date_today_s)) nb_services = cur.fetchone()[0] logging.debug("nb_services with same URL validated today=%d", nb_services) if ((spec == "Table Access Protocol") and (nb_services > 0)): # At least one TAP service with same URL validated today # Get the first one logging.info("Getting first such service") query = """ SELECT val_mode, result_vot, result_spec, nb_warn, nb_err, nb_fatal, nb_fail, days_same, id FROM services WHERE url = ? AND date = ? """ cur = db.execute_db(conn, query, (url, date_today_s)) service = cur.fetchone() prev_val_mode = service[0] prev_result_vot = service[1] prev_result_spec = service[2] prev_nb_warn = service[3] prev_nb_err = service[4] prev_nb_fatal = service[5] prev_nb_fail = service[6] prev_days_same = service[7] prev_ivoid = service[8] # Copy the previous results to the current service logging.info("Updating current service with service found") query = """ UPDATE services SET date=?, val_mode=?, result_vot=?, result_spec=?, nb_warn=?, nb_err=?, nb_fatal=?, nb_fail=?, days_same=? WHERE id=? AND url=? """ cur = db.execute_db( conn, query, (date_today_s, prev_val_mode, prev_result_vot, prev_result_spec, prev_nb_warn, prev_nb_err, prev_nb_fatal, prev_nb_fail, prev_days_same, ivoid, url)) conn.commit() # because of UPDATE if ( False ): # Copy the errors too - disabled 2018-04-09 to reduce time - because of all TAP VizieR services / webapp updated to take this into account logging.info("Copying errors") query = """ SELECT type,num,name,msg,section FROM errors WHERE id=? AND url=? AND date=? """ cur = db.execute_db(conn, query, (prev_ivoid, url, date_today_s)) errors = cur.fetchall() for error in errors: type = error[0] num = error[1] name = error[2] msg = error[3] section = error[4] upsert_error(conn, ivoid, url, date_today_s, type, num, name, msg, section) return else: # Run validator # Construct validator url: validator base URL vurl = validatorBaseURLs[spec] # add validator params #vurl += validatorParams[spec] vurl += params # add spec and spec version vurl += "&" + urllib.urlencode({"spec": spec + " " + specv}) # add service URL vurl += "&" + urllib.urlencode({"serviceURL": url}) # Set TAP validator timeout tap_timeout = timeout - 1 # try to make sure TAP validator timeouts before our socket timeout if (tap_timeout <= 0): tap_timeout = 1 if (spec == "Table Access Protocol" ): # TAP validator also needs the timeout vurl += "&" + urllib.urlencode({"timeout": tap_timeout}) vurl += "&" + urllib.urlencode({ "maxtable": "1" }) # added 2018-04-05 to reduce time taken by TAP validation #vurl="https://www.test123456.com/" # debug - test timeout logging.info("Calling validator URL: %s (timeout is %d secs)", vurl, timeout) # set the timeout for call to urllib2.urlopen - not necessary urllib2.urlopen has a timeout parameter # socket.setdefaulttimeout(float(timeout)) request = urllib2.Request(vurl) try: # NB: The optional timeout parameter specifies a timeout in seconds for blocking operations like the connection attempt # It seems that the timeout is useless here for timeouting the TAP validator : https://www.daniweb.com/programming/software-development/threads/182555/how-to-set-timeout-for-reading-from-urls-in-urllib # => the timeout there it is only for opening url. It wont give exception while reading. response = urllib2.urlopen(request, timeout=timeout) http_status = response.getcode() logging.debug("HTTP status: %d", http_status) except Exception as e: logging.error( "EXCEPTION %s while calling URL=%s. Using default results (-1).", e, vurl) # These are the results to use in case of timeout: results = { "result_vot": "", "result_spec": "", "nb_warn": -1, "nb_err": -1, "nb_fatal": -1, "nb_fail": -1, "warnings": [], "errors": [], "fatals": [], "fails": [] } # Update the service with the results update_service(conn, ivoid, url, results) except socket.timeout as e: logging.error( "EXCEPTION: socket timeout: %s while calling URL=%s. Using defaults results (-2).", e, vurl) # These are the results to use in case of *socket* timeout: results = { "result_vot": "", "result_spec": "", "nb_warn": -2, "nb_err": -2, "nb_fatal": -2, "nb_fail": -2, "warnings": [], "errors": [], "fatals": [], "fails": [] } # Update the service with the results update_service(conn, ivoid, url, results) else: # if no exception if (http_status == 200): try: # set a timer here to have a timeout during reading of data from URL, but tapvalidator.php does not stop correctly #t = Timer(timeout, response.close) #t.start() logging.debug("Reading data.") data = response.read( ) # get response in either XML or JSON format #t.cancel() logging.debug( "Reading data done." ) # we end up here in case tapvalidator.php times out. The unfinished JSON will make throw an exception in parse_tap_validator except Exception as e: # try to catch timeout exception... not sure it works logging.error( "EXCEPTION %s (timeout?) while reading data from URL=%s. Using default results (-1).", e, vurl) # These are the results to use in case of timeout: results = { "result_vot": "", "result_spec": "", "nb_warn": -1, "nb_err": -1, "nb_fatal": -1, "nb_fail": -1, "warnings": [], "errors": [], "fatals": [], "fails": [] } else: #logging.info(data) results = parse_validator(spec, data) # Update the service with the results update_service(conn, ivoid, url, results) else: # http_status!=200 logging.error("HTTP status is not 200 but %d. Giving up.", http_status) return
def update_service(conn, ivoid, url, results): ''' update the service in the sqlite3 DB :param conn: sqlite3 connection object :param ivoid: id of the service :param url: url of the service :param results: results object returned by parse_*_validator ''' logging.info("Updating sqlite3 db for service ivoid=%s url=%s", ivoid, url) # : %s",data) #print results new_result_vot = results["result_vot"] new_result_spec = results["result_spec"] new_nb_warn = results["nb_warn"] new_nb_err = results["nb_err"] new_nb_fatal = results["nb_fatal"] new_nb_fail = results["nb_fail"] # get today's date in format 2017-05-18 date_today = datetime.datetime.today() date_today_s = date_today.strftime('%Y-%m-%d') # Get previous results query = """ SELECT val_mode, result_vot, result_spec, nb_warn, nb_err, nb_fatal, nb_fail, date, days_same FROM services WHERE id=? AND url=? """ logging.info("Getting previous results") cur = db.execute_db(conn, query, (ivoid, url)) service = cur.fetchone() prev_val_mode = service[0] prev_result_vot = service[1] prev_result_vot = service[2] prev_nb_warn = service[3] prev_nb_err = service[4] prev_nb_fatal = service[5] prev_nb_fail = service[6] prev_date = service[7] prev_days_same = service[8] # Compute days same # datetime when service was last updated if (prev_date == None): # if service was never validated date_prev = date_today # then nb_days will be 0 else: date_prev = datetime.datetime.strptime(prev_date, '%Y-%m-%d') interval = date_today - date_prev nb_days = interval.days #logging.info(interval) logging.debug("The service was last updated %d days ago", nb_days) new_days_same = prev_days_same + nb_days logging.debug("Old days_same: %d New days_same: %d", prev_days_same, new_days_same) # create query to update the service query = """ UPDATE services SET date = ? ,val_mode='normal' ,result_vot=? ,result_spec=? ,nb_warn=? ,nb_err=? ,nb_fatal=? ,nb_fail=? ,days_same=? WHERE id=? AND url=? """ logging.info("Updating table services for service") cur = db.execute_db(conn, query, [ date_today_s, new_result_vot, new_result_spec, new_nb_warn, new_nb_err, new_nb_fatal, new_nb_fail, new_days_same, ivoid, url ]) conn.commit() # get today's date in format 2017-05-31 #date_today_s=datetime.date.today().strftime('%Y-%m-%d') # insert the warnings found num = 0 for warning in results["warnings"]: num = num + 1 logging.info("Upserting warning num=%d", num) upsert_error(conn, ivoid, url, date_today_s, "warning", num, warning["name"], warning["msg"], warning["section"]) # insert the errors found num = 0 for error in results["errors"]: num = num + 1 logging.info("Upserting error num=%d", num) upsert_error(conn, ivoid, url, date_today_s, "error", num, error["name"], error["msg"], error["section"]) # insert the fatals found num = 0 for fatal in results["fatals"]: num = num + 1 logging.info("Upserting fatal num=%d", num) upsert_error(conn, ivoid, url, date_today_s, "fatal", num, fatal["name"], fatal["msg"], fatal["section"]) # insert the failures found num = 0 for fail in results["fails"]: num = num + 1 logging.info("Upserting failure num=%d", num) upsert_error(conn, ivoid, url, date_today_s, "failure", num, fail["name"], fail["msg"], fail["section"]) return