def create_table_errors(conn):
"""
create the errors table if it does not exist
"""
# NB: the comments are kept by sqlite3 and can be accessed with command ".schema"
query_create = """
CREATE TABLE IF NOT EXISTS errors (
id TEXT NOT NULL /* resource ivoid */
,url TEXT NOT NULL /* access URL */
,date TEXT /* validation date */
,type TEXT /* error type ex: "error" "warning" "fatal" */
,num INT /* error number (there can be several errors with the same name) */
,name TEXT /* error name ex: "4.3.2" */
,msg TEXT /* error msg - verbose - for taplint */
,section TEXT /* error section - for taplint */
)
"""
cur_create = db.execute_db(conn, query_create, [], True)
#conn.commit() # no need, db.execute_db does it
query_create_index = """
CREATE UNIQUE INDEX IF NOT EXISTS pk ON errors (id,url,date,type,num,name)
"""
cur_create_index = db.execute_db(conn, query_create_index, [], True)
def create_table_services(conn):
"""
create the services table if it does not exist
"""
# NB: the comments are kept by sqlite3 and can be accessed with command ".schema"
query_create = """
CREATE TABLE IF NOT EXISTS services (
id TEXT NOT NULL /* resource ivoid */
,url TEXT NOT NULL /* access URL */
,title TEXT /* resource title */
,short_name TEXT /* resource short name */
,date_insert TEXT /* date row was inserted by query-rr.py = date when the service was first seen */
,date_update TEXT /* date row was updated by query-rr.py = last date when the service was seen */
,vor_status TEXT /* status of the resource, coming from the RR it will always be 'active' */
,vor_created TEXT /* date of creation of the resource */
,vor_updated TEXT /* date of update of the resource */
,contact_name TEXT /* 1st contact name for curation */
,contact_email TEXT /* 1st email for curation */
,provenance TEXT /* ivoid of registry where resource comes from */
,date TEXT /* validation date */
,standard_id TEXT /* standard_id of the capability */
,xsi_type TEXT /* type of the interface - should always be vs:paramhttp per the search() funtion implementation */
,spec TEXT /* specification of the service in natural language as in spec_from_standardid */
,specv TEXT /* version of the specification for the service */
,params TEXT /* parameters for the validator */
,val_mode TEXT /* validator mode ("not_run","normal","batch") */
,result_vot TEXT /* validator result for VOTable ("yes","no","") */
,result_spec TEXT /* validator result for spec ("yes","no","") */
,nb_warn INT /* validator nb of warnings */
,nb_err INT /* validator nb of errors */
,nb_fail INT /* validator nb of failures (for TAP validator taplint) */
,nb_fatal INT /* validator nb o f fatal errors */
,days_same INT DEFAULT 0 /* nb of days the result have been the same */
)
"""
cur_create = db.execute_db(conn, query_create, [], True)
#conn.commit() # no need, db.execute_db does it
query_create_index = """
CREATE UNIQUE INDEX IF NOT EXISTS pk ON services (id,url)
"""
cur_create_index = db.execute_db(conn, query_create_index, [], True)
def upsert_error(conn, ivoid, url, date, type, num, name, msg="", section=""):
'''
insert or update an error in the errors table
:param conn: sqlite3 DB connection object
:param ivoid: service id
:param url: service url
:param date: date
:param type: error type (warning, error, fatal, failure)
:param num: sequence number of error
:param name: name of error
:param msg: error msg
:param section: section for error
'''
logging.info("Upserting error")
# Check if the error already exists in the DB
# NB: this query can take a long time if no index (id,url) has been defined on table errors !
# => create index errors_idx on errors(id,url);
query = """
SELECT count(*) FROM errors
WHERE id = ? and url = ? and date = ? and type = ? and num = ? and name = ? and msg = ? and section = ?
"""
cur = db.execute_db(conn, query,
(ivoid, url, date, type, num, name, msg, section))
nb_errors = cur.fetchone()[0]
#logging.info("nb_errors = %d",nb_errors)
if (nb_errors == 0): # if the error does not already exist
# insert the error
query = """
INSERT INTO errors (id,url,date,type,num,name,msg,section)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
"""
cur = db.execute_db(conn, query,
(ivoid, url, date, type, num, name, msg, section))
conn.commit() # because INSERT
return
def main(argv):
'''
main program
:param argv: parameters
'''
program_version = "1.3"
#global logger
# Read program arguments
service_type = None # no default
db_file = None # no default
log_file = None # no default
try:
opts, args = getopt.getopt(argv, "h", ["type=", "db=", "log="])
except getopt.GetoptError as err:
print str(err)
usage()
sys.exit(2)
for o, a in opts:
if o in ("-h"):
usage()
sys.exit(0)
elif o in ("--type"):
service_type = a
elif o in ("--db"):
db_file = a
elif o in ("--log"):
log_file = a
else:
assert False, "unhandled option"
if (service_type == None):
print('ERROR: No service_type')
usage()
exit(2)
if (db_file == None):
print('ERROR: No db_file')
usage()
exit(2)
# Setup logging
# Try to use coloredlogs but does not work well
# Create a logger object.
#logger = logging.getLogger('val.py')
# By default the install() function installs a handler on the root logger,
# this means that log messages from your code and log messages from the
# libraries that you use will all show up on the terminal.
#coloredlogs.install(level='DEBUG',fmt='%(asctime)s %(filename)s %(levelname)s %(lineno)d %(processName)s %(funcName)s: %(message)s')
# Try to use colorlog - does not work
#colorlog.basicConfig(format='%(asctime)s %(filename)s %(levelname)s %(lineno)d %(processName)s %(funcName)s: %(message)s', level=logging.DEBUG)
#colorlog.info("Starting argv=%s",argv)
logging.basicConfig(
format=
'%(asctime)s %(filename)s %(levelname)s %(lineno)d %(processName)s %(funcName)s: %(message)s',
level=logging.DEBUG,
filename=log_file)
# Add colors - from https://stackoverflow.com/questions/384076/how-can-i-color-python-logging-output
logging.addLevelName(
logging.WARNING,
"\033[1;31m%s\033[1;0m" % logging.getLevelName(logging.WARNING))
logging.addLevelName(
logging.ERROR,
"\033[1;41m%s\033[1;0m" % logging.getLevelName(logging.ERROR))
logging.info("This is query-rr.py version %s. argv=%s", program_version,
argv)
# Try to open the DB file,
conn = db.open_db(db_file)
# create the tables if they don't exist
create_table_services(conn)
create_table_errors(conn)
# URL of RR to use
url_rr = "http://voparis-rr.obspm.fr/tap"
# Look for services with type service_type in the RR
logging.debug("Looking for services with type=%s in RR=%s", service_type,
url_rr)
services = search(baseurl=url_rr, servicetype=service_types[service_type])
# Get nb of services found
nb_services = len(services)
logging.debug("Nb of services found=%d", nb_services)
s = 0
for service in services:
s = s + 1
logging.info("Processing service %d/%d ivoid=%s url=%s standardid=%s",
s, nb_services, service.ivoid, service.access_url,
service.standard_id)
if (False): # display all service attributes
for a in service:
logging.debug("%s: %s", a, service[a])
# get today's date in format 2017-12-21
date_today = datetime.datetime.today()
date_today_s = date_today.strftime('%Y-%m-%d')
# Check if the service exists in the DB
query = """
SELECT count(*) FROM services WHERE id=? and url=?
"""
cur = db.execute_db(conn, query, (service.ivoid, service.access_url),
True)
nb_such_service = cur.fetchone()[0]
#logging.debug("Nb of such service found in the DB=%d",nb_such_service)
if (nb_such_service == 0
): # service does not exist in the DB yet => insert it
logging.debug("No such service found in the DB, inserting it")
query_insert = """
INSERT INTO services (id,url,date_insert)
VALUES (?, ?, ?)
"""
cur_insert = db.execute_db(
conn, query_insert,
(service.ivoid, service.access_url, date_today_s))
#conn.commit() # because INSERT # no need, db.execute_db does it
# Update the service with data from registry
if (True):
# Extract the part before the # in the standardid and the fragment (#something at the end of the standard id)
standardid = service.standard_id # in lowercase in the RR
index_pound = standardid.find(
'#') # index of char '#' in standardid
#logging.debug("index_pound=%d",index_pound)
fragment = None
if (index_pound !=
-1): # pound found, copy only the part before the pound
standardid_substring = standardid[:index_pound]
fragment = standardid[index_pound:]
else: # no pound found, copy the entire string
standardid_substring = standardid
logging.debug("Found standardid_substring=%s",
standardid_substring)
logging.debug("Found fragment=%s", fragment)
spec = spec_from_standardid[standardid_substring]
logging.debug("Found spec=%s", spec)
# Try to extract version of std used by service
# NB (per Markus after discussion in Santiago 2017-10)
# => the correct way is to check the end of standardid then if no version found, std_version
logging.debug("Determining spec version")
specv = None
if (
fragment != None
): # if a fragment was found, extract the spec from the fragment
logging.debug("standardid's fragment found")
specv_from_fragment = fragment[
7:] # extract "2.0" from "#query-2.0" => skip "#query-"
logging.debug("Found specv_from_fragment=%s",
specv_from_fragment)
if (
specv_from_fragment.replace(".", "", 1).isdigit()
): # https://stackoverflow.com/questions/4138202/using-isdigit-for-floats
logging.debug(
"specv_from_fragment can convert to float => using it")
specv = specv_from_fragment
if (specv == None): # version not found in fragment
logging.debug(
"No standardid's fragment found or version not found in fragment, checking std_version=%s",
service['std_version'])
if (service['std_version'] != ""):
logging.debug("std_version not empty, using it")
specv = service['std_version']
else: # if not found, use default version for this standard
logging.debug(
"std_version empty, using default specv from standardid"
)
specv = default_specv_from_standardid[standardid_substring]
logging.info("Found specv=%s", specv)
# default params for validation are extracted from the array val.validatorParams (in vla.py)
if ((spec == "Simple Image Access")
and (specv == "2.0")): # for SIAv2 we need this case
params = validatorParams[spec + " " + specv]
else:
params = validatorParams[spec]
# if some attributes are void string "" then set them to N/A
role_name = service['role_name']
if (role_name == ""):
role_name = "N/A"
email = service['email']
if (email == ""):
email = "N/A"
query_update = """
UPDATE services SET
date_update = ?
,vor_created = ?
,vor_updated = ?
,vor_status = ?
,provenance = ?
,standard_id = ?
,title = ?
,short_name = ?
,contact_name = ?
,contact_email = ?
,xsi_type = ?
,spec = ?
,specv = ?
,params = ?
WHERE id=? AND url=?
"""
logging.info("Updating table services for service")
cur_update = db.execute_db(
conn,
query_update,
[
date_today_s,
service[
'created'] # must be accessed like this and not by property because property not exposed in class RegistryResource
,
service['updated'] # idem etc.
,
'active' # we assume that if a service was found in the RR, it is active. Reason:
# See http://ivoa.net/documents/RegTAP/20171206/WD-RegTAP-1.1-20171206.html
# "The status attribute of vr:Resource is considered an implementation detail of the XML serialization and is not kept here.
# Neither inactive nor deleted records may be kept in the resource table. Since all other tables in the relational registry should keep
# a foreign key on the ivoid column, this implies that only metadata on active records is being kept in the relational registry.
# In other words, users can expect a resource to exist and work if they find it in a relational registry"
,
service[
'harvested_from'] # the provenance registry's ivoid
,
standardid # the standardid
,
service.res_title,
service.short_name,
role_name # contact name
,
email # contact email
,
service[
'intf_type'] # per the search() function's implementation this should always be 'vs:paramhttp'
,
spec,
specv,
params,
service.ivoid,
service.access_url
])
#conn.commit() # because UPDATE # no need, db.execute_db does it
# if True
# at the end, close the DB connection
logging.info("Done. Closing connection")
conn.close()
def main(argv):
'''
main program
:param argv: parameters
'''
program_version = "1.9"
#global logger
# Read program arguments
db_file = None # no default
nb_ps = 1 # nb of processes to use
timeout = 20 # timeout for validation of individual service, in secs
log_file = None # no default
try:
opts, args = getopt.getopt(argv, "h",
["db=", "ps=", "timeout=", "log="])
except getopt.GetoptError as err:
print str(err)
usage()
sys.exit(2)
for o, a in opts:
if o in ("-h"):
usage()
sys.exit(0)
elif o in ("--db"):
db_file = a
elif o in ("--ps"):
nb_ps = int(a)
elif o in ("--timeout"):
timeout = int(a)
if (timeout < 1):
logging.error("timeout must be greater or equal to 1")
sys.exit(1)
elif o in ("--log"):
log_file = a
else:
assert False, "unhandled option"
if (db_file == None):
print('ERROR: No db_file')
usage()
exit(2)
# Setup logging
# Try to use coloredlogs but does not work well
# Create a logger object.
#logger = logging.getLogger('val.py')
# By default the install() function installs a handler on the root logger,
# this means that log messages from your code and log messages from the
# libraries that you use will all show up on the terminal.
#coloredlogs.install(level='DEBUG',fmt='%(asctime)s %(filename)s %(levelname)s %(lineno)d %(processName)s %(funcName)s: %(message)s')
# Try to use colorlog - does not work
#colorlog.basicConfig(format='%(asctime)s %(filename)s %(levelname)s %(lineno)d %(processName)s %(funcName)s: %(message)s', level=logging.DEBUG)
#colorlog.info("Starting argv=%s",argv)
logging.basicConfig(
format=
'%(asctime)s %(filename)s %(levelname)s %(lineno)d %(processName)s %(funcName)s: %(message)s',
level=logging.DEBUG,
filename=log_file)
# Add colors - from https://stackoverflow.com/questions/384076/how-can-i-color-python-logging-output
logging.addLevelName(
logging.WARNING,
"\033[1;31m%s\033[1;0m" % logging.getLevelName(logging.WARNING))
logging.addLevelName(
logging.ERROR,
"\033[1;41m%s\033[1;0m" % logging.getLevelName(logging.ERROR))
logging.info("This is val.py version %s. argv=%s", program_version, argv)
# Configuration depending on hostname
hostname = socket.gethostname()
logging.info("Running on %s", hostname)
conn = db.open_db(db_file)
# Prepare where clause for extracting suitable services
#min_update_date = datetime.date.today() - datetime.timedelta(2) # today - 2 days
min_update_date = datetime.date.today(
) # 2018-04-18 changed to today => assume query-rr-*.py was run just before and the same day...
min_update_date_s = min_update_date.strftime('%Y-%m-%d')
logging.info("min_update_date_s is %s", min_update_date_s)
where = "date_update >='" + min_update_date_s + "'"
#where = where + " and id like '%vopdc%'" # debug: 3 services
#where = where + " and url like '%.au%'" # debug: 7 services
#where = where + " and id like '%irsa%'" # debug: 366 services
#where = where + " and id='ivo://vopdc.obspm/imcce/skybot'" # debug: 1 service with 1 error
#where = where + " and id='ivo://CDS.VizieR/J/AJ/127/1227'" # debug: 1 service with 2 errors
#where = where + " and id='ivo://vopdc.obspm/imcce/dynastvo/epn' and url='http://voparis-tap-planeto.obspm.fr/__system__/tap/run/tap'" # debug: TAP query
#where = where +" and url='http://camelot.star.le.ac.uk:8080/dsa-catalog/SubmitCone?DSACAT=ledas&DSATAB=a2rtraw&'" # debug: service for which the VO-Paris validator times out
# Count nb of suitable services in the db
query = "SELECT count(*) FROM services WHERE " + where
cur = db.execute_db(conn, query, [], True)
nb_services = cur.fetchone()[0]
logging.info("nb_services = %d", nb_services)
if (nb_services != 0):
if (nb_ps <= nb_services):
# Get suitable services
order = "id asc, url asc"
# query : get those columns only
query = "SELECT id,url,spec,specv,params FROM services WHERE " + where + " ORDER BY " + order
cur = db.execute_db(conn, query, [], True)
services = cur.fetchall()
conn.close()
#sys.exit(0)
#print(services[0]["id"]) # debug
#print(services[0])
#sys.exit(0)
#print(services[0])
# Slice the services array into nb_ps arrays of same size
sservices = numpy.array_split(services, nb_ps)
for i in range(len(sservices)):
logging.info("Slice # %d len=%d first ivoid=%s", i,
len(sservices[i]), (sservices[i][0][0]))
#sys.exit(0)
jobs = []
for i in range(nb_ps):
p = multiprocessing.Process(target=validate_services,
args=(sservices[i], timeout,
db_file))
jobs.append(p)
p.start()
#p.join()
else:
logging.error("nb_ps>nb_services. Try with a lower nb_ps")
sys.exit(1)
else:
logging.error("No suitable service found. Aborting.")
sys.exit(10)
def validate_service(conn, service, timeout):
'''
validate one service: calls validator and update the sqlite3 DB
:param conn: sqlite3 connection object
:param service: array containing attributes of the service per SQL query done
:param timeout: timeout for calling the validator
'''
# extract the service attributes, order is defined by SQL request done in main
ivoid = service[0]
url = service[1]
spec = service[2]
specv = service[3]
params = service[4]
logging.info("Processing service ivoid=%s url=%s spec=%s specv=%s", ivoid,
url, spec, specv)
# For TAP services, check if the url has already been validated today because
# there are many TAP services which have a different IVOID but the same URL
# get today's date in format 2017-05-18
date_today = datetime.datetime.today()
date_today_s = date_today.strftime('%Y-%m-%d')
# Check how many services with the same URL have been validated today
logging.info("Checking how many services with same URL validated today")
query = """
SELECT COUNT(*)
FROM services
WHERE url = ? AND date = ?
"""
cur = db.execute_db(conn, query, (url, date_today_s))
nb_services = cur.fetchone()[0]
logging.debug("nb_services with same URL validated today=%d", nb_services)
if ((spec == "Table Access Protocol")
and (nb_services >
0)): # At least one TAP service with same URL validated today
# Get the first one
logging.info("Getting first such service")
query = """
SELECT val_mode, result_vot, result_spec, nb_warn, nb_err, nb_fatal, nb_fail, days_same, id
FROM services
WHERE url = ? AND date = ?
"""
cur = db.execute_db(conn, query, (url, date_today_s))
service = cur.fetchone()
prev_val_mode = service[0]
prev_result_vot = service[1]
prev_result_spec = service[2]
prev_nb_warn = service[3]
prev_nb_err = service[4]
prev_nb_fatal = service[5]
prev_nb_fail = service[6]
prev_days_same = service[7]
prev_ivoid = service[8]
# Copy the previous results to the current service
logging.info("Updating current service with service found")
query = """
UPDATE services
SET date=?, val_mode=?, result_vot=?, result_spec=?, nb_warn=?, nb_err=?, nb_fatal=?, nb_fail=?, days_same=?
WHERE id=? AND url=?
"""
cur = db.execute_db(
conn, query,
(date_today_s, prev_val_mode, prev_result_vot, prev_result_spec,
prev_nb_warn, prev_nb_err, prev_nb_fatal, prev_nb_fail,
prev_days_same, ivoid, url))
conn.commit() # because of UPDATE
if (
False
): # Copy the errors too - disabled 2018-04-09 to reduce time - because of all TAP VizieR services / webapp updated to take this into account
logging.info("Copying errors")
query = """
SELECT type,num,name,msg,section
FROM errors
WHERE id=? AND url=? AND date=?
"""
cur = db.execute_db(conn, query, (prev_ivoid, url, date_today_s))
errors = cur.fetchall()
for error in errors:
type = error[0]
num = error[1]
name = error[2]
msg = error[3]
section = error[4]
upsert_error(conn, ivoid, url, date_today_s, type, num, name,
msg, section)
return
else: # Run validator
# Construct validator url: validator base URL
vurl = validatorBaseURLs[spec]
# add validator params
#vurl += validatorParams[spec]
vurl += params
# add spec and spec version
vurl += "&" + urllib.urlencode({"spec": spec + " " + specv})
# add service URL
vurl += "&" + urllib.urlencode({"serviceURL": url})
# Set TAP validator timeout
tap_timeout = timeout - 1
# try to make sure TAP validator timeouts before our socket timeout
if (tap_timeout <= 0): tap_timeout = 1
if (spec == "Table Access Protocol"
): # TAP validator also needs the timeout
vurl += "&" + urllib.urlencode({"timeout": tap_timeout})
vurl += "&" + urllib.urlencode({
"maxtable": "1"
}) # added 2018-04-05 to reduce time taken by TAP validation
#vurl="https://www.test123456.com/" # debug - test timeout
logging.info("Calling validator URL: %s (timeout is %d secs)", vurl,
timeout)
# set the timeout for call to urllib2.urlopen - not necessary urllib2.urlopen has a timeout parameter
# socket.setdefaulttimeout(float(timeout))
request = urllib2.Request(vurl)
try:
# NB: The optional timeout parameter specifies a timeout in seconds for blocking operations like the connection attempt
# It seems that the timeout is useless here for timeouting the TAP validator : https://www.daniweb.com/programming/software-development/threads/182555/how-to-set-timeout-for-reading-from-urls-in-urllib
# => the timeout there it is only for opening url. It wont give exception while reading.
response = urllib2.urlopen(request, timeout=timeout)
http_status = response.getcode()
logging.debug("HTTP status: %d", http_status)
except Exception as e:
logging.error(
"EXCEPTION %s while calling URL=%s. Using default results (-1).",
e, vurl)
# These are the results to use in case of timeout:
results = {
"result_vot": "",
"result_spec": "",
"nb_warn": -1,
"nb_err": -1,
"nb_fatal": -1,
"nb_fail": -1,
"warnings": [],
"errors": [],
"fatals": [],
"fails": []
}
# Update the service with the results
update_service(conn, ivoid, url, results)
except socket.timeout as e:
logging.error(
"EXCEPTION: socket timeout: %s while calling URL=%s. Using defaults results (-2).",
e, vurl)
# These are the results to use in case of *socket* timeout:
results = {
"result_vot": "",
"result_spec": "",
"nb_warn": -2,
"nb_err": -2,
"nb_fatal": -2,
"nb_fail": -2,
"warnings": [],
"errors": [],
"fatals": [],
"fails": []
}
# Update the service with the results
update_service(conn, ivoid, url, results)
else: # if no exception
if (http_status == 200):
try:
# set a timer here to have a timeout during reading of data from URL, but tapvalidator.php does not stop correctly
#t = Timer(timeout, response.close)
#t.start()
logging.debug("Reading data.")
data = response.read(
) # get response in either XML or JSON format
#t.cancel()
logging.debug(
"Reading data done."
) # we end up here in case tapvalidator.php times out. The unfinished JSON will make throw an exception in parse_tap_validator
except Exception as e: # try to catch timeout exception... not sure it works
logging.error(
"EXCEPTION %s (timeout?) while reading data from URL=%s. Using default results (-1).",
e, vurl)
# These are the results to use in case of timeout:
results = {
"result_vot": "",
"result_spec": "",
"nb_warn": -1,
"nb_err": -1,
"nb_fatal": -1,
"nb_fail": -1,
"warnings": [],
"errors": [],
"fatals": [],
"fails": []
}
else:
#logging.info(data)
results = parse_validator(spec, data)
# Update the service with the results
update_service(conn, ivoid, url, results)
else: # http_status!=200
logging.error("HTTP status is not 200 but %d. Giving up.",
http_status)
return
def update_service(conn, ivoid, url, results):
'''
update the service in the sqlite3 DB
:param conn: sqlite3 connection object
:param ivoid: id of the service
:param url: url of the service
:param results: results object returned by parse_*_validator
'''
logging.info("Updating sqlite3 db for service ivoid=%s url=%s", ivoid,
url) # : %s",data)
#print results
new_result_vot = results["result_vot"]
new_result_spec = results["result_spec"]
new_nb_warn = results["nb_warn"]
new_nb_err = results["nb_err"]
new_nb_fatal = results["nb_fatal"]
new_nb_fail = results["nb_fail"]
# get today's date in format 2017-05-18
date_today = datetime.datetime.today()
date_today_s = date_today.strftime('%Y-%m-%d')
# Get previous results
query = """
SELECT val_mode, result_vot, result_spec, nb_warn, nb_err, nb_fatal, nb_fail, date, days_same
FROM services
WHERE id=? AND url=?
"""
logging.info("Getting previous results")
cur = db.execute_db(conn, query, (ivoid, url))
service = cur.fetchone()
prev_val_mode = service[0]
prev_result_vot = service[1]
prev_result_vot = service[2]
prev_nb_warn = service[3]
prev_nb_err = service[4]
prev_nb_fatal = service[5]
prev_nb_fail = service[6]
prev_date = service[7]
prev_days_same = service[8]
# Compute days same
# datetime when service was last updated
if (prev_date == None): # if service was never validated
date_prev = date_today # then nb_days will be 0
else:
date_prev = datetime.datetime.strptime(prev_date, '%Y-%m-%d')
interval = date_today - date_prev
nb_days = interval.days
#logging.info(interval)
logging.debug("The service was last updated %d days ago", nb_days)
new_days_same = prev_days_same + nb_days
logging.debug("Old days_same: %d New days_same: %d", prev_days_same,
new_days_same)
# create query to update the service
query = """
UPDATE services SET
date = ?
,val_mode='normal'
,result_vot=?
,result_spec=?
,nb_warn=?
,nb_err=?
,nb_fatal=?
,nb_fail=?
,days_same=?
WHERE id=? AND url=?
"""
logging.info("Updating table services for service")
cur = db.execute_db(conn, query, [
date_today_s, new_result_vot, new_result_spec, new_nb_warn, new_nb_err,
new_nb_fatal, new_nb_fail, new_days_same, ivoid, url
])
conn.commit()
# get today's date in format 2017-05-31
#date_today_s=datetime.date.today().strftime('%Y-%m-%d')
# insert the warnings found
num = 0
for warning in results["warnings"]:
num = num + 1
logging.info("Upserting warning num=%d", num)
upsert_error(conn, ivoid, url, date_today_s, "warning", num,
warning["name"], warning["msg"], warning["section"])
# insert the errors found
num = 0
for error in results["errors"]:
num = num + 1
logging.info("Upserting error num=%d", num)
upsert_error(conn, ivoid, url, date_today_s, "error", num,
error["name"], error["msg"], error["section"])
# insert the fatals found
num = 0
for fatal in results["fatals"]:
num = num + 1
logging.info("Upserting fatal num=%d", num)
upsert_error(conn, ivoid, url, date_today_s, "fatal", num,
fatal["name"], fatal["msg"], fatal["section"])
# insert the failures found
num = 0
for fail in results["fails"]:
num = num + 1
logging.info("Upserting failure num=%d", num)
upsert_error(conn, ivoid, url, date_today_s, "failure", num,
fail["name"], fail["msg"], fail["section"])
return