def insert_images(images, sources, rootdir, cursor, mysql, step_function=None):
"""Inserts multiple images into a database using a single query
Parameters
----------
images : list
list of image dict representations
sources : list
tree of datasources supported by Helioviewer
rootdir : string
image archive root directory
cursor : mixed
database cursor
mysql : bool
whether or not MySQL syntax should be used
step_function : function
function to call after each insert query
"""
query = "INSERT IGNORE INTO images VALUES "
# Add images to SQL query
for i, img in enumerate(images):
# break up directory and filepath
directory, filename = os.path.split(img['filepath'])
path = "/" + os.path.relpath(directory, rootdir)
# Cast measurement to string
img["measurement"] = str(img["measurement"])
# Data Source
source = sources[img["observatory"]][img["instrument"]][img["detector"]][img["measurement"]]
# Enable datasource if it has not already been
if (not source['enabled']):
sources[img["observatory"]][img["instrument"]][img["detector"]][img["measurement"]]["enabled"] = True
enable_datasource(cursor, source['id'])
# insert into database
query += "(NULL, '%s', '%s', '%s', %d)," % (path, filename, img["date"], source['id'])
# Progressbar
if step_function and (i + 1) % __STEP_FXN_THROTTLE__ is 0:
step_function(filename)
# Remove trailing comma
query = query[:-1] + ";"
# Execute query
cursor.execute(query)
def insert_images(images, sources, rootdir, cursor, mysql, step_function=None, cursor_v2=None):
"""Inserts multiple images into a database using a single query
Parameters
----------
images : list
list of image dict representations
sources : list
tree of datasources supported by Helioviewer
rootdir : string
image archive root directory
cursor : mixed
database cursor
mysql : bool
whether or not MySQL syntax should be used
step_function : function
function to call after each insert query
"""
# TEMPORARY SOLUTION
# Because of database tables changes in Helioviewer v2 and Helioviewer v3
# we need have same data on both databases we need to insert image information into both databases
# separatly.
#
# To solve this we duplicated query with different table names and exetuning it to different databases.
#
query = "INSERT IGNORE INTO data VALUES "
query_v2 = "INSERT IGNORE INTO images VALUES "
# Add images to SQL query
for i, img in enumerate(images):
# break up directory and filepath
directory, filename = os.path.split(img['filepath'])
path = "/" + os.path.relpath(directory, rootdir)
prev = ""
source = sources
if img['observatory'] == "Hinode":
leafs = ["observatory", "instrument", "detector", "filter1", "filter2"]
else:
leafs = ["observatory", "instrument", "detector", "measurement"]
for leaf in leafs:
if img[leaf] != prev:
source = source[str(img[leaf])]
prev = img[leaf]
# Enable datasource if it has not already been
if (not source['enabled']):
# sources[img["observatory"]][img["instrument"]][img["detector"]][img["measurement"]]["enabled"] = True
enable_datasource(cursor, source['id'])
# insert into database
query += "(NULL, '%s', '%s', '%s', NULL, %d)," % (path, filename, img["date"], source['id'])
query_v2 += "(NULL, '%s', '%s', '%s', %d)," % (path, filename, img["date"], source['id'])
# Progressbar
if step_function and (i + 1) % __STEP_FXN_THROTTLE__ is 0:
step_function(filename)
# Remove trailing comma
query = query[:-1] + ";"
query_v2 = query_v2[:-1] + ";"
# Execute query
cursor.execute(query)
if cursor_v2:
cursor_v2.execute(query_v2)
def insert_images(images, sources, rootdir, db, cursor, mysql, step_function=None, cursor_v2=None):
"""Inserts multiple images into a database using a single query
Parameters
----------
images : list
list of image dict representations
sources : list
tree of datasources supported by Helioviewer
rootdir : string
image archive root directory
cursor : mixed
database cursor
mysql : bool
whether or not MySQL syntax should be used
step_function : function
function to call after each insert query
"""
# TEMPORARY SOLUTION
# Because of database tables changes in Helioviewer v2 and Helioviewer v3
# we need have same data on both databases we need to insert image information into both databases
# separatly.
#
# To solve this we duplicated query with different table names and exetuning it to different databases.
#
query = "INSERT IGNORE INTO data VALUES "
query_v2 = "INSERT IGNORE INTO images VALUES "
# Add images to SQL query
for i, img in enumerate(images):
# break up directory and filepath
directory, filename = os.path.split(img['filepath'])
path = "/" + os.path.relpath(directory, rootdir)
prev = ""
source = sources
if img['observatory'] == "Hinode":
leafs = ["observatory", "instrument", "detector", "filter1", "filter2"]
else:
leafs = ["observatory", "instrument", "detector", "measurement"]
for leaf in leafs:
if img[leaf] != prev:
source = source[str(img[leaf])]
prev = img[leaf]
# Enable datasource if it has not already been
if (not source['enabled']):
enable_datasource(cursor, source['id'])
groups = getImageGroup(source['id'])
# insert into database
queryStr = "(NULL, '%s', '%s', '%s', NULL, %d, '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', 1, %d, %d, %d),"
query += queryStr % (path, filename, img["date"], source['id'],
img["scale"], img["width"], img["height"], img["refPixelX"], img["refPixelY"], img["layeringOrder"],
img["DSUN_OBS"], img["SOLAR_R"], img["RADIUS"], img["CDELT1"], img["CDELT2"],
img["CRVAL1"], img["CRVAL2"], img["CRPIX1"], img["CRPIX2"], img["XCEN"], img["YCEN"],
img["CROTA1"], groups["groupOne"], groups["groupTwo"], groups["groupThree"])
query_v2 += "(NULL, '%s', '%s', '%s', %d)," % (path, filename, img["date"], source['id'])
# Progressbar
if step_function and (i + 1) % __STEP_FXN_THROTTLE__ is 0:
step_function(filename)
# Remove trailing comma
query = query[:-1] + ";"
query_v2 = query_v2[:-1] + ";"
# Commit enabling datasources
db.commit()
# Execute query
try:
cursor.execute(query)
except Exception as e:
print("Error: " + e.args[1])
if cursor_v2:
try:
cursor_v2.execute(query_v2)
except Exception as e:
print("Error: " + e.args[1])
