def get_tags(namespace,repo_name,registry=None,auth=None):
'''get_tags will return the tags for a repo using the Docker Version 2.0 Registry API
:param namespace: the namespace (eg, "library")
:param repo_name: the name for the repo (eg, "ubuntu")
:param registry: the docker registry to use (default will use index.docker.io)
:param auth: authorization header (default None)
'''
if registry == None:
registry = api_base
registry = add_http(registry) # make sure we have a complete url
base = "%s/%s/%s/%s/tags/list" %(registry,api_version,namespace,repo_name)
logger.info("Obtaining tags: %s", base)
token = get_token(registry=registry,
repo_name=repo_name,
namespace=namespace,
auth=auth)
response = api_get(base,headers=token)
try:
response = json.loads(response)
return response['tags']
except:
logger.error("Error obtaining tags: %s", base)
sys.exit(1)
def change_permissions(path,permission=None,recursive=True):
'''change_permissions will change all permissions of files
and directories. Recursive is default True, given a folder
:param path: path to change permissions for
:param permission: the permission from stat to add (default is stat.S_IWUSR)
:param recursive: do recursively (default is True)
'''
# Default permission to change is adding write
if permission == None:
permission = stat.S_IWUSR
# For a file, recursion is not relevant
if os.path.isfile(path):
logger.info("Changing permission of %s to %s",path,oct(permission))
change_permission(path,permission)
else:
# If the user wants recursive, use os.walk
logger.info("Changing permission of files and folders under %s to %s",path,oct(permission))
for root, dirs, files in os.walk(path, topdown=False, followlinks=False):
# Walking through directories
for name in dirs:
dir_path = os.path.join(root, name)
# Make sure it's a valid dir
if os.path.isdir(dir_path):
change_permission(dir_path, permission)
# Walking through files (and checking for symbolic links)
for name in files:
file_path = os.path.join(root, name)
# Make sure it's a valid file
if os.path.isfile(file_path) and not os.path.islink(file_path):
change_permission(file_path, permission)
def write_file(filename,content,mode="w"):
'''write_file will open a file, "filename" and write content, "content"
and properly close the file
'''
logger.info("Writing file %s with mode %s.",filename,mode)
with open(filename,mode) as filey:
filey.writelines(content)
return filename
def read_file(filename,mode="r"):
'''write_file will open a file, "filename" and write content, "content"
and properly close the file
'''
logger.info("Reading file %s with mode %s.",filename,mode)
with open(filename,mode) as filey:
content = filey.readlines()
return content
def create_runscript(cmd,base_dir):
'''create_runscript will write a bash script with command "cmd" into the base_dir
:param cmd: the command to write into the bash script
:param base_dir: the base directory to write the runscript to
'''
runscript = "%s/singularity" %(base_dir)
content = 'exec %s "$@"' %(cmd)
logger.info("Generating runscript at %s",runscript)
output_file = write_file(runscript,content)
return output_file
def write_json(json_obj,filename,mode="w",print_pretty=True):
'''write_json will (optionally,pretty print) a json object to file
:param json_obj: the dict to print to json
:param filename: the output file to write to
:param pretty_print: if True, will use nicer formatting
'''
logger.info("Writing json file %s with mode %s.",filename,mode)
with open(filename,mode) as filey:
if print_pretty == True:
filey.writelines(json.dumps(json_obj, indent=4, separators=(',', ': ')))
else:
filey.writelines(json.dumps(json_obj))
return filename
def get_image_name(manifest,extension='img.gz',use_commit=True):
'''get_image_name will return the image name for a manifest
:param manifest: the image manifest with 'image' as key with download link
:param use_commit: use the commit id to name the image (default) otherwise use md5sum
'''
image_url = os.path.basename(unquote(manifest['image']))
image_name = re.findall(".+[.]%s" %(extension),image_url)
if len(image_name) > 0:
image_name = image_name[0]
if use_commit == True:
image_name = "%s.img.gz" %(manifest["version"])
logger.info("Singularity Hub Image: %s", image_name)
return image_name
else:
logger.error("Singularity Hub Image not found with expected extension %s, exiting.",extension)
sys.exit(1)
def main():
'''main is a wrapper for the client to hand the parser to the executable functions
This makes it possible to set up a parser in test cases
'''
logger.info("\n*** STARTING PYTHON CLIENT PORTION ****")
parser = get_parser()
try:
(args,options) = parser.parse_args()
except:
logger.error("Input args to %s improperly set, exiting.", os.path.abspath(__file__))
parser.print_help()
sys.exit(0)
# Give the args to the main executable to run
run(args)
def run_command(cmd):
'''run_command uses subprocess to send a command to the terminal.
:param cmd: the command to send, should be a list for subprocess
'''
try:
logger.info("Running command %s with subprocess", " ".join(cmd))
process = subprocess.Popen(cmd,stdout=subprocess.PIPE)
except OSError as error:
logger.error("Error with subprocess: %s, returning None",error)
return None
output = process.communicate()[0]
if process.returncode != 0:
return None
return output
def get_layer(image_id,namespace,repo_name,download_folder=None,registry=None,auth=None):
'''get_layer will download an image layer (.tar.gz) to a specified download folder.
:param image_id: the (full) image id to get the manifest for, required
:param namespace: the namespace (eg, "library")
:param repo_name: the repo name, (eg, "ubuntu")
:param download_folder: if specified, download to folder. Otherwise return response with raw data (not recommended)
:param registry: the docker registry to use (default will use index.docker.io)
:param auth: authorization header (default None)
'''
if registry == None:
registry = api_base
registry = add_http(registry) # make sure we have a complete url
# The <name> variable is the namespace/repo_name
base = "%s/%s/%s/%s/blobs/%s" %(registry,api_version,namespace,repo_name,image_id)
logger.info("Downloading layers from %s", base)
# To get the image layers, we need a valid token to read the repo
token = get_token(registry=registry,
repo_name=repo_name,
namespace=namespace,
auth=auth)
if download_folder != None:
download_folder = "%s/%s.tar.gz" %(download_folder,image_id)
# Update user what we are doing
print("Downloading layer %s" %image_id)
try:
# Create temporary file with format .tar.gz.tmp.XXXXX
fd, tmp_file = tempfile.mkstemp(prefix=("%s.tmp." % download_folder))
os.close(fd)
response = api_get(base,headers=token,stream=tmp_file)
if isinstance(response, HTTPError):
logger.error("Error downloading layer %s, exiting.", base)
sys.exit(1)
os.rename(tmp_file, download_folder)
except:
logger.error("Removing temporary download file %s", tmp_file)
try:
os.remove(tmp_file)
except:
pass
sys.exit(1)
return download_folder
def get_manifest(repo_name,namespace,repo_tag="latest",registry=None,auth=None,headers=None):
'''get_manifest should return an image manifest for a particular repo and tag. The token is expected to
be from version 2.0 (function above)
:param repo_name: the name of the repo, eg "ubuntu"
:param namespace: the namespace for the image, default is "library"
:param repo_tag: the repo tag, default is "latest"
:param registry: the docker registry to use (default will use index.docker.io)
:param auth: authorization header (default None)
:param headers: dictionary of custom headers to add to token header (to get more specific manifest)
'''
if registry == None:
registry = api_base
registry = add_http(registry) # make sure we have a complete url
base = "%s/%s/%s/%s/manifests/%s" %(registry,api_version,namespace,repo_name,repo_tag)
logger.info("Obtaining manifest: %s", base)
# Format the token, and prepare a header
token = get_token(registry=registry,
repo_name=repo_name,
namespace=namespace,
auth=auth)
# Add ['Accept'] header to specify version 2 of manifest
if headers != None:
if token != None:
token.update(headers)
else:
token = headers
response = api_get(base,headers=token,default_header=True)
try:
response = json.loads(response)
except:
# If the call fails, give the user a list of acceptable tags
tags = get_tags(namespace=namespace,
repo_name=repo_name,
registry=registry,
auth=auth)
print("\n".join(tags))
logger.error("Error getting manifest for %s/%s:%s, exiting.", namespace,
repo_name,
repo_tag)
print("Error getting manifest for %s/%s:%s. Acceptable tags are listed above." %(namespace,repo_name,repo_tag))
sys.exit(1)
return response
def change_permission(file_path, permission=None):
'''change_permission changes a permission if the file does not have it
:param file_path the path to the file
:param permission: the stat permission to use
'''
if permission == None:
permission = stat.S_IWUSR
st = os.stat(file_path)
has_perm = has_permission(file_path, permission)
if not has_perm:
logger.info("Fixing permission on: %s", file_path)
try:
os.chmod(file_path, st.st_mode | permission)
except:
print("ERROR: Couldn't change permission on ", file_path)
sys.exit(1)
return has_permission(file_path, permission)
def change_permission(file_path,permission=None):
'''change_permission changes a permission if the file does not have it
:param file_path the path to the file
:param permission: the stat permission to use
'''
if permission == None:
permission = stat.S_IWUSR
st = os.stat(file_path)
has_perm = has_permission(file_path,permission)
if not has_perm:
logger.info("Fixing permission on: %s", file_path)
try:
os.chmod(file_path, st.st_mode | permission)
except:
print("ERROR: Couldn't change permission on ", file_path)
sys.exit(1)
return has_permission(file_path,permission)
def parse_image_uri(image, uri=None, default_namespace=None):
'''parse_image_uri will return a json structure with a repo name, tag, and
namespace.
:param image: the string provided on command line for the image name, eg: ubuntu:latest
:param uri: the uri (eg, docker:// to remove), default uses ""
:default_namespace: if not provided, will use "library"
:returns parsed: a json structure with repo_name, repo_tag, and namespace
'''
if default_namespace == None:
default_namespace = "library"
if uri == None:
uri = ""
# First split the docker image name by /
image = image.replace(uri, '')
# If the user provided a number (unique id for an image), return it
if is_number(image) == True:
logger.info("Numeric image ID %s%s found.", uri, image)
return int(image)
image = image.split('/')
# If there are two parts, we have namespace with repo (and maybe tab)
if len(image) >= 2:
namespace = image[0]
image = image[1]
# Otherwise, we must be using library namespace
else:
namespace = default_namespace
image = image[0]
# Now split the docker image name by :
image = image.split(':')
if len(image) == 2:
repo_name = image[0]
repo_tag = image[1]
# Otherwise, assume latest of an image
else:
repo_name = image[0]
repo_tag = "latest"
logger.info("Repo Name: %s", repo_name)
logger.info("Repo Tag: %s", repo_tag)
logger.info("Namespace: %s", namespace)
parsed = {
'repo_name': repo_name,
'repo_tag': repo_tag,
'namespace': namespace
}
return parsed
def get_manifest(repo_name,
namespace,
repo_tag="latest",
registry=None,
auth=None):
'''get_manifest should return an image manifest for a particular repo and tag. The token is expected to
be from version 2.0 (function above)
:param repo_name: the name of the repo, eg "ubuntu"
:param namespace: the namespace for the image, default is "library"
:param repo_tag: the repo tag, default is "latest"
:param registry: the docker registry to use (default will use index.docker.io)
:param auth: authorization header (default None)
'''
if registry == None:
registry = api_base
registry = add_http(registry) # make sure we have a complete url
base = "%s/%s/%s/%s/manifests/%s" % (registry, api_version, namespace,
repo_name, repo_tag)
logger.info("Obtaining manifest: %s", base)
# Format the token, and prepare a header
token = get_token(registry=registry,
repo_name=repo_name,
namespace=namespace,
auth=auth)
response = api_get(base, headers=token, default_header=True)
try:
response = json.loads(response)
except:
# If the call fails, give the user a list of acceptable tags
tags = get_tags(namespace=namespace,
repo_name=repo_name,
registry=registry,
auth=auth)
print("\n".join(tags))
logger.error("Error getting manifest for %s/%s:%s, exiting.",
namespace, repo_name, repo_tag)
print(
"Error getting manifest for %s/%s:%s. Acceptable tags are listed above."
% (namespace, repo_name, repo_tag))
sys.exit(1)
return response
def get_image_name(manifest, extension='img.gz', use_commit=True):
'''get_image_name will return the image name for a manifest
:param manifest: the image manifest with 'image' as key with download link
:param use_commit: use the commit id to name the image (default) otherwise use md5sum
'''
image_url = os.path.basename(unquote(manifest['image']))
image_name = re.findall(".+[.]%s" % (extension), image_url)
if len(image_name) > 0:
image_name = image_name[0]
if use_commit == True:
image_name = "%s.img.gz" % (manifest["version"])
logger.info("Singularity Hub Image: %s", image_name)
return image_name
else:
logger.error(
"Singularity Hub Image not found with expected extension %s, exiting.",
extension)
sys.exit(1)
def change_permissions(path, permission="0755", recursive=True):
'''change_permissions will use subprocess to change permissions of a file
or directory. Recursive is default True
:param path: path to change permissions for
:param permission: the permission level (default is 0755)
:param recursive: do recursively (default is True)
'''
if not isinstance(permission, str):
logger.warning(
"Please provide permission as a string, not number! Skipping.")
else:
permission = str(permission)
cmd = ["chmod", permission, "-R", path]
if recursive == False:
cmd = ["chmod", permission, path]
logger.info("Changing permission of %s to %s with command %s", path,
permission, " ".join(cmd))
return run_command(cmd)
def connectwithDB(dbpath):
try:
logger.info((messages.DB_TRY_CONNECTION).format(dbpath))
dbconn = sqlite3.connect(dbpath)
dbconn.commit()
except sqlite3.Error as e:
logger.critical((messages.DB_CONNECTION_FAIL).format(dbpath))
logger.critical(e, exc_info=True)
messagebox.showerror(title="CONNECTION FAILED",
message=messages.DB_CONN_FAIL)
except:
logger.error(messages.UNEXPECTED_ERROR)
logger.error("StackTrace : ", exc_info=True)
messagebox.showerror(title="CONNECTION FAILED",
message=messages.DB_CONN_FAIL)
if dbconn:
logger.info((messages.DB_CONNECTION_PASS).format(dbpath))
return dbconn
def callHomescreen(allframes):
utils.destroyframes(allframes)
# Commits all transactions done by updating or loading data into the DB
global dbconn
if dbconn:
dbconn.commit()
# Delete all global variables
global loadfilename
global recordDict
global exportfolder
exportfolder = EXPORTS
recordDict.clear()
loadfilename = None
logger.info(messages.CALL_HOME_SCREEN)
homescreen()
def createtable(dbconn, tablename, schemafile):
logger.info(messages.CREATE_TABLE.format(tablename, schemafile))
fielddict = {}
for line in open(schemafile):
if len(line) == 0 or line == '\n':
continue
if line[0] in ('!', '#'):
continue
field, datatype = line.split('=')
fielddict[field.strip()] = datatype.strip()
createquery = "CREATE TABLE IF NOT EXISTS " + tablename + "(" + os.linesep
fields = list(fielddict.keys())
datatypes = list(fielddict.values())
numfields = len(fielddict.keys())
for i in range(0, numfields - 1):
createquery = createquery + fields[i] + \
" " + datatypes[i] + "," + os.linesep
createquery = createquery + \
fields[numfields-1] + " " + datatypes[numfields-1]
createquery += (os.linesep + ");")
logger.info(messages.CREATE_TABLE_QUERY.format(createquery))
try:
createcursor = dbconn.cursor()
createcursor.execute(createquery)
dbconn.commit()
except sqlite3.Error as e:
logger.critical(messages.CREATE_TABLE_FAILED.format(tablename))
logger.critical(e, exc_info=True)
return False
except:
logger.error(messages.UNEXPECTED_ERROR)
logger.error("StackTrace : ", exc_info=True)
return False
return True
def get_config(manifest,spec="Entrypoint"):
'''get_config returns a particular spec (default is Entrypoint) from a manifest obtained with get_manifest.
:param manifest: the manifest obtained from get_manifest
:param spec: the key of the spec to return, default is "Entrypoint"
'''
cmd = None
if "history" in manifest:
for entry in manifest['history']:
if 'v1Compatibility' in entry:
entry = json.loads(entry['v1Compatibility'])
if "config" in entry:
if spec in entry["config"]:
cmd = entry["config"][spec]
# Standard is to include commands like ['/bin/sh']
if isinstance(cmd,list):
cmd = "\n".join(cmd)
logger.info("Found Docker command (%s) %s",spec,cmd)
return cmd
def main():
'''parse configuration options and produce configuration output file
'''
logger.info("\n*** STARTING PYTHON CONFIGURATION HELPER ****")
parser = get_parser()
try:
(args,options) = parser.parse_args()
except:
logger.error("Input args to %s improperly set, exiting.", os.path.abspath(__file__))
parser.print_help()
sys.exit(1)
# Check for required args
[check_required(parser,arg) for arg in [args.defaults,
args.infile,
args.outfile]]
# Run the configuration
configure(args)
def get_config(manifest, spec="Entrypoint"):
'''get_config returns a particular spec (default is Entrypoint) from a manifest obtained with get_manifest.
:param manifest: the manifest obtained from get_manifest
:param spec: the key of the spec to return, default is "Entrypoint"
'''
cmd = None
if "history" in manifest:
for entry in manifest['history']:
if 'v1Compatibility' in entry:
entry = json.loads(entry['v1Compatibility'])
if "config" in entry:
if spec in entry["config"]:
cmd = entry["config"][spec]
# Standard is to include commands like ['/bin/sh']
if isinstance(cmd, list):
cmd = "\n".join(cmd)
logger.info("Found Docker command (%s) %s", spec, cmd)
return cmd
def get_layer(image_id,
namespace,
repo_name,
download_folder=None,
registry=None,
auth=None):
'''get_layer will download an image layer (.tar.gz) to a specified download folder.
:param image_id: the (full) image id to get the manifest for, required
:param namespace: the namespace (eg, "library")
:param repo_name: the repo name, (eg, "ubuntu")
:param download_folder: if specified, download to folder. Otherwise return response with raw data (not recommended)
:param registry: the docker registry to use (default will use index.docker.io)
:param auth: authorization header (default None)
'''
if registry == None:
registry = api_base
registry = add_http(registry) # make sure we have a complete url
# The <name> variable is the namespace/repo_name
base = "%s/%s/%s/%s/blobs/%s" % (registry, api_version, namespace,
repo_name, image_id)
logger.info("Downloading layers from %s", base)
# To get the image layers, we need a valid token to read the repo
token = get_token(registry=registry,
repo_name=repo_name,
namespace=namespace,
auth=auth)
if download_folder != None:
download_folder = "%s/%s.tar.gz" % (download_folder, image_id)
# Update user what we are doing
print("Downloading layer %s" % image_id)
response = api_get(base, headers=token, stream=download_folder)
if isinstance(response, HTTPError):
logger.error("Error downloading layer %s, exiting.", base)
sys.exit(1)
return response
def read_digests(manifest):
'''read_layers will return a list of layers from a manifest. The function is
intended to work with both version 1 and 2 of the schema
:param manifest: the manifest to read_layers from
'''
digests = []
# https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-2.md#image-manifest
if 'layers' in manifest:
layer_key = 'layers'
digest_key = 'digest'
logger.info('Image manifest version 2.2 found.')
# https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-1.md#example-manifest
elif 'fsLayers' in manifest:
layer_key = 'fsLayers'
digest_key = 'blobSum'
logger.info('Image manifest version 2.1 found.')
else:
logger.error(
'Improperly formed manifest, layers or fsLayers must be present')
sys.exit(1)
for layer in manifest[layer_key]:
if digest_key in layer:
if layer[digest_key] not in digests:
logger.info("Adding digest %s", layer[digest_key])
digests.append(layer[digest_key])
return digests
def createDB(dataLabel, dbname):
# create a new db and pop an info/error message with the Path
global currtable
global dbconn
global currdb
if dbconn != None:
logger.info(messages.DB_DETAILS_CLEAN.format(currdb, currtable))
dbconn.close()
currdb = None
currtable = None
if (dbname != "" and dbname != None):
serverdir = PROJECT_ROOT + os.path.sep + SERVER
if not (dbname.endswith(".db")):
dbname = dbname + ".db"
logger.info(messages.USER_GIVEN_DBNAME.format(dbname))
dbpath = Path(serverdir + os.path.sep + dbname)
dbconn = utils.connectwithDB(dbpath)
if dbconn:
currdb = dbname
logger.info(messages.CURRDB_VALUE.format(currdb))
dataLabel.configure(text="Successfully connected to " + currdb +
" database")
else:
dataLabel.configure(text="Currently not connected to any Database")
logger.warning(messages.INVALID_DB_NAME_ENTERED.format(dbname))
messagebox.showinfo(title='INVALID DB NAME',
message=messages.INVALID_DB_NAME)
def sqltoexcel(dbconn, currtable, exportfilepath):
schemaList = getSchemaFromTable(dbconn, currtable)
logger.info(messages.EXPORT_STARTER.format(exportfilepath, schemaList))
# from openpyxl import Workbook
wb = Workbook()
# grab the active worksheet
ws = wb.active
# Appending Schema Row first
ws.append(schemaList)
try:
cur = dbconn.cursor()
cur.execute("SELECT * from " + currtable + ";")
results = cur.fetchall()
totalrows = len(results)
exportedrows = 0
for eachrow in results:
ws.append(eachrow)
exportedrows += 1
# Save the file
wb.save(exportfilepath)
dbconn.commit()
except sqlite3.Error as e:
logger.critical(
messages.DATA_EXPORT_FAILED.format(currtable, exportfilepath))
logger.critical(e, exc_info=True)
return False
except:
logger.error(messages.UNEXPECTED_ERROR)
logger.error("StackTrace : ", exc_info=True)
return False
return (totalrows == exportedrows), exportedrows
def getStudentRecord(dbconn, currtable, studentRoll, studentName):
searchQuery = formSearchQuery(currtable, studentRoll, studentName)
logger.info(messages.STUDENT_SEARCH_QUERY.format(searchQuery))
try:
searchcursor = dbconn.cursor()
searchcursor.execute(searchQuery)
studentRecords = searchcursor.fetchall()
if (len(studentRecords) == 0):
logger.warning(messages.RECORD_NOT_FOUND)
messagebox.showerror(title="RECORD_NOT_FOUND",
message=messages.RECORD_NOT_FOUND)
return None
if (len(studentRecords) > 1):
messagebox.showerror(title="DUPLICATE_RECORDS",
message=messages.DUPLICATE_RECORDS)
return None
studentRecord = studentRecords[0]
logger.info(messages.OBTAINED_RECORD.format(str(studentRecord)))
return studentRecord
except sqlite3.Error as e:
messagebox.showerror(title="SEARCH_FAILED",
message=messages.SEARCH_FAILED)
logger.critical(messages.SEARCH_FAILED.format(searchQuery))
logger.critical(e, exc_info=True)
return None
except:
messagebox.showerror(title="UNEXPECTED_ERROR",
message=messages.UNEXPECTED_ERROR)
logger.error(messages.UNEXPECTED_ERROR)
logger.error("StackTrace : ", exc_info=True)
return None
def exceltosql(dbconn, currtable, loadfilename, fieldmapfile):
logger.info(
messages.START_LOADING.format(currtable, loadfilename, fieldmapfile))
wb = load_workbook(filename=loadfilename, read_only=True)
ws = wb.active
colrowiter = ws.iter_rows(max_row=1, values_only=True)
excelfields = next(colrowiter)
dbfieldlist, irrelevantcolindices = getdbfields(excelfields, fieldmapfile)
logger.info(messages.DB_FIELD_LIST.format(dbfieldlist))
rowstoload = 0
insertedrows = 0
for datarow in ws.iter_rows(values_only=True, min_row=2):
if not all(v is None for v in datarow):
rowstoload += 1
datalist = list(datarow)
rowtoinsert = preparevalues(datalist, irrelevantcolindices,
dbfieldlist, currtable, rowstoload)
if rowtoinsert != False:
insertedrows = rowinserter(rowtoinsert, dbconn, currtable,
dbfieldlist, insertedrows)
logger.info(messages.ROW_DIFF.format(rowstoload, insertedrows))
return insertedrows, rowstoload
def read_digests(manifest):
'''read_layers will return a list of layers from a manifest. The function is
intended to work with both version 1 and 2 of the schema
:param manifest: the manifest to read_layers from
'''
digests = []
# https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-2.md#image-manifest
if 'layers' in manifest:
layer_key = 'layers'
digest_key = 'digest'
logger.info('Image manifest version 2.2 found.')
# https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-1.md#example-manifest
elif 'fsLayers' in manifest:
layer_key = 'fsLayers'
digest_key = 'blobSum'
logger.info('Image manifest version 2.1 found.')
else:
logger.error('Improperly formed manifest, layers or fsLayers must be present')
sys.exit(1)
for layer in manifest[layer_key]:
if digest_key in layer:
if layer[digest_key] not in digests:
logger.info("Adding digest %s",layer[digest_key])
digests.append(layer[digest_key])
return digests
def configure(args):
# Get fullpath to each file, and concurrently check that exists
defaultfile = get_fullpath(args.defaults) # ../src/lib/config_defaults.h
infile = get_fullpath(args.infile) # singularity.conf.in
# Find define statements
define_re = re.compile("#define ([A-Z_]+) (.*)")
# Read in input and default files
defaultfile = read_file(defaultfile)
data = "".join(read_file(infile))
# Lookup for values we want replaced
lookup = {'0':'no',
'1':'yes'}
defaults = {}
# Read in defaults to dictionary
for line in defaultfile:
match = define_re.match(line)
if match:
key, value = match.groups()
# Maintain the original default set by user
defaults[key] = value
# Use parsed value for final config
new_value = value.replace('"', '')
if new_value in lookup:
new_value = lookup[new_value]
data = data.replace("@" + key + "@", new_value)
# Write to output file
outfile = "%s.tmp" %(args.outfile)
write_file(outfile,data)
os.rename(outfile, args.outfile)
logger.info("*** FINISHED PYTHON CONFIGURATION HELPER ****\n")
def chooseFile(filedisplay):
global loadfilename
if loadfilename != None:
logger.info(messages.LOADFILE_CLEAR.format(loadfilename))
loadfilename = None
chosenfile = filedialog.askopenfilename(initialdir=Path(PROJECT_ROOT),
title="Select an Excel file",
filetypes=(("Excel workbook",
"*.xlsx"), ))
if chosenfile == None or chosenfile == "":
logger.warning(messages.NO_EXCEL_FILE_SELECTED)
messagebox.showinfo(title="NO_EXCEL_FILE_SELECTED",
message=messages.NO_EXCEL_FILE_SELECTED)
loadfilename = None
else:
loadfilename = chosenfile
logger.info(messages.LOADFILE_VALUE.format(loadfilename))
filedisplay.configure(
text="The Student data will be imported from \n " + loadfilename)
def selectDB(dataLabel):
# open the DB. and open an connection. Store it as a global variable. Show error message
global currtable
global dbconn
global currdb
if dbconn != None:
logger.info(messages.DB_DETAILS_CLEAN.format(currdb, currtable))
dbconn.close()
currdb = None
currtable = None
serverdir = PROJECT_ROOT + os.path.sep + SERVER
filepattern = serverdir + os.path.sep + "*.db"
if not (glob.glob(filepattern)):
logger.warning(messages.NO_DB_EXISTS)
messagebox.showerror(title="No DATABASE PRESENT",
message=messages.NO_DB_EXISTS)
return
else:
dbpath = filedialog.askopenfilename(initialdir=Path(SERVER),
title="Select an Database file",
filetypes=(("SQLite3 Databases",
"*.db"), ))
if dbpath == '':
logger.warn(messages.INVALID_DB_SELECTION)
messagebox.showinfo(title="INVALID_DB_SELECTION",
message=messages.INVALID_DB_SELECTION)
dataLabel.configure(text="Currently not connected to any Database")
return
else:
dbname = os.path.basename(Path(dbpath))
dbconn = utils.connectwithDB(dbpath)
if dbconn:
currdb = dbname
logger.info(messages.CURRDB_VALUE.format(currdb))
dataLabel.configure(text="Successfully connected to " + currdb +
" database")
def change_permissions(path, permission=None, recursive=True):
'''change_permissions will change all permissions of files
and directories. Recursive is default True, given a folder
:param path: path to change permissions for
:param permission: the permission from stat to add (default is stat.S_IWUSR)
:param recursive: do recursively (default is True)
'''
# Default permission to change is adding write
if permission == None:
permission = stat.S_IWUSR
# For a file, recursion is not relevant
if os.path.isfile(path):
logger.info("Changing permission of %s to %s", path, oct(permission))
change_permission(path, permission)
else:
# If the user wants recursive, use os.walk
logger.info("Changing permission of files and folders under %s to %s",
path, oct(permission))
for root, dirs, files in os.walk(path,
topdown=False,
followlinks=False):
# Walking through directories
for name in dirs:
dir_path = os.path.join(root, name)
# Make sure it's a valid dir
if os.path.isdir(dir_path):
change_permission(dir_path, permission)
# Walking through files (and checking for symbolic links)
for name in files:
file_path = os.path.join(root, name)
# Make sure it's a valid file
if os.path.isfile(file_path) and not os.path.islink(file_path):
change_permission(file_path, permission)
def backup(prjroot, storagefolder, backupfolder):
today = datetime.date.today()
today = (str(today)).replace('-', '_')
onlydbs = [
f for f in os.listdir(storagefolder)
if os.path.isfile(os.path.join(storagefolder, f)) and f.endswith(".db")
]
currdatefolder = Path(backupfolder + os.path.sep + today)
if not os.path.exists(currdatefolder):
os.mkdir(currdatefolder)
try:
for eachdb in onlydbs:
logger.info(messages.CURRENT_BACKUP_DB.format(eachdb))
storagedbpath = Path(storagefolder + os.path.sep + eachdb)
storagedbconn = sqlite3.connect(storagedbpath)
backupdbpath = Path(backupfolder + os.path.sep + today +
os.path.sep + eachdb)
backupdbconn = sqlite3.connect(backupdbpath)
logger.info(messages.BACKUP_TARGET.format(backupdbpath._str))
storagedbconn.backup(backupdbconn, pages=0, progress=progress)
storagedbconn.commit()
backupdbconn.commit()
storagedbconn.close()
backupdbconn.close()
logger.info(messages.BACKUP_SUCCESS.format(eachdb))
except sqlite3.Error as e:
logger.critical(messages.BACKUP_CREATION_FAILED.format(eachdb))
logger.critical(e, exc_info=True)
return False
except:
logger.error(messages.UNEXPECTED_ERROR)
logger.error("Exception : ", exc_info=True)
return False
finally:
if storagedbconn:
storagedbconn.close()
if backupdbconn:
backupdbconn.close()
def run(args):
# Find root filesystem location
if args.rootfs != None:
singularity_rootfs = args.rootfs
else:
singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None)
if singularity_rootfs == None and args.shub == None:
logger.error(
"root file system not specified OR defined as environmental variable, exiting!"
)
sys.exit(1)
if singularity_rootfs != None:
logger.info("Root file system defined as %s", singularity_rootfs)
# Does the registry require authentication?
auth = None
if args.username is not None and args.password is not None:
auth = basic_auth_header(args.username, args.password)
logger.info("Username for registry authentication: %s", args.username)
# Does the user want to download a Singularity image?
if args.shub != None:
image = args.shub
manifest = get_shub_manifest(image)
if args.pull_folder == None:
cache_base = get_cache(subfolder="shub",
disable_cache=args.disable_cache)
else:
cache_base = args.pull_folder
# The image name is the md5 hash, download if it's not there
image_name = get_image_name(manifest)
image_file = "%s/%s" % (cache_base, image_name)
if not os.path.exists(image_file):
image_file = download_image(manifest=manifest,
download_folder=cache_base)
else:
print("Image already exists at %s, skipping download." %
image_file)
logger.info("Singularity Hub Image Download: %s", image_file)
# If singularity_rootfs is provided, write metadata to it
if singularity_rootfs != None:
logger.debug(
"Writing SINGULARITY_RUNDIR and SINGULARITY_IMAGE to %s",
singularity_rootfs)
write_file("%s/SINGULARITY_RUNDIR" % singularity_rootfs,
os.path.dirname(image_file))
write_file("%s/SINGULARITY_IMAGE" % singularity_rootfs, image_file)
# Do we have a docker image specified?
elif args.docker != None:
# Does the user want to override default Entrypoint and use CMD as runscript?
includecmd = args.includecmd
logger.info("Including Docker command as Runscript? %s", includecmd)
image = args.docker
logger.info("Docker image: %s", image)
# Input Parsing ----------------------------
# Parse image name, repo name, and namespace
image = parse_image_uri(image=image, uri="docker://")
namespace = image['namespace']
repo_name = image['repo_name']
repo_tag = image['repo_tag']
# Tell the user the namespace, repo name and tag
logger.info("Docker image path: %s/%s:%s", namespace, repo_name,
repo_tag)
# IMAGE METADATA -------------------------------------------
# Use Docker Registry API (version 2.0) to get images ids, manifest
# Get an image manifest - has image ids to parse, and will be
# used later to get Cmd
manifest = get_manifest(repo_name=repo_name,
namespace=namespace,
repo_tag=repo_tag,
registry=args.registry,
auth=auth)
# Get images from manifest using version 2.0 of Docker Registry API
images = get_images(manifest=manifest)
# DOWNLOAD LAYERS -------------------------------------------
# Each is a .tar.gz file, obtained from registry with curl
# Get the cache (or temporary one) for docker
cache_base = get_cache(subfolder="docker",
disable_cache=args.disable_cache)
layers = []
for image_id in images:
# Download the layer, if we don't have it
targz = "%s/%s.tar.gz" % (cache_base, image_id)
if not os.path.exists(targz):
targz = get_layer(image_id=image_id,
namespace=namespace,
repo_name=repo_name,
download_folder=cache_base,
registry=args.registry,
auth=auth)
layers.append(targz) # in case we want a list at the end
# Extract image and remove tar
output = extract_tar(targz, singularity_rootfs)
if output is None:
logger.error("Error extracting image: %s", targz)
sys.exit(1)
if args.disable_cache == True:
os.remove(targz)
# If the user wants to include the CMD as runscript, generate it here
if includecmd == True:
spec = "Cmd"
else:
spec = "Entrypoint"
cmd = get_config(manifest, spec=spec)
# Only add runscript if command is defined
if cmd != None:
print("Adding Docker %s as Singularity runscript..." %
(spec.upper()))
print(cmd)
runscript = create_runscript(cmd=cmd, base_dir=singularity_rootfs)
# When we finish, clean up images
if args.disable_cache == True:
shutil.rmtree(cache_base)
logger.info("*** FINISHING DOCKER BOOTSTRAP PYTHON PORTION ****\n")
def main():
logger.info("\n*** STARTING DOCKER BOOTSTRAP PYTHON PORTION ****")
parser = argparse.ArgumentParser(
description="bootstrap Docker images for Singularity containers")
# Name of the docker image, required
parser.add_argument(
"--docker",
dest='docker',
help=
"name of Docker image to bootstrap, in format library/ubuntu:latest",
type=str,
default=None)
# root file system of singularity image
parser.add_argument("--rootfs",
dest='rootfs',
help="the path for the root filesystem to extract to",
type=str,
default=None)
# Docker registry (default is registry-1.docker.io
parser.add_argument(
"--registry",
dest='registry',
help="the registry path to use, to replace registry-1.docker.io",
type=str,
default=None)
# Flag to add the Docker CMD as a runscript
parser.add_argument(
"--cmd",
dest='includecmd',
action="store_true",
help=
"boolean to specify that CMD should be used instead of ENTRYPOINT as the runscript.",
default=False)
parser.add_argument("--username",
dest='username',
help="username for registry authentication",
default=None)
parser.add_argument("--password",
dest='password',
help="password for registry authentication",
default=None)
# Flag to disable cache
parser.add_argument("--no-cache",
dest='disable_cache',
action="store_true",
help="boolean to specify disabling the cache.",
default=False)
try:
args = parser.parse_args()
except:
logger.error("Input args to %s improperly set, exiting.",
os.path.abspath(__file__))
parser.print_help()
sys.exit(0)
# Find root filesystem location
if args.rootfs != None:
singularity_rootfs = args.rootfs
logger.info("Root file system defined by command line variable as %s",
singularity_rootfs)
else:
singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None)
if singularity_rootfs == None:
logger.error(
"root file system not specified OR defined as environmental variable, exiting!"
)
sys.exit(1)
logger.info("Root file system defined by env variable as %s",
singularity_rootfs)
# Does the registry require authentication?
auth = None
if args.username is not None and args.password is not None:
auth = basic_auth_header(args.username, args.password)
logger.info("Username for registry authentication: %s", args.username)
# Does the user want to override default Entrypoint and use CMD as runscript?
includecmd = args.includecmd
logger.info("Including Docker command as Runscript? %s", includecmd)
# Do we have a docker image specified?
if args.docker != None:
image = args.docker
logger.info("Docker image: %s", image)
# INPUT PARSING -------------------------------------------
# Parse image name, repo name, and namespace
# First split the docker image name by /
image = image.split('/')
# If there are two parts, we have namespace with repo (and maybe tab)
if len(image) == 2:
namespace = image[0]
image = image[1]
# Otherwise, we must be using library namespace
else:
namespace = "library"
image = image[0]
# Now split the docker image name by :
image = image.split(':')
if len(image) == 2:
repo_name = image[0]
repo_tag = image[1]
# Otherwise, assume latest of an image
else:
repo_name = image[0]
repo_tag = "latest"
# Tell the user the namespace, repo name and tag
logger.info("Docker image path: %s/%s:%s", namespace, repo_name,
repo_tag)
# IMAGE METADATA -------------------------------------------
# Use Docker Registry API (version 2.0) to get images ids, manifest
# Get an image manifest - has image ids to parse, and will be
# used later to get Cmd
manifest = get_manifest(repo_name=repo_name,
namespace=namespace,
repo_tag=repo_tag,
registry=args.registry,
auth=auth)
# Get images from manifest using version 2.0 of Docker Registry API
images = get_images(manifest=manifest,
registry=args.registry,
auth=auth)
# DOWNLOAD LAYERS -------------------------------------------
# Each is a .tar.gz file, obtained from registry with curl
# Get the cache (or temporary one) for docker
cache_base = get_cache(subfolder="docker",
disable_cache=args.disable_cache)
layers = []
for image_id in images:
# Download the layer, if we don't have it
targz = "%s/%s.tar.gz" % (cache_base, image_id)
if not os.path.exists(targz):
targz = get_layer(image_id=image_id,
namespace=namespace,
repo_name=repo_name,
download_folder=cache_base,
registry=args.registry,
auth=auth)
layers.append(targz) # in case we want a list at the end
# @chrisfilo suggestion to try compiling into one tar.gz
# Extract image and remove tar
extract_tar(targz, singularity_rootfs)
if args.disable_cache == True:
os.remove(targz)
# If the user wants to include the CMD as runscript, generate it here
if includecmd == True:
spec = "Cmd"
else:
spec = "Entrypoint"
cmd = get_config(manifest, spec=spec)
# Only add runscript if command is defined
if cmd != None:
print("Adding Docker %s as Singularity runscript..." %
(spec.upper()))
print(cmd)
runscript = create_runscript(cmd=cmd, base_dir=singularity_rootfs)
# When we finish, clean up images
if args.disable_cache == True:
shutil.rmtree(cache_base)
logger.info("*** FINISHING DOCKER BOOTSTRAP PYTHON PORTION ****\n")
quitbutton = Button(lowerframe,
text="Quit",
bg="#decccc",
font=("Verdana", 15),
bd=5,
relief="raised",
command=quitapplication)
quitbutton.place(relx=0.53, rely=0.60, relwidth=0.40, relheight=0.2)
allframes = []
allframes.append(upperframe)
allframes.append(lowerframe)
allframes.append(comboframe)
allframes.append(dataframe)
######################################### INITIALIZERS #################################################################
logger.info(messages.APPLN_START)
utils.verifySetup(PROJECT_ROOT, SERVER, CONFIG, BACKUPS, EXPORTS)
# Entry point for the program, where homescreen is displayed first
homescreen()
# Basic Root Window Initializations
Logo = utils.resource_path(ATHLETE_ICON_FILE)
window.iconphoto(True, PhotoImage(file=Logo))
window.title('MSSSDO v1.0')
window.geometry("750x850")
window.protocol("WM_DELETE_WINDOW", quitapplication)
window.mainloop()
def updateStudentData(heightEntry, speedEntry, enduranceEntry, strengthEntry,
explosiveEntry, agilityEntry, dataLabel):
global recordDict
height = heightEntry.get()
fiftytime = speedEntry.get()
eighthundredtime = enduranceEntry.get()
shotputdist = strengthEntry.get()
longjumpdist = explosiveEntry.get()
agilitytime = agilityEntry.get()
logger.info(
messages.UPDATE_VALUES.format(height, fiftytime, eighthundredtime,
shotputdist, longjumpdist, agilitytime))
if (height == "" and fiftytime == "" and eighthundredtime == ""
and shotputdist == "" and longjumpdist == ""
and agilitytime == ""):
messagebox.showerror(title="EMPTY_VALUES",
message=messages.EMPTY_VALUES)
return
if not recordDict:
messagebox.showerror(title="NO_STUDENT_CHOSEN",
message=messages.NO_STUDENT_CHOSEN)
return
dynamicRecordDict = recordDict.copy()
updateQueries, recordDict = utils.getUpdateQueries(
height, fiftytime, eighthundredtime, shotputdist, longjumpdist,
agilitytime, dynamicRecordDict)
logger.info(messages.UPDATE_QUERY_NUM.format(len(updateQueries)))
if len(updateQueries) < 1:
logger.info(messages.NO_UPDATE_QUERIES)
del dynamicRecordDict
return
finalupdatequery = utils.formUpdateQuery(currtable, updateQueries,
dynamicRecordDict)
logger.info(messages.UPDATE_QUERY.format(finalupdatequery))
result = utils.updateTable(dbconn, finalupdatequery)
if result:
logger.info(messages.SPORT_UPDATE_PASSED)
totalscoreupdatequery = utils.updateTotalScore(currtable,
dynamicRecordDict)
logger.info(messages.UPDATE_QUERY.format(totalscoreupdatequery))
combinedresult = utils.updateTable(dbconn, totalscoreupdatequery)
else:
logger.error(messages.SPORT_UPDATE_FAILED)
combinedresult = False
if combinedresult:
logger.info(messages.UPDATE_RECORD_PASS)
dataLabel.config(text="Student Record was updated Successfully")
recordDict.update(dynamicRecordDict)
else:
messagebox.showerror(title="UPDATE_FAILED",
message=messages.UPDATE_FAILED)
del dynamicRecordDict
return
def run(args):
# Find root filesystem location
if args.rootfs != None:
singularity_rootfs = args.rootfs
else:
singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None)
if singularity_rootfs == None and args.shub == None:
logger.error("root file system not specified OR defined as environmental variable, exiting!")
sys.exit(1)
if singularity_rootfs != None:
logger.info("Root file system defined as %s", singularity_rootfs)
# Does the registry require authentication?
auth = None
if args.username is not None and args.password is not None:
auth = basic_auth_header(args.username, args.password)
logger.info("Username for registry authentication: %s", args.username)
# Does the user want to download a Singularity image?
if args.shub != None:
image_id = int(args.shub)
manifest = get_shub_manifest(image_id)
cache_base = get_cache(subfolder="shub",
disable_cache = args.disable_cache)
# The image name is the md5 hash, download if it's not there
image_name = get_image_name(manifest)
image_file = "%s/%s" %(cache_base,image_name)
if not os.path.exists(image_file):
image_file = download_image(manifest=manifest,
download_folder=cache_base)
else:
print("Image already exists at %s, skipping download." %image_file)
logger.info("Singularity Hub Image Download: %s", image_file)
# If singularity_rootfs is provided, write metadata to it
if singularity_rootfs != None:
logger.debug("Writing SINGULARITY_RUNDIR and SINGULARITY_IMAGE to %s",singularity_rootfs)
write_file("%s/SINGULARITY_RUNDIR" %singularity_rootfs, os.path.dirname(image_file))
write_file("%s/SINGULARITY_IMAGE" %singularity_rootfs, image_file)
# Do we have a docker image specified?
elif args.docker != None:
# Does the user want to override default Entrypoint and use CMD as runscript?
includecmd = args.includecmd
logger.info("Including Docker command as Runscript? %s", includecmd)
image = args.docker
logger.info("Docker image: %s", image)
# Input Parsing ----------------------------
# Parse image name, repo name, and namespace
# First split the docker image name by /
image = image.split('/')
# If there are two parts, we have namespace with repo (and maybe tab)
if len(image) == 2:
namespace = image[0]
image = image[1]
# Otherwise, we must be using library namespace
else:
namespace = "library"
image = image[0]
# Now split the docker image name by :
image = image.split(':')
if len(image) == 2:
repo_name = image[0]
repo_tag = image[1]
# Otherwise, assume latest of an image
else:
repo_name = image[0]
repo_tag = "latest"
# Tell the user the namespace, repo name and tag
logger.info("Docker image path: %s/%s:%s", namespace,repo_name,repo_tag)
# IMAGE METADATA -------------------------------------------
# Use Docker Registry API (version 2.0) to get images ids, manifest
# Get an image manifest - has image ids to parse, and will be
# used later to get Cmd
manifest = get_manifest(repo_name=repo_name,
namespace=namespace,
repo_tag=repo_tag,
registry=args.registry,
auth=auth)
# Get images from manifest using version 2.0 of Docker Registry API
images = get_images(repo_name=repo_name,
namespace=namespace,
registry=args.registry,
auth=auth)
# DOWNLOAD LAYERS -------------------------------------------
# Each is a .tar.gz file, obtained from registry with curl
# Get the cache (or temporary one) for docker
cache_base = get_cache(subfolder="docker",
disable_cache = args.disable_cache)
layers = []
for image_id in images:
# Download the layer, if we don't have it
targz = "%s/%s.tar.gz" %(cache_base,image_id)
if not os.path.exists(targz):
targz = get_layer(image_id=image_id,
namespace=namespace,
repo_name=repo_name,
download_folder=cache_base,
registry=args.registry,
auth=auth)
layers.append(targz) # in case we want a list at the end
# Extract image and remove tar
output = extract_tar(targz,singularity_rootfs)
if output is None:
logger.error("Error extracting image: %s", targz)
sys.exit(1)
if args.disable_cache == True:
os.remove(targz)
# If the user wants to include the CMD as runscript, generate it here
if includecmd == True:
spec="Cmd"
else:
spec="Entrypoint"
cmd = get_config(manifest,spec=spec)
# Only add runscript if command is defined
if cmd != None:
print("Adding Docker %s as Singularity runscript..." %(spec.upper()))
print(cmd)
runscript = create_runscript(cmd=cmd,
base_dir=singularity_rootfs)
# When we finish, clean up images
if args.disable_cache == True:
shutil.rmtree(cache_base)
logger.info("*** FINISHING DOCKER BOOTSTRAP PYTHON PORTION ****\n")
