def get_tags(namespace,repo_name,registry=None,auth=None): '''get_tags will return the tags for a repo using the Docker Version 2.0 Registry API :param namespace: the namespace (eg, "library") :param repo_name: the name for the repo (eg, "ubuntu") :param registry: the docker registry to use (default will use index.docker.io) :param auth: authorization header (default None) ''' if registry == None: registry = api_base registry = add_http(registry) # make sure we have a complete url base = "%s/%s/%s/%s/tags/list" %(registry,api_version,namespace,repo_name) logger.info("Obtaining tags: %s", base) token = get_token(registry=registry, repo_name=repo_name, namespace=namespace, auth=auth) response = api_get(base,headers=token) try: response = json.loads(response) return response['tags'] except: logger.error("Error obtaining tags: %s", base) sys.exit(1)
def change_permissions(path,permission=None,recursive=True): '''change_permissions will change all permissions of files and directories. Recursive is default True, given a folder :param path: path to change permissions for :param permission: the permission from stat to add (default is stat.S_IWUSR) :param recursive: do recursively (default is True) ''' # Default permission to change is adding write if permission == None: permission = stat.S_IWUSR # For a file, recursion is not relevant if os.path.isfile(path): logger.info("Changing permission of %s to %s",path,oct(permission)) change_permission(path,permission) else: # If the user wants recursive, use os.walk logger.info("Changing permission of files and folders under %s to %s",path,oct(permission)) for root, dirs, files in os.walk(path, topdown=False, followlinks=False): # Walking through directories for name in dirs: dir_path = os.path.join(root, name) # Make sure it's a valid dir if os.path.isdir(dir_path): change_permission(dir_path, permission) # Walking through files (and checking for symbolic links) for name in files: file_path = os.path.join(root, name) # Make sure it's a valid file if os.path.isfile(file_path) and not os.path.islink(file_path): change_permission(file_path, permission)
def write_file(filename,content,mode="w"): '''write_file will open a file, "filename" and write content, "content" and properly close the file ''' logger.info("Writing file %s with mode %s.",filename,mode) with open(filename,mode) as filey: filey.writelines(content) return filename
def read_file(filename,mode="r"): '''write_file will open a file, "filename" and write content, "content" and properly close the file ''' logger.info("Reading file %s with mode %s.",filename,mode) with open(filename,mode) as filey: content = filey.readlines() return content
def create_runscript(cmd,base_dir): '''create_runscript will write a bash script with command "cmd" into the base_dir :param cmd: the command to write into the bash script :param base_dir: the base directory to write the runscript to ''' runscript = "%s/singularity" %(base_dir) content = 'exec %s "$@"' %(cmd) logger.info("Generating runscript at %s",runscript) output_file = write_file(runscript,content) return output_file
def write_json(json_obj,filename,mode="w",print_pretty=True): '''write_json will (optionally,pretty print) a json object to file :param json_obj: the dict to print to json :param filename: the output file to write to :param pretty_print: if True, will use nicer formatting ''' logger.info("Writing json file %s with mode %s.",filename,mode) with open(filename,mode) as filey: if print_pretty == True: filey.writelines(json.dumps(json_obj, indent=4, separators=(',', ': '))) else: filey.writelines(json.dumps(json_obj)) return filename
def get_image_name(manifest,extension='img.gz',use_commit=True): '''get_image_name will return the image name for a manifest :param manifest: the image manifest with 'image' as key with download link :param use_commit: use the commit id to name the image (default) otherwise use md5sum ''' image_url = os.path.basename(unquote(manifest['image'])) image_name = re.findall(".+[.]%s" %(extension),image_url) if len(image_name) > 0: image_name = image_name[0] if use_commit == True: image_name = "%s.img.gz" %(manifest["version"]) logger.info("Singularity Hub Image: %s", image_name) return image_name else: logger.error("Singularity Hub Image not found with expected extension %s, exiting.",extension) sys.exit(1)
def main(): '''main is a wrapper for the client to hand the parser to the executable functions This makes it possible to set up a parser in test cases ''' logger.info("\n*** STARTING PYTHON CLIENT PORTION ****") parser = get_parser() try: (args,options) = parser.parse_args() except: logger.error("Input args to %s improperly set, exiting.", os.path.abspath(__file__)) parser.print_help() sys.exit(0) # Give the args to the main executable to run run(args)
def run_command(cmd): '''run_command uses subprocess to send a command to the terminal. :param cmd: the command to send, should be a list for subprocess ''' try: logger.info("Running command %s with subprocess", " ".join(cmd)) process = subprocess.Popen(cmd,stdout=subprocess.PIPE) except OSError as error: logger.error("Error with subprocess: %s, returning None",error) return None output = process.communicate()[0] if process.returncode != 0: return None return output
def get_layer(image_id,namespace,repo_name,download_folder=None,registry=None,auth=None): '''get_layer will download an image layer (.tar.gz) to a specified download folder. :param image_id: the (full) image id to get the manifest for, required :param namespace: the namespace (eg, "library") :param repo_name: the repo name, (eg, "ubuntu") :param download_folder: if specified, download to folder. Otherwise return response with raw data (not recommended) :param registry: the docker registry to use (default will use index.docker.io) :param auth: authorization header (default None) ''' if registry == None: registry = api_base registry = add_http(registry) # make sure we have a complete url # The <name> variable is the namespace/repo_name base = "%s/%s/%s/%s/blobs/%s" %(registry,api_version,namespace,repo_name,image_id) logger.info("Downloading layers from %s", base) # To get the image layers, we need a valid token to read the repo token = get_token(registry=registry, repo_name=repo_name, namespace=namespace, auth=auth) if download_folder != None: download_folder = "%s/%s.tar.gz" %(download_folder,image_id) # Update user what we are doing print("Downloading layer %s" %image_id) try: # Create temporary file with format .tar.gz.tmp.XXXXX fd, tmp_file = tempfile.mkstemp(prefix=("%s.tmp." % download_folder)) os.close(fd) response = api_get(base,headers=token,stream=tmp_file) if isinstance(response, HTTPError): logger.error("Error downloading layer %s, exiting.", base) sys.exit(1) os.rename(tmp_file, download_folder) except: logger.error("Removing temporary download file %s", tmp_file) try: os.remove(tmp_file) except: pass sys.exit(1) return download_folder
def get_manifest(repo_name,namespace,repo_tag="latest",registry=None,auth=None,headers=None): '''get_manifest should return an image manifest for a particular repo and tag. The token is expected to be from version 2.0 (function above) :param repo_name: the name of the repo, eg "ubuntu" :param namespace: the namespace for the image, default is "library" :param repo_tag: the repo tag, default is "latest" :param registry: the docker registry to use (default will use index.docker.io) :param auth: authorization header (default None) :param headers: dictionary of custom headers to add to token header (to get more specific manifest) ''' if registry == None: registry = api_base registry = add_http(registry) # make sure we have a complete url base = "%s/%s/%s/%s/manifests/%s" %(registry,api_version,namespace,repo_name,repo_tag) logger.info("Obtaining manifest: %s", base) # Format the token, and prepare a header token = get_token(registry=registry, repo_name=repo_name, namespace=namespace, auth=auth) # Add ['Accept'] header to specify version 2 of manifest if headers != None: if token != None: token.update(headers) else: token = headers response = api_get(base,headers=token,default_header=True) try: response = json.loads(response) except: # If the call fails, give the user a list of acceptable tags tags = get_tags(namespace=namespace, repo_name=repo_name, registry=registry, auth=auth) print("\n".join(tags)) logger.error("Error getting manifest for %s/%s:%s, exiting.", namespace, repo_name, repo_tag) print("Error getting manifest for %s/%s:%s. Acceptable tags are listed above." %(namespace,repo_name,repo_tag)) sys.exit(1) return response
def change_permission(file_path, permission=None): '''change_permission changes a permission if the file does not have it :param file_path the path to the file :param permission: the stat permission to use ''' if permission == None: permission = stat.S_IWUSR st = os.stat(file_path) has_perm = has_permission(file_path, permission) if not has_perm: logger.info("Fixing permission on: %s", file_path) try: os.chmod(file_path, st.st_mode | permission) except: print("ERROR: Couldn't change permission on ", file_path) sys.exit(1) return has_permission(file_path, permission)
def change_permission(file_path,permission=None): '''change_permission changes a permission if the file does not have it :param file_path the path to the file :param permission: the stat permission to use ''' if permission == None: permission = stat.S_IWUSR st = os.stat(file_path) has_perm = has_permission(file_path,permission) if not has_perm: logger.info("Fixing permission on: %s", file_path) try: os.chmod(file_path, st.st_mode | permission) except: print("ERROR: Couldn't change permission on ", file_path) sys.exit(1) return has_permission(file_path,permission)
def parse_image_uri(image, uri=None, default_namespace=None): '''parse_image_uri will return a json structure with a repo name, tag, and namespace. :param image: the string provided on command line for the image name, eg: ubuntu:latest :param uri: the uri (eg, docker:// to remove), default uses "" :default_namespace: if not provided, will use "library" :returns parsed: a json structure with repo_name, repo_tag, and namespace ''' if default_namespace == None: default_namespace = "library" if uri == None: uri = "" # First split the docker image name by / image = image.replace(uri, '') # If the user provided a number (unique id for an image), return it if is_number(image) == True: logger.info("Numeric image ID %s%s found.", uri, image) return int(image) image = image.split('/') # If there are two parts, we have namespace with repo (and maybe tab) if len(image) >= 2: namespace = image[0] image = image[1] # Otherwise, we must be using library namespace else: namespace = default_namespace image = image[0] # Now split the docker image name by : image = image.split(':') if len(image) == 2: repo_name = image[0] repo_tag = image[1] # Otherwise, assume latest of an image else: repo_name = image[0] repo_tag = "latest" logger.info("Repo Name: %s", repo_name) logger.info("Repo Tag: %s", repo_tag) logger.info("Namespace: %s", namespace) parsed = { 'repo_name': repo_name, 'repo_tag': repo_tag, 'namespace': namespace } return parsed
def get_manifest(repo_name, namespace, repo_tag="latest", registry=None, auth=None): '''get_manifest should return an image manifest for a particular repo and tag. The token is expected to be from version 2.0 (function above) :param repo_name: the name of the repo, eg "ubuntu" :param namespace: the namespace for the image, default is "library" :param repo_tag: the repo tag, default is "latest" :param registry: the docker registry to use (default will use index.docker.io) :param auth: authorization header (default None) ''' if registry == None: registry = api_base registry = add_http(registry) # make sure we have a complete url base = "%s/%s/%s/%s/manifests/%s" % (registry, api_version, namespace, repo_name, repo_tag) logger.info("Obtaining manifest: %s", base) # Format the token, and prepare a header token = get_token(registry=registry, repo_name=repo_name, namespace=namespace, auth=auth) response = api_get(base, headers=token, default_header=True) try: response = json.loads(response) except: # If the call fails, give the user a list of acceptable tags tags = get_tags(namespace=namespace, repo_name=repo_name, registry=registry, auth=auth) print("\n".join(tags)) logger.error("Error getting manifest for %s/%s:%s, exiting.", namespace, repo_name, repo_tag) print( "Error getting manifest for %s/%s:%s. Acceptable tags are listed above." % (namespace, repo_name, repo_tag)) sys.exit(1) return response
def get_image_name(manifest, extension='img.gz', use_commit=True): '''get_image_name will return the image name for a manifest :param manifest: the image manifest with 'image' as key with download link :param use_commit: use the commit id to name the image (default) otherwise use md5sum ''' image_url = os.path.basename(unquote(manifest['image'])) image_name = re.findall(".+[.]%s" % (extension), image_url) if len(image_name) > 0: image_name = image_name[0] if use_commit == True: image_name = "%s.img.gz" % (manifest["version"]) logger.info("Singularity Hub Image: %s", image_name) return image_name else: logger.error( "Singularity Hub Image not found with expected extension %s, exiting.", extension) sys.exit(1)
def change_permissions(path, permission="0755", recursive=True): '''change_permissions will use subprocess to change permissions of a file or directory. Recursive is default True :param path: path to change permissions for :param permission: the permission level (default is 0755) :param recursive: do recursively (default is True) ''' if not isinstance(permission, str): logger.warning( "Please provide permission as a string, not number! Skipping.") else: permission = str(permission) cmd = ["chmod", permission, "-R", path] if recursive == False: cmd = ["chmod", permission, path] logger.info("Changing permission of %s to %s with command %s", path, permission, " ".join(cmd)) return run_command(cmd)
def connectwithDB(dbpath): try: logger.info((messages.DB_TRY_CONNECTION).format(dbpath)) dbconn = sqlite3.connect(dbpath) dbconn.commit() except sqlite3.Error as e: logger.critical((messages.DB_CONNECTION_FAIL).format(dbpath)) logger.critical(e, exc_info=True) messagebox.showerror(title="CONNECTION FAILED", message=messages.DB_CONN_FAIL) except: logger.error(messages.UNEXPECTED_ERROR) logger.error("StackTrace : ", exc_info=True) messagebox.showerror(title="CONNECTION FAILED", message=messages.DB_CONN_FAIL) if dbconn: logger.info((messages.DB_CONNECTION_PASS).format(dbpath)) return dbconn
def callHomescreen(allframes): utils.destroyframes(allframes) # Commits all transactions done by updating or loading data into the DB global dbconn if dbconn: dbconn.commit() # Delete all global variables global loadfilename global recordDict global exportfolder exportfolder = EXPORTS recordDict.clear() loadfilename = None logger.info(messages.CALL_HOME_SCREEN) homescreen()
def createtable(dbconn, tablename, schemafile): logger.info(messages.CREATE_TABLE.format(tablename, schemafile)) fielddict = {} for line in open(schemafile): if len(line) == 0 or line == '\n': continue if line[0] in ('!', '#'): continue field, datatype = line.split('=') fielddict[field.strip()] = datatype.strip() createquery = "CREATE TABLE IF NOT EXISTS " + tablename + "(" + os.linesep fields = list(fielddict.keys()) datatypes = list(fielddict.values()) numfields = len(fielddict.keys()) for i in range(0, numfields - 1): createquery = createquery + fields[i] + \ " " + datatypes[i] + "," + os.linesep createquery = createquery + \ fields[numfields-1] + " " + datatypes[numfields-1] createquery += (os.linesep + ");") logger.info(messages.CREATE_TABLE_QUERY.format(createquery)) try: createcursor = dbconn.cursor() createcursor.execute(createquery) dbconn.commit() except sqlite3.Error as e: logger.critical(messages.CREATE_TABLE_FAILED.format(tablename)) logger.critical(e, exc_info=True) return False except: logger.error(messages.UNEXPECTED_ERROR) logger.error("StackTrace : ", exc_info=True) return False return True
def get_config(manifest,spec="Entrypoint"): '''get_config returns a particular spec (default is Entrypoint) from a manifest obtained with get_manifest. :param manifest: the manifest obtained from get_manifest :param spec: the key of the spec to return, default is "Entrypoint" ''' cmd = None if "history" in manifest: for entry in manifest['history']: if 'v1Compatibility' in entry: entry = json.loads(entry['v1Compatibility']) if "config" in entry: if spec in entry["config"]: cmd = entry["config"][spec] # Standard is to include commands like ['/bin/sh'] if isinstance(cmd,list): cmd = "\n".join(cmd) logger.info("Found Docker command (%s) %s",spec,cmd) return cmd
def main(): '''parse configuration options and produce configuration output file ''' logger.info("\n*** STARTING PYTHON CONFIGURATION HELPER ****") parser = get_parser() try: (args,options) = parser.parse_args() except: logger.error("Input args to %s improperly set, exiting.", os.path.abspath(__file__)) parser.print_help() sys.exit(1) # Check for required args [check_required(parser,arg) for arg in [args.defaults, args.infile, args.outfile]] # Run the configuration configure(args)
def get_config(manifest, spec="Entrypoint"): '''get_config returns a particular spec (default is Entrypoint) from a manifest obtained with get_manifest. :param manifest: the manifest obtained from get_manifest :param spec: the key of the spec to return, default is "Entrypoint" ''' cmd = None if "history" in manifest: for entry in manifest['history']: if 'v1Compatibility' in entry: entry = json.loads(entry['v1Compatibility']) if "config" in entry: if spec in entry["config"]: cmd = entry["config"][spec] # Standard is to include commands like ['/bin/sh'] if isinstance(cmd, list): cmd = "\n".join(cmd) logger.info("Found Docker command (%s) %s", spec, cmd) return cmd
def get_layer(image_id, namespace, repo_name, download_folder=None, registry=None, auth=None): '''get_layer will download an image layer (.tar.gz) to a specified download folder. :param image_id: the (full) image id to get the manifest for, required :param namespace: the namespace (eg, "library") :param repo_name: the repo name, (eg, "ubuntu") :param download_folder: if specified, download to folder. Otherwise return response with raw data (not recommended) :param registry: the docker registry to use (default will use index.docker.io) :param auth: authorization header (default None) ''' if registry == None: registry = api_base registry = add_http(registry) # make sure we have a complete url # The <name> variable is the namespace/repo_name base = "%s/%s/%s/%s/blobs/%s" % (registry, api_version, namespace, repo_name, image_id) logger.info("Downloading layers from %s", base) # To get the image layers, we need a valid token to read the repo token = get_token(registry=registry, repo_name=repo_name, namespace=namespace, auth=auth) if download_folder != None: download_folder = "%s/%s.tar.gz" % (download_folder, image_id) # Update user what we are doing print("Downloading layer %s" % image_id) response = api_get(base, headers=token, stream=download_folder) if isinstance(response, HTTPError): logger.error("Error downloading layer %s, exiting.", base) sys.exit(1) return response
def read_digests(manifest): '''read_layers will return a list of layers from a manifest. The function is intended to work with both version 1 and 2 of the schema :param manifest: the manifest to read_layers from ''' digests = [] # https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-2.md#image-manifest if 'layers' in manifest: layer_key = 'layers' digest_key = 'digest' logger.info('Image manifest version 2.2 found.') # https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-1.md#example-manifest elif 'fsLayers' in manifest: layer_key = 'fsLayers' digest_key = 'blobSum' logger.info('Image manifest version 2.1 found.') else: logger.error( 'Improperly formed manifest, layers or fsLayers must be present') sys.exit(1) for layer in manifest[layer_key]: if digest_key in layer: if layer[digest_key] not in digests: logger.info("Adding digest %s", layer[digest_key]) digests.append(layer[digest_key]) return digests
def createDB(dataLabel, dbname): # create a new db and pop an info/error message with the Path global currtable global dbconn global currdb if dbconn != None: logger.info(messages.DB_DETAILS_CLEAN.format(currdb, currtable)) dbconn.close() currdb = None currtable = None if (dbname != "" and dbname != None): serverdir = PROJECT_ROOT + os.path.sep + SERVER if not (dbname.endswith(".db")): dbname = dbname + ".db" logger.info(messages.USER_GIVEN_DBNAME.format(dbname)) dbpath = Path(serverdir + os.path.sep + dbname) dbconn = utils.connectwithDB(dbpath) if dbconn: currdb = dbname logger.info(messages.CURRDB_VALUE.format(currdb)) dataLabel.configure(text="Successfully connected to " + currdb + " database") else: dataLabel.configure(text="Currently not connected to any Database") logger.warning(messages.INVALID_DB_NAME_ENTERED.format(dbname)) messagebox.showinfo(title='INVALID DB NAME', message=messages.INVALID_DB_NAME)
def sqltoexcel(dbconn, currtable, exportfilepath): schemaList = getSchemaFromTable(dbconn, currtable) logger.info(messages.EXPORT_STARTER.format(exportfilepath, schemaList)) # from openpyxl import Workbook wb = Workbook() # grab the active worksheet ws = wb.active # Appending Schema Row first ws.append(schemaList) try: cur = dbconn.cursor() cur.execute("SELECT * from " + currtable + ";") results = cur.fetchall() totalrows = len(results) exportedrows = 0 for eachrow in results: ws.append(eachrow) exportedrows += 1 # Save the file wb.save(exportfilepath) dbconn.commit() except sqlite3.Error as e: logger.critical( messages.DATA_EXPORT_FAILED.format(currtable, exportfilepath)) logger.critical(e, exc_info=True) return False except: logger.error(messages.UNEXPECTED_ERROR) logger.error("StackTrace : ", exc_info=True) return False return (totalrows == exportedrows), exportedrows
def getStudentRecord(dbconn, currtable, studentRoll, studentName): searchQuery = formSearchQuery(currtable, studentRoll, studentName) logger.info(messages.STUDENT_SEARCH_QUERY.format(searchQuery)) try: searchcursor = dbconn.cursor() searchcursor.execute(searchQuery) studentRecords = searchcursor.fetchall() if (len(studentRecords) == 0): logger.warning(messages.RECORD_NOT_FOUND) messagebox.showerror(title="RECORD_NOT_FOUND", message=messages.RECORD_NOT_FOUND) return None if (len(studentRecords) > 1): messagebox.showerror(title="DUPLICATE_RECORDS", message=messages.DUPLICATE_RECORDS) return None studentRecord = studentRecords[0] logger.info(messages.OBTAINED_RECORD.format(str(studentRecord))) return studentRecord except sqlite3.Error as e: messagebox.showerror(title="SEARCH_FAILED", message=messages.SEARCH_FAILED) logger.critical(messages.SEARCH_FAILED.format(searchQuery)) logger.critical(e, exc_info=True) return None except: messagebox.showerror(title="UNEXPECTED_ERROR", message=messages.UNEXPECTED_ERROR) logger.error(messages.UNEXPECTED_ERROR) logger.error("StackTrace : ", exc_info=True) return None
def exceltosql(dbconn, currtable, loadfilename, fieldmapfile): logger.info( messages.START_LOADING.format(currtable, loadfilename, fieldmapfile)) wb = load_workbook(filename=loadfilename, read_only=True) ws = wb.active colrowiter = ws.iter_rows(max_row=1, values_only=True) excelfields = next(colrowiter) dbfieldlist, irrelevantcolindices = getdbfields(excelfields, fieldmapfile) logger.info(messages.DB_FIELD_LIST.format(dbfieldlist)) rowstoload = 0 insertedrows = 0 for datarow in ws.iter_rows(values_only=True, min_row=2): if not all(v is None for v in datarow): rowstoload += 1 datalist = list(datarow) rowtoinsert = preparevalues(datalist, irrelevantcolindices, dbfieldlist, currtable, rowstoload) if rowtoinsert != False: insertedrows = rowinserter(rowtoinsert, dbconn, currtable, dbfieldlist, insertedrows) logger.info(messages.ROW_DIFF.format(rowstoload, insertedrows)) return insertedrows, rowstoload
def read_digests(manifest): '''read_layers will return a list of layers from a manifest. The function is intended to work with both version 1 and 2 of the schema :param manifest: the manifest to read_layers from ''' digests = [] # https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-2.md#image-manifest if 'layers' in manifest: layer_key = 'layers' digest_key = 'digest' logger.info('Image manifest version 2.2 found.') # https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-1.md#example-manifest elif 'fsLayers' in manifest: layer_key = 'fsLayers' digest_key = 'blobSum' logger.info('Image manifest version 2.1 found.') else: logger.error('Improperly formed manifest, layers or fsLayers must be present') sys.exit(1) for layer in manifest[layer_key]: if digest_key in layer: if layer[digest_key] not in digests: logger.info("Adding digest %s",layer[digest_key]) digests.append(layer[digest_key]) return digests
def configure(args): # Get fullpath to each file, and concurrently check that exists defaultfile = get_fullpath(args.defaults) # ../src/lib/config_defaults.h infile = get_fullpath(args.infile) # singularity.conf.in # Find define statements define_re = re.compile("#define ([A-Z_]+) (.*)") # Read in input and default files defaultfile = read_file(defaultfile) data = "".join(read_file(infile)) # Lookup for values we want replaced lookup = {'0':'no', '1':'yes'} defaults = {} # Read in defaults to dictionary for line in defaultfile: match = define_re.match(line) if match: key, value = match.groups() # Maintain the original default set by user defaults[key] = value # Use parsed value for final config new_value = value.replace('"', '') if new_value in lookup: new_value = lookup[new_value] data = data.replace("@" + key + "@", new_value) # Write to output file outfile = "%s.tmp" %(args.outfile) write_file(outfile,data) os.rename(outfile, args.outfile) logger.info("*** FINISHED PYTHON CONFIGURATION HELPER ****\n")
def chooseFile(filedisplay): global loadfilename if loadfilename != None: logger.info(messages.LOADFILE_CLEAR.format(loadfilename)) loadfilename = None chosenfile = filedialog.askopenfilename(initialdir=Path(PROJECT_ROOT), title="Select an Excel file", filetypes=(("Excel workbook", "*.xlsx"), )) if chosenfile == None or chosenfile == "": logger.warning(messages.NO_EXCEL_FILE_SELECTED) messagebox.showinfo(title="NO_EXCEL_FILE_SELECTED", message=messages.NO_EXCEL_FILE_SELECTED) loadfilename = None else: loadfilename = chosenfile logger.info(messages.LOADFILE_VALUE.format(loadfilename)) filedisplay.configure( text="The Student data will be imported from \n " + loadfilename)
def selectDB(dataLabel): # open the DB. and open an connection. Store it as a global variable. Show error message global currtable global dbconn global currdb if dbconn != None: logger.info(messages.DB_DETAILS_CLEAN.format(currdb, currtable)) dbconn.close() currdb = None currtable = None serverdir = PROJECT_ROOT + os.path.sep + SERVER filepattern = serverdir + os.path.sep + "*.db" if not (glob.glob(filepattern)): logger.warning(messages.NO_DB_EXISTS) messagebox.showerror(title="No DATABASE PRESENT", message=messages.NO_DB_EXISTS) return else: dbpath = filedialog.askopenfilename(initialdir=Path(SERVER), title="Select an Database file", filetypes=(("SQLite3 Databases", "*.db"), )) if dbpath == '': logger.warn(messages.INVALID_DB_SELECTION) messagebox.showinfo(title="INVALID_DB_SELECTION", message=messages.INVALID_DB_SELECTION) dataLabel.configure(text="Currently not connected to any Database") return else: dbname = os.path.basename(Path(dbpath)) dbconn = utils.connectwithDB(dbpath) if dbconn: currdb = dbname logger.info(messages.CURRDB_VALUE.format(currdb)) dataLabel.configure(text="Successfully connected to " + currdb + " database")
def change_permissions(path, permission=None, recursive=True): '''change_permissions will change all permissions of files and directories. Recursive is default True, given a folder :param path: path to change permissions for :param permission: the permission from stat to add (default is stat.S_IWUSR) :param recursive: do recursively (default is True) ''' # Default permission to change is adding write if permission == None: permission = stat.S_IWUSR # For a file, recursion is not relevant if os.path.isfile(path): logger.info("Changing permission of %s to %s", path, oct(permission)) change_permission(path, permission) else: # If the user wants recursive, use os.walk logger.info("Changing permission of files and folders under %s to %s", path, oct(permission)) for root, dirs, files in os.walk(path, topdown=False, followlinks=False): # Walking through directories for name in dirs: dir_path = os.path.join(root, name) # Make sure it's a valid dir if os.path.isdir(dir_path): change_permission(dir_path, permission) # Walking through files (and checking for symbolic links) for name in files: file_path = os.path.join(root, name) # Make sure it's a valid file if os.path.isfile(file_path) and not os.path.islink(file_path): change_permission(file_path, permission)
def backup(prjroot, storagefolder, backupfolder): today = datetime.date.today() today = (str(today)).replace('-', '_') onlydbs = [ f for f in os.listdir(storagefolder) if os.path.isfile(os.path.join(storagefolder, f)) and f.endswith(".db") ] currdatefolder = Path(backupfolder + os.path.sep + today) if not os.path.exists(currdatefolder): os.mkdir(currdatefolder) try: for eachdb in onlydbs: logger.info(messages.CURRENT_BACKUP_DB.format(eachdb)) storagedbpath = Path(storagefolder + os.path.sep + eachdb) storagedbconn = sqlite3.connect(storagedbpath) backupdbpath = Path(backupfolder + os.path.sep + today + os.path.sep + eachdb) backupdbconn = sqlite3.connect(backupdbpath) logger.info(messages.BACKUP_TARGET.format(backupdbpath._str)) storagedbconn.backup(backupdbconn, pages=0, progress=progress) storagedbconn.commit() backupdbconn.commit() storagedbconn.close() backupdbconn.close() logger.info(messages.BACKUP_SUCCESS.format(eachdb)) except sqlite3.Error as e: logger.critical(messages.BACKUP_CREATION_FAILED.format(eachdb)) logger.critical(e, exc_info=True) return False except: logger.error(messages.UNEXPECTED_ERROR) logger.error("Exception : ", exc_info=True) return False finally: if storagedbconn: storagedbconn.close() if backupdbconn: backupdbconn.close()
def run(args): # Find root filesystem location if args.rootfs != None: singularity_rootfs = args.rootfs else: singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None) if singularity_rootfs == None and args.shub == None: logger.error( "root file system not specified OR defined as environmental variable, exiting!" ) sys.exit(1) if singularity_rootfs != None: logger.info("Root file system defined as %s", singularity_rootfs) # Does the registry require authentication? auth = None if args.username is not None and args.password is not None: auth = basic_auth_header(args.username, args.password) logger.info("Username for registry authentication: %s", args.username) # Does the user want to download a Singularity image? if args.shub != None: image = args.shub manifest = get_shub_manifest(image) if args.pull_folder == None: cache_base = get_cache(subfolder="shub", disable_cache=args.disable_cache) else: cache_base = args.pull_folder # The image name is the md5 hash, download if it's not there image_name = get_image_name(manifest) image_file = "%s/%s" % (cache_base, image_name) if not os.path.exists(image_file): image_file = download_image(manifest=manifest, download_folder=cache_base) else: print("Image already exists at %s, skipping download." % image_file) logger.info("Singularity Hub Image Download: %s", image_file) # If singularity_rootfs is provided, write metadata to it if singularity_rootfs != None: logger.debug( "Writing SINGULARITY_RUNDIR and SINGULARITY_IMAGE to %s", singularity_rootfs) write_file("%s/SINGULARITY_RUNDIR" % singularity_rootfs, os.path.dirname(image_file)) write_file("%s/SINGULARITY_IMAGE" % singularity_rootfs, image_file) # Do we have a docker image specified? elif args.docker != None: # Does the user want to override default Entrypoint and use CMD as runscript? includecmd = args.includecmd logger.info("Including Docker command as Runscript? %s", includecmd) image = args.docker logger.info("Docker image: %s", image) # Input Parsing ---------------------------- # Parse image name, repo name, and namespace image = parse_image_uri(image=image, uri="docker://") namespace = image['namespace'] repo_name = image['repo_name'] repo_tag = image['repo_tag'] # Tell the user the namespace, repo name and tag logger.info("Docker image path: %s/%s:%s", namespace, repo_name, repo_tag) # IMAGE METADATA ------------------------------------------- # Use Docker Registry API (version 2.0) to get images ids, manifest # Get an image manifest - has image ids to parse, and will be # used later to get Cmd manifest = get_manifest(repo_name=repo_name, namespace=namespace, repo_tag=repo_tag, registry=args.registry, auth=auth) # Get images from manifest using version 2.0 of Docker Registry API images = get_images(manifest=manifest) # DOWNLOAD LAYERS ------------------------------------------- # Each is a .tar.gz file, obtained from registry with curl # Get the cache (or temporary one) for docker cache_base = get_cache(subfolder="docker", disable_cache=args.disable_cache) layers = [] for image_id in images: # Download the layer, if we don't have it targz = "%s/%s.tar.gz" % (cache_base, image_id) if not os.path.exists(targz): targz = get_layer(image_id=image_id, namespace=namespace, repo_name=repo_name, download_folder=cache_base, registry=args.registry, auth=auth) layers.append(targz) # in case we want a list at the end # Extract image and remove tar output = extract_tar(targz, singularity_rootfs) if output is None: logger.error("Error extracting image: %s", targz) sys.exit(1) if args.disable_cache == True: os.remove(targz) # If the user wants to include the CMD as runscript, generate it here if includecmd == True: spec = "Cmd" else: spec = "Entrypoint" cmd = get_config(manifest, spec=spec) # Only add runscript if command is defined if cmd != None: print("Adding Docker %s as Singularity runscript..." % (spec.upper())) print(cmd) runscript = create_runscript(cmd=cmd, base_dir=singularity_rootfs) # When we finish, clean up images if args.disable_cache == True: shutil.rmtree(cache_base) logger.info("*** FINISHING DOCKER BOOTSTRAP PYTHON PORTION ****\n")
def main(): logger.info("\n*** STARTING DOCKER BOOTSTRAP PYTHON PORTION ****") parser = argparse.ArgumentParser( description="bootstrap Docker images for Singularity containers") # Name of the docker image, required parser.add_argument( "--docker", dest='docker', help= "name of Docker image to bootstrap, in format library/ubuntu:latest", type=str, default=None) # root file system of singularity image parser.add_argument("--rootfs", dest='rootfs', help="the path for the root filesystem to extract to", type=str, default=None) # Docker registry (default is registry-1.docker.io parser.add_argument( "--registry", dest='registry', help="the registry path to use, to replace registry-1.docker.io", type=str, default=None) # Flag to add the Docker CMD as a runscript parser.add_argument( "--cmd", dest='includecmd', action="store_true", help= "boolean to specify that CMD should be used instead of ENTRYPOINT as the runscript.", default=False) parser.add_argument("--username", dest='username', help="username for registry authentication", default=None) parser.add_argument("--password", dest='password', help="password for registry authentication", default=None) # Flag to disable cache parser.add_argument("--no-cache", dest='disable_cache', action="store_true", help="boolean to specify disabling the cache.", default=False) try: args = parser.parse_args() except: logger.error("Input args to %s improperly set, exiting.", os.path.abspath(__file__)) parser.print_help() sys.exit(0) # Find root filesystem location if args.rootfs != None: singularity_rootfs = args.rootfs logger.info("Root file system defined by command line variable as %s", singularity_rootfs) else: singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None) if singularity_rootfs == None: logger.error( "root file system not specified OR defined as environmental variable, exiting!" ) sys.exit(1) logger.info("Root file system defined by env variable as %s", singularity_rootfs) # Does the registry require authentication? auth = None if args.username is not None and args.password is not None: auth = basic_auth_header(args.username, args.password) logger.info("Username for registry authentication: %s", args.username) # Does the user want to override default Entrypoint and use CMD as runscript? includecmd = args.includecmd logger.info("Including Docker command as Runscript? %s", includecmd) # Do we have a docker image specified? if args.docker != None: image = args.docker logger.info("Docker image: %s", image) # INPUT PARSING ------------------------------------------- # Parse image name, repo name, and namespace # First split the docker image name by / image = image.split('/') # If there are two parts, we have namespace with repo (and maybe tab) if len(image) == 2: namespace = image[0] image = image[1] # Otherwise, we must be using library namespace else: namespace = "library" image = image[0] # Now split the docker image name by : image = image.split(':') if len(image) == 2: repo_name = image[0] repo_tag = image[1] # Otherwise, assume latest of an image else: repo_name = image[0] repo_tag = "latest" # Tell the user the namespace, repo name and tag logger.info("Docker image path: %s/%s:%s", namespace, repo_name, repo_tag) # IMAGE METADATA ------------------------------------------- # Use Docker Registry API (version 2.0) to get images ids, manifest # Get an image manifest - has image ids to parse, and will be # used later to get Cmd manifest = get_manifest(repo_name=repo_name, namespace=namespace, repo_tag=repo_tag, registry=args.registry, auth=auth) # Get images from manifest using version 2.0 of Docker Registry API images = get_images(manifest=manifest, registry=args.registry, auth=auth) # DOWNLOAD LAYERS ------------------------------------------- # Each is a .tar.gz file, obtained from registry with curl # Get the cache (or temporary one) for docker cache_base = get_cache(subfolder="docker", disable_cache=args.disable_cache) layers = [] for image_id in images: # Download the layer, if we don't have it targz = "%s/%s.tar.gz" % (cache_base, image_id) if not os.path.exists(targz): targz = get_layer(image_id=image_id, namespace=namespace, repo_name=repo_name, download_folder=cache_base, registry=args.registry, auth=auth) layers.append(targz) # in case we want a list at the end # @chrisfilo suggestion to try compiling into one tar.gz # Extract image and remove tar extract_tar(targz, singularity_rootfs) if args.disable_cache == True: os.remove(targz) # If the user wants to include the CMD as runscript, generate it here if includecmd == True: spec = "Cmd" else: spec = "Entrypoint" cmd = get_config(manifest, spec=spec) # Only add runscript if command is defined if cmd != None: print("Adding Docker %s as Singularity runscript..." % (spec.upper())) print(cmd) runscript = create_runscript(cmd=cmd, base_dir=singularity_rootfs) # When we finish, clean up images if args.disable_cache == True: shutil.rmtree(cache_base) logger.info("*** FINISHING DOCKER BOOTSTRAP PYTHON PORTION ****\n")
quitbutton = Button(lowerframe, text="Quit", bg="#decccc", font=("Verdana", 15), bd=5, relief="raised", command=quitapplication) quitbutton.place(relx=0.53, rely=0.60, relwidth=0.40, relheight=0.2) allframes = [] allframes.append(upperframe) allframes.append(lowerframe) allframes.append(comboframe) allframes.append(dataframe) ######################################### INITIALIZERS ################################################################# logger.info(messages.APPLN_START) utils.verifySetup(PROJECT_ROOT, SERVER, CONFIG, BACKUPS, EXPORTS) # Entry point for the program, where homescreen is displayed first homescreen() # Basic Root Window Initializations Logo = utils.resource_path(ATHLETE_ICON_FILE) window.iconphoto(True, PhotoImage(file=Logo)) window.title('MSSSDO v1.0') window.geometry("750x850") window.protocol("WM_DELETE_WINDOW", quitapplication) window.mainloop()
def updateStudentData(heightEntry, speedEntry, enduranceEntry, strengthEntry, explosiveEntry, agilityEntry, dataLabel): global recordDict height = heightEntry.get() fiftytime = speedEntry.get() eighthundredtime = enduranceEntry.get() shotputdist = strengthEntry.get() longjumpdist = explosiveEntry.get() agilitytime = agilityEntry.get() logger.info( messages.UPDATE_VALUES.format(height, fiftytime, eighthundredtime, shotputdist, longjumpdist, agilitytime)) if (height == "" and fiftytime == "" and eighthundredtime == "" and shotputdist == "" and longjumpdist == "" and agilitytime == ""): messagebox.showerror(title="EMPTY_VALUES", message=messages.EMPTY_VALUES) return if not recordDict: messagebox.showerror(title="NO_STUDENT_CHOSEN", message=messages.NO_STUDENT_CHOSEN) return dynamicRecordDict = recordDict.copy() updateQueries, recordDict = utils.getUpdateQueries( height, fiftytime, eighthundredtime, shotputdist, longjumpdist, agilitytime, dynamicRecordDict) logger.info(messages.UPDATE_QUERY_NUM.format(len(updateQueries))) if len(updateQueries) < 1: logger.info(messages.NO_UPDATE_QUERIES) del dynamicRecordDict return finalupdatequery = utils.formUpdateQuery(currtable, updateQueries, dynamicRecordDict) logger.info(messages.UPDATE_QUERY.format(finalupdatequery)) result = utils.updateTable(dbconn, finalupdatequery) if result: logger.info(messages.SPORT_UPDATE_PASSED) totalscoreupdatequery = utils.updateTotalScore(currtable, dynamicRecordDict) logger.info(messages.UPDATE_QUERY.format(totalscoreupdatequery)) combinedresult = utils.updateTable(dbconn, totalscoreupdatequery) else: logger.error(messages.SPORT_UPDATE_FAILED) combinedresult = False if combinedresult: logger.info(messages.UPDATE_RECORD_PASS) dataLabel.config(text="Student Record was updated Successfully") recordDict.update(dynamicRecordDict) else: messagebox.showerror(title="UPDATE_FAILED", message=messages.UPDATE_FAILED) del dynamicRecordDict return
def run(args): # Find root filesystem location if args.rootfs != None: singularity_rootfs = args.rootfs else: singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None) if singularity_rootfs == None and args.shub == None: logger.error("root file system not specified OR defined as environmental variable, exiting!") sys.exit(1) if singularity_rootfs != None: logger.info("Root file system defined as %s", singularity_rootfs) # Does the registry require authentication? auth = None if args.username is not None and args.password is not None: auth = basic_auth_header(args.username, args.password) logger.info("Username for registry authentication: %s", args.username) # Does the user want to download a Singularity image? if args.shub != None: image_id = int(args.shub) manifest = get_shub_manifest(image_id) cache_base = get_cache(subfolder="shub", disable_cache = args.disable_cache) # The image name is the md5 hash, download if it's not there image_name = get_image_name(manifest) image_file = "%s/%s" %(cache_base,image_name) if not os.path.exists(image_file): image_file = download_image(manifest=manifest, download_folder=cache_base) else: print("Image already exists at %s, skipping download." %image_file) logger.info("Singularity Hub Image Download: %s", image_file) # If singularity_rootfs is provided, write metadata to it if singularity_rootfs != None: logger.debug("Writing SINGULARITY_RUNDIR and SINGULARITY_IMAGE to %s",singularity_rootfs) write_file("%s/SINGULARITY_RUNDIR" %singularity_rootfs, os.path.dirname(image_file)) write_file("%s/SINGULARITY_IMAGE" %singularity_rootfs, image_file) # Do we have a docker image specified? elif args.docker != None: # Does the user want to override default Entrypoint and use CMD as runscript? includecmd = args.includecmd logger.info("Including Docker command as Runscript? %s", includecmd) image = args.docker logger.info("Docker image: %s", image) # Input Parsing ---------------------------- # Parse image name, repo name, and namespace # First split the docker image name by / image = image.split('/') # If there are two parts, we have namespace with repo (and maybe tab) if len(image) == 2: namespace = image[0] image = image[1] # Otherwise, we must be using library namespace else: namespace = "library" image = image[0] # Now split the docker image name by : image = image.split(':') if len(image) == 2: repo_name = image[0] repo_tag = image[1] # Otherwise, assume latest of an image else: repo_name = image[0] repo_tag = "latest" # Tell the user the namespace, repo name and tag logger.info("Docker image path: %s/%s:%s", namespace,repo_name,repo_tag) # IMAGE METADATA ------------------------------------------- # Use Docker Registry API (version 2.0) to get images ids, manifest # Get an image manifest - has image ids to parse, and will be # used later to get Cmd manifest = get_manifest(repo_name=repo_name, namespace=namespace, repo_tag=repo_tag, registry=args.registry, auth=auth) # Get images from manifest using version 2.0 of Docker Registry API images = get_images(repo_name=repo_name, namespace=namespace, registry=args.registry, auth=auth) # DOWNLOAD LAYERS ------------------------------------------- # Each is a .tar.gz file, obtained from registry with curl # Get the cache (or temporary one) for docker cache_base = get_cache(subfolder="docker", disable_cache = args.disable_cache) layers = [] for image_id in images: # Download the layer, if we don't have it targz = "%s/%s.tar.gz" %(cache_base,image_id) if not os.path.exists(targz): targz = get_layer(image_id=image_id, namespace=namespace, repo_name=repo_name, download_folder=cache_base, registry=args.registry, auth=auth) layers.append(targz) # in case we want a list at the end # Extract image and remove tar output = extract_tar(targz,singularity_rootfs) if output is None: logger.error("Error extracting image: %s", targz) sys.exit(1) if args.disable_cache == True: os.remove(targz) # If the user wants to include the CMD as runscript, generate it here if includecmd == True: spec="Cmd" else: spec="Entrypoint" cmd = get_config(manifest,spec=spec) # Only add runscript if command is defined if cmd != None: print("Adding Docker %s as Singularity runscript..." %(spec.upper())) print(cmd) runscript = create_runscript(cmd=cmd, base_dir=singularity_rootfs) # When we finish, clean up images if args.disable_cache == True: shutil.rmtree(cache_base) logger.info("*** FINISHING DOCKER BOOTSTRAP PYTHON PORTION ****\n")