Beispiel #1
0
def get_tags(namespace,repo_name,registry=None,auth=None):
    '''get_tags will return the tags for a repo using the Docker Version 2.0 Registry API
    :param namespace: the namespace (eg, "library")
    :param repo_name: the name for the repo (eg, "ubuntu")
    :param registry: the docker registry to use (default will use index.docker.io)
    :param auth: authorization header (default None)
    '''
    if registry == None:
        registry = api_base
    registry = add_http(registry) # make sure we have a complete url

    base = "%s/%s/%s/%s/tags/list" %(registry,api_version,namespace,repo_name)
    logger.info("Obtaining tags: %s", base)

    token = get_token(registry=registry,
                      repo_name=repo_name,
                      namespace=namespace,
                      auth=auth)

    response = api_get(base,headers=token)
    try:
        response = json.loads(response)
        return response['tags']
    except:
        logger.error("Error obtaining tags: %s", base)
        sys.exit(1)
Beispiel #2
0
def api_get_pagination(url):
    '''api_pagination is a wrapper for "api_get" that will also handle pagination
   :param url: the url to retrieve:
   :param default_header: include default_header (above)
   :param headers: headers to add (default is None)
   '''
    done = False
    results = []
    while not done:
        response = api_get(url=url)
        try:
            response = json.loads(response)
        except:
            logger.error("Error parsing response for url %s, exiting.", url)
            sys.exit(1)

        # If we have a next url
        if "next" in response:
            url = response["next"]
        else:
            done = True

        # Add new call to the results
        results = results + response['results']
    return results
Beispiel #3
0
def dockerfile_to_singularity(dockerfile_path, output_dir=None):
    '''dockerfile_to_singularity will return a Singularity build file based on
    a provided Dockerfile. If output directory is not specified, the string
    will be returned. Otherwise, a file called Singularity will be written to 
    output_dir
    :param dockerfile_path: the path to the Dockerfile
    :param output_dir: the output directory to write the Singularity file to
    '''
    if os.path.basename(dockerfile_path) == "Dockerfile":
        spec = read_file(dockerfile_path)
        # Use a common mapping
        mapping = get_mapping()

        # Put into dict of keys (section titles) and list of commands (values)
        sections = organize_sections(lines=spec, mapping=mapping)

        # We have to, by default, add the Docker bootstrap
        sections["bootstrap"] = ["docker"]

        # Put into one string based on "order" variable in mapping
        build_file = print_sections(sections=sections, mapping=mapping)
        if output_dir != None:
            write_file("%s/Singularity" % (output_dir), build_file)
            print("Singularity spec written to %s" % (output_dir))
        return build_file

    # If we make it here, something didn't work
    logger.error("Could not find %s, exiting.", dockerfile_path)
    return sys.exit(1)
Beispiel #4
0
def verifySetup(prjroot, SERVER, CONFIG, BACKUPS, EXPORTS):

    try:
        serverpath = Path(prjroot + os.path.sep + SERVER)
        if not os.path.exists(serverpath):
            os.mkdir(serverpath)
            logger.info(messages.SERVER_FOLDER_CREATE)

        configpath = Path(prjroot + os.path.sep + CONFIG)
        if not os.path.exists(configpath):
            os.mkdir(configpath)
            logger.info(messages.CONFIG_FOLDER_CREATE)

        backuppath = Path(prjroot + os.path.sep + BACKUPS)
        if not os.path.exists(backuppath):
            os.mkdir(backuppath)
            logger.info(messages.BACKUPS_FOLDER_CREATE)

        exportpath = Path(prjroot + os.path.sep + EXPORTS)
        if not os.path.exists(exportpath):
            os.mkdir(exportpath)
            logger.info(messages.EXPORTS_FOLDER_CREATE)
    except:
        logger.error(messages.UNEXPECTED_ERROR)
        logger.error("StackTrace : ", exc_info=True)
        return False
Beispiel #5
0
def rowinserter(rowtoinsert, dbconn, currtable, dbfieldlist, numrows):

    try:
        updatecursor = dbconn.cursor()

        insertquery = "INSERT INTO " + currtable + " " + str(
            tuple(dbfieldlist)) + " VALUES (" + "?, " * (len(dbfieldlist) -
                                                         1) + "?" + ")"

        updatecursor.execute(insertquery, rowtoinsert)
        dbconn.commit()

    except sqlite3.IntegrityError as ie:
        logger.critical(
            messages.RECORD_EXISTS.format(
                currtable, rowtoinsert[dbfieldlist.index("Roll_Number")]))
        logger.critical(ie, exc_info=True)
        return numrows
    except sqlite3.Error as e:
        logger.critical(messages.INSERT_QUERY_FAILED.format(insertquery))
        logger.critical(e, exc_info=True)
        return numrows
    except:
        logger.error(messages.UNEXPECTED_ERROR)
        logger.error("StackTrace : ", exc_info=True)
        return numrows

    return numrows + 1
Beispiel #6
0
def get_tags(namespace, repo_name, registry=None, auth=None):
    '''get_tags will return the tags for a repo using the Docker Version 2.0 Registry API
    :param namespace: the namespace (eg, "library")
    :param repo_name: the name for the repo (eg, "ubuntu")
    :param registry: the docker registry to use (default will use index.docker.io)
    :param auth: authorization header (default None)
    '''
    if registry == None:
        registry = api_base
    registry = add_http(registry)  # make sure we have a complete url

    base = "%s/%s/%s/%s/tags/list" % (registry, api_version, namespace,
                                      repo_name)
    logger.info("Obtaining tags: %s", base)

    token = get_token(registry=registry,
                      repo_name=repo_name,
                      namespace=namespace,
                      auth=auth)

    response = api_get(base, headers=token)
    try:
        response = json.loads(response)
        return response['tags']
    except:
        logger.error("Error obtaining tags: %s", base)
        sys.exit(1)
Beispiel #7
0
def dockerfile_to_singularity(dockerfile_path, output_dir=None):
    '''dockerfile_to_singularity will return a Singularity build file based on
    a provided Dockerfile. If output directory is not specified, the string
    will be returned. Otherwise, a file called Singularity will be written to 
    output_dir
    :param dockerfile_path: the path to the Dockerfile
    :param output_dir: the output directory to write the Singularity file to
    '''
    if os.path.basename(dockerfile_path) == "Dockerfile":
        spec = read_file(dockerfile_path)
        # Use a common mapping
        mapping = get_mapping()
   
        # Put into dict of keys (section titles) and list of commands (values)
        sections = organize_sections(lines=spec,
                                     mapping=mapping)

        # We have to, by default, add the Docker bootstrap
        sections["bootstrap"] = ["docker"]

        # Put into one string based on "order" variable in mapping
        build_file = print_sections(sections=sections,
                                    mapping=mapping)
        if output_dir != None:
            write_file("%s/Singularity" %(output_dir),build_file)
            print("Singularity spec written to %s" %(output_dir))
        return build_file

    # If we make it here, something didn't work
    logger.error("Could not find %s, exiting.", dockerfile_path)
    return sys.exit(1)
Beispiel #8
0
def getSchemaFromTable(conn, currtable):

    templist = []
    try:
        cur = conn.cursor()

        schemaPragma = "PRAGMA table_info('" + currtable + "')"
        logger.info(messages.SCHEMA_QUERY.format(schemaPragma))

        cur.execute(schemaPragma)
        schema = cur.fetchall()

        for row in schema:
            templist.append(row[1])

        st = " "
        logger.info(
            messages.FETCHED_SCHEMA.format(currtable, st.join(templist)))

    except sqlite3.Error as e:
        logger.critical(messages.SCHEMA_PRAGMA_FAILED.format(schemaPragma))
        logger.critical(e, exc_info=True)
    except:
        logger.error(messages.UNEXPECTED_ERROR)
        logger.error("StackTrace : ", exc_info=True)
    finally:
        return templist
Beispiel #9
0
def api_get_pagination(url):
   '''api_pagination is a wrapper for "api_get" that will also handle pagination
   :param url: the url to retrieve:
   :param default_header: include default_header (above)
   :param headers: headers to add (default is None)
   '''
   done = False
   results = []
   while not done:
       response = api_get(url=url)
       try:
           response = json.loads(response)
       except:
           logger.error("Error parsing response for url %s, exiting.", url)        
           sys.exit(1)

       # If we have a next url
       if "next" in response:
           url = response["next"]
       else:
           done = True
 
       # Add new call to the results
       results = results + response['results']
   return results
Beispiel #10
0
def read_digests(manifest):
    '''read_layers will return a list of layers from a manifest. The function is
    intended to work with both version 1 and 2 of the schema
    :param manifest: the manifest to read_layers from
    '''

    digests = []

    # https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-2.md#image-manifest
    if 'layers' in manifest:
        layer_key = 'layers'
        digest_key = 'digest'
        logger.info('Image manifest version 2.2 found.')

    # https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-1.md#example-manifest
    elif 'fsLayers' in manifest:
        layer_key = 'fsLayers'
        digest_key = 'blobSum'
        logger.info('Image manifest version 2.1 found.')

    else:
        logger.error(
            'Improperly formed manifest, layers or fsLayers must be present')
        sys.exit(1)

    for layer in manifest[layer_key]:
        if digest_key in layer:
            if layer[digest_key] not in digests:
                logger.info("Adding digest %s", layer[digest_key])
                digests.append(layer[digest_key])
    return digests
Beispiel #11
0
def get_images(repo_name=None,
               namespace=None,
               manifest=None,
               repo_tag="latest",
               registry=None,
               auth=None):
    '''get_images is a wrapper for get_manifest, but it additionally parses the repo_name and tag's
    images and returns the complete ids
    :param repo_name: the name of the repo, eg "ubuntu"
    :param namespace: the namespace for the image, default is "library"
    :param repo_tag: the repo tag, default is "latest"
    :param registry: the docker registry url, default will use index.docker.io
    '''

    # Get full image manifest, using version 2.0 of Docker Registry API
    if manifest == None:
        if repo_name != None and namespace != None:
            manifest = get_manifest(repo_name=repo_name,
                                    namespace=namespace,
                                    repo_tag=repo_tag,
                                    registry=registry,
                                    auth=auth)
        else:
            logger.error(
                "No namespace and repo name OR manifest provided, exiting.")
            sys.exit(1)

    digests = []
    if 'fsLayers' in manifest:
        for fslayer in manifest['fsLayers']:
            if 'blobSum' in fslayer:
                if fslayer['blobSum'] not in digests:
                    logger.info("Adding digest %s", fslayer['blobSum'])
                    digests.append(fslayer['blobSum'])
    return digests
Beispiel #12
0
def read_digests(manifest):
    '''read_layers will return a list of layers from a manifest. The function is
    intended to work with both version 1 and 2 of the schema
    :param manifest: the manifest to read_layers from
    '''

    digests = []

    # https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-2.md#image-manifest
    if 'layers' in manifest:
        layer_key = 'layers'
        digest_key = 'digest'
        logger.info('Image manifest version 2.2 found.')

    # https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-1.md#example-manifest
    elif 'fsLayers' in manifest:
        layer_key = 'fsLayers'
        digest_key = 'blobSum'
        logger.info('Image manifest version 2.1 found.')

    else:
        logger.error('Improperly formed manifest, layers or fsLayers must be present')
        sys.exit(1)

    for layer in manifest[layer_key]:
        if digest_key in layer:
            if layer[digest_key] not in digests:
                logger.info("Adding digest %s",layer[digest_key])
                digests.append(layer[digest_key])
    return digests
Beispiel #13
0
def get_layer(image_id,namespace,repo_name,download_folder=None,registry=None,auth=None):
    '''get_layer will download an image layer (.tar.gz) to a specified download folder.
    :param image_id: the (full) image id to get the manifest for, required
    :param namespace: the namespace (eg, "library")
    :param repo_name: the repo name, (eg, "ubuntu")
    :param download_folder: if specified, download to folder. Otherwise return response with raw data (not recommended)
    :param registry: the docker registry to use (default will use index.docker.io)
    :param auth: authorization header (default None)
    '''
    if registry == None:
        registry = api_base
    registry = add_http(registry) # make sure we have a complete url

    # The <name> variable is the namespace/repo_name
    base = "%s/%s/%s/%s/blobs/%s" %(registry,api_version,namespace,repo_name,image_id)
    logger.info("Downloading layers from %s", base)
    
    # To get the image layers, we need a valid token to read the repo
    token = get_token(registry=registry,
                      repo_name=repo_name,
                      namespace=namespace,
                      auth=auth)

    if download_folder != None:
        download_folder = "%s/%s.tar.gz" %(download_folder,image_id)

        # Update user what we are doing
        print("Downloading layer %s" %image_id)

    response = api_get(base,headers=token,stream=download_folder)
    if isinstance(response, HTTPError):
        logger.error("Error downloading layer %s, exiting.", base)
        sys.exit(1)

    return response
Beispiel #14
0
def get_images(repo_name=None,namespace=None,manifest=None,repo_tag="latest",registry=None,auth=None):
    '''get_images is a wrapper for get_manifest, but it additionally parses the repo_name and tag's
    images and returns the complete ids
    :param repo_name: the name of the repo, eg "ubuntu"
    :param namespace: the namespace for the image, default is "library"
    :param repo_tag: the repo tag, default is "latest"
    :param registry: the docker registry url, default will use index.docker.io
    '''

    # Get full image manifest, using version 2.0 of Docker Registry API
    if manifest == None:
        if repo_name != None and namespace != None:

            # Custom header to specify we want a list of the version 2 schema, meaning the correct order of digests returned (base to child)
            headers = {"Accept":'application/vnd.docker.distribution.manifest.v2+json,application/vnd.docker.distribution.manifest.list.v2+json'}
            manifest = get_manifest(repo_name=repo_name,
                                    namespace=namespace,
                                    repo_tag=repo_tag,
                                    registry=registry,
                                    headers=headers,
                                    auth=auth)
        else:
            logger.error("No namespace and repo name OR manifest provided, exiting.")
            sys.exit(1)

    digests = read_digests(manifest)
    return digests
Beispiel #15
0
def get_manifest(repo_name,
                 namespace,
                 repo_tag="latest",
                 registry=None,
                 auth=None,
                 headers=None):
    '''get_manifest should return an image manifest for a particular repo and tag. The token is expected to
    be from version 2.0 (function above)
    :param repo_name: the name of the repo, eg "ubuntu"
    :param namespace: the namespace for the image, default is "library"
    :param repo_tag: the repo tag, default is "latest"
    :param registry: the docker registry to use (default will use index.docker.io)
    :param auth: authorization header (default None)
    :param headers: dictionary of custom headers to add to token header (to get more specific manifest)
    '''
    if registry == None:
        registry = api_base
    registry = add_http(registry)  # make sure we have a complete url

    base = "%s/%s/%s/%s/manifests/%s" % (registry, api_version, namespace,
                                         repo_name, repo_tag)
    logger.info("Obtaining manifest: %s", base)

    # Format the token, and prepare a header
    token = get_token(registry=registry,
                      repo_name=repo_name,
                      namespace=namespace,
                      auth=auth)

    # Add ['Accept'] header to specify version 2 of manifest
    if headers != None:
        if token != None:
            token.update(headers)
        else:
            token = headers

    response = api_get(base, headers=token, default_header=True)
    try:
        response = json.loads(response)
    except:
        # If the call fails, give the user a list of acceptable tags
        tags = get_tags(namespace=namespace,
                        repo_name=repo_name,
                        registry=registry,
                        auth=auth)
        print("\n".join(tags))
        logger.error("Error getting manifest for %s/%s:%s, exiting.",
                     namespace, repo_name, repo_tag)
        print(
            "Error getting manifest for %s/%s:%s. Acceptable tags are listed above."
            % (namespace, repo_name, repo_tag))
        sys.exit(1)

    return response
Beispiel #16
0
def create_folders(path):
    '''create_folders attempts to get the same functionality as mkdir -p
    :param path: the path to create.
    '''
    try:
        os.makedirs(path)
    except OSError as e:
        if e.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            logger.error("Error creating path %s, exiting.",path)
            sys.exit(1)
Beispiel #17
0
def create_folders(path):
    '''create_folders attempts to get the same functionality as mkdir -p
    :param path: the path to create.
    '''
    try:
        os.makedirs(path)
    except OSError as e:
        if e.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            logger.error("Error creating path %s, exiting.", path)
            sys.exit(1)
Beispiel #18
0
def api_get(url,
            data=None,
            default_header=True,
            headers=None,
            stream=None,
            return_response=False):
    '''api_get gets a url to the api with appropriate headers, and any optional data
    :param data: a dictionary of key:value items to add to the data args variable
    :param url: the url to get
    :param stream: The name of a file to stream the response to. If defined, will stream
    default is None (will not stream)
    :returns response: the requests response object
    '''
    headers = parse_headers(default_header=default_header, headers=headers)

    # Does the user want to stream a response?
    do_stream = False
    if stream != None:
        do_stream = True

    if data != None:
        args = urlencode(data)
        request = Request(url=url, data=args, headers=headers)
    else:
        request = Request(url=url, headers=headers)

    try:
        response = urlopen(request)

    # If we have an HTTPError, try to follow the response
    except HTTPError as error:
        return error

    # Does the call just want to return the response?
    if return_response == True:
        return response

    if do_stream == False:
        return response.read().decode('utf-8')

    chunk_size = 1 << 20
    with open(stream, 'wb') as filey:
        while True:
            chunk = response.read(chunk_size)
            if not chunk:
                break
            try:
                filey.write(chunk)
            except:  #PermissionError
                logger.error("Cannot write to %s, exiting", stream)
                sys.exit(1)

    return stream
Beispiel #19
0
def api_get(url,data=None,default_header=True,headers=None,stream=None,return_response=False):
    '''api_get gets a url to the api with appropriate headers, and any optional data
    :param data: a dictionary of key:value items to add to the data args variable
    :param url: the url to get
    :param stream: The name of a file to stream the response to. If defined, will stream
    default is None (will not stream)
    :returns response: the requests response object
    '''
    headers = parse_headers(default_header=default_header,
                            headers=headers)

    # Does the user want to stream a response?
    do_stream = False
    if stream != None:
        do_stream = True

    if data != None:
        args = urlencode(data)
        request = Request(url=url, 
                          data=args, 
                          headers=headers) 
    else:
        request = Request(url=url, 
                          headers=headers) 

    try:
        response = urlopen(request)

    # If we have an HTTPError, try to follow the response
    except HTTPError as error:
        return error

    # Does the call just want to return the response?
    if return_response == True:
        return response

    if do_stream == False:
        return response.read().decode('utf-8')
       
    chunk_size = 1 << 20
    with open(stream, 'wb') as filey:
        while True:
            chunk = response.read(chunk_size)
            if not chunk: 
                break
            try:
                filey.write(chunk)
            except: #PermissionError
                logger.error("Cannot write to %s, exiting",stream)
                sys.exit(1)

    return stream
Beispiel #20
0
def run_command(cmd):
    '''run_command uses subprocess to send a command to the terminal.
    :param cmd: the command to send, should be a list for subprocess
    '''
    try:
        logger.info("Running command %s with subprocess", " ".join(cmd))
        process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
        output, err = process.communicate()
    except OSError as error:
        logger.error("Error with subprocess: %s, returning None", error)
        return None

    return output
Beispiel #21
0
def backup(prjroot, storagefolder, backupfolder):

    today = datetime.date.today()
    today = (str(today)).replace('-', '_')

    onlydbs = [
        f for f in os.listdir(storagefolder)
        if os.path.isfile(os.path.join(storagefolder, f)) and f.endswith(".db")
    ]

    currdatefolder = Path(backupfolder + os.path.sep + today)
    if not os.path.exists(currdatefolder):
        os.mkdir(currdatefolder)

    try:
        for eachdb in onlydbs:

            logger.info(messages.CURRENT_BACKUP_DB.format(eachdb))
            storagedbpath = Path(storagefolder + os.path.sep + eachdb)
            storagedbconn = sqlite3.connect(storagedbpath)

            backupdbpath = Path(backupfolder + os.path.sep + today +
                                os.path.sep + eachdb)
            backupdbconn = sqlite3.connect(backupdbpath)
            logger.info(messages.BACKUP_TARGET.format(backupdbpath._str))

            storagedbconn.backup(backupdbconn, pages=0, progress=progress)

            storagedbconn.commit()
            backupdbconn.commit()

            storagedbconn.close()
            backupdbconn.close()

            logger.info(messages.BACKUP_SUCCESS.format(eachdb))
    except sqlite3.Error as e:
        logger.critical(messages.BACKUP_CREATION_FAILED.format(eachdb))
        logger.critical(e, exc_info=True)
        return False
    except:
        logger.error(messages.UNEXPECTED_ERROR)
        logger.error("Exception : ", exc_info=True)
        return False
    finally:
        if storagedbconn:
            storagedbconn.close()
        if backupdbconn:
            backupdbconn.close()
Beispiel #22
0
def get_image_name(manifest,extension='img.gz',use_commit=True):
    '''get_image_name will return the image name for a manifest
    :param manifest: the image manifest with 'image' as key with download link
    :param use_commit: use the commit id to name the image (default) otherwise use md5sum
    '''
    image_url = os.path.basename(unquote(manifest['image']))
    image_name = re.findall(".+[.]%s" %(extension),image_url)
    if len(image_name) > 0:
        image_name = image_name[0]
        if use_commit == True:
            image_name = "%s.img.gz" %(manifest["version"])            
        logger.info("Singularity Hub Image: %s", image_name)
        return image_name
    else:
        logger.error("Singularity Hub Image not found with expected extension %s, exiting.",extension)
        sys.exit(1)
Beispiel #23
0
def run_command(cmd):
    '''run_command uses subprocess to send a command to the terminal.
    :param cmd: the command to send, should be a list for subprocess
    '''
    try:
        logger.info("Running command %s with subprocess", " ".join(cmd))
        process = subprocess.Popen(cmd,stdout=subprocess.PIPE)
    except OSError as error:
        logger.error("Error with subprocess: %s, returning None",error)
        return None

    output = process.communicate()[0]
    if process.returncode != 0:
        return None

    return output
Beispiel #24
0
def updateTable(dbconn, updatequery):

    try:
        cur = dbconn.cursor()
        cur.execute(updatequery)
        dbconn.commit()
    except sqlite3.Error as e:
        logger.critical(messages.UPDATE_TABLE_FAILED.format(updatequery))
        logger.critical(e, exc_info=True)
        return False
    except:
        logger.error(messages.UNEXPECTED_ERROR)
        logger.error("Exception : ", exc_info=True)
        return False

    return True
Beispiel #25
0
def main():
    '''main is a wrapper for the client to hand the parser to the executable functions
    This makes it possible to set up a parser in test cases
    '''
    logger.info("\n*** STARTING PYTHON CLIENT PORTION ****")
    parser = get_parser()
    
    try:
        (args,options) = parser.parse_args()
    except:
        logger.error("Input args to %s improperly set, exiting.", os.path.abspath(__file__))
        parser.print_help()
        sys.exit(0)

    # Give the args to the main executable to run
    run(args)
Beispiel #26
0
def get_layer(image_id,namespace,repo_name,download_folder=None,registry=None,auth=None):
    '''get_layer will download an image layer (.tar.gz) to a specified download folder.
    :param image_id: the (full) image id to get the manifest for, required
    :param namespace: the namespace (eg, "library")
    :param repo_name: the repo name, (eg, "ubuntu")
    :param download_folder: if specified, download to folder. Otherwise return response with raw data (not recommended)
    :param registry: the docker registry to use (default will use index.docker.io)
    :param auth: authorization header (default None)
    '''
    if registry == None:
        registry = api_base
    registry = add_http(registry) # make sure we have a complete url

    # The <name> variable is the namespace/repo_name
    base = "%s/%s/%s/%s/blobs/%s" %(registry,api_version,namespace,repo_name,image_id)
    logger.info("Downloading layers from %s", base)
    
    # To get the image layers, we need a valid token to read the repo
    token = get_token(registry=registry,
                      repo_name=repo_name,
                      namespace=namespace,
                      auth=auth)

    if download_folder != None:
        download_folder = "%s/%s.tar.gz" %(download_folder,image_id)

        # Update user what we are doing
        print("Downloading layer %s" %image_id)

    try:
        # Create temporary file with format .tar.gz.tmp.XXXXX
        fd, tmp_file = tempfile.mkstemp(prefix=("%s.tmp." % download_folder))
        os.close(fd)
        response = api_get(base,headers=token,stream=tmp_file)
        if isinstance(response, HTTPError):
            logger.error("Error downloading layer %s, exiting.", base)
            sys.exit(1)
        os.rename(tmp_file, download_folder)
    except:
        logger.error("Removing temporary download file %s", tmp_file)
        try:
            os.remove(tmp_file)
        except:
            pass
        sys.exit(1)

    return download_folder
Beispiel #27
0
def get_manifest(repo_name,namespace,repo_tag="latest",registry=None,auth=None,headers=None):
    '''get_manifest should return an image manifest for a particular repo and tag. The token is expected to
    be from version 2.0 (function above)
    :param repo_name: the name of the repo, eg "ubuntu"
    :param namespace: the namespace for the image, default is "library"
    :param repo_tag: the repo tag, default is "latest"
    :param registry: the docker registry to use (default will use index.docker.io)
    :param auth: authorization header (default None)
    :param headers: dictionary of custom headers to add to token header (to get more specific manifest)
    '''
    if registry == None:
        registry = api_base
    registry = add_http(registry) # make sure we have a complete url

    base = "%s/%s/%s/%s/manifests/%s" %(registry,api_version,namespace,repo_name,repo_tag)
    logger.info("Obtaining manifest: %s", base)
    
    # Format the token, and prepare a header
    token = get_token(registry=registry,
                      repo_name=repo_name,
                      namespace=namespace,
                      auth=auth)

    # Add ['Accept'] header to specify version 2 of manifest
    if headers != None:
        if token != None:
            token.update(headers)
        else:
            token = headers

    response = api_get(base,headers=token,default_header=True)
    try:
        response = json.loads(response)
    except:
        # If the call fails, give the user a list of acceptable tags
        tags = get_tags(namespace=namespace,
                        repo_name=repo_name,
                        registry=registry,
                        auth=auth)
        print("\n".join(tags))
        logger.error("Error getting manifest for %s/%s:%s, exiting.", namespace,
                                                                       repo_name,
                                                                       repo_tag)
        print("Error getting manifest for %s/%s:%s. Acceptable tags are listed above." %(namespace,repo_name,repo_tag))
        sys.exit(1)

    return response
Beispiel #28
0
def main():
    '''main is a wrapper for the client to hand the parser to the executable functions
    This makes it possible to set up a parser in test cases
    '''
    logger.info("\n*** STARTING PYTHON CLIENT PORTION ****")
    parser = get_parser()

    try:
        (args, options) = parser.parse_args()
    except:
        logger.error("Input args to %s improperly set, exiting.",
                     os.path.abspath(__file__))
        parser.print_help()
        sys.exit(0)

    # Give the args to the main executable to run
    run(args)
Beispiel #29
0
def get_image_name(manifest, extension='img.gz', use_commit=True):
    '''get_image_name will return the image name for a manifest
    :param manifest: the image manifest with 'image' as key with download link
    :param use_commit: use the commit id to name the image (default) otherwise use md5sum
    '''
    image_url = os.path.basename(unquote(manifest['image']))
    image_name = re.findall(".+[.]%s" % (extension), image_url)
    if len(image_name) > 0:
        image_name = image_name[0]
        if use_commit == True:
            image_name = "%s.img.gz" % (manifest["version"])
        logger.info("Singularity Hub Image: %s", image_name)
        return image_name
    else:
        logger.error(
            "Singularity Hub Image not found with expected extension %s, exiting.",
            extension)
        sys.exit(1)
Beispiel #30
0
def get_fullpath(file_path,required=True):
    '''get_fullpath checks if a file exists, and returns the
    full path to it if it does. If required is true, an error is triggered.
    :param file_path: the path to check
    :param required: is the file required? If True, will exit with error
    '''
    file_path = os.path.abspath(file_path)
    if os.path.exists(file_path):
        return file_path

    # If file is required, we exit
    if required == True:
        logger.error("Cannot find file %s, exiting.",file_path)
        sys.exit(1)

    # If file isn't required and doesn't exist, return None
    logger.warning("Cannot find file %s",file_path)
    return None
Beispiel #31
0
def get_fullpath(file_path, required=True):
    '''get_fullpath checks if a file exists, and returns the
    full path to it if it does. If required is true, an error is triggered.
    :param file_path: the path to check
    :param required: is the file required? If True, will exit with error
    '''
    file_path = os.path.abspath(file_path)
    if os.path.exists(file_path):
        return file_path

    # If file is required, we exit
    if required == True:
        logger.error("Cannot find file %s, exiting.", file_path)
        sys.exit(1)

    # If file isn't required and doesn't exist, return None
    logger.warning("Cannot find file %s", file_path)
    return None
Beispiel #32
0
def connectwithDB(dbpath):
    try:
        logger.info((messages.DB_TRY_CONNECTION).format(dbpath))
        dbconn = sqlite3.connect(dbpath)
        dbconn.commit()
    except sqlite3.Error as e:
        logger.critical((messages.DB_CONNECTION_FAIL).format(dbpath))
        logger.critical(e, exc_info=True)
        messagebox.showerror(title="CONNECTION FAILED",
                             message=messages.DB_CONN_FAIL)
    except:
        logger.error(messages.UNEXPECTED_ERROR)
        logger.error("StackTrace : ", exc_info=True)
        messagebox.showerror(title="CONNECTION FAILED",
                             message=messages.DB_CONN_FAIL)

    if dbconn:
        logger.info((messages.DB_CONNECTION_PASS).format(dbpath))
        return dbconn
Beispiel #33
0
def createtable(dbconn, tablename, schemafile):

    logger.info(messages.CREATE_TABLE.format(tablename, schemafile))

    fielddict = {}
    for line in open(schemafile):
        if len(line) == 0 or line == '\n':
            continue
        if line[0] in ('!', '#'):
            continue
        field, datatype = line.split('=')
        fielddict[field.strip()] = datatype.strip()

    createquery = "CREATE TABLE IF NOT EXISTS " + tablename + "(" + os.linesep

    fields = list(fielddict.keys())
    datatypes = list(fielddict.values())
    numfields = len(fielddict.keys())
    for i in range(0, numfields - 1):
        createquery = createquery + fields[i] + \
            " " + datatypes[i] + "," + os.linesep

    createquery = createquery + \
        fields[numfields-1] + " " + datatypes[numfields-1]
    createquery += (os.linesep + ");")

    logger.info(messages.CREATE_TABLE_QUERY.format(createquery))

    try:
        createcursor = dbconn.cursor()
        createcursor.execute(createquery)
        dbconn.commit()
    except sqlite3.Error as e:
        logger.critical(messages.CREATE_TABLE_FAILED.format(tablename))
        logger.critical(e, exc_info=True)
        return False
    except:
        logger.error(messages.UNEXPECTED_ERROR)
        logger.error("StackTrace : ", exc_info=True)
        return False

    return True
Beispiel #34
0
def main():
    '''parse configuration options and produce configuration output file
    '''
    logger.info("\n*** STARTING PYTHON CONFIGURATION HELPER ****")
    parser = get_parser()

    try:
        (args,options) = parser.parse_args()
    except:
        logger.error("Input args to %s improperly set, exiting.", os.path.abspath(__file__))
        parser.print_help()
        sys.exit(1)

    # Check for required args
    [check_required(parser,arg) for arg in [args.defaults,
                                            args.infile,
                                            args.outfile]]

    # Run the configuration
    configure(args)
Beispiel #35
0
def getStudentRecord(dbconn, currtable, studentRoll, studentName):

    searchQuery = formSearchQuery(currtable, studentRoll, studentName)
    logger.info(messages.STUDENT_SEARCH_QUERY.format(searchQuery))

    try:
        searchcursor = dbconn.cursor()
        searchcursor.execute(searchQuery)

        studentRecords = searchcursor.fetchall()

        if (len(studentRecords) == 0):
            logger.warning(messages.RECORD_NOT_FOUND)
            messagebox.showerror(title="RECORD_NOT_FOUND",
                                 message=messages.RECORD_NOT_FOUND)
            return None

        if (len(studentRecords) > 1):
            messagebox.showerror(title="DUPLICATE_RECORDS",
                                 message=messages.DUPLICATE_RECORDS)
            return None

        studentRecord = studentRecords[0]

        logger.info(messages.OBTAINED_RECORD.format(str(studentRecord)))

        return studentRecord

    except sqlite3.Error as e:
        messagebox.showerror(title="SEARCH_FAILED",
                             message=messages.SEARCH_FAILED)
        logger.critical(messages.SEARCH_FAILED.format(searchQuery))
        logger.critical(e, exc_info=True)
        return None
    except:
        messagebox.showerror(title="UNEXPECTED_ERROR",
                             message=messages.UNEXPECTED_ERROR)
        logger.error(messages.UNEXPECTED_ERROR)
        logger.error("StackTrace : ", exc_info=True)
        return None
Beispiel #36
0
def sqltoexcel(dbconn, currtable, exportfilepath):

    schemaList = getSchemaFromTable(dbconn, currtable)

    logger.info(messages.EXPORT_STARTER.format(exportfilepath, schemaList))

    # from openpyxl import Workbook
    wb = Workbook()
    # grab the active worksheet
    ws = wb.active
    # Appending Schema Row first
    ws.append(schemaList)

    try:
        cur = dbconn.cursor()
        cur.execute("SELECT * from " + currtable + ";")
        results = cur.fetchall()

        totalrows = len(results)
        exportedrows = 0

        for eachrow in results:
            ws.append(eachrow)
            exportedrows += 1

        # Save the file
        wb.save(exportfilepath)
        dbconn.commit()

    except sqlite3.Error as e:
        logger.critical(
            messages.DATA_EXPORT_FAILED.format(currtable, exportfilepath))
        logger.critical(e, exc_info=True)
        return False
    except:
        logger.error(messages.UNEXPECTED_ERROR)
        logger.error("StackTrace : ", exc_info=True)
        return False

    return (totalrows == exportedrows), exportedrows
Beispiel #37
0
def get_token(namespace, repo_name, registry=None, auth=None):
    '''get_token uses HTTP basic authentication to get a token for Docker registry API V2 operations
    :param namespace: the namespace for the image
    :param repo_name: the name of the repo, eg "ubuntu"
    :param registry: the docker registry to use
    :param auth: authorization header (default None)
    :: note
            # https://docs.docker.com/registry/spec/auth/token/
    '''
    if registry == None:
        registry = api_base
    registry = add_http(registry)  # make sure we have a complete url

    # Check if we need a token at all by probing the tags/list endpoint.  This
    # is an arbitrary choice, ideally we should always attempt without a token
    # and then retry with a token if we received a 401.
    base = "%s/%s/%s/%s/tags/list" % (registry, api_version, namespace,
                                      repo_name)
    response = api_get(base, default_header=False)
    if not isinstance(response, HTTPError):
        # No token required for registry.
        return None

    if response.code != 401 or not response.headers.has_key(
            "WWW-Authenticate"):
        logger.error("Authentication error for registry %s, exiting.",
                     registry)
        sys.exit(1)

    challenge = response.headers["WWW-Authenticate"]
    match = re.match(
        '^Bearer\s+realm="([^"]+)",service="([^"]+)",scope="([^"]+)"\s*$',
        challenge)
    if not match:
        logger.error(
            "Unrecognized authentication challenge from registry %s, exiting.",
            registry)
        sys.exit(1)

    realm = match.group(1)
    service = match.group(2)
    scope = match.group(3)

    base = "%s?service=%s&scope=%s" % (realm, service, scope)
    headers = dict()
    if auth is not None:
        headers.update(auth)

    response = api_get(base, default_header=False, headers=headers)
    try:
        token = json.loads(response)["token"]
        token = {"Authorization": "Bearer %s" % (token)}
        return token
    except:
        logger.error("Error getting token for repository %s/%s, exiting.",
                     namespace, repo_name)
        sys.exit(1)
Beispiel #38
0
def get_images(repo_name=None,
               namespace=None,
               manifest=None,
               repo_tag="latest",
               registry=None,
               auth=None):
    '''get_images is a wrapper for get_manifest, but it additionally parses the repo_name and tag's
    images and returns the complete ids
    :param repo_name: the name of the repo, eg "ubuntu"
    :param namespace: the namespace for the image, default is "library"
    :param repo_tag: the repo tag, default is "latest"
    :param registry: the docker registry url, default will use index.docker.io
    '''

    # Get full image manifest, using version 2.0 of Docker Registry API
    if manifest == None:
        if repo_name != None and namespace != None:

            # Custom header to specify we want a list of the version 2 schema, meaning the correct order of digests returned (base to child)
            headers = {
                "Accept":
                'application/vnd.docker.distribution.manifest.v2+json,application/vnd.docker.distribution.manifest.list.v2+json'
            }
            manifest = get_manifest(repo_name=repo_name,
                                    namespace=namespace,
                                    repo_tag=repo_tag,
                                    registry=registry,
                                    headers=headers,
                                    auth=auth)
        else:
            logger.error(
                "No namespace and repo name OR manifest provided, exiting.")
            sys.exit(1)

    digests = read_digests(manifest)
    return digests
Beispiel #39
0
def get_token(namespace,repo_name,registry=None,auth=None):
    '''get_token uses HTTP basic authentication to get a token for Docker registry API V2 operations
    :param namespace: the namespace for the image
    :param repo_name: the name of the repo, eg "ubuntu"
    :param registry: the docker registry to use
    :param auth: authorization header (default None)
    :: note
            # https://docs.docker.com/registry/spec/auth/token/
    '''
    if registry == None:
        registry = api_base
    registry = add_http(registry) # make sure we have a complete url

    # Check if we need a token at all by probing the tags/list endpoint.  This
    # is an arbitrary choice, ideally we should always attempt without a token
    # and then retry with a token if we received a 401.
    base = "%s/%s/%s/%s/tags/list" %(registry,api_version,namespace,repo_name)
    response = api_get(base, default_header=False)
    if not isinstance(response, HTTPError):
        # No token required for registry.
        return None

    if response.code != 401 or "WWW-Authenticate" not in response.headers:
        logger.error("Authentication error for registry %s, exiting.", registry)
        sys.exit(1)

    challenge = response.headers["WWW-Authenticate"]
    match = re.match('^Bearer\s+realm="([^"]+)",service="([^"]+)",scope="([^"]+)"\s*$', challenge)
    if not match:
        logger.error("Unrecognized authentication challenge from registry %s, exiting.", registry)
        sys.exit(1)

    realm = match.group(1)
    service = match.group(2)
    scope = match.group(3)

    base = "%s?service=%s&scope=%s" % (realm, service, scope)
    headers = dict()
    if auth is not None:
        headers.update(auth)

    response = api_get(base,default_header=False,headers=headers)
    try:
        token = json.loads(response)["token"]
        token = {"Authorization": "Bearer %s" %(token) }
        return token
    except:
        logger.error("Error getting token for repository %s/%s, exiting.", namespace,repo_name)
        sys.exit(1)
Beispiel #40
0
def main():

    logger.info("\n*** STARTING DOCKER BOOTSTRAP PYTHON PORTION ****")

    parser = argparse.ArgumentParser(
        description="bootstrap Docker images for Singularity containers")

    # Name of the docker image, required
    parser.add_argument(
        "--docker",
        dest='docker',
        help=
        "name of Docker image to bootstrap, in format library/ubuntu:latest",
        type=str,
        default=None)

    # root file system of singularity image
    parser.add_argument("--rootfs",
                        dest='rootfs',
                        help="the path for the root filesystem to extract to",
                        type=str,
                        default=None)

    # Docker registry (default is registry-1.docker.io
    parser.add_argument(
        "--registry",
        dest='registry',
        help="the registry path to use, to replace registry-1.docker.io",
        type=str,
        default=None)

    # Flag to add the Docker CMD as a runscript
    parser.add_argument(
        "--cmd",
        dest='includecmd',
        action="store_true",
        help=
        "boolean to specify that CMD should be used instead of ENTRYPOINT as the runscript.",
        default=False)

    parser.add_argument("--username",
                        dest='username',
                        help="username for registry authentication",
                        default=None)

    parser.add_argument("--password",
                        dest='password',
                        help="password for registry authentication",
                        default=None)

    # Flag to disable cache
    parser.add_argument("--no-cache",
                        dest='disable_cache',
                        action="store_true",
                        help="boolean to specify disabling the cache.",
                        default=False)

    try:
        args = parser.parse_args()
    except:
        logger.error("Input args to %s improperly set, exiting.",
                     os.path.abspath(__file__))
        parser.print_help()
        sys.exit(0)

    # Find root filesystem location
    if args.rootfs != None:
        singularity_rootfs = args.rootfs
        logger.info("Root file system defined by command line variable as %s",
                    singularity_rootfs)
    else:
        singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None)
        if singularity_rootfs == None:
            logger.error(
                "root file system not specified OR defined as environmental variable, exiting!"
            )
            sys.exit(1)
        logger.info("Root file system defined by env variable as %s",
                    singularity_rootfs)

    # Does the registry require authentication?
    auth = None
    if args.username is not None and args.password is not None:
        auth = basic_auth_header(args.username, args.password)
        logger.info("Username for registry authentication: %s", args.username)

    # Does the user want to override default Entrypoint and use CMD as runscript?
    includecmd = args.includecmd
    logger.info("Including Docker command as Runscript? %s", includecmd)

    # Do we have a docker image specified?
    if args.docker != None:
        image = args.docker
        logger.info("Docker image: %s", image)

        # INPUT PARSING -------------------------------------------
        # Parse image name, repo name, and namespace

        # First split the docker image name by /
        image = image.split('/')

        # If there are two parts, we have namespace with repo (and maybe tab)
        if len(image) == 2:
            namespace = image[0]
            image = image[1]

        # Otherwise, we must be using library namespace
        else:
            namespace = "library"
            image = image[0]

        # Now split the docker image name by :
        image = image.split(':')
        if len(image) == 2:
            repo_name = image[0]
            repo_tag = image[1]

        # Otherwise, assume latest of an image
        else:
            repo_name = image[0]
            repo_tag = "latest"

        # Tell the user the namespace, repo name and tag
        logger.info("Docker image path: %s/%s:%s", namespace, repo_name,
                    repo_tag)

        # IMAGE METADATA -------------------------------------------
        # Use Docker Registry API (version 2.0) to get images ids, manifest

        # Get an image manifest - has image ids to parse, and will be
        # used later to get Cmd
        manifest = get_manifest(repo_name=repo_name,
                                namespace=namespace,
                                repo_tag=repo_tag,
                                registry=args.registry,
                                auth=auth)

        # Get images from manifest using version 2.0 of Docker Registry API
        images = get_images(manifest=manifest,
                            registry=args.registry,
                            auth=auth)

        #  DOWNLOAD LAYERS -------------------------------------------
        # Each is a .tar.gz file, obtained from registry with curl

        # Get the cache (or temporary one) for docker
        cache_base = get_cache(subfolder="docker",
                               disable_cache=args.disable_cache)

        layers = []
        for image_id in images:

            # Download the layer, if we don't have it
            targz = "%s/%s.tar.gz" % (cache_base, image_id)

            if not os.path.exists(targz):
                targz = get_layer(image_id=image_id,
                                  namespace=namespace,
                                  repo_name=repo_name,
                                  download_folder=cache_base,
                                  registry=args.registry,
                                  auth=auth)

            layers.append(targz)  # in case we want a list at the end
            # @chrisfilo suggestion to try compiling into one tar.gz

            # Extract image and remove tar
            extract_tar(targz, singularity_rootfs)
            if args.disable_cache == True:
                os.remove(targz)

        # If the user wants to include the CMD as runscript, generate it here
        if includecmd == True:
            spec = "Cmd"
        else:
            spec = "Entrypoint"

        cmd = get_config(manifest, spec=spec)

        # Only add runscript if command is defined
        if cmd != None:
            print("Adding Docker %s as Singularity runscript..." %
                  (spec.upper()))
            print(cmd)
            runscript = create_runscript(cmd=cmd, base_dir=singularity_rootfs)

        # When we finish, clean up images
        if args.disable_cache == True:
            shutil.rmtree(cache_base)

        logger.info("*** FINISHING DOCKER BOOTSTRAP PYTHON PORTION ****\n")
Beispiel #41
0
def run(args):

    # Find root filesystem location
    if args.rootfs != None:
       singularity_rootfs = args.rootfs
    else:
       singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None)
       if singularity_rootfs == None and args.shub == None: 
           logger.error("root file system not specified OR defined as environmental variable, exiting!")
           sys.exit(1)
    
    if singularity_rootfs != None:
        logger.info("Root file system defined as %s", singularity_rootfs)

    # Does the registry require authentication?
    auth = None
    if args.username is not None and args.password is not None:
        auth = basic_auth_header(args.username, args.password)
        logger.info("Username for registry authentication: %s", args.username)


    # Does the user want to download a Singularity image?
    if args.shub != None:
        image_id = int(args.shub)
        manifest = get_shub_manifest(image_id)

        cache_base = get_cache(subfolder="shub", 
                               disable_cache = args.disable_cache)

        # The image name is the md5 hash, download if it's not there
        image_name = get_image_name(manifest)
        image_file = "%s/%s" %(cache_base,image_name)
        if not os.path.exists(image_file):
            image_file = download_image(manifest=manifest,
                                        download_folder=cache_base)
        else:
            print("Image already exists at %s, skipping download." %image_file)
        logger.info("Singularity Hub Image Download: %s", image_file)
       
        # If singularity_rootfs is provided, write metadata to it
        if singularity_rootfs != None:
            logger.debug("Writing SINGULARITY_RUNDIR and SINGULARITY_IMAGE to %s",singularity_rootfs)
            write_file("%s/SINGULARITY_RUNDIR" %singularity_rootfs, os.path.dirname(image_file))
            write_file("%s/SINGULARITY_IMAGE" %singularity_rootfs, image_file)

    # Do we have a docker image specified?
    elif args.docker != None:

        # Does the user want to override default Entrypoint and use CMD as runscript?
        includecmd = args.includecmd
        logger.info("Including Docker command as Runscript? %s", includecmd)

        image = args.docker
        logger.info("Docker image: %s", image)

        # Input Parsing ----------------------------
        # Parse image name, repo name, and namespace

        # First split the docker image name by /
        image = image.split('/')

        # If there are two parts, we have namespace with repo (and maybe tab)
        if len(image) == 2:
            namespace = image[0]
            image = image[1]

        # Otherwise, we must be using library namespace
        else:
            namespace = "library"
            image = image[0]

        # Now split the docker image name by :
        image = image.split(':')
        if len(image) == 2:
            repo_name = image[0]
            repo_tag = image[1]

        # Otherwise, assume latest of an image
        else:
            repo_name = image[0]
            repo_tag = "latest"

        # Tell the user the namespace, repo name and tag
        logger.info("Docker image path: %s/%s:%s", namespace,repo_name,repo_tag)


        # IMAGE METADATA -------------------------------------------
        # Use Docker Registry API (version 2.0) to get images ids, manifest

        # Get an image manifest - has image ids to parse, and will be
        # used later to get Cmd
        manifest = get_manifest(repo_name=repo_name,
                                namespace=namespace,
                                repo_tag=repo_tag,
                                registry=args.registry,
                                auth=auth)

        # Get images from manifest using version 2.0 of Docker Registry API
        images = get_images(repo_name=repo_name,
                            namespace=namespace,
                            registry=args.registry,
                            auth=auth)
        
       
        #  DOWNLOAD LAYERS -------------------------------------------
        # Each is a .tar.gz file, obtained from registry with curl

        # Get the cache (or temporary one) for docker
        cache_base = get_cache(subfolder="docker", 
                               disable_cache = args.disable_cache)

        layers = []
        for image_id in images:

            # Download the layer, if we don't have it
            targz = "%s/%s.tar.gz" %(cache_base,image_id)
 
            if not os.path.exists(targz):
                targz = get_layer(image_id=image_id,
                                  namespace=namespace,
                                  repo_name=repo_name,
                                  download_folder=cache_base,
                                  registry=args.registry,
                                  auth=auth)

            layers.append(targz) # in case we want a list at the end

            # Extract image and remove tar
            output = extract_tar(targz,singularity_rootfs)
            if output is None:
                logger.error("Error extracting image: %s", targz)
                sys.exit(1)
            if args.disable_cache == True:
                os.remove(targz)
               
     
        # If the user wants to include the CMD as runscript, generate it here
        if includecmd == True:
            spec="Cmd"
        else:
            spec="Entrypoint"

        cmd = get_config(manifest,spec=spec)

        # Only add runscript if command is defined
        if cmd != None:
            print("Adding Docker %s as Singularity runscript..." %(spec.upper()))
            print(cmd)
            runscript = create_runscript(cmd=cmd,
                                         base_dir=singularity_rootfs)

        # When we finish, clean up images
        if args.disable_cache == True:
            shutil.rmtree(cache_base)


        logger.info("*** FINISHING DOCKER BOOTSTRAP PYTHON PORTION ****\n")
Beispiel #42
0
def run(args):

    # Find root filesystem location
    if args.rootfs != None:
        singularity_rootfs = args.rootfs
    else:
        singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None)
        if singularity_rootfs == None and args.shub == None:
            logger.error(
                "root file system not specified OR defined as environmental variable, exiting!"
            )
            sys.exit(1)

    if singularity_rootfs != None:
        logger.info("Root file system defined as %s", singularity_rootfs)

    # Does the registry require authentication?
    auth = None
    if args.username is not None and args.password is not None:
        auth = basic_auth_header(args.username, args.password)
        logger.info("Username for registry authentication: %s", args.username)

    # Does the user want to download a Singularity image?
    if args.shub != None:
        image = args.shub
        manifest = get_shub_manifest(image)
        if args.pull_folder == None:
            cache_base = get_cache(subfolder="shub",
                                   disable_cache=args.disable_cache)
        else:
            cache_base = args.pull_folder

        # The image name is the md5 hash, download if it's not there
        image_name = get_image_name(manifest)
        image_file = "%s/%s" % (cache_base, image_name)
        if not os.path.exists(image_file):
            image_file = download_image(manifest=manifest,
                                        download_folder=cache_base)
        else:
            print("Image already exists at %s, skipping download." %
                  image_file)
        logger.info("Singularity Hub Image Download: %s", image_file)

        # If singularity_rootfs is provided, write metadata to it
        if singularity_rootfs != None:
            logger.debug(
                "Writing SINGULARITY_RUNDIR and SINGULARITY_IMAGE to %s",
                singularity_rootfs)
            write_file("%s/SINGULARITY_RUNDIR" % singularity_rootfs,
                       os.path.dirname(image_file))
            write_file("%s/SINGULARITY_IMAGE" % singularity_rootfs, image_file)

    # Do we have a docker image specified?
    elif args.docker != None:

        # Does the user want to override default Entrypoint and use CMD as runscript?
        includecmd = args.includecmd
        logger.info("Including Docker command as Runscript? %s", includecmd)

        image = args.docker
        logger.info("Docker image: %s", image)

        # Input Parsing ----------------------------
        # Parse image name, repo name, and namespace

        image = parse_image_uri(image=image, uri="docker://")
        namespace = image['namespace']
        repo_name = image['repo_name']
        repo_tag = image['repo_tag']

        # Tell the user the namespace, repo name and tag
        logger.info("Docker image path: %s/%s:%s", namespace, repo_name,
                    repo_tag)

        # IMAGE METADATA -------------------------------------------
        # Use Docker Registry API (version 2.0) to get images ids, manifest

        # Get an image manifest - has image ids to parse, and will be
        # used later to get Cmd
        manifest = get_manifest(repo_name=repo_name,
                                namespace=namespace,
                                repo_tag=repo_tag,
                                registry=args.registry,
                                auth=auth)

        # Get images from manifest using version 2.0 of Docker Registry API
        images = get_images(manifest=manifest)

        #  DOWNLOAD LAYERS -------------------------------------------
        # Each is a .tar.gz file, obtained from registry with curl

        # Get the cache (or temporary one) for docker
        cache_base = get_cache(subfolder="docker",
                               disable_cache=args.disable_cache)

        layers = []
        for image_id in images:

            # Download the layer, if we don't have it
            targz = "%s/%s.tar.gz" % (cache_base, image_id)

            if not os.path.exists(targz):
                targz = get_layer(image_id=image_id,
                                  namespace=namespace,
                                  repo_name=repo_name,
                                  download_folder=cache_base,
                                  registry=args.registry,
                                  auth=auth)

            layers.append(targz)  # in case we want a list at the end

            # Extract image and remove tar
            output = extract_tar(targz, singularity_rootfs)
            if output is None:
                logger.error("Error extracting image: %s", targz)
                sys.exit(1)
            if args.disable_cache == True:
                os.remove(targz)

        # If the user wants to include the CMD as runscript, generate it here
        if includecmd == True:
            spec = "Cmd"
        else:
            spec = "Entrypoint"

        cmd = get_config(manifest, spec=spec)

        # Only add runscript if command is defined
        if cmd != None:
            print("Adding Docker %s as Singularity runscript..." %
                  (spec.upper()))
            print(cmd)
            runscript = create_runscript(cmd=cmd, base_dir=singularity_rootfs)

        # When we finish, clean up images
        if args.disable_cache == True:
            shutil.rmtree(cache_base)

        logger.info("*** FINISHING DOCKER BOOTSTRAP PYTHON PORTION ****\n")
Beispiel #43
0
def updateStudentData(heightEntry, speedEntry, enduranceEntry, strengthEntry,
                      explosiveEntry, agilityEntry, dataLabel):

    global recordDict

    height = heightEntry.get()
    fiftytime = speedEntry.get()
    eighthundredtime = enduranceEntry.get()
    shotputdist = strengthEntry.get()
    longjumpdist = explosiveEntry.get()
    agilitytime = agilityEntry.get()

    logger.info(
        messages.UPDATE_VALUES.format(height, fiftytime, eighthundredtime,
                                      shotputdist, longjumpdist, agilitytime))

    if (height == "" and fiftytime == "" and eighthundredtime == ""
            and shotputdist == "" and longjumpdist == ""
            and agilitytime == ""):
        messagebox.showerror(title="EMPTY_VALUES",
                             message=messages.EMPTY_VALUES)
        return

    if not recordDict:
        messagebox.showerror(title="NO_STUDENT_CHOSEN",
                             message=messages.NO_STUDENT_CHOSEN)
        return

    dynamicRecordDict = recordDict.copy()

    updateQueries, recordDict = utils.getUpdateQueries(
        height, fiftytime, eighthundredtime, shotputdist, longjumpdist,
        agilitytime, dynamicRecordDict)
    logger.info(messages.UPDATE_QUERY_NUM.format(len(updateQueries)))

    if len(updateQueries) < 1:
        logger.info(messages.NO_UPDATE_QUERIES)
        del dynamicRecordDict
        return

    finalupdatequery = utils.formUpdateQuery(currtable, updateQueries,
                                             dynamicRecordDict)
    logger.info(messages.UPDATE_QUERY.format(finalupdatequery))

    result = utils.updateTable(dbconn, finalupdatequery)

    if result:
        logger.info(messages.SPORT_UPDATE_PASSED)
        totalscoreupdatequery = utils.updateTotalScore(currtable,
                                                       dynamicRecordDict)
        logger.info(messages.UPDATE_QUERY.format(totalscoreupdatequery))

        combinedresult = utils.updateTable(dbconn, totalscoreupdatequery)
    else:
        logger.error(messages.SPORT_UPDATE_FAILED)
        combinedresult = False

    if combinedresult:
        logger.info(messages.UPDATE_RECORD_PASS)
        dataLabel.config(text="Student Record was updated Successfully")
        recordDict.update(dynamicRecordDict)
    else:
        messagebox.showerror(title="UPDATE_FAILED",
                             message=messages.UPDATE_FAILED)

    del dynamicRecordDict
    return