Exemplo n.º 1
0
def check_for_updates(quiet=False):
    """Check for updates to datasets.

    This updates the HOME_DIR scripts directory with the latest script versions
    """
    try:
        # open version.txt for current release branch and get script versions
        version_file = requests.get(REPOSITORY + "version.txt").text
        version_file = version_file.splitlines()[1:]

        # read scripts from the repository and the checksums from the version.txt
        scripts = []
        for line in version_file:
            scripts.append(line.strip('\n').split(','))

        # create script directory if not available
        if not os.path.isdir(SCRIPT_WRITE_PATH):
            os.makedirs(SCRIPT_WRITE_PATH)

        for script in tqdm(scripts, unit='files', desc='Downloading scripts'):
            script_name = script[0]
            if len(script) > 1:
                script_version = script[1]
            else:
                script_version = None

            path_script_name = os.path.normpath(
                os.path.join(HOME_DIR, "scripts", script_name))
            if not file_exists(path_script_name):
                _download_from_repository(
                    "scripts/" + script_name,
                    os.path.normpath(
                        os.path.join(SCRIPT_WRITE_PATH, script_name)))

            need_to_download = False
            try:
                file_object, pathname, desc = imp.find_module(
                    ''.join(script_name.split('.')[:-1]), [SCRIPT_WRITE_PATH])
                new_module = imp.load_module(script_name, file_object,
                                             pathname, desc)
                m = str(new_module.SCRIPT.version)
                need_to_download = parse_version(
                    str(script_version)) > parse_version(m)
            except:
                pass
            if need_to_download:
                try:
                    os.remove(
                        os.path.normpath(
                            os.path.join(HOME_DIR, "scripts", script_name)))
                    _download_from_repository(
                        "scripts/" + script_name,
                        os.path.normpath(
                            os.path.join(SCRIPT_WRITE_PATH, script_name)))
                except Exception as e:
                    print(e)
                    pass
    except:
        raise
Exemplo n.º 2
0
def check_for_updates(repo=REPOSITORY):
    """Check for updates to datasets.

    This updates the HOME_DIR scripts directory with the latest script versions
    """
    try:
        # open version.txt for current release branch and get script versions
        version_file = requests.get(repo + "version.txt").text
        version_file = version_file.splitlines()[1:]

        # read scripts from the repository and the checksums from the version.txt
        scripts = []
        for line in version_file:
            scripts.append(line.strip('\n').split(','))

        # create script directory if not available
        if not os.path.isdir(SCRIPT_WRITE_PATH):
            print('No scripts are currently available. Creating scripts folder...')
            os.makedirs(SCRIPT_WRITE_PATH)

        scripts_type = 'upstream'
        if repo == RETRIEVER_REPOSITORY:
            scripts_type = 'default'
        desc = 'Downloading {} scripts'.format(scripts_type)
        for script in tqdm(scripts, unit='files', desc=desc):
            script_name = script[0]
            if len(script) > 1:
                script_version = script[1]
            else:
                script_version = None

            script_home_path = os.path.normpath(
                os.path.join(HOME_DIR, "scripts", script_name))
            download_path = os.path.normpath(os.path.join(SCRIPT_WRITE_PATH, script_name))
            if not file_exists(script_home_path):
                _download_from_repository("scripts/" + script_name, download_path, repo)
            need_to_download = False
            try:
                file_object, pathname, desc = imp.find_module(
                    ''.join(script_name.split('.')[:-1]), [SCRIPT_WRITE_PATH])
                new_module = imp.load_module(script_name, file_object, pathname, desc)
                m = str(new_module.SCRIPT.version)
                need_to_download = parse_version(str(script_version)) > parse_version(m)
            except:
                pass
            if need_to_download:
                try:
                    os.remove(script_home_path)
                    _download_from_repository("scripts/" + script_name, download_path,
                                              repo)
                except Exception as e:
                    print(e)
    except:
        raise

    if repo == RETRIEVER_REPOSITORY:
        return
    check_for_updates(RETRIEVER_REPOSITORY)
Exemplo n.º 3
0
def check_for_updates(quiet=False):
    """Check for updates to datasets.

    This updates the HOME_DIR scripts directory with the latest script versions
    """
    try:
        # open version.txt for current release branch and get script versions
        version_file = urllib.request.urlopen(REPOSITORY + "version.txt")
        version_file.readline()

        # read scripts from the repository and the checksums from the version.txt
        scripts = []
        for line in version_file:
            scripts.append(line.decode().strip('\n').split(','))

        total_script_count = len(scripts)

        # create script directory if not available
        if not os.path.isdir(SCRIPT_WRITE_PATH):
            os.makedirs(SCRIPT_WRITE_PATH)

        if not quiet:
            print("Downloading scripts...")
            _update_progressbar(0.0 / float(total_script_count))
        for index, script in enumerate(scripts):
            script_name = script[0]
            if len(script) > 1:
                script_version = script[1]
            else:
                script_version = None

            path_script_name = os.path.normpath(os.path.join(HOME_DIR, "scripts", script_name))
            if not file_exists(path_script_name):
                _download_from_repository("scripts/" + script_name,
                                          os.path.normpath(os.path.join(SCRIPT_WRITE_PATH, script_name)))

            need_to_download = False
            try:
                file_object, pathname, desc = imp.find_module(''.join(script_name.split('.')[:-1]), [SCRIPT_WRITE_PATH])
                new_module = imp.load_module(script_name, file_object, pathname, desc)
                m = str(new_module.SCRIPT.version)
                need_to_download = parse_version(str(script_version)) > parse_version(m)
            except:
                pass
            if need_to_download:
                try:
                    os.remove(os.path.normpath(os.path.join(HOME_DIR, "scripts", script_name)))
                    _download_from_repository("scripts/" + script_name,
                                              os.path.normpath(os.path.join(SCRIPT_WRITE_PATH, script_name)))
                except Exception as e:
                    print(e)
                    pass
            if not quiet:
                _update_progressbar(float(index + 1) / float(total_script_count))
    except:
        raise
    if not quiet:
        print("\nThe retriever is up-to-date")
Exemplo n.º 4
0
def check_for_updates(quiet=False):
    """Check for updates to datasets.

    This updates the HOME_DIR scripts directory with the latest script versions
    """
    try:
        # open version.txt for current release branch and get script versions
        version_file = requests.get(REPOSITORY + "version.txt").text
        version_file = version_file.splitlines()[1:]

        # read scripts from the repository and the checksums from the version.txt
        scripts = []
        for line in version_file:
            scripts.append(line.strip('\n').split(','))

        total_script_count = len(scripts)

        # create script directory if not available
        if not os.path.isdir(SCRIPT_WRITE_PATH):
            os.makedirs(SCRIPT_WRITE_PATH)

        for script in tqdm(scripts, unit='files', desc='Downloading scripts'):
            script_name = script[0]
            if len(script) > 1:
                script_version = script[1]
            else:
                script_version = None

            path_script_name = os.path.normpath(os.path.join(HOME_DIR, "scripts", script_name))
            if not file_exists(path_script_name):
                _download_from_repository("scripts/" + script_name,
                                          os.path.normpath(os.path.join(SCRIPT_WRITE_PATH, script_name)))

            need_to_download = False
            try:
                file_object, pathname, desc = imp.find_module(''.join(script_name.split('.')[:-1]), [SCRIPT_WRITE_PATH])
                new_module = imp.load_module(script_name, file_object, pathname, desc)
                m = str(new_module.SCRIPT.version)
                need_to_download = parse_version(str(script_version)) > parse_version(m)
            except:
                pass
            if need_to_download:
                try:
                    os.remove(os.path.normpath(os.path.join(HOME_DIR, "scripts", script_name)))
                    _download_from_repository("scripts/" + script_name,
                                              os.path.normpath(os.path.join(SCRIPT_WRITE_PATH, script_name)))
                except Exception as e:
                    print(e)
                    pass
    except:
        raise
Exemplo n.º 5
0
    def run(self):
        try:
            running_from = os.path.basename(sys.argv[0])

            # NOTE: exe auto-update functionality has been temporarily disabled
            # since the binaries were moved to AWS.

            if False:  #running_from[-4:] == ".exe":
                if os.path.isfile('retriever_old.exe'
                                  ) and running_from != 'retriever_old.exe':
                    try:
                        os.remove('retriever_old.exe')
                    except:
                        pass

                # Windows: open master branch version file to find out most recent executable version
                try:
                    version_file = urllib.urlopen(MASTER_BRANCH +
                                                  "version.txt")
                except IOError:
                    print "Couldn't open version.txt from repository"
                    return

                latest = version_file.readline().strip('\n')

                if more_recent(latest, VERSION):
                    import wx

                    msg = "You're running version " + VERSION + "."
                    msg += '\n\n'
                    msg += "Version " + latest + " is available. Do you want to upgrade?"
                    choice = wx.MessageDialog(None, msg, "Update", wx.YES_NO)
                    if choice.ShowModal() == wx.ID_YES:
                        print "Updating to latest version. Please wait..."
                        try:
                            if not "_old" in running_from:
                                os.rename(
                                    running_from,
                                    '.'.join(running_from.split('.')[:-1]) +
                                    "_old." + running_from.split('.')[-1])
                        except:
                            pass

                        download_from_repository("windows/" + executable_name +
                                                 ".exe",
                                                 executable_name + ".exe",
                                                 repo=REPO_URL + latest + "/")

                        sys.stdout = sys.__stdout__

                        wx.MessageBox(
                            "Update complete. The program will now restart.")

                        os.execv(executable_name + ".exe", sys.argv)

                        sys.exit()

                version_file.close()

            # open version.txt for current release branch and get script versions
            version_file = urllib.urlopen(REPOSITORY + "version.txt")
            version_file.readline()
            scripts = []
            for line in version_file:
                scripts.append(line.strip('\n').split(','))

            # get script files
            if not os.path.isdir(SCRIPT_WRITE_PATH):
                os.makedirs(SCRIPT_WRITE_PATH)
            for script in scripts:
                script_name = script[0]
                if len(script) > 1:
                    script_version = script[1]
                else:
                    script_version = None

                if not file_exists(os.path.join("scripts", script_name)):
                    # File doesn't exist: download it
                    print "Downloading script: " + script_name
                    download_from_repository(
                        "scripts/" + script_name,
                        os.path.join(SCRIPT_WRITE_PATH, script_name))
                elif script_version:
                    # File exists: import and check MD5 sum
                    need_to_download = False
                    try:
                        file, pathname, desc = imp.find_module(
                            ''.join(script_name.split('.')[:-1]), ["scripts"])
                        new_module = imp.load_module(script_name, file,
                                                     pathname, desc)
                        m = md5()
                        m.update(''.join(
                            getsourcelines(new_module)[0]).replace(
                                "\r\n", "\n"))
                        m = m.hexdigest()
                        need_to_download = script_version != m
                    except:
                        pass

                    if need_to_download:
                        try:
                            os.remove(os.path.join("scripts", script_name))
                            download_from_repository(
                                "scripts/" + script_name,
                                os.path.join(SCRIPT_WRITE_PATH, script_name))
                        except Exception as e:
                            print e
                            pass
        except:
            raise
            return
Exemplo n.º 6
0
    def run(self):
        try:
            running_from = os.path.basename(sys.argv[0])
            
            # NOTE: exe auto-update functionality has been temporarily disabled 
            # since the binaries were moved to AWS.

            if False:#running_from[-4:] == ".exe":
                if os.path.isfile('retriever_old.exe') and running_from != 'retriever_old.exe':
                    try:
                        os.remove('retriever_old.exe')
                    except:
                        pass

                # Windows: open master branch version file to find out most recent executable version            
                try:
                    version_file = urllib.urlopen(MASTER_BRANCH + "version.txt")
                except IOError:
                    print "Couldn't open version.txt from repository"
                    return
                    
                latest = version_file.readline().strip('\n')
                    
                if more_recent(latest, VERSION):
                    import wx

                    msg = "You're running version " + VERSION + "."
                    msg += '\n\n'
                    msg += "Version " + latest + " is available. Do you want to upgrade?"
                    choice = wx.MessageDialog(None, msg, "Update", wx.YES_NO)
                    if choice.ShowModal() == wx.ID_YES:
                        print "Updating to latest version. Please wait..."
                        try:
                            if not "_old" in running_from:
                                os.rename(running_from,
                                          '.'.join(running_from.split('.')[:-1])
                                          + "_old." + running_from.split('.')[-1])
                        except:
                            pass
                                
                        download_from_repository("windows/" + executable_name + ".exe",
                                                 executable_name + ".exe",
                                                 repo=REPO_URL + latest + "/")

                        sys.stdout = sys.__stdout__

                        wx.MessageBox("Update complete. The program will now restart.")

                        os.execv(executable_name + ".exe", sys.argv)
                            
                        sys.exit()

                version_file.close()

            # open version.txt for current release branch and get script versions
            version_file = urllib.urlopen(REPOSITORY + "version.txt")
            version_file.readline()
            scripts = []
            for line in version_file:
                scripts.append(line.strip('\n').split(','))
            
            # get script files
            if not os.path.isdir(SCRIPT_WRITE_PATH):
                os.makedirs(SCRIPT_WRITE_PATH)
            for script in scripts:
                script_name = script[0]
                if len(script) > 1:
                    script_version = script[1]
                else:
                    script_version = None

                if not file_exists(os.path.join("scripts", script_name)):
                    # File doesn't exist: download it
                    print "Downloading script: " + script_name
                    download_from_repository("scripts/" + script_name,
                                             os.path.join(SCRIPT_WRITE_PATH, script_name))
                elif script_version:
                    # File exists: import and check MD5 sum
                    need_to_download = False
                    try:
                        file, pathname, desc = imp.find_module(''.join(script_name.split('.')[:-1]), 
                                                               ["scripts"])
                        new_module = imp.load_module(script_name, file, pathname, desc)
                        m = md5()
                        m.update(''.join(getsourcelines(new_module)[0]).replace("\r\n", "\n"))
                        m = m.hexdigest()
                        need_to_download = script_version != m
                    except:            
                        pass
                                
                    if need_to_download:
                        try:
                            os.remove(os.path.join("scripts", script_name))
                            download_from_repository("scripts/" + script_name,
                                                     os.path.join(SCRIPT_WRITE_PATH, script_name))
                        except Exception as e:
                            print e
                            pass
        except:
            raise
            return
Exemplo n.º 7
0
    def run(self):
        try:
            running_from = os.path.basename(sys.argv[0])

            # NOTE: exe auto-update functionality has been temporarily disabled
            # since the binaries were moved to AWS.

            # open version.txt for current release branch and get script
            # versions
            version_file = urllib.urlopen(REPOSITORY + "version.txt")
            version_file.readline()

            # read scripts from the repository and the checksums from the
            # version.txt
            scripts = []
            print("Downloading scripts...")
            for line in version_file:
                scripts.append(line.strip('\n').split(','))

            total_script_count = len(scripts)

            # create script directory if not available
            if not os.path.isdir(SCRIPT_WRITE_PATH):
                os.makedirs(SCRIPT_WRITE_PATH)

            update_progressbar(0.0 / float(total_script_count))
            for index, script in enumerate(scripts):
                script_name = script[0]
                if len(script) > 1:
                    script_version = script[1]
                else:
                    script_version = None

                path_script_name = os.path.normpath(
                    os.path.join(HOME_DIR, "scripts", script_name))

                if not file_exists(path_script_name):
                    download_from_repository(
                        "scripts/" + script_name,
                        os.path.normpath(
                            os.path.join(SCRIPT_WRITE_PATH, script_name)))

                # check MD5sum based on the script version to download the right scripts
                # if the MD5sum doesn't match need_to_download is set to True
                need_to_download = False

                try:
                    file, pathname, desc = imp.find_module(
                        ''.join(script_name.split('.')[:-1]), ["scripts"])
                    new_module = imp.load_module(script_name, file, pathname,
                                                 desc)
                    m = md5()
                    m.update(''.join(getsourcelines(new_module)[0]).replace(
                        "\r\n", "\n"))
                    m = m.hexdigest()
                    need_to_download = script_version != m
                except:
                    pass

                if need_to_download:
                    try:
                        os.remove(
                            os.path.normpath(
                                os.path.join(HOME_DIR, "scripts",
                                             script_name)))
                        download_from_repository(
                            "scripts/" + script_name,
                            os.path.normpath(
                                os.path.join(SCRIPT_WRITE_PATH, script_name)))
                    except Exception as e:
                        print(e)
                        pass
                update_progressbar(
                    float(index + 1) / float(total_script_count))
        except:
            raise
            return
Exemplo n.º 8
0
def check_for_updates(quiet=False):
    """Check for updates to datasets.

    This updates the HOME_DIR scripts directory with the latest script versions
    """
    try:
        # open version.txt for current release branch and get script versions
        version_file = urllib.request.urlopen(REPOSITORY + "version.txt")
        version_file.readline()

        # read scripts from the repository and the checksums from the version.txt
        scripts = []
        for line in version_file:
            scripts.append(line.decode().strip('\n').split(','))

        total_script_count = len(scripts)

        # create script directory if not available
        if not os.path.isdir(SCRIPT_WRITE_PATH):
            os.makedirs(SCRIPT_WRITE_PATH)

        if not quiet:
            print("Downloading scripts...")
            _update_progressbar(0.0 / float(total_script_count))
        for index, script in enumerate(scripts):
            script_name = script[0]
            if len(script) > 1:
                script_version = script[1]
            else:
                script_version = None

            path_script_name = os.path.normpath(
                os.path.join(HOME_DIR, "scripts", script_name))
            if not file_exists(path_script_name):
                _download_from_repository(
                    "scripts/" + script_name,
                    os.path.normpath(
                        os.path.join(SCRIPT_WRITE_PATH, script_name)))

            need_to_download = False
            try:
                file_object, pathname, desc = imp.find_module(
                    ''.join(script_name.split('.')[:-1]), [SCRIPT_WRITE_PATH])
                new_module = imp.load_module(script_name, file_object,
                                             pathname, desc)
                m = str(new_module.SCRIPT.version)
                need_to_download = parse_version(
                    str(script_version)) > parse_version(m)
            except:
                pass
            if need_to_download:
                try:
                    os.remove(
                        os.path.normpath(
                            os.path.join(HOME_DIR, "scripts", script_name)))
                    _download_from_repository(
                        "scripts/" + script_name,
                        os.path.normpath(
                            os.path.join(SCRIPT_WRITE_PATH, script_name)))
                except Exception as e:
                    print(e)
                    pass
            if not quiet:
                _update_progressbar(
                    float(index + 1) / float(total_script_count))
    except:
        raise
    if not quiet:
        print("\nThe retriever is up-to-date")