Ejemplo n.º 1
0
def check_credentials(request):
    if request.method == 'POST':
        try:
            data = json.loads(request.body)
            response = {}
            creds = Credential.objects.filter(site_user_name=request.user)
            if len(creds) != 0:
                for c in creds:
                    try:
                        if c.service == 'esgf':
                            import pyesgf
                            from pyesgf.logon import LogonManager
                            lm = LogonManager()
                            lm.logon_with_openid(
                                c.service_user_name, c.password)
                            if lm.is_logged_on():
                                response[s] = 'success'
                                print 'esgf log in successful'
                            else:
                                print 'esgf log in failed'
                                response[s] = 'fail'
                        if c.service == 'velo':
                            user, password = c.service_user_name, c.password
                            velo_creds = {
                                "velo_user": user,
                                "velo_pass": password,
                                "command": "init"
                            }
                            result = velo_request(velo_creds)
                            # TODO: Extract values out to CAPITAL_NAMED_CONSTANTS
                            if result == "Success":
                                print "velo login successful"
                            else:
                                print "velo login failed"
                                response[s] = "fail"

                        if c.service == 'github':
                            import github3
                            from github3 import login
                            gh = login(c.site_user_name, password=c.password)
                            if gh.user() == c.site_user_name:
                                print 'Github login successful'
                                response[s] = 'success'
                            else:
                                print 'Github login failure'
                                response[s] = 'fail'

                        if c.service == 'jira':
                            print 'Working on jira....'
                    except:
                        print_debug(c)
                        return HttpResponse(status=500)

            return HttpResponse(json.dumps(response))

        except Exception as e:
            print_debug(e)
            return HttpResponse(status=500)
    else:
        return HttpResponse(status=404)
Ejemplo n.º 2
0
def test_logon_openid():
    _clear_creds()
    _load_creds(certificates_tarball='pcmdi9-certs.tar.gz')
    lm = LogonManager(esgf_dir)
    lm.logon_with_openid(TEST_OPENID, TEST_PASSWORD, interactive=False)

    assert lm.is_logged_on()
Ejemplo n.º 3
0
def test_logon_openid():
    _clear_creds()
    _load_creds(certificates_tarball='pcmdi9-certs.tar.gz')
    lm = LogonManager(esgf_dir)
    lm.logon_with_openid(TEST_OPENID, TEST_PASSWORD, interactive=False)

    assert lm.is_logged_on()
Ejemplo n.º 4
0
def test_logon():
    _clear_creds()
    _load_creds(certificates_tarball='pcmdi9-certs.tar.gz')
    lm = LogonManager(esgf_dir)
    lm.logon(TEST_USER, TEST_PASSWORD, TEST_MYPROXY)

    assert lm.is_logged_on()
Ejemplo n.º 5
0
def test_logon():
    _clear_creds()
    _load_creds(certificates_tarball='pcmdi9-certs.tar.gz')
    lm = LogonManager(esgf_dir)
    lm.logon(TEST_USER, TEST_PASSWORD, TEST_MYPROXY)

    assert lm.is_logged_on()
Ejemplo n.º 6
0
def logon(request):
    credential = {
        'username': request.GET.get('username'),
        'password': request.GET.get('password')
    }
    if not credential['username']:
        print "[-] No username in logon request"
        return HttpResponse(status=403)
    elif not credential['password']:
        print "[-] No password in logon request"
        return HttpResponse(status=403)

    lm = LogonManager()
    bootstrap = False
    if not os.path.exists(ESGF_CREDENTIALS):
        bootstrap = True
    try:
        lm.logon_with_openid(credential['username'], credential['password'], bootstrap=bootstrap)
    except Exception as e:
        print_message('Unable to log in user {}'.format(credential.get('username')))
        return HttpResponse(status=403)
    if lm.is_logged_on():
        return HttpResponse(status=200)
    else:
        return HttpResponse(status=403)
Ejemplo n.º 7
0
    def test_logon(self, extra_args=None):
        _clear_creds(self.esgf_dir)
        _load_creds(self.esgf_dir, certificates_tarball='pcmdi9-certs.tar.gz')

        if not extra_args: extra_args = {}
        lm = LogonManager(self.esgf_dir, **extra_args)
        lm.logon(TEST_USER, TEST_PASSWORD, TEST_MYPROXY)

        assert lm.is_logged_on()
Ejemplo n.º 8
0
    def test_logon(self, extra_args=None):
        _clear_creds(self.esgf_dir)
        _load_creds(self.esgf_dir, certificates_tarball='pcmdi9-certs.tar.gz')

        if not extra_args: extra_args = {}
        lm = LogonManager(self.esgf_dir, **extra_args)
        lm.logon(TEST_USER, TEST_PASSWORD, TEST_MYPROXY)

        assert lm.is_logged_on()
Ejemplo n.º 9
0
def logon(openid, password):
    ''' Login on ESGF with credentials, returns connection object '''
    lm=LogonManager()
    try:
        lm.logon_with_openid(openid, password, bootstrap=True, update_trustroots=True)
    except:
        e = sys.exc_info()[1]
        print("Logon Error: ",  e)
    return lm 
Ejemplo n.º 10
0
    def test_logon_openid(self):
        _clear_creds(self.esgf_dir)
        _load_creds(self.esgf_dir, certificates_tarball='pcmdi9-certs.tar.gz')
        lm = LogonManager(self.esgf_dir)

        # NOTE: for many users the OpenID lookup might not provide the username
        #       in which case this test will fail because it needs interactive
        #       prompting for a username.
        lm.logon_with_openid(TEST_OPENID, TEST_PASSWORD, interactive=False)

        assert lm.is_logged_on()
Ejemplo n.º 11
0
    def test_logon_openid(self):
        _clear_creds(self.esgf_dir)
        _load_creds(self.esgf_dir, certificates_tarball='pcmdi9-certs.tar.gz')
        lm = LogonManager(self.esgf_dir)

        # NOTE: for many users the OpenID lookup might not provide the username
        #       in which case this test will fail because it needs interactive
        #       prompting for a username.
        lm.logon_with_openid(TEST_OPENID, TEST_PASSWORD, interactive=False)

        assert lm.is_logged_on()
Ejemplo n.º 12
0
 def auth(self):
     '''
     Authenticate with the auth server specified on object creation.
     '''
     # Check that we're logged on
     lm = LogonManager()
     log.debug('Logon manager started')
     if not lm.is_logged_on():
         log.debug(self.username, self.password, self.auth_server)
         lm.logon(self.username, self.password, self.auth_server)
     if not lm.is_logged_on():
         raise Exception('NOAUTH')
Ejemplo n.º 13
0
def myproxy_logon_with_openid(openid, password=None, interactive=False, outdir=None):
    """
    Tries to get MyProxy parameters from OpenID and calls :meth:`logon`.

    :param openid: OpenID used to login at ESGF node.
    """
    outdir = outdir or os.curdir
    username, hostname, port = parse_openid(openid)
    lm = LogonManager(esgf_dir=outdir, dap_config=os.path.join(outdir, 'dodsrc'))
    lm.logoff()
    lm.logon(username=username, password=password, hostname=hostname,
             bootstrap=True, update_trustroots=False, interactive=interactive)
    return os.path.join(outdir, ESGF_CREDENTIALS)
Ejemplo n.º 14
0
def logon(openid, password):
    '''
    Function to retrieve a short-term X.509 certificate that can be used to authenticate with ESGF.
    The certificate is written in the location ~/.esg/credentials.pem.
    The trusted CA certificates are written in the directory ~/.esg/certificates.
    '''
    
    # Must configure the DN of the JPL MyProxy server if using a JPL openid
    if "esg-datanode.jpl.nasa.gov" in openid:  
        os.environ['MYPROXY_SERVER_DN'] = JPL_MYPROXY_SERVER_DN
        
    lm = LogonManager()
    lm.logon_with_openid(openid,password)
    return lm.is_logged_on()
Ejemplo n.º 15
0
def check_credentials(request):
    if request.method == 'POST':
        try:
            data = json.loads(request.body)
            response = {}
            for s in data:
                if s == 'esgf':
                    import pyesgf
                    from pyesgf.logon import LogonManager

                    lm = LogonManager()
                    lm.logon_with_openid(data[s]['username'], data[s]['password'])
                    if lm.is_logged_on() != True:
                        response[s] = 'failed'
                    else:
                        response[s] = 'success'
                if s == 'velo':
                    lib_path = os.path.abspath(os.path.join('apps', 'velo'))
                    sys.path.append(lib_path)
                    import VeloAPI

                    velo_api = VeloAPI.Velo()
                    velo_api.start_jvm()
                    '''
                    Using the test credentials for the time being, simply uncomment to
                    use the users credentials
                    velo_api.init_velo(data[s]['username'], data[s]['password'])
                    '''
                    # res = velo_api.init_velo("acmetest", "acmetest")
                    response[s] = 'success'
                    '''
                    if res.logged_on() != True:
                        resonse[s] = 'failed'
                    else:
                        response[s] = 'success
                    '''
            return HttpResponse(json.dumps(response))

        except Exception as e:
            import traceback
            print '1', e.__doc__
            print '2', sys.exc_info()
            print '3', sys.exc_info()[0]
            print '4', sys.exc_info()[1]
            print '5', traceback.tb_lineno(sys.exc_info()[2])
            ex_type, ex, tb = sys.exc_info()
            print '6', traceback.print_tb(tb)
            return HttpResponse(status=500)
    else:
        return HttpResponse(status=404)
Ejemplo n.º 16
0
def logon(openid, password):
    """
    Function to retrieve a short-term X.509 certificate that can be used to authenticate with ESGF.
    The certificate is written in the location ~/.esg/credentials.pem.
    The trusted CA certificates are written in the directory ~/.esg/certificates.
    """
    # Must configure the DN of the JPL MyProxy server if using a JPL openid
    if JPL_HOSTNAME in openid:
        os.environ['MYPROXY_SERVER_DN'] = JPL_MYPROXY_SERVER_DN

    logon_manager = LogonManager()

    logon_manager.logon_with_openid(openid, password, bootstrap=True)

    return logon_manager.is_logged_on()
Ejemplo n.º 17
0
    def test_config3(self):
        # Create the config when one already exists
        # with the BEGIN section in it

        lines = ['# Welcome to my config file', 'SOME_OPT=foo', '']
        preamble = '\n'.join(lines)

        lines = ['', '# Some more config here', 'OTHER_OPT=bar', '']
        postamble = '\n'.join(lines)

        config = '''\
{0}
# BEGIN <<< Managed by esgf-pyclient >>>
CURL.VERBOSE=0
CURL.COOKIEJAR=/tmp/foo/certificates/.dods_cookies
CURL.SSL.VALIDATE=1
CURL.SSL.CERTIFICATE=/tmp/foo/certificates/credentials.pem
CURL.SSL.KEY=/tmp/foo/certificates/credentials.pem
CURL.SSL.CAPATH=/tmp/foo/certificates/certificates
# END <<< Managed by esgf-pyclient >>>

{1}
'''.format(preamble, postamble)

        self.init_config(config)

        LogonManager(self.esgf_dir, dap_config=self.dap_config)
        config1 = self.read_config()

        print(config1)
        assert self.check_preamble(preamble, config1)
        assert self.check_postamble(postamble, config1)
Ejemplo n.º 18
0
def test_logoff():
    lm = LogonManager(esgf_dir)

    # Only re-logon if credentials are not valid
    if not lm.is_logged_on():
        lm.logon(TEST_USER, TEST_PASSWORD, TEST_MYPROXY, bootstrap=True)

    assert lm.is_logged_on()
    lm.logoff()

    assert not op.exists(op.join(esgf_dir, 'credentials.pem'))
    assert not lm.is_logged_on()
    assert lm.state == lm.STATE_NO_CREDENTIALS
Ejemplo n.º 19
0
def logon(host=DKRZ_HOST):
    """logon to ESGF Host."""
    from pyesgf.logon import LogonManager

    print("logon to: ".format(host))
    lm = LogonManager()
    lm.logoff()
    lm.logon(hostname=host, interactive=True, bootstrap=True)
    print("logged on: {}".format(lm.is_logged_on()))
    return lm.is_logged_on()
Ejemplo n.º 20
0
    def test_config1(self):
        # Create the config file from scratch
        LogonManager(self.esgf_dir, dap_config=self.dap_config)
        config = self.read_config()

        print(config)
        assert re.match(r'\s*^# BEGIN {0}$.*^# END {0}$'
                        .format(DAP_CONFIG_MARKER),
                        config, re.M | re.S)
Ejemplo n.º 21
0
    def test_config2(self):
        # Create the config when one already exists.  Check it is retained.
        lines = ['# Welcome to my config file', 'SOME_OPT=foo', '']
        preamble = '\n'.join(lines)
        self.init_config(preamble)

        LogonManager(self.esgf_dir, dap_config=self.dap_config)
        config = self.read_config()

        print(config)
        assert self.check_preamble(preamble, config)
Ejemplo n.º 22
0
def test_logoff():
    lm = LogonManager(esgf_dir)

    # Only re-logon if credentials are not valid
    if not lm.is_logged_on():
        lm.logon(TEST_USER, TEST_PASSWORD, TEST_MYPROXY, bootstrap=True)

    assert lm.is_logged_on()
    lm.logoff()

    assert not op.exists(op.join(esgf_dir, 'credentials.pem'))
    assert not lm.is_logged_on()
    assert lm.state == lm.STATE_NO_CREDENTIALS
Ejemplo n.º 23
0
def download(request):
    credential = {
        'username': '******',
        'password': '******'
    }
    lm = LogonManager()
    bootstrap = False
    if not os.path.exists(ESGF_CREDENTIALS):
        bootstrap = True
    try:
        print "logging in"
        lm.logon_with_openid(credential['username'], credential['password'], bootstrap=bootstrap)
    except Exception as e:
        # print_debug(e)
        return HttpResponse(status=403)
    if lm.is_logged_on():
        print "...login success"
        try:
            if not os.path.exists('/tmp'):
                os.mkdir('/tmp')
            url = request.GET.get('url')
            # esgf_download(request.GET.get('url'))
            print "[+] Downloading " + url
            # response = urllib2.urlopen(url)
            print "opening connection to server"
            response = requests.get(url, verify='/Users/sterling/.esg/credentials.pem',stream=True)
            response.raw.decode_content = True
            with open("someClimateFile.nc", 'wb') as f:
                    shutil.copyfileobj(response.raw, f)
            print "...download success"
            return HttpResponse(status=200)
        except Exception as e:
            # print_debug(e)
            return HttpResponse(status=400)
    else:
        return HttpResponse(status=403)
    return HttpResponse(status=200)
Ejemplo n.º 24
0
def logon(username=None, password=None, hostname=None, interactive=False, outdir=None):
    """
    Logon to MyProxy and fetch proxy certificate.
    """
    outdir = outdir or os.curdir
    lm = LogonManager(esgf_dir=outdir, dap_config=os.path.join(outdir, 'dodsrc'))
    lm.logoff()
    # logon
    lm.logon(username=username, password=password, hostname=hostname,
             bootstrap=True, update_trustroots=False, interactive=interactive)
    return os.path.join(outdir, ESGF_CREDENTIALS)
Ejemplo n.º 25
0
    def test_open_url(self, TEST_SERVICE):
        import netCDF4

        LogonManager(self.esgf_dir, dap_config=self.dap_config)
        print(('Using dap_config at %s' % self.dap_config))

        conn = SearchConnection(TEST_SERVICE, distrib=False)

        # !TODO: replace with request for specific dataset
        ctx = conn.new_context(project='CMIP5')
        results = ctx.search()

        r1 = results[0]
        f_ctx = r1.file_context()

        file_results = f_ctx.search()

        opendap_url = file_results[0].opendap_url
        print(('OPeNDAP URL is %s' % opendap_url))

        ds = netCDF4.Dataset(opendap_url)
        print((list(ds.variables.keys())))
Ejemplo n.º 26
0
def myproxy_logon_with_openid(openid,
                              password=None,
                              interactive=False,
                              outdir=None):
    """
    Tries to get MyProxy parameters from OpenID and calls :meth:`logon`.

    :param openid: OpenID used to login at ESGF node.
    """
    outdir = outdir or os.curdir
    username, hostname, port = parse_openid(openid)
    lm = LogonManager(esgf_dir=outdir,
                      dap_config=os.path.join(outdir, 'dodsrc'))
    lm.logoff()
    lm.logon(username=username,
             password=password,
             hostname=hostname,
             bootstrap=True,
             update_trustroots=False,
             interactive=interactive)
    return os.path.join(outdir, ESGF_CREDENTIALS)
Ejemplo n.º 27
0
def logon(username=None,
          password=None,
          hostname=None,
          interactive=False,
          outdir=None):
    """
    Logon to MyProxy and fetch proxy certificate.
    """
    outdir = outdir or os.curdir
    # use myproxy patch
    # TODO: update to myproxyclient 2.x
    from phoenix.patch import patch_myproxy_client
    patch_myproxy_client()
    # end patch
    lm = LogonManager(esgf_dir=outdir,
                      dap_config=os.path.join(outdir, 'dodsrc'))
    lm.logoff()
    lm.logon(username=username,
             password=password,
             hostname=hostname,
             bootstrap=True,
             update_trustroots=False,
             interactive=interactive)
    return os.path.join(outdir, ESGF_CREDENTIALS)
Ejemplo n.º 28
0
def dataset_download(message, data, user):
    print 'got a dataset_download request'
    username = data.get('params').get('openid_username')
    password = data.get('params').get('openid_password')
    search_string = data.get('params').get('search_string')
    nodes = data.get('params').get('nodes')
    data_type = data.get('params').get('data_type')
    data_name = data.get('params').get('data_name').replace(' ', '_')
    if not username:
        print_message('No username given')
        return -1
    if not password:
        print_message('No password given')
        return -1
    if not search_string:
        print_message('No search_string given')
        return -1
    if not nodes:
        print_message('No nodes given')
        return -1
    if not data_type:
        print_message('No data_type given')
        return -1
    if not data_name:
        print_message('No data_name given')
        return -1

    lm = LogonManager()
    lm.logon_with_openid(username, password, bootstrap=True)
    if not lm.is_logged_on():
        print_message('User {user} is not logged in during download request'.format(user=user))
        return -1
    for node in nodes:
        try:
            path = os.path.abspath(os.path.dirname(__file__)) + '/../../'
            print '[+] searching {node} for {string}'.format(node=node, string=search_string)
            conn_string = 'http://{node}{suffix}'.format(node=node, suffix=ESGF_SEARCH_SUFFIX)
            conn = SearchConnection(conn_string, distrib=True)
            context = conn.new_context(**search_string)
            rs = context.search()
            print_message('got reply from {node}'.format(node=node))
            if len(rs) == 0:
                continue

            script_text = context.get_download_script()
            script_path = ''
            if data_type == 'observation':
                script_path = path + 'userdata/' + user + '/observations/' + data_name
            elif data_type == 'model':
                script_path = path + 'userdata/' + user + '/model_output/' + data_name

            script_name = '{path}/{name}_download_script.sh'.format(path=script_path, name=data_name)
            if not os.path.exists(script_path):
                print_message('creating directory {}'.format(script_path))
                os.makedirs(script_path)
            try:
                with open(script_name, 'w') as script:
                    script.write(script_text)
                    script.close()
            except Exception, e:
                print_debug(e)

            word_count_cmd = 'cat {script_name} | grep \\.nc.*http:// | wc -l'.format(path=script_path, script_name=script_name)
            print_message('running command {}'.format(word_count_cmd))
            p = subprocess.check_output(word_count_cmd, shell=True)
            number_of_downloads = int(p)
            percent_complete = 0
            number_complete = 0.0
            print_message('Number of .nc files to download: {}'.format(number_of_downloads))

            try:
                subprocess.call(['chmod', '+x', script_name])
                p = subprocess.Popen('exec {name} > output_status.txt 2>&1'.format(name=script_name),
                                     shell=True,
                                     cwd=script_path,
                                     preexec_fn=os.setsid)
                should_break = False
                for line in tail('-f', '{path}/output_status.txt'.format(path=script_path), _iter=True):
                    update_message = None
                    if 'ERROR' in line:
                        os.killpg(os.getpgid(p.pid), signal.SIGTERM)
                        print_message('error downloading file')
                        update_message = {
                            'text': json.dumps({
                                'user': user,
                                'data_name': data_name,
                                'percent_complete': percent_complete,
                                'message': line,
                                'destination': 'esgf_download_status'
                            })
                        }
                    if 'saved' in line or 'Saving to' in line:
                        number_complete += 1
                        percent_complete = number_complete / number_of_downloads * 100
                        print_message('percent complete: {}%'.format(percent_complete), 'ok')
                        update_message = {
                            'text': json.dumps({
                                'user': user,
                                'data_name': data_name,
                                'percent_complete': percent_complete,
                                'message': 'downloading',
                                'destination': 'esgf_download_status'
                            })
                        }
                    if percent_complete > 99:
                        print_message('Download complete', 'ok')
                        update_message = {
                            'text': json.dumps({
                                'user': user,
                                'data_name': data_name,
                                'percent_complete': 100.0,
                                'message': 'complete',
                                'destination': 'esgf_download_status'
                            })
                        }
                        should_break = True
                    if update_message:
                        print_message("sending to active group {}".format(update_message))
                        Group('active').send(update_message)
                    if should_break:
                        break
                out, err = p.communicate()
                print out, err
            except Exception, e:
                print_debug(e)
                return -1
            break
Ejemplo n.º 29
0
 def test_no_logon(self):
     _clear_creds(self.esgf_dir)
     lm = LogonManager(self.esgf_dir)
     assert not lm.is_logged_on()
Ejemplo n.º 30
0
def test_bootstrap():
    _clear_creds()
    lm = LogonManager(esgf_dir)
    lm.logon(TEST_USER, TEST_PASSWORD, TEST_MYPROXY, bootstrap=True)

    assert lm.is_logged_on()
Ejemplo n.º 31
0
def test_no_logon():
    _clear_creds()
    lm = LogonManager(esgf_dir)
    assert lm.is_logged_on() == False
Ejemplo n.º 32
0
def test_expired():
    _load_creds('expired.pem')
    lm = LogonManager(esgf_dir)
    assert lm.state == lm.STATE_EXPIRED_CREDENTIALS
Ejemplo n.º 33
0
def grid(request):
    ''' For demo purposes this is loading a local file '''
    from xml.etree.ElementTree import parse
    import requests
    from StringIO import StringIO

    try:
        r = requests.get('http://pcmdi9.llnl.gov/esgf-node-manager/registration.xml')
        f = StringIO(r.content)
        out = open('scripts/registration.xml', 'w')
        out.write(f.read())
        out.close()
    except Exception as e:
        print repr(e)
        return HttpResponse(status=404)
    tree = parse('scripts/registration.xml')

    node_name_list = []
    node_peer_list = []
    node_url_list = []
    node_location_list = []
    for node in tree.getroot():
        attrs = node.attrib
        node_name_list.append(attrs["shortName"])
        node_peer_list.append(attrs["adminPeer"])
        node_url_list.append(attrs["hostname"])
        for child in node:
            if child.tag[-11:] == "GeoLocation":
                node_location_list.append(child.attrib["city"])
    node_list = zip(node_peer_list, node_url_list, node_name_list, node_location_list)

    creds = Credential.objects.filter(site_user_name=request.user)
    if len(creds) != 0:
        for c in creds:
            try:
                if c.service == 'esgf':
                    import pyesgf
                    from pyesgf import LogonManager
                    lm = LogonManager()
                    lm.logon_with_openid(c.service_user_name, c.password)
                    if lm.is_logged_on():
                        request.session['esgf_login'] = lm
                        print 'esgf log in successful'
                if c.service == 'velo':
                    lib_path = os.path.abspath(os.path.join('apps', 'velo'))
                    sys.path.append(lib_path)
                    import VeloAPI

                    velo_api = VeloAPI.Velo()
                    velo_api.start_jvm()
                    res = velo_api.init('acmetest', 'acmetest')
                    request.session['velo_login'] = res
                    print 'velo log in successful'
                    '''
                    For production, uncomment
                    res = velo_api.init(c.service_user_name, c.password)
                    '''
            except:
                import traceback
                print '1', e.__doc__
                print '2', sys.exc_info()
                print '3', sys.exc_info()[0]
                print '4', sys.exc_info()[1]
                print '5', traceback.tb_lineno(sys.exc_info()[2])
                ex_type, ex, tb = sys.exc_info()
                print '6', traceback.print_tb(tb)
                return HttpResponse(status=500)

    return HttpResponse(render_template(request, "web_fe/grid.html", {'nodes': node_list}))
Ejemplo n.º 34
0
conn = SearchConnection('http://esgf-data.dkrz.de/esg-search', distrib=False) #connect to a server

# ===========================================================================================
# The first time this need to be run
# Note: This never worked in the HPC, so I ran it in my computer and then copied the entire 
# content of my ~/esg folder to the same folder in the home in the HPC
#
#OPENID = 'https://esgf-data.dkrz.de/esgf-idp/openid/******'
#lm.logon_with_openid(openid=OPENID, password=None, bootstrap=True)
#lm.is_logged_on()
#
# ===========================================================================================

#loggin to ESGF
lm = LogonManager()
#lm.logoff()
#lm.logon_with_openid('https://esgf-data.dkrz.de/esgf-idp/openid/XXXXX', password= '******') 
#lm.is_logged_on()


var6 = ['ta300'
        ,'ta500'
        ,'ta700'
        ,'ta850'
        ,'ua850'
        ,'ua300'
        ,'va850'
        ,'va300'
        ,'uas'
        ,'vas'
Ejemplo n.º 35
0
 def test_no_logon(self):
     _clear_creds(self.esgf_dir)
     lm = LogonManager(self.esgf_dir)
     assert not lm.is_logged_on()
Ejemplo n.º 36
0
"""Example for plotting all orographies with OoPlot by accessing the ESGF database.
"""

import cordex.plot as crxplt

from pyesgf.search import SearchConnection
from pyesgf.logon import LogonManager

# logon to ESGF node
print('logon to ESGF')
lm = LogonManager()
lm.logoff()
lm.logon(hostname='esgf-data.dkrz.de', interactive=True, bootstrap=True)
print('logged on: {}'.format(lm.is_logged_on()))


def plot_orog(filename, output):
    """plots the orog variable to output file.
    """
    var = 'orog'
    crxplt.contour2(filename, var, output)


# search CORDEX project for REMO2015 fx orog variables
conn = SearchConnection('http://esgf-data.dkrz.de/esg-search', distrib=False)
ctx = conn.new_context(project='CORDEX',
                       experiment='evaluation',
                       time_frequency='fx',
                       rcm_name='REMO2015',
                       variable='orog')
result = ctx.search()
Ejemplo n.º 37
0
def main():
    #--- input user credential --
    username='******'
    password='******'
    #----------------------------
    
    #--- input search criteria --
    project='ISIMIP2b'
    model='GFDL-ESM2M'
    impact_model='WaterGAP2'
    experiment ='rcp45'
    variable='Discharge'

    # Loging in
    lm = LogonManager()
    lm.logoff()
    lm.is_logged_on()
    OPENID = 'https://esgf-data.dkrz.de/esgf-idp/openid/'+username
    myproxy_host = 'esgf-data.dkrz.de'
    lm.logon_with_openid(openid=OPENID, password=password, bootstrap=True)
    lm.logon(hostname=myproxy_host,interactive=False, username=username,password=password,bootstrap=True)
    lm.is_logged_on()
    
    # Open connection with potsam node (ISIMIP) 
    conn = SearchConnection('http://esg.pik-potsdam.de/esg-search', distrib=False)
    
    # Search datasets
    # can do general searches e.g., search for all datasets in ISIMIP2b with experiment='rcp45'. This will return all instances in ctx
    ctx = conn.new_context(
        project=project,
        model=model,
        impact_model=impact_model,
        experiment=experiment,
        variable_long_name=variable)
    
    # list number of counts
    print('Founds '+str(ctx.hit_count)+' matching datasets')
    
    # grab search results and display them
    a=ctx.search()
    cnt=1
    for i in a :
       print('['+str(cnt)+']'+'   ----->  '+i.dataset_id)
       print('- - - - - - - - - - - - - - - - - - - - - - - -')
       cnt = cnt + 1
    
    # Ask user to choose a dataset or to download all 
    num = input("Which one should I download master? [Type -1 for all, 3 for third listed dataset.]")
    
    # Case where user select a specific dataset
    if num != -1:
        print("Downloading dataset "+str(num)+".")
        wget_makeNrun(a[num-1].file_context())
    # case where user selects all
    elif num == -1:
        print("Downloading all "+str(ctx.hit_count)+" datasets.")
        for i in a:
            print("Downloading all datasets returned in search.")
            wget_makeNrun(i.file_context())
    return 0
Ejemplo n.º 38
0
    'hurs',
    'tasmin',
    'tasmax',
    'uas',
    'vas',
    'tas',
    'pr',
    'ps',
    'rsds',
    'rlds']  # ,'sfcWind'
expers = ['rcp26', 'historical', 'rcp85']



# logon manager
lm = LogonManager()
if not lm.is_logged_on():

    lm.logoff()
    lm.is_logged_on()

    lm.logon_with_openid(openid=openid, password=None, bootstrap=True)
    lm.is_logged_on()

    lm.logon(hostname='esgf-data.dkrz.de', interactive=True, bootstrap=True)
    lm.is_logged_on()


conn = SearchConnection('https://esgf-data.dkrz.de/esg-search', distrib=True)

for exper in expers:
Ejemplo n.º 39
0
def test_no_logon():
    _clear_creds()
    lm = LogonManager(esgf_dir)
    assert lm.is_logged_on() == False
Ejemplo n.º 40
0
def download_ESGF_data(Open_ID, password, server, project, experiment,
                       time_frequency, variable, domain, path_output):
    """Esta función nos permite descargar masivamente mediante WGET los diferentes ficheros netcdf que contienen los servidrores de ESGF sobre cambio climático.
    
    El servidor por defecto que se va a utilizar es: https://esgf-data.dkrz.de/projects/esgf-dkrz/.
    
    Parámetros:
    ---------------------
    Open_ID         : string. ID de tu usuario para acceder a la base de datos correspondiente del servidor
    password        : string. Contraseña correspondiente a la ID
    server          : string. Servidor del que se desea descargar la información. Ejemplo: https://esgf-data.dkrz.de/esg-search
    project         : string. Proyecto dentro del servidor del que se quiere descargar los datos. Ejemplo: CORDEX, CMIP5, CMIP6
    experiment      : string. Escenarios de cambio climático. Ejemplo: historical, rcp26, rcp45, rcp85
    time_frequency  : string. Frecuencia de la base de datos que se quiere. Ejemplo: 1hr, 6hr, day, mon
    variable        : string. Variable que se desea descargar: tasmax, tasmin, pr 
    domain          : string. En el caso de que se desee descargar CORDEX, se debe de incluir el nombre de la malla. Ejemplo: EUR-11
    path_output     : string. Directorio donde se desean guardar los ficheros
    
    Salidas:
    ----------------------
    Ficheros netcdf para cada uno de los escenarios y modelos solicitados
    
    """
    dir_file = __file__
    os.chdir(dir_file[:-16])
    print(dir_file)
    conn = SearchConnection('https://' + server + '/esg-search', distrib=True)
    lm = LogonManager()
    lm.logoff()
    lm.is_logged_on()
    lm.logon_with_openid(Open_ID, password, bootstrap=True)
    lm.is_logged_on()
    if project == 'CORDEX':
        ctx = conn.new_context(
            project=project,
            experiment=experiment,
            time_frequency=time_frequency,
            variable=variable,
            domain=domain,
        )
    else:
        ctx = conn.new_context(
            project=project,
            experiment=experiment,
            time_frequency=time_frequency,
            variable=variable,
        )

    with open('wget-plantilla-ESGF.sh', "r+") as out_file:
        lines = out_file.readlines()

    for ct in tqdm.tqdm(range(ctx.hit_count)):
        files_list = list()
        result = ctx.search()[ct]
        lines[22] = "openId='" + Open_ID + "'\n"
        lines[
            23] = "search_url='https://" + server + "/esg-search/wget/?distrib=false&dataset_id=" + result.dataset_id + "'\n"
        lines_first = lines[:27]
        lines_end = lines[28:]

        files = result.file_context().search()
        ntcf_ds = list()
        ntcf_name = list()
        for file in files:
            try:
                if variable in file.opendap_url:
                    files_list.append("'" + file.filename + "'" + ' ' + "'" +
                                      file.download_url + "'" + ' ' + "'" +
                                      file.checksum_type + "'" + ' ' + "'" +
                                      file.checksum + "'" + '\n')
            except:
                continue
        if len(files_list) == 0:
            continue
        else:
            with open(path_output + "Download.sh", "w") as fh:
                for line in (lines_first + files_list + lines_end):
                    fh.write(line)

            conn = SearchConnection('https://' + server + '/esg-search',
                                    distrib=True)
            lm = LogonManager()
            lm.logoff()
            lm.is_logged_on()

            lm.logon_with_openid(Open_ID, password)
            lm.is_logged_on()
            os.chdir(path_output)
            os.system('bash ' + path_output + 'Download.sh' + ' H ' + Open_ID +
                      ' ' + password)
Ejemplo n.º 41
0
 def test_expired(self):
     _load_creds(self.esgf_dir, credentials_file='expired.pem')
     lm = LogonManager(self.esgf_dir)
     assert lm.state == lm.STATE_EXPIRED_CREDENTIALS
Ejemplo n.º 42
0
def check_credentials(request):
    if request.method == 'POST':
        try:
            data = json.loads(request.body)
            response = {}
            creds = Credential.objects.filter(site_user_name=request.user)
            velo_started = False
            if len(creds) != 0:
                for c in creds:
                    try:
                        if c.service == 'esgf':
                            import pyesgf
                            from pyesgf.logon import LogonManager
                            lm = LogonManager()
                            lm.logon_with_openid(
                                c.service_user_name, c.password)
                            if lm.is_logged_on():
                                response[s] = 'success'
                                print 'esgf log in successful'
                            else:
                                print 'esgf log in failed'
                                response[s] = 'fail'
                        if c.service == 'velo':
                            lib_path = os.path.abspath(
                                os.path.join('apps', 'velo'))
                            sys.path.append(lib_path)

                            '''
                            For production, replace below with:
                            rm = velo_api.init(c.service_user_name, c.password)
                            '''
                            rm = velo_api.init('acmetest', 'acmetest')
                            if rm.getRepositoryUrlBase() == 'u\'http://acmetest.ornl.gov:80/alfresco\'':
                                response[s] = 'success'
                                velo_api.shutdown_jvm()
                                print 'velo log in successful'
                            else:
                                velo_api.shutdown_jvm()
                                response[s] = 'fail'
                                print 'Error in velo initialization', rm.getRepositoryUrlBase()

                        if c.service == 'github':
                            import github3
                            from github3 import login
                            gh = login(c.site_user_name, password=c.password)
                            if gh.user() == c.site_user_name:
                                print 'Github login successful'
                                response[s] = 'success'
                            else:
                                print 'Github login failure'
                                response[s] = 'fail'

                        if c.service == 'jira':
                            print 'Working on jira....'
                    except:
                        if velo_started:
                            velo_api.shutdown_jvm()
                        import traceback
                        print '1', e.__doc__
                        print '2', sys.exc_info()
                        print '3', sys.exc_info()[0]
                        print '4', sys.exc_info()[1]
                        print '5', traceback.tb_lineno(sys.exc_info()[2])
                        ex_type, ex, tb = sys.exc_info()
                        print '6', traceback.print_tb(tb)
                        return HttpResponse(status=500)

            return HttpResponse(json.dumps(response))

        except Exception as e:
            import traceback
            print '1', e.__doc__
            print '2', sys.exc_info()
            print '3', sys.exc_info()[0]
            print '4', sys.exc_info()[1]
            print '5', traceback.tb_lineno(sys.exc_info()[2])
            ex_type, ex, tb = sys.exc_info()
            print '6', traceback.print_tb(tb)
            return HttpResponse(status=500)
    else:
        return HttpResponse(status=404)
Ejemplo n.º 43
0
def test_bootstrap():
    _clear_creds()
    lm = LogonManager(esgf_dir)
    lm.logon(TEST_USER, TEST_PASSWORD, TEST_MYPROXY, bootstrap=True)

    assert lm.is_logged_on()