예제 #1
0
파일: artemis.py 프로젝트: STBadman/heliopy
 def download_func(remote_base_url, local_base_dir,
                   directory, fname, remote_fname, extension):
     remote_url = remote_base_url + str(directory)
     # Now load remotely
     util.load(fname + extension,
               local_base_dir / directory,
               remote_url)
예제 #2
0
파일: wind.py 프로젝트: ikitcheng/heliopy
 def download_func(remote_base_url, local_base_dir, directory, fname,
                   extension):
     remote_url = '{}{}'.format(remote_base_url, directory)
     util.load(fname + extension,
               local_base_dir / directory,
               remote_url,
               guessversion=True)
예제 #3
0
def _merged_fromascii(probe, year, doy, try_download):
    """
    Read in a single day of merged data.

    Data is loaded from orignal ascii files, and saved to a hdf file for faster
    access after first read in.

    Parameters
    ----------
    probe : int, string
        Helios probe to import data from. Must be 1 or 2.
    year : int
        Year
    doy : int
        Day of year

    Returns
    -------
    data : DataFrame
        Merged data set
    """
    probe = _check_probe(probe)
    local_dir = _merged_localdir(probe)
    remote_url = ('ftp://cdaweb.gsfc.nasa.gov/pub/data/helios/'
                  'helios{}/merged/he{}_40sec'.format(probe, probe))
    filename = _merged_fname(probe, year, doy) + '.dat'
    asciiloc = os.path.join(local_dir, filename)

    # Make sure file is downloaded
    util.load(filename, local_dir, remote_url, try_download=try_download)

    # Load data
    data = pd.read_table(asciiloc, delim_whitespace=True)

    # Process data
    data['year'] = data['year'].astype(int)
    # Convert date info to datetime
    data['Time'] = pd.to_datetime(data['year'], format='%Y') + \
        pd.to_timedelta(data['day'] - 1, unit='d') + \
        pd.to_timedelta(data['hour'], unit='h') + \
        pd.to_timedelta(data['min'], unit='m') + \
        pd.to_timedelta(data['sec'], unit='s')

    data = data.drop(['year', 'day', 'hour', 'min', 'sec', 'dechr'], axis=1)
    # Set zero values to nans
    data.replace(0.0, np.nan, inplace=True)

    # Save data to a hdf store
    if use_hdf:
        _save_hdf(data, _merged_localdir(probe),
                  _merged_fname(probe, year, doy))
    return (data)
예제 #4
0
def _mag_hires_helper(year, doy, local_dir, url, coords):
    fname = str(year)[2:] + doy + '_FGM_' + coords

    hdf_fname = '{}_{}.hdf'.format(year, doy)
    hdfloc = local_dir / hdf_fname
    if hdfloc.exists():
        return pd.read_hdf(hdfloc)

    f = util.load(fname + '.TAB', local_dir, url)
    if f is None:
        raise RuntimeError(
            'No file named {} exits on remote server'.format(fname))
    elif 'error_message' in f.readline():
        location = f.name
        f.close()
        os.remove(f.name)
        raise RuntimeError(
            'No file named {} exits on remote server'.format(fname))

    df = pd.read_table(f,
                       names=['Time', 'Bx', 'By', 'Bz'],
                       delim_whitespace=True,
                       parse_dates=[0],
                       index_col=0)
    f.close()
    if use_hdf:
        df.to_hdf(hdfloc, key='data', mode='w')

    return df
예제 #5
0
파일: imp.py 프로젝트: ikitcheng/heliopy
def mitplasma_h0(probe, starttime, endtime):
    """
    Import mit h0 plasma data.

    Parameters
    ----------
    probe : string
        Probe number.
    starttime : datetime
        Start of interval.
    endtime : datetime
        End of interval.

    Returns
    -------
    data : DataFrame
        Requested data.
    """
    data = []
    dtimes = util._daysplitinterval(starttime, endtime)
    # Loop through years
    for dtime in dtimes:
        date = dtime[0]
        intervalstring = str(date.year) +\
            str(date.month).zfill(2) +\
            str(date.day).zfill(2)
        filename = 'i' + probe + '_h0_mitplasma_' + intervalstring + '_v01.cdf'
        # Location of file relative to local directory or remote url
        relative_loc = 'imp' + probe + '/plasma_mit/mitplasma_h0/' +\
            str(date.year)

        local_dir = os.path.join(imp_dir, relative_loc)
        remote_url = imp_url + relative_loc

        cdf = util.load(filename, local_dir, remote_url)
        keys = {
            'EW_flowangle_best': 'EW_flowangle_best',
            'EW_flowangle_mom': 'EW_flowangle_best',
            'Epoch': 'Time',
            'Flow_elevation_thresh': 'Flow_elevation_thresh',
            'Flow_elevation_threshsp': 'Flow_elevation_threshsp',
            'Region': 'Region',
            'V_fit': 'V_fit',
            'V_mom': 'V_fit',
            'mode': 'mode',
            'protonV_thermal_fit': 'protonV_thermal_fit',
            'protonV_thermal_mom': 'protonV_thermal_fit',
            'proton_density_fit': 'proton_density_fit',
            'proton_density_mom': 'proton_density_mom',
            'xyzgse': ['x_gse', 'y_gse', 'z_gse'],
            'ygsm': 'ygsm',
            'zgsm': 'zgsm'
        }
        thisdata = util.cdf2df(cdf, 'Epoch', keys)
        data.append(thisdata)

    data = pd.concat(data)
    data = data[(data['Time'] > starttime) & (data['Time'] < endtime)]
    data.index.name = 'Time'
    return data
예제 #6
0
 def download_func(remote_base_url, local_base_dir,
                   directory, fname, extension):
     def check_exists():
         # Because the version might be different to the one we guess, work
         # out the downloaded filename
         for f in os.listdir(os.path.join(local_base_dir, directory)):
             if (f[:-6] == (fname + extension)[:-6]):
                 # Return filename with '.cdf' stripped off the end
                 return f[:-4]
     if check_exists() is not None:
         return check_exists()
     # Now load remotely
     util.load(fname + extension,
               os.path.join(local_base_dir, directory),
               remote_base_url + directory, guessversion=True)
     if check_exists() is not None:
         return check_exists()
예제 #7
0
파일: imp.py 프로젝트: ikitcheng/heliopy
def mag320ms(probe, startTime, endTime):
    """
    Import 320ms cadence magnetic field data.

    Parameters
    ----------
        probe : string
            Probe number.
        starttime : datetime
            Start of interval.
        endtime : datetime
            End of interval.

    Returns
    -------
        data : DataFrame
            Requested data.
    """
    data = []
    dtimes = util._daysplitinterval(startTime, endTime)
    # Loop through years
    for dtime in dtimes:
        date = dtime[0]
        intervalstring = str(date.year) +\
            str(date.month).zfill(2) +\
            str(date.day).zfill(2)
        filename = 'i8_320msec_mag_' + intervalstring + '_v01.cdf'
        hdffname = filename[:-3] + '.hdf'
        # Location of file relative to local directory or remote url
        relative_loc = 'imp' + probe + '/mag/mag_320msec_cdaweb/' +\
            str(date.year)

        local_dir = os.path.join(imp_dir, relative_loc)
        hdffile = os.path.join(local_dir, hdffname)
        if os.path.exists(hdffile):
            thisdata = pd.read_hdf(hdffile)
            data.append(thisdata)
            continue

        remote_url = imp_url + relative_loc
        cdf = util.load(filename, local_dir, remote_url)
        keys = {
            'B': '|B|',
            'BX': 'Bx',
            'BY': 'By',
            'BZ': 'Bz',
            'Epoch': 'Time'
        }
        thisdata = util.cdf2df(cdf, 'Epoch', keys)
        data.append(thisdata)
        if use_hdf:
            thisdata.to_hdf(hdffile, key='merged', mode='w')

    data = pd.concat(data)
    data = data[(data['Time'] > startTime) & (data['Time'] < endTime)]
    data = data.drop(columns='Time')
    data.index.name = 'Time'
    return data
예제 #8
0
def fgm_hires(starttime, endtime):
    """
    Import high resolution fluxgate magnetometer data.

    Parameters
    ----------
    starttime : datetime
        Start of interval
    endtime : datetime
        End of interval

    Returns
    -------
    data : DataFrame
        Requested data
    """
    fgm_options = url_options
    readargs = {
        'names':
        ['year', 'doy', 'hour', 'minute', 'second', 'Bx', 'By', 'Bz', '|B|'],
        'delim_whitespace':
        True
    }

    data = []
    dtimes = util._daysplitinterval(starttime, endtime)
    # Loop through years
    for dtime in dtimes:
        date = dtime[0]
        yearstr = date.strftime('%Y')
        fgm_options['FILE_NAME'] = ('U' + yearstr[-2:] + date.strftime('%j') +
                                    'SH.ASC')
        # Local locaiton to download to
        local_dir = ulysses_dir / 'fgm' / 'hires' / yearstr
        local_file = local_dir / fgm_options['FILE_NAME']
        local_hdf = local_file.with_suffix('.hdf')
        # If we have already saved a hdf file
        if local_hdf.exists():
            thisdata = pd.read_hdf(local_hdf)
        else:
            # Put together remote url
            fgm_options['FILE_PATH'] = '/ufa/HiRes/VHM-FGM/' + yearstr
            remote_url = ulysses_url
            for key in fgm_options:
                remote_url += key + '=' + fgm_options[key] + '&'
            f = util.load(fgm_options['FILE_NAME'], local_dir, remote_url)

            # Read in data
            thisdata = pd.read_table(f, **readargs)
            # Process data/time
            thisdata = _convert_ulysses_time(thisdata)
            if use_hdf:
                thisdata.to_hdf(local_hdf, 'fgm_hires')
        data.append(thisdata)
    return util.timefilter(data, starttime, endtime)
예제 #9
0
파일: ace.py 프로젝트: ikitcheng/heliopy
    def download_func(remote_base_url, local_base_dir, directory, fname,
                      extension):
        def check_exists():
            # Because the version might be different to the one we guess, work
            # out the downloaded filename
            for f in (local_base_dir / directory).iterdir():
                fstr = str(f.name)
                if (fstr[:-6] == (fname + extension)[:-6]):
                    # Return filename with '.cdf' stripped off the end
                    return fstr[:-4]

        if check_exists() is not None:
            return check_exists()

        # Now load remotely
        util.load(fname + extension,
                  local_base_dir / directory,
                  remote_base_url + str(directory),
                  guessversion=True)
        return check_exists()
예제 #10
0
def _mag_ness_fromascii(probe, year, doy, try_download=True):
    """
    Read in a single day of 6 second magnetic field data.

    Data is read from orignal ascii files, and saved to a hdf file for faster
    access after the first read.

    Parameters
    ----------
    probe : int, string
        Helios probe to import data from. Must be 1 or 2.
    year : int
        Year
    doy : int
        Day of year

    Returns
    -------
    data : DataFrame
        6 second magnetic field data set
    """
    probe = _check_probe(probe)
    local_dir = _ness_localdir(probe, year)
    remote_url = ('ftp://spdf.sci.gsfc.nasa.gov/pub/data/helios/helios' +
                  probe + '/mag/6sec_ness/' + str(year) + '/')
    fname = _ness_fname(probe, year, doy) + '.asc'
    f = util.load(fname, local_dir, remote_url, try_download=try_download)

    # Read in data
    headings = [
        'probe', 'year', 'doy', 'hour', 'minute', 'second', 'naverage', 'Bx',
        'By', 'Bz', '|B|', 'sigma_Bx', 'sigma_By', 'sigma_Bz'
    ]

    colspecs = [(1, 2), (2, 4), (4, 7), (7, 9), (9, 11), (11, 13), (13, 15),
                (15, 22), (22, 29), (29, 36), (36, 42), (42, 48), (48, 54),
                (54, 60)]
    data = pd.read_fwf(f, names=headings, header=None, colspecs=colspecs)

    # Process data
    data['year'] += 1900
    # Convert date info to datetime
    data['Time'] = pd.to_datetime(data['year'], format='%Y') + \
        pd.to_timedelta(data['doy'] - 1, unit='d') + \
        pd.to_timedelta(data['hour'], unit='h') + \
        pd.to_timedelta(data['minute'], unit='m') + \
        pd.to_timedelta(data['second'], unit='s')
    data = data.drop(['year', 'doy', 'hour', 'minute', 'second'], axis=1)
    data = data.set_index('Time', drop=False)

    # Save data to a hdf store
    if use_hdf:
        _save_hdf(data, local_dir, _ness_fname(probe, year, doy))
    return (data)
예제 #11
0
파일: wind.py 프로젝트: vigneshtdev/heliopy
    def processing_func(local_dir, fname):
        distkeys = []
        for i in range(0, 13):
            distkeys.append('f_pitch_E' + str(i).zfill(2))
        anglelabels = []
        for i in range(0, 30):
            anglelabels.append((i + 0.5) * np.pi / 30)
        timekey = 'Epoch'
        energykey = 'Ve'

        cdf = util.load(fname, local_dir, '')
        df = util.pitchdist_cdf2df(cdf, distkeys, energykey, timekey,
                                   anglelabels)
        return df
예제 #12
0
파일: wind.py 프로젝트: vigneshtdev/heliopy
    def processing_func(directory, fname):
        cdf = util.load(fname, directory, '')

        epoch_dict = {'h0': 'Epoch3', 'h2': 'Epoch'}
        mag_dict = {'h0': 'B3GSE', 'h2': 'BGSE'}

        epoch_key = epoch_dict[version]
        mag_key = mag_dict[version]

        keys = {mag_key: ['Bx_gse', 'By_gse', 'Bz_gse'], epoch_key: 'Time'}
        badvalues = {'Bx_gse': -1e+31, 'By_gse': -1e+31, 'Bz_gse': -1e+31}
        df = util.cdf2df(cdf,
                         index_key=epoch_key,
                         keys=keys,
                         badvalues=badvalues)
        return df
예제 #13
0
파일: mms.py 프로젝트: vigneshtdev/heliopy
def fgm_survey(probe, starttime, endtime):
    """
    Import fgm survey mode magnetic field data.

    Parameters
    ----------
    probe : string
        Probe number, must be 1, 2, 3, or 4
    starttime : datetime
        Interval start time.
    endtime : datetime
        Interval end time.

    Returns
    -------
    data : DataFrame
        Imported data.
    """
    # Directory relative to main MMS data directory
    relative_dir = os.path.join('mms' + probe, 'fgm', 'srvy', 'l2')

    daylist = util._daysplitinterval(starttime, endtime)
    data = []
    for day in daylist:
        date = day[0]
        this_relative_dir = os.path.join(relative_dir, str(date.year),
                                         str(date.month).zfill(2))
        filename = 'mms{}_fgm_srvy_l2_{}{:02}{:02}_v4.18.0.cdf'.format(
            probe, date.year, date.month, date.day)

        # Absolute path to local directory for this data file
        local_dir = os.path.join(mms_dir, this_relative_dir)
        util._checkdir(local_dir)

        remote_url = remote_mms_dir + this_relative_dir
        # Load cdf file
        cdf = util.load(filename, local_dir, remote_url)

        # Convert cdf to dataframe
        keys = {
            'mms' + probe + '_fgm_b_gsm_srvy_l2': ['Bx', 'By', 'Bz', 'Br'],
            'Epoch': 'Time'
        }
        df = util.cdf2df(cdf, 'Epoch', keys)
        data.append(df)

    return util.timefilter(data, starttime, endtime)
예제 #14
0
def _swics(starttime, endtime, names, product):
    swics_options = url_options
    readargs = {'names': names, 'delim_whitespace': True}

    data = []
    dtimes = util._daysplitinterval(starttime, endtime)
    # Loop through years
    for year in range(starttime.year, endtime.year + 1):
        swics_options['FILE_NAME'] = '{}{}.dat'.format(product, str(year)[-2:])
        # Local locaiton to download to
        local_dir = os.path.join(ulysses_dir, 'swics')
        local_file = os.path.join(local_dir, swics_options['FILE_NAME'])
        local_hdf = local_file[:-4] + '.hdf'
        # If we have already saved a hdf file
        if os.path.exists(local_hdf):
            thisdata = pd.read_hdf(local_hdf)
        else:
            # Put together remote url
            swics_options['FILE_PATH'] = '/ufa/HiRes/data/swics'
            remote_url = ulysses_url
            for key in swics_options:
                remote_url += key + '=' + swics_options[key] + '&'
            f = util.load(swics_options['FILE_NAME'], local_dir, remote_url)
            if f is None:
                print('File {}/{} not available\n'.format(
                    remote_url, filename))
                continue

            # Read in data
            thisdata = pd.read_table(f, **readargs)
            # Process data/time
            thisdata = _convert_ulysses_time(thisdata)
            if use_hdf:
                thisdata.to_hdf(local_hdf, 'swics')
        data.append(thisdata)
    return util.timefilter(data, starttime, endtime)
예제 #15
0
파일: wind.py 프로젝트: ikitcheng/heliopy
def threedp_sfpd(starttime, endtime):
    """
    Import 'sfpd' wind data.

    12 second energetic electron pitch-angle energy spectra from the foil SST

    Parameters
    ----------
    starttime : datetime
        Interval start time.
    endtime : datetime
        Interval end time.

    Returns
    -------
    data : DataFrame
    """
    # Directory relative to main WIND data directory
    relative_dir = path.Path('3dp') / '3dp_sfpd'

    daylist = util._daysplitinterval(starttime, endtime)
    data = []
    mag = []
    for (date, _, _) in daylist:
        this_relative_dir = relative_dir / str(date.year)
        # Absolute path to local directory for this data file
        local_dir = wind_dir / this_relative_dir
        filename = 'wi_sfpd_3dp_{:{dfmt}}_v02'.format(date, dfmt='%Y%m%d')
        hdfname = filename + '.hdf'
        hdfloc = local_dir / hdfname
        if hdfloc.exists():
            df = pd.read_hdf(hdfloc)
            data.append(df)
            continue

        util._checkdir(local_dir)
        remote_url = remote_wind_dir + str(this_relative_dir)
        cdf = util.load(filename + '.cdf',
                        local_dir,
                        remote_url,
                        guessversion=True)
        if cdf is None:
            print('File {}/{} not available\n'.format(remote_url, filename))
            continue

        data_today = []
        # Loop through each timestamp to build up fluxes
        for i, time in enumerate(cdf['Epoch'][...]):
            energies = cdf['ENERGY'][i, :]
            angles = cdf['PANGLE'][i, :]
            fluxes = cdf['FLUX'][i, :, :]
            magfield = cdf['MAGF'][i, :]
            index = pd.MultiIndex.from_product(
                ([time], energies, angles),
                names=['Time', 'Energy', 'Pitch angle'])
            df = pd.DataFrame(fluxes.ravel(), index=index, columns=['Flux'])
            df['Bx'] = magfield[0]
            df['By'] = magfield[1]
            df['Bz'] = magfield[2]
            data_today.append(df)
        data_today = pd.concat(data_today)
        data_today = data_today.sort_index()

        if use_hdf:
            data_today.to_hdf(hdfloc, 'sfpd', mode='w')
        data.append(data_today)

    data = util.timefilter(data, starttime, endtime)
    data = data.reset_index(level=['Energy', 'Pitch angle'])
    return data
예제 #16
0
파일: wind.py 프로젝트: ikitcheng/heliopy
def threedp_pm(starttime, endtime):
    """
    Import 'pm' wind data.

    3 second time resolution solar wind proton and alpha particle moments from
    the PESA LOW sensor, computed on-board the spacecraft

    Parameters
    ----------
    starttime : datetime
        Interval start time.
    endtime : datetime
        Interval end time.

    Returns
    -------
    data : DataFrame
    """
    # Directory relative to main WIND data directory
    relative_dir = path.Path('3dp') / '3dp_pm'

    daylist = util._daysplitinterval(starttime, endtime)
    data = []
    for day in daylist:
        date = day[0]
        this_relative_dir = relative_dir / str(day[0].year)
        # Absolute path to local directory for this data file
        local_dir = wind_dir / this_relative_dir
        filename = 'wi_pm_3dp_' +\
            str(date.year) +\
            str(date.month).zfill(2) +\
            str(date.day).zfill(2) +\
            '_v05.cdf'
        hdfname = filename[:-4] + 'hdf'
        hdfloc = local_dir / hdfname
        if hdfloc.exists():
            df = pd.read_hdf(hdfloc)
            data.append(df)
            continue

        util._checkdir(local_dir)
        remote_url = remote_wind_dir + str(this_relative_dir)
        cdf = util.load(filename, local_dir, remote_url, guessversion=True)
        if cdf is None:
            print('File {}/{} not available\n'.format(remote_url, filename))
            continue

        keys = {
            'A_DENS': 'n_a',
            'A_TEMP': 'T_a',
            'A_VELS': ['va_x', 'va_y', 'va_z'],
            'P_DENS': 'n_p',
            'P_TEMP': 'T_p',
            'P_VELS': ['vp_x', 'vp_y', 'vp_z'],
            'Epoch': 'Time'
        }
        df = util.cdf2df(cdf, index_key='Epoch', keys=keys)
        if use_hdf:
            df.to_hdf(hdfloc, 'pm', mode='w')
        data.append(df)

    return util.timefilter(data, starttime, endtime)
예제 #17
0
def fpi_dis_moms(probe, mode, starttime, endtime):
    """
    Import fpi ion distribution moment data.

    Parameters
    ----------
    probe : string
        Probe number, must be 1, 2, 3, or 4
    mode : string
        Data mode, must be 'fast' or 'brst'
    starttime : datetime
        Interval start time.
    endtime : datetime
        Interval end time.

    Returns
    -------
    data : DataFrame
        Imported data.
    """
    valid_modes = ['fast', 'brst']
    if mode not in valid_modes:
        raise RuntimeError('Mode must be either fast or brst')
    # Directory relative to main MMS data directory
    relative_dir = os.path.join('mms' + probe, 'fpi', mode, 'l2', 'dis-moms')

    daylist = util._daysplitinterval(starttime, endtime)
    data = []
    for day in daylist:
        date = day[0]
        starthour = day[1].hour
        endhour = day[2].hour + 1
        # fips fast data product has files every two hours, so get nearest two
        # hour stamps
        starthour -= np.mod(starthour, 2)
        endhour += np.mod(endhour, 2)
        for h in range(starthour, endhour, 2):
            this_relative_dir = os.path.join(relative_dir,
                                             str(date.year),
                                             str(date.month).zfill(2))
            filename = ('mms{}_fpi_{}_l2_dis-moms_'
                        '{}{:02}{:02}{:02}0000_v3.3.0.cdf').format(
                            probe, mode, date.year, date.month, date.day, h)

            # Absolute path to local directory for this data file
            local_dir = os.path.join(mms_dir, this_relative_dir)
            util._checkdir(local_dir)

            remote_url = remote_mms_dir + this_relative_dir
            # Load cdf file
            try:
                cdf = util.load(filename, local_dir, remote_url)
            except urllib.error.HTTPError as e:
                if str(e) == 'HTTP Error 404: Not Found':
                    print('No data available for hours', str(h) + '-' +
                          str(h + 2), 'on', date.strftime('%d/%m/%Y'))
                    continue
                else:
                    raise

            probestr = 'mms' + probe + '_'
            # Convert cdf to dataframe
            keys = {'Epoch': 'Time',
                    probestr + 'dis_bulkv_gse_fast': ['bulkv_x',
                                                      'bulkv_y',
                                                      'bulkv_z'],
                    probestr + 'dis_heatq_gse_fast': ['heatq_x',
                                                      'heatq_y',
                                                      'heatq_z'],
                    probestr + 'dis_numberdensity_fast': 'n',
                    probestr + 'dis_temppara_fast': 'T_par',
                    probestr + 'dis_tempperp_fast': 'T_perp'}
            df = util.cdf2df(cdf, 'Epoch', keys)
            data.append(df)

    return util.timefilter(data, starttime, endtime)
예제 #18
0
파일: wind.py 프로젝트: vigneshtdev/heliopy
 def processing_func(directory, fname):
     cdf = util.load(fname, directory, '')
     return util.cdf2df(cdf, 'Epoch', badvalues=badvalues)
예제 #19
0
def mag_1min(starttime, endtime, coords):
    """
    Import 1 minute magnetic field from Cassini.

    See http://pds-ppi.igpp.ucla.edu/search/view/?f=yes&id=pds://PPI/CO-E_SW_J_S-MAG-4-SUMM-1MINAVG-V1.0
    for more information.

    Cassini Orbiter Magnetometer Calibrated MAG data in 1 minute averages
    available covering the period 1999-08-16 (DOY 228) to 2016-12-31 (DOY 366).
    The data are provided in RTN coordinates throughout the mission, with
    Earth, Jupiter, and Saturn centered coordinates for the respective
    flybys of those planets.

    Parameters
    ----------
    starttime : datetime
        Interval start time.
    endtime : datetime
        Interval end time.
    coords : strings
        Requested coordinate system. Must be one of
        ``['KRTP', 'KSM', 'KSO', 'RTN']``

    Returns
    -------
    data : DataFrame
        Requested data
    """
    valid_coords = ['KRTP', 'KSM', 'KSO', 'RTN']
    if coords not in valid_coords:
        raise ValueError('coords must be one of {}'.format(valid_coords))
    base_url = ('http://pds-ppi.igpp.ucla.edu/ditdos/download?'
                'id=pds://PPI/CO-E_SW_J_S-MAG-4-SUMM-1MINAVG-V1.0/DATA')

    data = []
    for year in starttime.year, endtime.year:
        url = '{}/{}'.format(base_url, year)
        local_dir = os.path.join(cassini_dir, 'mag', '1min')

        fname = '{}_FGM_{}_1M'.format(year, coords)

        hdfloc = os.path.join(local_dir, fname + '.hdf')
        if os.path.isfile(hdfloc):
            df = pd.read_hdf(hdfloc)
            data.append(df)
            continue

        f = util.load(fname + '.TAB', local_dir, url)
        if 'error_message' in f.readline():
            f.close()
            os.remove(os.path.join(local_dir, fname + '.TAB'))
            continue

        df = pd.read_table(f,
                           names=[
                               'Time', 'Bx', 'By', 'Bz', '|B|', 'X', 'Y', 'Z',
                               'Local hour', 'n points'
                           ],
                           delim_whitespace=True,
                           parse_dates=[0],
                           index_col=0)
        data.append(df)

        if use_hdf:
            df.to_hdf(hdfloc, key='data', mode='w')

    return util.timefilter(data, starttime, endtime)
예제 #20
0
def mag_rtn(starttime, endtime):
    """
    Import magnetic field in RTN coordinates from Messenger.

    Parameters
    ----------
    starttime : datetime
        Interval start time.
    endtime : datetime
        Interval end time.

    Returns
    -------
    data : DataFrame
    """
    # Directory relative to main WIND data directory
    relative_dir = 'rtn'

    daylist = util._daysplitinterval(starttime, endtime)
    data = []
    for day in daylist:
        date = day[0]

        this_relative_dir = os.path.join(relative_dir, str(date.year))
        hdffile = 'messenger_mag_rtn_' +\
            str(date.year) +\
            str(date.month).zfill(2) +\
            str(date.day).zfill(2) +\
            '_v01.hdf'
        hdfloc = os.path.join(mess_dir, this_relative_dir, hdffile)
        # Try to load hdf file
        if os.path.isfile(hdfloc):
            df = pd.read_hdf(hdfloc)
            data.append(df)
            continue

        filename = hdffile[:-4] + '.cdf'
        # Absolute path to local directory for this data file
        local_dir = os.path.join(mess_dir, this_relative_dir)
        util._checkdir(local_dir)

        remote_url = os.path.join(remote_mess_dir, this_relative_dir)

        cdf = util.load(filename, local_dir, remote_url, guessversion=True)
        if cdf is None:
            print('File {}/{} not available\n'.format(remote_url, filename))
            continue

        keys = {
            'B_normal': 'Bn',
            'B_radial': 'Br',
            'B_tangential': 'Bt',
            'Epoch': 'Time',
            'azimuth_ecliptic': 'sc_Az',
            'latitude_ecliptic': 'sc_Lat',
            'radialDistance': 'sc_r',
            'MissionElapsedTime': 'mission_time'
        }
        df = util.cdf2df(cdf, index_key='Epoch', keys=keys)

        if use_hdf:
            hdffile = filename[:-4] + '.hdf'
            df.to_hdf(hdfloc, key='data', mode='w')
        data.append(df)

    return util.timefilter(data, starttime, endtime)
예제 #21
0
 def download_func(remote_base_url, local_base_dir, directory, fname,
                   extension):
     remote_url = '{}{}'.format(remote_base_url, directory)
     util.load(fname + extension, os.path.join(local_base_dir, directory),
               remote_url)
예제 #22
0
파일: ace.py 프로젝트: vigneshtdev/heliopy
 def processing_func(local_dir, local_fname):
     cdf = util.load(local_fname, local_dir, '')
     return util.cdf2df(cdf,
                        index_key='Epoch',
                        keys=keys,
                        badvalues=badvalues)
예제 #23
0
def _fourHz_fromascii(probe, year, doy, try_download=True):
    """
    Read in a single day of 4Hz magnetic field data.

    Parameters
    ----------
    probe : int, string
        Helios probe to import data from. Must be 1 or 2.
    year : int
        Year
    doy : int
        Day of year

    Returns
    -------
    data : DataFrame
        4Hz magnetic field data set
    """
    probe = _check_probe(probe)
    local_dir = _4hz_localdir(probe)
    fname_prefix = _4hz_filename(probe, year, doy)
    # For some reason the last number in the filename is the hour at which
    # data starts from on that day... this means a loop to check each hour
    asciiloc = None
    fname = None
    for i in range(0, 24):
        testloc = os.path.join(local_dir,
                               fname_prefix + str(i).zfill(2) + '.asc')
        if os.path.isfile(testloc):
            asciiloc = testloc
            break
    if asciiloc is not None:
        if os.name == 'nt':
            splitchar = '\\'
        else:
            splitchar = '/'
        fname = asciiloc.split(splitchar)[-1]
        remote_url = None
    elif try_download is not False:
        ftpsite = 'apollo.ssl.berkeley.edu'
        remote_dir = ('pub/helios-data/E2_experiment/'
                      'Data_Cologne_Nov2016_bestdata/'
                      'HR/helios{}'.format(probe))
        remote_url = 'ftp://' + ftpsite + '/' + remote_dir

        fname = None
        with FTP(ftpsite) as ftp:
            ftp.login()
            filenames = ftp.nlst(remote_dir)
            for filename in filenames:
                if fname_prefix in filename:
                    fname = filename
                    break
    if fname is None:
        raise ValueError('No mag data available locally or remotely')

    asciiloc = util.load(fname, local_dir, remote_url)

    # Read in data
    headings = ['Time', 'Bx', 'By', 'Bz', '|B|']
    cols = [0, 4, 5, 6, 7]
    data = pd.read_table(asciiloc,
                         names=headings,
                         header=None,
                         usecols=cols,
                         delim_whitespace=True)

    # Convert date info to datetime
    data['Time'] = pd.to_datetime(data['Time'], format='%Y-%m-%dT%H:%M:%S')
    data = data.set_index('Time', drop=False)

    # Save data to a hdf store
    if use_hdf:
        fname = _4hz_filename(probe, year, doy)
        _save_hdf(data, local_dir, fname)
    return (data)
예제 #24
0
def merged(probe, starttime, endtime, verbose=False):
    """
    Import merged plasma data. See
    ftp://cdaweb.gsfc.nasa.gov/pub/data/imp/imp8/merged/00readme.txt
    for information on variables.

    Parameters
    ----------
    probe : string
        Probe number.
    starttime : datetime
        Start of interval.
    endtime : datetime
        End of interval.
    verbose : bool, optional
        If ``True``, print information whilst loading. Default is
        ``False``.

    Returns
    -------
    data : DataFrame
        Requested data.
    """
    _check_probe(probe, ['8'])
    data = []
    startyear = starttime.year
    endyear = endtime.year
    # Loop through years
    for year in range(startyear, endyear + 1):
        if year == startyear:
            startmonth = starttime.month
        else:
            startmonth = 1

        if year == endyear:
            endmonth = endtime.month
        else:
            endmonth = 12

        # Loop through months
        for month in range(startmonth, endmonth + 1):
            if verbose:
                print('Loading IMP merged probe {}, {:02d}/{}'.format(
                    probe, month, year))
            intervalstring = str(year) + str(month).zfill(2)
            filename = 'imp_min_merge' + intervalstring + '.asc'
            # Location of file relative to local directory or remote url
            relative_loc = os.path.join('imp' + probe, 'merged')

            local_dir = os.path.join(imp_dir, relative_loc)
            hdffile = os.path.join(local_dir, filename[:-4] + '.hdf')
            if os.path.isfile(hdffile):
                data.append(pd.read_hdf(hdffile))
                continue

            remote_url = imp_url + relative_loc
            f = util.load(filename, local_dir, remote_url)
            if f is None:
                print('File {}/{} not available\n'.format(
                    remote_url, filename))
                continue
            readargs = {
                'names': [
                    'Year', 'doy', 'Hour', 'Minute', 'sw_flag', 'x_gse',
                    'y_gse', 'z_gse', 'y_gsm', 'z_gsm', 'Nm', 'FCm', 'DWm',
                    '<|B|>', '|<B>|', '<B_lat>', '<B_long>', 'Bx_gse',
                    'By_gse', 'Bz_gse', 'By_gsm', 'Bz_gsm', 'sigma|B|',
                    'sigma B', 'sigma B_x', 'sigma B_y', 'sigma B_z',
                    'plas_reg', 'Npp', 'FCp', 'DWp', 'v_fit', 'vx_fit_gse',
                    'vy_fit_gse', 'vz_fit_gse', 'vlong_fit', 'vlat_fit',
                    'np_fit', 'Tp_fit', 'v_mom', 'vx_mom_gse', 'vy_mom_gse',
                    'vz_mom_gse', 'vlong_mom', 'vlat_mom', 'np_mom', 'Tp_mom'
                ],
                'na_values': [
                    '9999', '999', '99', '99', '9', '9999.99', '9999.99',
                    '9999.99', '9999.99', '9999.99', '9', '99', '9.99',
                    '9999.99', '9999.99', '9999.99', '9999.99', '9999.99',
                    '9999.99', '9999.99', '9999.99', '9999.99', '9999.99',
                    '9999.99', '9999.99', '9999.99', '9999.99', '9', '9', '99',
                    '9.99', '9999.9', '9999.9', '9999.9', '9999.9', '9999.9',
                    '9999.9', '9999.9', '9999999.', '9999.9', '9999.9',
                    '9999.9', '9999.9', '9999.9', '9999.9', '9999.9',
                    '9999999.'
                ],
                'delim_whitespace':
                True
            }
            # Read in data
            thisdata = pd.read_table(f, **readargs)
            thisdata['Time'] = (
                pd.to_datetime(thisdata['Year'], format='%Y') +
                pd.to_timedelta(thisdata['doy'] - 1, unit='d') +
                pd.to_timedelta(thisdata['Hour'], unit='h') +
                pd.to_timedelta(thisdata['Minute'], unit='m'))
            if use_hdf:
                thisdata.to_hdf(hdffile, key='distparams', mode='w')
            data.append(thisdata)

    return util.timefilter(data, starttime, endtime)
예제 #25
0
파일: imp.py 프로젝트: ikitcheng/heliopy
def mag15s(probe, starttime, endtime, verbose=False):
    """
    Import 15s cadence magnetic field data.

    Parameters
    ----------
    probe : string
        Probe number.
    starttime : datetime
        Start of interval.
    endtime : datetime
        End of interval.
    verbose : bool, optional
        If ``True``, print information whilst loading. Default is ``False``.

    Returns
    -------
        data : DataFrame
            Requested data.
    """
    data = []
    dtimes = util._daysplitinterval(starttime, endtime)
    # Loop through years
    for dtime in dtimes:
        startdt = dtime[0]
        year = startdt.year
        doy = util.dtime2doy(startdt)
        if verbose:
            print('Loading IMP 15s mag probe {}, {:03d}/{}'.format(
                probe, doy, year))
        filename = '{}{:03d}_imp{}_mag_15s_v3.asc'.format(year, doy, probe)
        hdffname = filename[:-3] + 'hdf'
        # Location of file relative to local directory or remote url
        relative_loc = os.path.join('imp{}'.format(probe), 'mag',
                                    '15s_ascii_v3', str(year))

        local_dir = os.path.join(imp_dir, relative_loc)
        hdffile = os.path.join(local_dir, hdffname)
        if os.path.exists(hdffile):
            thisdata = pd.read_hdf(hdffile)
            data.append(thisdata)
            continue

        remote_url = imp_url + relative_loc
        f = util.load(filename, local_dir, remote_url)
        readargs = {
            'names': [
                'Year', 'doy', 'Second', 'Source flag', 'n points', 'x gse',
                'y gse', 'z gse', 'y gsm', 'z gsm', '|B|', 'Bx gse', 'By gse',
                'Bz gse', 'By gsm', 'Bz gsm', 'Bxx gse', 'Byy gse', 'Bzz gse',
                'Byx gse', 'Bzx gse', 'Bzy gse', 'Time shift', 'sw flag'
            ],
            'na_values': [
                '9999', '999', '99', '9', '999', '99.99', '99.99', '99.99',
                '99.99', '99.99', '9999.99', '9999.99', '9999.99', '9999.99',
                '9999.99', '9999.99', '9999.99', '9999.99', '9999.99',
                '9999.99', '9999.99', '9999.99', '999.9', '9'
            ],
            'delim_whitespace':
            True
        }
        # Read in data
        thisdata = pd.read_table(f, **readargs)
        thisdata['Time'] = (pd.to_datetime(thisdata['Year'], format='%Y') +
                            pd.to_timedelta(thisdata['doy'] - 1, unit='d') +
                            pd.to_timedelta(thisdata['Second'], unit='s'))
        thisdata = thisdata.set_index('Time', drop=False)
        thisdata = thisdata.drop(['Year', 'doy', 'Second'], 1)
        if use_hdf:
            thisdata.to_hdf(hdffile, key='distparams', mode='w')
        data.append(thisdata)
    return util.timefilter(data, starttime, endtime)
예제 #26
0
def swoops_ions(starttime, endtime):
    """
    Import SWOOPS ion data.

    Parameters
    ----------
    starttime : datetime
        Start of interval
    endtime : datetime
        End of interval

    Returns
    -------
    data : DataFrame
        Requested data
    """
    swoops_options = url_options
    readargs = {
        'names': [
            'year', 'doy', 'hour', 'minute', 'second', 'r', 'hlat', 'hlon',
            'n_p', 'n_a', 'T_p_large', 'T_p_small', 'v_r', 'v_t', 'v_n',
            'iqual'
        ],
        'delim_whitespace':
        True
    }

    data = []
    months_loaded = []
    dtimes = util._daysplitinterval(starttime, endtime)
    # Loop through individual days
    for dtime in dtimes:
        thisdate = dtime[0]
        # Get first day of the month
        first_day = date(thisdate.year, thisdate.month, 1)
        # Check if this month's data already loaded
        if first_day in months_loaded:
            continue
        doy = first_day.strftime('%j')

        swoops_options['FILE_NAME'] = ('u' + first_day.strftime('%y') + doy +
                                       'bam.dat')
        swoops_options['FILE_PATH'] =\
            ('/ufa/stageIngestArea/swoops/ions/bamion' +
             first_day.strftime('%y') + '.zip_files')

        # Put together url for this days data
        remote_url = ulysses_url
        for key in swoops_options:
            remote_url += key + '=' + swoops_options[key] + '&'
        # Local locaiton to download to
        local_dir = os.path.join(ulysses_dir, 'swoops', 'ions',
                                 first_day.strftime('%Y'))

        # Load data
        try:
            f = util.load(swoops_options['FILE_NAME'], local_dir, remote_url)
        except HTTPError:
            print('No SWOOPS ion data available for date %s' % first_day)
            continue

        # Read in data
        thisdata = pd.read_table(f, **readargs)
        # Process data/time
        thisdata = _convert_ulysses_time(thisdata)
        data.append(thisdata)
        months_loaded.append(first_day)

    return util.timefilter(data, starttime, endtime)
예제 #27
0
def fgm(probe, rate, coords, starttime, endtime):
    """
    Import fgm magnetic field data from THEMIS.

    Parameters
    ----------
        probe : string
            Alowed values are [a, b, c, d, e].
        rate : string
            Date rate to return. Allowed values are [e, h, l, s].
        coords : string
            Magnetic field co-ordinate system. Allowed values are
            [dsl, gse, gsm, ssl]. NOTE: Add link to co-ordinate system
            descriptions.
        starttime : datetime
            Interval start time.
        endtime : datetime
            Interval end time.

    Returns
    -------
        data : DataFrame
    """
    valid_rates = ['e', 'h', 'l', 's']
    valid_coords = ['dsl', 'gse', 'gsm', 'ssl']
    _validate_probe(probe)
    if rate not in valid_rates:
        raise ValueError(('rate argument %s is not in list of allowed'
                          'rates: %s') % (rate, valid_rates))
    if coords not in valid_coords:
        raise ValueError(('coords argument %s is not in list of allowed'
                          'co-ordinate systems: %s') % (rate, valid_rates))

    # Directory relative to main THEMIS data directory
    relative_dir = os.path.join('th' + probe, 'l2', 'fgm')

    daylist = util._daysplitinterval(starttime, endtime)
    data = []
    for day in daylist:
        date = day[0]
        this_relative_dir = os.path.join(relative_dir, str(date.year))
        filename = 'th' + probe + '_l2_fgm_' +\
            str(date.year) +\
            str(date.month).zfill(2) +\
            str(date.day).zfill(2) +\
            '_v01.cdf'
        # Absolute path to local directory for this data file
        local_dir = os.path.join(themis_dir, this_relative_dir)
        util._checkdir(local_dir)

        remote_url = remote_themis_dir + this_relative_dir
        cdf = util.load(filename, local_dir, remote_url)
        if cdf is None:
            print('File {}/{} not available\n'.format(remote_url, filename))
            continue

        probestr = 'th' + probe
        ratestr = '_fg' + rate + '_'
        keys = {
            probestr + ratestr + 'btotal':
            '|B|',
            probestr + ratestr + coords:
            ['Bx_' + coords, 'By_' + coords, 'Bz_' + coords],
            probestr + ratestr + 'time':
            'Time'
        }
        df = util.cdf2df(cdf,
                         probestr + ratestr + 'time',
                         keys,
                         dtimeindex=False)
        df = df.set_index(pd.to_datetime(df.index.values, unit='s'))
        df['Time'] = df.index.values
        data.append(df)

    data = pd.concat(data)
    data = data[(data['Time'] > starttime) & (data['Time'] < endtime)]
    return data