Beispiel #1
0
def collect_year(year, data_source = 'NCDC'):
    """ Collect the GSOD data file for all locations for the specified
    year. Look locally for the tar file first. If it is not there, and its gzip
    version is not either, use the ftp connection to retrieve it from data
    source.
    """
    filename = info2filepath(year)
    local_folderpath = os.path.join("Data", "GSOD", "gsod_"+str(year))
    local_filepath = os.path.join(local_folderpath, filename)
    if not os.path.isdir(local_folderpath):
        # Folder not already present
        os.mkdir(local_folderpath)
    if count_op_files(local_folderpath) < 10:
        # probably not all the data files are present
        if not os.path.exists(local_filepath):
            # tar file not present either: download it!
            if data_source == 'NCDC':
                remote_location = str(year)
            print("Retrieving archive %s... This may take several minutes." 
                  % local_filepath)
            remote_target = os.path.join(remote_location, filename)
            retrieve_file(data_source, remote_target, local_filepath)
        untar(local_filepath)
    try:
        panda = datafolder2pandas(local_folderpath)
    except MemoryError:
        # For years where there is a large amount of data, it is not possible to
        # load everything in memory
        # FIXME: load the data in a memory mapped/pytable stored pandas in this
        # case? Clarify because the memory error is thrown by mmap. It may be
        # doing this already, but be running into mmap limitations?
        warnings.warn("The year %s contains too much data to be loaded into a "
                      "single object in memory")
        panda = None
    return panda
Beispiel #2
0
def collect_year_at_loc(year, location_WMO, location_WBAN, data_source = 'NCDC', 
                        internet_connected = True):
    """ Collect the data GSOD data file for specified location and specified
    year. Look locally for the file first. If it is not there, and its gzip
    version is not either, untar the file if it is present and has not been
    untared, or use the ftp connection to retrieve it from data source.
    """
    filename = info2filepath(year, location_WMO, location_WBAN)
    folder_location = os.path.join("Data", "GSOD", "gsod_"+str(year))
    filepath = os.path.join(folder_location, filename)
    print "Attempting to collect %s..." % filepath
    filepath_found = True
    
    if not os.path.exists(filepath):
        zipped_filepath = filepath+".gz"
        if os.path.exists(zipped_filepath):
            unzip(zipped_filepath)
        elif os.path.exists(os.path.join(folder_location,
                                         "gsod_"+str(year)+".tar")):
            # Possible not to rely on outside servers: untar the file if there
            # are no op.gz or op files. If not it means that the file is
            # missing.
            there_are_op_files = False
            for filename in os.listdir(folder_location):
                if os.path.splitext(filename)[1] in [".op", ".op.gz"]:
                    there_are_op_files = True
                    break
            if not there_are_op_files:
                untar(os.path.join(folder_location, "gsod_"+str(year)+".tar"))
            if os.path.isfile(zipped_filepath):
                unzip(zipped_filepath)
            else:
                warnings.warn("File %s is missing from the dataset: skipping "
                              "this location." % zipped_filepath)
                filepath_found = False
        elif internet_connected:
            target_folder = "Data/GSOD/gsod_"+str(year)
            if not os.path.exists(target_folder):
                print "Creating locally the folder %s." % target_folder
                os.mkdir(target_folder)
            # Download the file from NCDC
            if data_source == 'NCDC':
                remote_location = str(year)
            remote_target = os.path.join(remote_location, filename+".gz")
            retrieve_file(data_source, remote_target, zipped_filepath)
            if os.path.isfile(zipped_filepath):
                unzip(zipped_filepath)
            else:
                filepath_found = False
        else:
            filepath_found = False
        
    if filepath_found:
        return datafile2pandas(filepath)
Beispiel #3
0
                    filepath_found = True
                except KeyError, e:
                    log.warn("File %s is missing from the dataset: skipping "
                             "this location." % zipped_filepath)
        elif internet_connected:
            target_folder = "Data/GSOD/"
            if not os.path.exists(target_folder):
                log.info("Creating locally the folder %s." % target_folder)
                os.mkdir(target_folder)
            # Download the file from NCDC
            if data_source == 'NCDC':
                remote_location = str(year)
            remote_target = remote_location + '/' + filename + ".gz"
            #zch
            print 'remote', remote_target
            retrieve_file(data_source, remote_target, zipped_filepath)
            if os.path.isfile(zipped_filepath):
                filepath = gzip.open(zipped_filepath)
                filepath_found = True
    if filepath_found:
        return datafile2pandas(filepath)


def count_op_files(folder):
    return len([
        filename for filename in os.listdir(folder)
        if os.path.splitext(filename)[1] in [".op", ".gz"]
    ])


def collect_year(year, data_source='NCDC'):
                    filepath_found = True
                except KeyError, e:
                    log.warn("File %s is missing from the dataset: skipping "
                              "this location." % zipped_filepath)
        elif internet_connected:
            target_folder = "Data/GSOD/"
            if not os.path.exists(target_folder):
                log.info("Creating locally the folder %s." % target_folder)
                os.mkdir(target_folder)
            # Download the file from NCDC
            if data_source == 'NCDC':
                remote_location = str(year)
            remote_target = remote_location + '/' + filename + ".gz"
            #zch
            print 'remote', remote_target
            retrieve_file(data_source, remote_target, zipped_filepath)
            if os.path.isfile(zipped_filepath):
                filepath = gzip.open(zipped_filepath)
                filepath_found = True
    if filepath_found:
        return datafile2pandas(filepath)


def count_op_files(folder):
    return len([filename for filename in os.listdir(folder)
                if os.path.splitext(filename)[1] in [".op", ".gz"]])


def collect_year(year, data_source='NCDC'):
    """ Collect the GSOD data file for all locations for the specified
    year. Look locally for the tar file first. If it is not there, and its gzip