示例#1
0
def cleanHouse():
    # Grab a reference to the existing logger.
    # This only works if the script calling this function has
    # already called mosHelper.setUpTheLogger().
    module_logger = logging.getLogger('mosgraphics.cleanHouse')

    dictDirNames = mosHelper.getDirNames()

    # Can probably rewrite mosplots.calc_dates based on the work here. Perhaps
    # in the ample free time with which all forecasters are blessed. Maybe use
    # xrange instead, also?
    #
    # Alright, here's the deal. The UTC date may be in the future compared to the
    # local date at times, so we'll toss in some negative numbers just to be sure the
    # early day runs (00z, 06z) won't get accidentally deleted.
    # Next, grab the current YYYY-MM-DD (local time) and use that as a starting point
    # from which to calculate which files to keep. Some of the filenames may not exist
    # yet, but that's OK. The important thing is that they won't be deleted. Yeah, that
    # makes total sense...
    hrsToKeep = {}
    hrsToKeep['MEX'] = [-24, -12, 0, 12, 24, 36, 48, 60, 72, 84, 96, 108, 120, 132, 144, 156, 168, 180, 192, 204, 216]
    hrsToKeep['MAV'] = [-24, -18, -12, -6, 0, 6, 12, 18, 24, 30, 36, 42, 48, 54, 60, 66, 72]
    hrsToKeep['MET'] = [-36, -24, -12, 0, 12, 24, 36, 48, 60, 72, 84]
    #hrsToKeep['ECE'] = [-24, 0, 24, 48, 72, 96, 120, 144, 168, 192, 216]
    #hrsToKeep['ECS'] = [-24, 0, 24, 48, 72, 96]

    # Now. You're looking at now, sir. Everything that happens now, is happening now.
    # What happened to then?
    # We passed then.
    # When?
    # Just now. We're at now now.     
    rightnow = dt.datetime.now()

    # But not anymore! Choose 0 o'clock as a baseline. It makes the math easier.
    nowish = dt.datetime(year = rightnow.year, month = rightnow.month, day = rightnow.day, hour = 0)

    keyIter = hrsToKeep.iterkeys()

    # Loop over time to create filenames to keep
    for key in keyIter:
        keepfiles = []
        for hr in hrsToKeep[key]:
            mostype = key.lower()
            goback = dt.timedelta(hours = hr)
            prev = nowish - goback
            Y = prev.strftime('%Y')
            M = prev.strftime('%m')
            D = prev.strftime('%d')
            H = prev.strftime('%H')
            appendme = mosHelper.makeFilenames(mostype, 'ABCD', Y, M, D, H)['raw']
            keepfiles.append(appendme)
            
        keepfiles = set(keepfiles)
        
        # get the contents of the raw files directory for this mostype
        rawfiles = set(mosHelper.listRawFiles(mostype))
        
        # Suppose set1 = ([f1, f2, f3, etc.]) contains the filenames to keep, and
        # set2 = ([f0, f1, f2, f3, f4, f5, f6]) has the names of all raw files.
        # Then set2.difference(set1) is the set of files to delete.
        delme = rawfiles.difference(keepfiles)

        module_logger.info('%s are marked for deletion from %s', delme, mostype.upper())

        for fn in delme:
            fullname = os.path.join(dictDirNames['raw'], fn)
            os.remove(fullname)
示例#2
0
def GrabEm():

    # Grab a reference to the existing logger.
    # This only works if the script calling this function has
    # already called mosHelper.setUpTheLogger().
    module_logger = logging.getLogger('mosgraphics.GrabEm')

    dictDirNames = mosHelper.getDirNames()

    moslist = ['MET', 'MEX', 'MAV']

    # Use this to keep track of which files were written and still need
    # to be processed with mosHelper.parseStations.
    rawfiles = []
    
    for mosname in moslist:
        tempObj = MOS(mosname)
        tempObj.set_filethresh()
        
        module_logger.info('Asking MDL for the {}'.format(mosname))
        status = tempObj.check_primary()

        if status is not 1:
            module_logger.warning('Lost the connection with MDL. File was not downloaded.')
            module_logger.info('Trying another server')
            status2 = tempObj.check_backup()
            
            if status2 is not 1:
                module_logger.warning('Bummer. Unable to download {}'.format(mosname))
            elif status2 is 1:
                module_logger.info('Success!')
            else:
                module_logger.info('? ? ? ? ? ?')

        if tempObj.fileurls is not None:
            for furl, localfilename in zip(tempObj.fileurls, tempObj.localfnames):
                # If there already exists a file with the intended localfilename,
                # check to see if it has an appropriate size. If the file seems
                # too small, then try downloading it again. Otherwise, don't bother
                # because it's probably OK.
                existingRawFiles = mosHelper.listRawFiles(mosname)
                if localfilename in existingRawFiles:
                    module_logger.info('{} already exists on disk.'.format(localfilename))
                    fpath = os.path.join(dictDirNames['raw'], localfilename)

                    # If the file size is too small, then something went wrong the
                    # last time the file was downloaded. Try to download it again
                    # now so that it will be available for the next script run.
                    thresh = tempObj.filethresh * 1000
                    if os.path.getsize(fpath) > thresh:
                        module_logger.info('Skipping. It\'s probably OK.')
                        # 'continue': the current iteration of the loop terminates
                        # and execution continues with the next iteration of the loop.
                        continue
                    else:
                        module_logger.info('Downloading. The copy on disk seems too small.')

                # Note that in order to reach this part of the script, the file
                # size must pass the above if-else.
                response = urllib2.urlopen(furl)
                contents = response.read()
                response.close()

                module_logger.info('Writing to %s', localfilename)
                fullname = os.path.join(dictDirNames['raw'], localfilename)
                output = open(fullname, 'w')
                output.write(contents)
                output.close()
                rawfiles.append(localfilename)
                    
        else:
            module_logger.info('A rolling stone gathers no {} MOS.'.format(mosname))

    return(rawfiles)