コード例 #1
0
ファイル: aggregate.py プロジェクト: govtmirror/ncETL
def fetch(d, rfc, basedir, dryrun=False):
    logging.info("Aggregating month %s for RFC %s", d.strftime('%Y-%m'),
                 rfc.upper())

    destDir = os.path.join(basedir, 'archive/all_rfc/harvest')
    try:
        if dryrun:
            _varName, rfcId = fetchAggregate.discoverMetadata(rfc)
            filename = fetchAggregate.makeArchiveName(d, rfcId, destDir)
            agg = "Would fetch aggregate %s for %s to %s" % (
                rfc, d.strftime('%Y-%m'), filename)
            logging.info(agg)
        else:
            agg = fetchAggregate.fetchAggregate(rfc,
                                                d.strftime('%Y-%m'),
                                                destDir=destDir)
            logging.info("Wrote aggregate as %s", agg)
            os.system("ncks -O --fix_rec_dmn " + agg + " " + agg)
            logging.info(
                "Fixed time dimension of %s with 'ncks -O --fix_rec_dmn'", agg)
    except requests.exceptions.RequestException as e:
        logging.warn("Failed to fetch for %s because %s", rfc, e)
        agg = e

    return agg
コード例 #2
0
ファイル: aggregate.py プロジェクト: rhayes-cida/ncETL
def fetch(d, rfc, basedir, dryrun=False):
    logging.info("Aggregating month %s for RFC %s", d.strftime('%Y-%m'), rfc.upper())
    
    destDir = os.path.join(basedir,'archive/all_rfc/harvest')
    try:
        if dryrun:
            _varName, rfcId = fetchAggregate.discoverMetadata(rfc)
            filename = fetchAggregate.makeArchiveName(d, rfcId, destDir)
            agg = "Would fetch aggregate %s for %s to %s" % (rfc, d.strftime('%Y-%m'), filename)
            logging.info(agg)
        else: 
            agg = fetchAggregate.fetchAggregate(rfc, d.strftime('%Y-%m'), destDir=destDir)
            logging.info("Wrote aggregate as %s", agg)
    except requests.exceptions.RequestException as e:
        logging.warn("Failed to fetch for %s because %s", rfc, e)
        agg = e
        
    return agg
コード例 #3
0
ファイル: aggregate.py プロジェクト: USGS-CIDA/ncETL
def fetch(d, rfc, basedir, dryrun=False):
    logging.info("Aggregating month %s for RFC %s", d.strftime("%Y-%m"), rfc.upper())

    destDir = os.path.join(basedir, "archive/all_rfc/harvest")
    try:
        if dryrun:
            _varName, rfcId = fetchAggregate.discoverMetadata(rfc)
            filename = fetchAggregate.makeArchiveName(d, rfcId, destDir)
            agg = "Would fetch aggregate %s for %s to %s" % (rfc, d.strftime("%Y-%m"), filename)
            logging.info(agg)
        else:
            agg = fetchAggregate.fetchAggregate(rfc, d.strftime("%Y-%m"), destDir=destDir)
            logging.info("Wrote aggregate as %s", agg)
            os.system("ncks -O --fix_rec_dmn " + agg + " " + agg)
            logging.info("Fixed time dimension of %s with 'ncks -O --fix_rec_dmn'", agg)
    except requests.exceptions.RequestException as e:
        logging.warn("Failed to fetch for %s because %s", rfc, e)
        agg = e

    return agg
コード例 #4
0
ファイル: allCodes.py プロジェクト: USGS-CIDA/ncETL
import fetchAggregate

rfcs = ['tsju','ktua','kstr','krsa','korn','krha','kkrf','kmsr','ktar','kptr','ktir','kalr','kfwr']
for rfc in rfcs:
    varName, rfcCode = fetchAggregate.discoverMetadata(rfc)
    print rfc, rfcCode, varName

コード例 #5
0
ファイル: allCodes.py プロジェクト: rhayes-cida/ncETL
import fetchAggregate

rfcs = [
    'tsju', 'ktua', 'kstr', 'krsa', 'korn', 'krha', 'kkrf', 'kmsr', 'ktar',
    'kptr', 'ktir', 'kalr', 'kfwr'
]
for rfc in rfcs:
    varName, rfcCode = fetchAggregate.discoverMetadata(rfc)
    print rfc, rfcCode, varName