def fetch(d, rfc, basedir, dryrun=False):
logging.info("Aggregating month %s for RFC %s", d.strftime('%Y-%m'),
rfc.upper())
destDir = os.path.join(basedir, 'archive/all_rfc/harvest')
try:
if dryrun:
_varName, rfcId = fetchAggregate.discoverMetadata(rfc)
filename = fetchAggregate.makeArchiveName(d, rfcId, destDir)
agg = "Would fetch aggregate %s for %s to %s" % (
rfc, d.strftime('%Y-%m'), filename)
logging.info(agg)
else:
agg = fetchAggregate.fetchAggregate(rfc,
d.strftime('%Y-%m'),
destDir=destDir)
logging.info("Wrote aggregate as %s", agg)
os.system("ncks -O --fix_rec_dmn " + agg + " " + agg)
logging.info(
"Fixed time dimension of %s with 'ncks -O --fix_rec_dmn'", agg)
except requests.exceptions.RequestException as e:
logging.warn("Failed to fetch for %s because %s", rfc, e)
agg = e
return agg
def fetch(d, rfc, basedir, dryrun=False):
logging.info("Aggregating month %s for RFC %s", d.strftime('%Y-%m'), rfc.upper())
destDir = os.path.join(basedir,'archive/all_rfc/harvest')
try:
if dryrun:
_varName, rfcId = fetchAggregate.discoverMetadata(rfc)
filename = fetchAggregate.makeArchiveName(d, rfcId, destDir)
agg = "Would fetch aggregate %s for %s to %s" % (rfc, d.strftime('%Y-%m'), filename)
logging.info(agg)
else:
agg = fetchAggregate.fetchAggregate(rfc, d.strftime('%Y-%m'), destDir=destDir)
logging.info("Wrote aggregate as %s", agg)
except requests.exceptions.RequestException as e:
logging.warn("Failed to fetch for %s because %s", rfc, e)
agg = e
return agg
def fetch(d, rfc, basedir, dryrun=False):
logging.info("Aggregating month %s for RFC %s", d.strftime("%Y-%m"), rfc.upper())
destDir = os.path.join(basedir, "archive/all_rfc/harvest")
try:
if dryrun:
_varName, rfcId = fetchAggregate.discoverMetadata(rfc)
filename = fetchAggregate.makeArchiveName(d, rfcId, destDir)
agg = "Would fetch aggregate %s for %s to %s" % (rfc, d.strftime("%Y-%m"), filename)
logging.info(agg)
else:
agg = fetchAggregate.fetchAggregate(rfc, d.strftime("%Y-%m"), destDir=destDir)
logging.info("Wrote aggregate as %s", agg)
os.system("ncks -O --fix_rec_dmn " + agg + " " + agg)
logging.info("Fixed time dimension of %s with 'ncks -O --fix_rec_dmn'", agg)
except requests.exceptions.RequestException as e:
logging.warn("Failed to fetch for %s because %s", rfc, e)
agg = e
return agg
import fetchAggregate
rfcs = ['tsju','ktua','kstr','krsa','korn','krha','kkrf','kmsr','ktar','kptr','ktir','kalr','kfwr']
for rfc in rfcs:
varName, rfcCode = fetchAggregate.discoverMetadata(rfc)
print rfc, rfcCode, varName
import fetchAggregate
rfcs = [
'tsju', 'ktua', 'kstr', 'krsa', 'korn', 'krha', 'kkrf', 'kmsr', 'ktar',
'kptr', 'ktir', 'kalr', 'kfwr'
]
for rfc in rfcs:
varName, rfcCode = fetchAggregate.discoverMetadata(rfc)
print rfc, rfcCode, varName