def save_file(url):
"""!
Save the given url
@param url URL to save
@return Path that URL was saved to
"""
return common.save_http(DIR_DATA, url)
def run_fires(site, region):
url = site + '/jobs/archive'
try:
p = getPage(url)
except Exception as e:
logging.error("Can't load {}".format(url))
logging.error(e)
return None
a = p.findAll('a')
zips = [x.get('href') for x in a if x.get('href').endswith('.zip')]
fires = sorted(set([x[x.rindex('/') + 1:x.index('_')] for x in zips]))
times = {}
recent = {}
simtimes = {}
dates = []
totaltime = 0
dir_download = common.ensure_dir(os.path.join(DIR, region))
dir_ext = common.ensure_dir(os.path.join(EXT_DIR, region))
logging.debug("Checking {} fires".format(len(fires)))
for f in fires:
times[f] = [
datetime.datetime.strptime(x[x.rindex('_') + 1:x.rindex('.')],
'%Y%m%d%H%M%S%f') for x in zips
if x[x.rindex('/') + 1:x.index('_')] == f
]
recent[f] = {
'time':
max(times[f]),
'url': [
x for x in zips
if x[x.rindex('/') +
1:x.index('_')] == f and datetime.datetime.strptime(
x[x.rindex('_') +
1:x.rindex('.')], '%Y%m%d%H%M%S%f') == max(times[f])
][0],
}
logging.debug('{}: {}'.format(f, recent[f]['time']))
z = common.save_http(dir_download,
site + recent[f]['url'],
ignore_existing=True)
cur_dir = os.path.join(dir_ext, os.path.basename(z)[:-4])
common.unzip(z, cur_dir)
fgmj = os.path.join(cur_dir, 'job.fgmj')
if os.path.exists(fgmj):
try:
t0 = timeit.default_timer()
log_name = firestarr.do_run(fgmj)
t1 = timeit.default_timer()
if log_name is not None:
simtimes[f] = t1 - t0
totaltime = totaltime + simtimes[f]
logging.info("Took {}s to run {}".format(simtimes[f], f))
d = os.path.basename(os.path.dirname(log_name))[:8]
if d not in dates:
dates.append(d)
except Exception as e:
logging.error(e)
return simtimes, totaltime, dates
def save_file(partial_url):
"""!
Save the given url
@param partial_url Partial url to use for determining name
@return Path saved to when using URL to retrieve
"""
out_file = os.path.join(save_dir, save_as.format(weather_index.name))
if os.path.isfile(out_file):
# HACK: no timestamp so don't download if exists
# logging.debug("Have {}".format(out_file))
return out_file
# logging.debug("Downloading {}".format(out_file))
try:
common.save_http(save_dir, partial_url, save_as=out_file)
except:
# get rid of file that's there if there was an error
common.try_remove(out_file)
raise
return out_file
def save_and_read(self, url):
"""!
Save a URL and then return the contents of the saved file
@param url URL to save from
@return String with entire contents of downloaded file
"""
## Name of file to save to
filename = os.path.join(self.DIR_DATA, os.path.basename(url))
# HACK: set filename but then set it again for no reason if we need to download
if not self.no_download:
filename = common.save_http(self.DIR_DATA, url)
return self.read_file(filename)
def save_file(filename):
"""!
Save the given url
@param filename URL to save
@return Path that file was saved to
"""
print(filename)
url = '/'.join(
[common.CONFIG.get('FireGUARD', 'reanalysis_server'), filename])
if url.startswith('ftp'):
return common.save_ftp(
DIR_DATA,
url,
user=common.CONFIG.get('FireGUARD', 'reanalysis_server_user'),
password=common.CONFIG.get('FireGUARD',
'reanalysis_server_password'),
ignore_existing=True)
else:
return common.save_http(DIR_DATA, url, ignore_existing=True)
def save_other(url):
return common.save_http(DIR_DATA, url)
def to_download(url):
return common.save_http(DOWNLOAD_DIR, url, ignore_existing=True)