def download_station_list(redownload=False):
"""
:type redownload: bool
:rtype:
"""
filepath = station_list_file()
if not os.path.exists(filepath) or redownload:
try:
input_file = open(
urllib.request.urlretrieve(ISD_STATION_LIST_URL)[0], 'rb')
with open(filepath, "wb") as ofile:
for line in input_file:
ofile.write(line)
except urllib.error.URLError as ex:
getLogger(__name__).warn(
'Could not download WBAN station list file : %s (%s)' %
(ISD_STATION_LIST_URL, ex.reason))
os.remove(filepath)
return False
except Exception as ex:
getLogger(__name__).warn(
'Could not download WBAN station list file : %s (%s)' %
(ISD_STATION_LIST_URL, ex))
os.remove(filepath)
return False
return os.path.abspath(filepath)
def download_isd_stations(redownload=False):
"""
:type redownload: bool
:rtype:
"""
station_list_file = isd_station_list_file()
if os.path.exists(station_list_file):
ctime = _created_time(station_list_file)
if (datetime.datetime.now() - ctime).days <= 7:
return os.path.abspath(station_list_file)
if not os.path.exists(station_list_file) or redownload:
try:
with open(
urllib.request.urlretrieve(ISD_STATION_LIST_URL)[0],
'rb') as input_file:
with open(station_list_file, "wb") as ofile:
for line in input_file:
ofile.write(line)
except urllib.error.URLError as ex:
getLogger(__name__).warning(
'Could not download ISD station list file : %s (%s)' %
(ISD_STATION_LIST_URL, ex.reason))
os.remove(station_list_file)
return False
except Exception as ex:
getLogger(__name__).warning(
'Could not download ISD station list file : %s (%s)' %
(ISD_STATION_LIST_URL, ex))
os.remove(station_list_file)
return False
return os.path.abspath(station_list_file)
def download(year, month):
"""
:type year: int or str
:type month: int or str
:rtype: str or bool
"""
fileurl = _dataurl(year, month)
try:
with ZipFile(urllib.request.urlretrieve(fileurl)[0],
'r') as input_file:
output_dir = _output_dir(year)
output_file = os.path.join(output_dir, _filename(year, month))
with input_file.open('%s%02dhourly.txt' %
(year, int(month))) as input_fp:
with open(output_file, "wb") as output_fp:
for line in input_fp:
output_fp.write(line)
except urllib.error.URLError as ex:
getLogger(__name__).warn('Could not download QCLCD data : %s (%s)' %
(fileurl, ex.reason))
return False
except Exception as ex:
getLogger(__name__).warn(
'Could not download QCLCD data file : %s (%s)' % (fileurl, ex))
return False
return os.path.abspath(output_file)
def file_download(remote_file, n_try=1, timeout=None):
"""
:type remote_file: str
:type n_try: int
:type timeout: int
:rtype: (str, Union(url_error.URLError, ConnectionResetError))
"""
logger = getLogger(__name__)
timeout = timeout or DEFAULT_TIMEOUT
try:
with get_url_opener(remote_file) as res:
return res.read(), None
except url_error.HTTPError as e:
logger.debug(
'Could not get the remote file : %s (reason=%s, code=%s)'.format(
remote_file, str(e.reason), e.code))
return None, e
except url_error.URLError as e:
logger.critical(
'Could not connect to the remote file : %s (reason=%s)'.format(
remote_file, str(e.reason)))
return None, e
except ConnectionResetError as e:
logger.critical('HTTP Connection has been reset : %s (code=%s)'.format(
remote_file, e.errno))
if n_try <= MAX_TRY_NUM:
logger.critical('Retrying...')
time.sleep(1)
return file_download(remote_file, n_try=n_try + 1, timeout=timeout)
return None, e
def download(usaf, wban, year):
"""
:type usaf: int or str
:type wban: int or str
:type year: int or str
:rtype: str or bool
"""
fileurl = _dataurl(usaf, wban, year)
output_dir = _output_dir(year)
output_file = os.path.join(output_dir, _filename(usaf, wban, year))
if os.path.exists(output_file):
os.remove(output_file)
# if os.path.exists(output_file):
# ctime = _created_time(output_file)
# if (datetime.datetime.now() - ctime).days <= 1:
# return output_file
try:
input_file = gzip.open(urllib.request.urlretrieve(fileurl)[0])
with open(output_file, "wb") as of:
for line in input_file:
of.write(line)
except urllib.error.URLError as ex:
getLogger(__name__).warn('Could not download ISD file : %s (%s)' %
(fileurl, ex.reason))
if os.path.exists(output_file):
os.remove(output_file)
return False
except Exception as ex:
getLogger(__name__).warn('Could not download ISD file : %s (%s)' %
(fileurl, ex))
if os.path.exists(output_file):
os.remove(output_file)
return False
#save_as_daily_data(output_file)
return os.path.abspath(output_file)
def print_prof_data():
logger = getLogger(__name__)
logger.info('Profile Results ----------------------')
for fname, data in PROF_DATA.items():
max_time = max(data[1])
avg_time = sum(data[1]) / len(data[1])
logger.info("")
logger.info(" + function : %s()" % fname)
logger.info(" - called : %d times" % data[0])
logger.info(
" - execution time : max=%.3fs, avg=%.3fs, total=%.3fs" %
(max_time, avg_time, avg_time * data[0]))
logger.info('-' * 40)
def location(host, guest):
"""
:type host: pyticas.ttypes.Route
:type guest: pyticas.ttypes.Route
:rtype: (pyticas_tetres.ttypes.LOC_TYPE, float, float)
"""
logger = getLogger(__name__)
s1 = host.rnodes[0]
e1 = host.rnodes[-1]
s2 = guest.rnodes[0]
e2 = guest.rnodes[-1]
# distance map from the most upstream of corridor
# for a route through multiple corridor
corrs1 = route.corridors(host)
corrs2 = route.corridors(guest)
corrs = corrs1 if len(corrs1) >= len(corrs2) else corrs2
mmap = {}
last_mp = 0
for corr in corrs:
t_mmap = geo.get_mile_point_map(corr.rnodes)
for rn_name, mp in t_mmap.items():
t_mmap[rn_name] = mp + last_mp
last_mp = t_mmap[corr.rnodes[-1].name]
mmap.update(t_mmap)
s1mp = mmap.get(s1.name, None)
e1mp = mmap.get(e1.name, None)
s2mp = mmap.get(s2.name, None)
e2mp = mmap.get(e2.name, None)
if None in [s1mp, e1mp, s2mp, e2mp]:
#logger.warn('Could not make distance map for %s and %s' % (host, guest))
return (None, None, None)
distance = s1mp - s2mp
if s1mp >= e2mp:
return (LOC_TYPE.UP, distance, e2mp - s1mp)
if e1mp <= s2mp:
return (LOC_TYPE.DOWN, distance, s2mp - e1mp)
if s1mp <= s2mp and e1mp >= e2mp:
return (LOC_TYPE.INSIDE, distance, 0)
if s1mp >= s2mp and e1mp <= e2mp:
return (LOC_TYPE.WRAP, distance, 0)
if s1mp >= s2mp and e1mp >= e2mp and s1mp <= e2mp:
return (LOC_TYPE.UP_OVERLAPPED, distance, 0)
if s1mp <= s2mp and e1mp <= e2mp and e1mp >= s2mp:
return (LOC_TYPE.DOWN_OVERLAPPED, distance, 0)
logger.warn('Could not decide location type for %s and %s' % (host, guest))
return (None, None, None)
def _json_decoder(args):
""" Return object that is unserialized
:type args: dict
:rtype: object
"""
# for `datetime.datetime`
if '__type__' in args and args['__type__'] == 'datetime':
inst = datetime.datetime.strptime(args['datetime'],
'%Y-%m-%d %H:%M:%S')
elif '__type__' in args and args['__type__'] == 'date':
inst = datetime.datetime.strptime(args['date'], '%Y-%m-%d').date()
elif '__type__' in args and args['__type__'] == 'time':
inst = datetime.datetime.strptime(args['time'], '%H:%M:%S').time()
elif '__type__' in args and args['__type__'] == 'numpy.ndarray':
inst = numpy.array(args['list'])
elif '__type__' in args and args['__type__'] in numpy_types:
inst = eval(args['__type__'])(args['item'])
elif '__enum__' in args:
if '.' in args["__enum__"]:
name, member = args["__enum__"].split(".")
else:
member = args["__enum__"]
name = 'ValidState'
enumObj = _find_serializable_class(name)
return getattr(enumObj, member)
# for `InfraObject` class
elif '_obj_type_' in args and 'name' in args:
_obj_type_ = args.pop('_obj_type_')
getter = 'get_%s' % _obj_type_.lower()
from pyticas.infra import Infra
inst = getattr(Infra.get_infra(args.get('infra_cfg_date', '')),
getter)(args['name'])
# for `Serializable` class
elif '__class__' in args:
class_name = args.pop('__class__')
module_name = args.pop('__module__', None)
try:
module = import_module(module_name)
cls = getattr(module, class_name)
# if module is not found by changing module name or path
except Exception as ex:
tb.traceback(ex)
getLogger(__name__).error('fail to unserialize for %s.%s' %
(module_name, class_name))
cls = _find_serializable_class(class_name)
if cls and hasattr(cls, 'unserialize'):
args = dict((key, value) for key, value in args.items())
inst = cls.unserialize(args)
else:
inst = args
else:
inst = args
if hasattr(inst, '__unserialized__'):
inst.__unserialized__()
return inst