def handle(self,
_channel=CHANNEL.WEBSPHERE_MQ,
ts_format='YYYYMMDDHHmmssSS'):
request = loads(self.request.raw_request)
msg = request['msg']
service_name = request['service_name']
# Make MQ-level attributes easier to handle
correlation_id = unhexlify(
msg['correlation_id']) if msg['correlation_id'] else None
expiration = datetime_from_ms(
msg['expiration']) if msg['expiration'] else None
timestamp = '{}{}'.format(msg['put_date'], msg['put_time'])
timestamp = arrow_get(timestamp,
ts_format).replace(tzinfo='UTC').datetime
data = payload_from_request(self.cid, msg['text'],
request['data_format'], None)
self.invoke(service_name,
data,
_channel,
wmq_ctx={
'msg_id': unhexlify(msg['msg_id']),
'correlation_id': correlation_id,
'timestamp': timestamp,
'put_time': msg['put_time'],
'put_date': msg['put_date'],
'expiration': expiration,
'reply_to': msg['reply_to'],
})
def _get_data_from_sliceable(self,
sliceable,
query_ctx,
_time_keys=time_keys):
max_chars = self.request.input.get('max_chars') or 30
out = []
now = self.time.utcnow(needs_format=False)
start = query_ctx.cur_page * query_ctx.page_size
stop = start + query_ctx.page_size
for idx, item in enumerate(sliceable[start:stop]):
# Internally, time is kept as doubles so we need to convert it to a datetime object or null it out.
for name in _time_keys:
_value = item[name]
if _value:
item[name] = arrow_get(_value)
else:
item[name] = None
del _value
# Compute expiry since the last operation + the time left to expiry
expiry = item.pop('expiry')
if expiry:
item['expiry_op'] = int(expiry)
item['expiry_left'] = int(
(item['expires_at'] - now).total_seconds())
else:
item['expiry_op'] = None
item['expiry_left'] = None
# Now that we have worked with all the time keys needed, we can serialize them to the ISO-8601 format.
for name in _time_keys:
if item[name]:
item[name] = item[name].isoformat()
# Shorten the value if it's possible, if it's not something else than a string/unicode
value = item['value']
if isinstance(value, basestring):
len_value = len(value)
chars_omitted = len_value - max_chars
chars_omitted = chars_omitted if chars_omitted > 0 else 0
if chars_omitted:
value = value[:max_chars]
item['value'] = value
item['chars_omitted'] = chars_omitted
item['cache_id'] = self.request.input.cache_id
item['server'] = '{} ({})'.format(self.server.name,
self.server.pid)
out.append(item)
return SearchResults(None, out, None, len(sliceable))
def __init__(self, obj: Dict[str, Any]) -> None: # noqa: D102
super().__init__(obj)
self.__status = Status(obj)
self.__data = {} # type: Dict[str, Any]
for name in obj.keys():
self.__data[name] = obj.get(name, None)
if self.__data["last_date"] is not None:
self.__data["last_date"] = arrow_get(
self.__data["last_date"],
"DD.MM.YYYY HH:mm:ss",
tzinfo=tz.gettz("Europe/Moscow")).datetime
if self.__data["send_date"] is not None:
self.__data["send_date"] = arrow_get(
self.__data["send_date"],
"DD.MM.YYYY HH:mm:ss",
tzinfo=tz.gettz("Europe/Moscow")).datetime
for name in ["send_timestamp", "last_timestamp"]:
del self.__data[name]
def handle(self, _channel=CHANNEL.IBM_MQ, ts_format='YYYYMMDDHHmmssSS'):
request = loads(self.request.raw_request)
msg = request['msg']
service_name = request['service_name']
# Make MQ-level attributes easier to handle
correlation_id = unhexlify(
msg['correlation_id']) if msg['correlation_id'] else None
expiration = datetime_from_ms(
msg['expiration']) if msg['expiration'] else None
timestamp = '{}{}'.format(msg['put_date'], msg['put_time'])
timestamp = arrow_get(timestamp,
ts_format).replace(tzinfo='UTC').datetime
# Extract MQMD
mqmd = msg['mqmd']
mqmd = b64decode(mqmd)
mqmd = pickle_loads(mqmd)
# Find the message's CCSID
request_ccsid = mqmd.CodedCharSetId
# Try to find an encoding matching the CCSID,
# if not found, use the default one.
try:
encoding = CCSIDConfig.encoding_map[request_ccsid]
except KeyError:
encoding = CCSIDConfig.default_encoding
# Encode the input Unicode data into bytes
msg['text'] = msg['text'].encode(encoding)
# Extract the business payload
data = payload_from_request(self.server.json_parser, self.cid,
msg['text'], request['data_format'], None)
# Invoke the target service
self.invoke(service_name,
data,
_channel,
wmq_ctx={
'msg_id': unhexlify(msg['msg_id']),
'correlation_id': correlation_id,
'timestamp': timestamp,
'put_time': msg['put_time'],
'put_date': msg['put_date'],
'expiration': expiration,
'reply_to': msg['reply_to'],
'data': data,
'mqmd': mqmd
})
def create_user_from_dict(dictionary: Dict[str, any]):
"""Method to create Tweet from dictionary."""
return User(arrow_get(dictionary['created_at']), str(dictionary['id_str']),
str(dictionary['rest_id_str']), dictionary['default_profile'],
dictionary['default_profile_image'], dictionary['description'],
dictionary['favourites_count'], dictionary['followers_count'],
dictionary['friends_count'],
dictionary['has_custom_timelines'], dictionary['listed_count'],
dictionary['location'], dictionary['media_count'],
dictionary['name'], dictionary['pinned_tweet_ids_str'],
dictionary['profile_banner_url'],
dictionary['profile_image_url_https'], dictionary['protected'],
dictionary['screen_name'], dictionary['statuses_count'],
dictionary['verified'])
def _get_last_notified(self, key):
with self.lock():
value = self.kvdb.conn.get(key)
return arrow_get(value) if value else None
def create_current_obs_graphs(self):
location_objects = []
for i in range(0, len(self.loc_names)):
root_folder = "C:\Users\Nathan\Documents\Storm Chasing\Chases\\"
date_path = arrow_now().format('YYYY-MM-DD')
for j in range(0, len(self.obs_list)):
if self.loc_names[i] == self.obs_list[j].code:
code = self.obs_list[j].code
loc_name = self.obs_list[j].loc_name
lat = self.obs_list[j].lat
lon = self.obs_list[j].lon
height = self.obs_list[j].height
break
with open(root_folder + date_path + "\Observations\\" +
self.loc_names[i] + ".csv") as f:
for line in f:
line = line.split(',')
location_objects.append(
ObservationLocation(code, loc_name, lat, lon, height))
location_objects[len(location_objects) - 1].time = line[0]
location_objects[len(location_objects) - 1].temp = line[1]
location_objects[len(location_objects) - 1].dew = line[2]
location_objects[len(location_objects) - 1].rain = line[3]
location_objects[len(location_objects) -
1].pressure = line[4]
location_objects[len(location_objects) - 1].lcl = line[5]
location_objects[len(location_objects) -
1].rel_hum = line[6]
location_objects[len(location_objects) -
1].wind_vel = line[7]
location_objects[len(location_objects) - 1].wind_dir = str(
line[8]).replace('\n', '')
print
t = 0
dpi_int = 40
fig = plt.figure(figsize=(16, 9))
for k in self.loc_names:
loc_obs_temp = []
loc_obs_dew = []
loc_obs_rain = []
loc_obs_pressure = []
loc_obs_lcl = []
loc_obs_rel_hum = []
loc_obs_wind_vel = []
loc_obs_wind_dir = []
for j in location_objects:
if k == j.code:
loc_obs_temp.append([j.time, j.temp])
loc_obs_dew.append([j.time, j.dew])
loc_obs_rain.append([j.time, j.rain])
loc_obs_pressure.append([j.time, j.pressure])
loc_obs_lcl.append([j.time, j.lcl])
loc_obs_rel_hum.append([j.time, j.rel_hum])
loc_obs_wind_vel.append([j.time, j.wind_vel])
loc_obs_wind_dir.append([j.time, j.wind_dir])
if True:
print
print str(
int(round(100 * float(t) / float(len(self.loc_names)),
0))) + "% Done"
t += 1
print "Beginning to create graphs for", k
if len(loc_obs_temp) > 1:
# Code to create graphs go here.
# Create temperature graph first
graph_type = "temperature"
file_short_list = []
for f in listdir(root_folder + date_path +
"\Observations\\"):
if re_match(k + "-" + graph_type, f):
file_short_list.append(f)
if len(file_short_list) > 1:
file_short_list.sort()
file_short_list.remove(file_short_list[-1])
for g in file_short_list:
os_remove(root_folder + date_path +
"\Observations\\" + g)
print "Creating", graph_type, "graph"
x = []
y = []
for m in loc_obs_temp:
try:
y.append(float(m[1]))
x.append(datetime.fromtimestamp(float(m[0])))
except ValueError:
continue
try:
time_now = arrow_get(x[len(x) - 1]).format('HH-mm')
with open(root_folder + date_path + "\Observations\\" +
k + "-" + graph_type + "-" + time_now +
".png"):
print "Graph Already Exists"
except IOError:
ax = fig.add_subplot(111)
ax.plot(x, y)
for xy in zip(x, y):
ax.annotate(xy[1], xy=xy, textcoords='data')
buf = (max(y) - min(y)) * .1
ax.get_yaxis().get_major_formatter().set_useOffset(
False)
ax.axis(
[x[0], x[len(x) - 1],
min(y) - buf,
max(y) + buf])
fig.tight_layout(pad=3)
print "Saving"
fig.savefig(
root_folder + date_path + "\Observations\\" + k +
"-" + graph_type + "-" + time_now + ".png",
dpi=dpi_int,
)
fig.clf()
except IndexError:
pass
# Create dew points graph
graph_type = "dew-point"
file_short_list = []
for f in listdir(root_folder + date_path +
"\Observations\\"):
if re_match(k + "-" + graph_type, f):
file_short_list.append(f)
if len(file_short_list) > 1:
file_short_list.sort()
file_short_list.remove(file_short_list[-1])
for g in file_short_list:
os_remove(root_folder + date_path +
"\Observations\\" + g)
print "Creating", graph_type, "graph"
x = []
y = []
for m in loc_obs_dew:
try:
y.append(float(m[1]))
x.append(datetime.fromtimestamp(float(m[0])))
except ValueError:
continue
try:
time_now = arrow_get(x[len(x) - 1]).format('HH-mm')
with open(root_folder + date_path + "\Observations\\" +
k + "-" + graph_type + "-" + time_now +
".png"):
print "Graph Already Exists"
except IOError:
ax = fig.add_subplot(111)
ax.plot(x, y)
for xy in zip(x, y):
ax.annotate(xy[1], xy=xy, textcoords='data')
buf = (max(y) - min(y)) * .1
ax.get_yaxis().get_major_formatter().set_useOffset(
False)
ax.axis(
[x[0], x[len(x) - 1],
min(y) - buf,
max(y) + buf])
fig.tight_layout(pad=3)
print "Saving"
fig.savefig(root_folder + date_path +
"\Observations\\" + k + "-" + graph_type +
"-" + time_now + ".png",
dpi=dpi_int)
fig.clf()
except IndexError:
pass
# Create rain points graph
graph_type = "rain"
file_short_list = []
for f in listdir(root_folder + date_path +
"\Observations\\"):
if re_match(k + "-" + graph_type, f):
file_short_list.append(f)
if len(file_short_list) > 1:
file_short_list.sort()
file_short_list.remove(file_short_list[-1])
for g in file_short_list:
os_remove(root_folder + date_path +
"\Observations\\" + g)
print "Creating", graph_type, "graph"
x = []
y = []
for m in loc_obs_rain:
try:
y.append(float(m[1]))
x.append(datetime.fromtimestamp(float(m[0])))
except ValueError:
continue
try:
time_now = arrow_get(x[len(x) - 1]).format('HH-mm')
with open(root_folder + date_path + "\Observations\\" +
k + "-" + graph_type + "-" + time_now +
".png"):
print "Graph Already Exists"
except IOError:
ax = fig.add_subplot(111)
ax.plot(x, y)
for xy in zip(x, y):
ax.annotate(xy[1], xy=xy, textcoords='data')
buf = (max(y) - min(y)) * .1
ax.get_yaxis().get_major_formatter().set_useOffset(
False)
ax.axis(
[x[0], x[len(x) - 1],
min(y) - buf,
max(y) + buf])
fig.tight_layout(pad=3)
print "Saving"
fig.savefig(root_folder + date_path +
"\Observations\\" + k + "-" + graph_type +
"-" + time_now + ".png",
dpi=dpi_int)
fig.clf()
except IndexError:
pass
# Create Pressure Graph
graph_type = "pressure"
file_short_list = []
for f in listdir(root_folder + date_path +
"\Observations\\"):
if re_match(k + "-" + graph_type, f):
file_short_list.append(f)
if len(file_short_list) > 1:
file_short_list.sort()
file_short_list.remove(file_short_list[-1])
for g in file_short_list:
os_remove(root_folder + date_path +
"\Observations\\" + g)
print "Creating", graph_type, "graph"
x = []
y = []
for m in loc_obs_pressure:
try:
y.append(float(m[1]))
x.append(datetime.fromtimestamp(float(m[0])))
except ValueError:
continue
try:
time_now = arrow_get(x[len(x) - 1]).format('HH-mm')
with open(root_folder + date_path + "\Observations\\" +
k + "-" + graph_type + "-" + time_now +
".png"):
print "Graph Already Exists"
except IOError:
ax = fig.add_subplot(111)
ax.plot(x, y)
for xy in zip(x, y):
ax.annotate(xy[1], xy=xy, textcoords='data')
buf = (max(y) - min(y)) * .1
ax.get_yaxis().get_major_formatter().set_useOffset(
False)
ax.axis(
[x[0], x[len(x) - 1],
min(y) - buf,
max(y) + buf])
fig.tight_layout(pad=3)
print "Saving"
fig.savefig(root_folder + date_path +
"\Observations\\" + k + "-" + graph_type +
"-" + time_now + ".png",
dpi=dpi_int)
fig.clf()
except IndexError:
pass
# Create LCL Graph
graph_type = "lcl"
file_short_list = []
for f in listdir(root_folder + date_path +
"\Observations\\"):
if re_match(k + "-" + graph_type, f):
file_short_list.append(f)
if len(file_short_list) > 1:
file_short_list.sort()
file_short_list.remove(file_short_list[-1])
for g in file_short_list:
os_remove(root_folder + date_path +
"\Observations\\" + g)
print "Creating", graph_type, "graph"
x = []
y = []
for m in loc_obs_lcl:
try:
y.append(float(m[1]))
x.append(datetime.fromtimestamp(float(m[0])))
except ValueError:
continue
try:
time_now = arrow_get(x[len(x) - 1]).format('HH-mm')
with open(root_folder + date_path + "\Observations\\" +
k + "-" + graph_type + "-" + time_now +
".png"):
print "Graph Already Exists"
except IOError:
ax = fig.add_subplot(111)
ax.plot(x, y)
for xy in zip(x, y):
ax.annotate(xy[1], xy=xy, textcoords='data')
buf = (max(y) - min(y)) * .1
ax.get_yaxis().get_major_formatter().set_useOffset(
False)
ax.axis(
[x[0], x[len(x) - 1],
min(y) - buf,
max(y) + buf])
fig.tight_layout(pad=3)
print "Saving"
fig.savefig(root_folder + date_path +
"\Observations\\" + k + "-" + graph_type +
"-" + time_now + ".png",
dpi=dpi_int)
fig.clf()
except IndexError:
pass
# Create relative humidity Graph
graph_type = "rel_hum"
file_short_list = []
for f in listdir(root_folder + date_path +
"\Observations\\"):
if re_match(k + "-" + graph_type, f):
file_short_list.append(f)
if len(file_short_list) > 1:
file_short_list.sort()
file_short_list.remove(file_short_list[-1])
for g in file_short_list:
os_remove(root_folder + date_path +
"\Observations\\" + g)
print "Creating", graph_type, "graph"
x = []
y = []
for m in loc_obs_rel_hum:
try:
y.append(float(m[1]))
x.append(datetime.fromtimestamp(float(m[0])))
except ValueError:
continue
try:
time_now = arrow_get(x[len(x) - 1]).format('HH-mm')
with open(root_folder + date_path + "\Observations\\" +
k + "-" + graph_type + "-" + time_now +
".png"):
print "Graph Already Exists"
except IOError:
ax = fig.add_subplot(111)
ax.plot(x, y)
for xy in zip(x, y):
ax.annotate(xy[1], xy=xy, textcoords='data')
buf = (max(y) - min(y)) * .1
ax.get_yaxis().get_major_formatter().set_useOffset(
False)
ax.axis(
[x[0], x[len(x) - 1],
min(y) - buf,
max(y) + buf])
fig.tight_layout(pad=3)
print "Saving"
fig.savefig(root_folder + date_path +
"\Observations\\" + k + "-" + graph_type +
"-" + time_now + ".png",
dpi=dpi_int)
fig.clf()
except IndexError:
pass
# Create wind speed graph
graph_type = "wind"
file_short_list = []
for f in listdir(root_folder + date_path +
"\Observations\\"):
if re_match(k + "-" + graph_type, f):
file_short_list.append(f)
if len(file_short_list) > 1:
file_short_list.sort()
file_short_list.remove(file_short_list[-1])
for g in file_short_list:
os_remove(root_folder + date_path +
"\Observations\\" + g)
print "Creating", graph_type, "graph"
x = []
y = []
for m in loc_obs_wind_vel:
try:
y.append(float(m[1]))
x.append(datetime.fromtimestamp(float(m[0])))
except ValueError:
continue
try:
time_now = arrow_get(x[len(x) - 1]).format('HH-mm')
with open(root_folder + date_path + "\Observations\\" +
k + "-" + graph_type + "-" + time_now +
".png"):
print "Graph Already Exists"
except IOError:
ax = fig.add_subplot(111)
ax.plot(x, y)
for xy in zip(x, y):
ax.annotate(xy[1], xy=xy, textcoords='data')
buf = (max(y) - min(y)) * .1
ax.get_yaxis().get_major_formatter().set_useOffset(
False)
ax.axis(
[x[0], x[len(x) - 1],
min(y) - buf,
max(y) + buf])
fig.tight_layout(pad=3)
print "Saving"
fig.savefig(root_folder + date_path +
"\Observations\\" + k + "-" + graph_type +
"-" + time_now + ".png",
dpi=dpi_int)
fig.clf()
except IndexError:
pass
def create_current_overlays(self):
root_folder = "C:\Users\Nathan\Documents\Storm Chasing\Chases\\"
date_path = arrow_now().format('YYYY-MM-DD')
config_filename = root_folder + date_path + "\Observations\\" + "range.cfg"
min_temp = "-"
max_temp = "-"
min_dew = "-"
max_dew = "-"
min_pressure = "-"
max_pressure = "-"
min_rain = "-"
max_rain = "-"
min_rel_hum = "-"
max_rel_hum = "-"
min_lcl = "-"
max_lcl = "-"
min_wind_vel = "-"
max_wind_vel = "-"
with open(config_filename, 'r') as f:
line = f.readline()
line_list = line.split(',')
min_temp = float(line_list[0])
max_temp = float(line_list[1].replace('\n', ''))
line = f.readline()
line_list = line.split(',')
min_dew = float(line_list[0])
max_dew = float(line_list[1].replace('\n', ''))
line = f.readline()
line_list = line.split(',')
min_rain = float(line_list[0])
max_rain = float(line_list[1].replace('\n', ''))
line = f.readline()
line_list = line.split(',')
min_pressure = float(line_list[0])
max_pressure = float(line_list[1].replace('\n', ''))
line = f.readline()
line_list = line.split(',')
min_rel_hum = float(line_list[0])
max_rel_hum = float(line_list[1].replace('\n', ''))
line = f.readline()
line_list = line.split(',')
min_lcl = float(line_list[0])
max_lcl = float(line_list[1].replace('\n', ''))
line = f.readline()
line_list = line.split(',')
min_wind_vel = float(line_list[0])
max_wind_vel = float(line_list[1].replace('\n', ''))
print
# Temperature Graph
if True:
graph_type = "temp"
time_now = arrow_get(self.obs_list[36].time,
'HH:mmA').format('HH-mm')
graph_filename = root_folder + date_path + "\Observations\\" + "vic-" + graph_type + "-" + time_now + ".png"
try:
with open(graph_filename, 'r') as f:
print "Graph already created"
except IOError:
x = []
y = []
z = []
for i in range(0, len(self.obs_list)):
if self.obs_list[i].temp == "-":
continue
y.append(self.obs_list[i].lat)
x.append(self.obs_list[i].lon)
z.append(self.obs_list[i].temp)
resolution = 100
xi = linspace(141, 150, resolution)
yi = linspace(-39, -34, resolution)
zi = griddata((x, y),
z, (xi[None, :], yi[:, None]),
method='cubic')
print min(z), max(z)
w = 15
h = 12.5
fig = plt.figure(figsize=(w, h),
facecolor='w',
frameon=False,
edgecolor="Yellow")
ax = fig.add_subplot(111)
ax.contourf(xi,
yi,
zi,
120,
cmap=plt.cm.jet,
vmin=min_temp,
vmax=max_temp)
C = ax.contour(xi, yi, zi, 20, linewidths=0.5, colors='k')
ax.clabel(C, inline=1, fontsize=6, fmt='%1.0f')
ax.scatter(x, y, marker='o', c='b', s=.1)
ax.axis([141, 150, -39, -34])
ax.set_axis_off()
fig.subplots_adjust(left=0,
right=1,
bottom=0,
top=1,
hspace=0,
wspace=0)
fig.tight_layout(pad=0)
print "Start Save for", graph_type
print graph_filename
self.SaveFigureAsImage(graph_filename, fig=fig)
print "End Save"
# Dew Point Graph
if True:
graph_type = "dew"
time_now = arrow_get(self.obs_list[36].time,
'HH:mmA').format('HH-mm')
graph_filename = root_folder + date_path + "\Observations\\" + "vic-" + graph_type + "-" + time_now + ".png"
try:
with open(graph_filename, 'r') as f:
print "Graph already created"
except IOError:
x = []
y = []
z = []
for i in range(0, len(self.obs_list)):
if self.obs_list[i].dew == "-":
continue
y.append(self.obs_list[i].lat)
x.append(self.obs_list[i].lon)
z.append(self.obs_list[i].dew)
resolution = 100
xi = linspace(141, 150, resolution)
yi = linspace(-39, -34, resolution)
zi = griddata((x, y),
z, (xi[None, :], yi[:, None]),
method='cubic')
w = 15
h = 12.5
fig = plt.figure(figsize=(w, h),
facecolor='w',
frameon=False,
edgecolor="Yellow")
ax = fig.add_subplot(111)
ax.contourf(xi,
yi,
zi,
15,
cmap=plt.cm.jet,
vmin=min_dew,
vmax=max_dew)
C = ax.contour(xi, yi, zi, 20, linewidths=0.5, colors='k')
ax.clabel(C, inline=1, fontsize=6, fmt='%1.0f')
ax.scatter(x, y, marker='o', c='b', s=.1)
ax.axis([141, 150, -39, -34])
ax.set_axis_off()
fig.subplots_adjust(left=0,
right=1,
bottom=0,
top=1,
hspace=0,
wspace=0)
fig.tight_layout(pad=0)
print "Start Save for", graph_type
print graph_filename
self.SaveFigureAsImage(graph_filename, fig=fig)
print "End Save"
# Rain Graph
if True:
graph_type = "rain"
time_now = arrow_get(self.obs_list[36].time,
'HH:mmA').format('HH-mm')
graph_filename = root_folder + date_path + "\Observations\\" + "vic-" + graph_type + "-" + time_now + ".png"
try:
with open(graph_filename, 'r') as f:
print "Graph already created"
except IOError:
x = []
y = []
z = []
for i in range(0, len(self.obs_list)):
if self.obs_list[i].rain == "-":
continue
y.append(self.obs_list[i].lat)
x.append(self.obs_list[i].lon)
z.append(self.obs_list[i].rain)
resolution = 100
xi = linspace(141, 150, resolution)
yi = linspace(-39, -34, resolution)
zi = griddata((x, y),
z, (xi[None, :], yi[:, None]),
method='cubic')
w = 15
h = 12.5
fig = plt.figure(figsize=(w, h),
facecolor='w',
frameon=False,
edgecolor="Yellow")
ax = fig.add_subplot(111)
ax.contourf(xi,
yi,
zi,
15,
cmap=plt.cm.jet,
vmin=min_rain,
vmax=max_rain)
C = ax.contour(xi, yi, zi, 20, linewidths=0.5, colors='k')
ax.clabel(C, inline=1, fontsize=6, fmt='%1.0f')
ax.scatter(x, y, marker='o', c='b', s=.1)
ax.axis([141, 150, -39, -34])
ax.set_axis_off()
fig.subplots_adjust(left=0,
right=1,
bottom=0,
top=1,
hspace=0,
wspace=0)
fig.tight_layout(pad=0)
print "Start Save for", graph_type
print graph_filename
self.SaveFigureAsImage(graph_filename, fig=fig)
print "End Save"
# Pressure Point Graph
if True:
graph_type = "pressure"
time_now = arrow_get(self.obs_list[36].time,
'HH:mmA').format('HH-mm')
graph_filename = root_folder + date_path + "\Observations\\" + "vic-" + graph_type + "-" + time_now + ".png"
try:
with open(graph_filename, 'r') as f:
print "Graph already created"
except IOError:
x = []
y = []
z = []
for i in range(0, len(self.obs_list)):
if self.obs_list[i].pressure == "-":
continue
y.append(self.obs_list[i].lat)
x.append(self.obs_list[i].lon)
z.append(self.obs_list[i].pressure)
resolution = 100
xi = linspace(141, 150, resolution)
yi = linspace(-39, -34, resolution)
zi = griddata((x, y),
z, (xi[None, :], yi[:, None]),
method='cubic')
w = 15
h = 12.5
fig = plt.figure(figsize=(w, h),
facecolor='w',
frameon=False,
edgecolor="Yellow")
ax = fig.add_subplot(111)
ax.contourf(xi,
yi,
zi,
15,
cmap=plt.cm.jet,
vmin=min_pressure,
vmax=max_pressure)
C = ax.contour(xi, yi, zi, 20, linewidths=0.5, colors='k')
ax.clabel(C, inline=1, fontsize=6, fmt='%1.0f')
ax.scatter(x, y, marker='o', c='b', s=.1)
ax.axis([141, 150, -39, -34])
ax.set_axis_off()
fig.subplots_adjust(left=0,
right=1,
bottom=0,
top=1,
hspace=0,
wspace=0)
fig.tight_layout(pad=0)
print "Start Save for", graph_type
print graph_filename
self.SaveFigureAsImage(graph_filename, fig=fig)
print "End Save"
# Relative Humidity Graph
if True:
graph_type = "rel_hum"
time_now = arrow_get(self.obs_list[36].time,
'HH:mmA').format('HH-mm')
graph_filename = root_folder + date_path + "\Observations\\" + "vic-" + graph_type + "-" + time_now + ".png"
try:
with open(graph_filename, 'r') as f:
print "Graph already created"
except IOError:
x = []
y = []
z = []
for i in range(0, len(self.obs_list)):
if self.obs_list[i].rel_hum == "-":
continue
y.append(self.obs_list[i].lat)
x.append(self.obs_list[i].lon)
z.append(self.obs_list[i].rel_hum)
resolution = 100
xi = linspace(141, 150, resolution)
yi = linspace(-39, -34, resolution)
zi = griddata((x, y),
z, (xi[None, :], yi[:, None]),
method='cubic')
w = 15
h = 12.5
fig = plt.figure(figsize=(w, h),
facecolor='w',
frameon=False,
edgecolor="Yellow")
ax = fig.add_subplot(111)
ax.contourf(xi,
yi,
zi,
15,
cmap=plt.cm.jet,
vmin=min_rel_hum,
vmax=max_rel_hum)
C = ax.contour(xi, yi, zi, 20, linewidths=0.5, colors='k')
ax.clabel(C, inline=1, fontsize=6, fmt='%1.0f')
ax.scatter(x, y, marker='o', c='b', s=.1)
ax.axis([141, 150, -39, -34])
ax.set_axis_off()
fig.subplots_adjust(left=0,
right=1,
bottom=0,
top=1,
hspace=0,
wspace=0)
fig.tight_layout(pad=0)
print "Start Save for", graph_type
print graph_filename
self.SaveFigureAsImage(graph_filename, fig=fig)
print "End Save"
# LCL Graph
if True:
graph_type = "lcl"
time_now = arrow_get(self.obs_list[36].time,
'HH:mmA').format('HH-mm')
graph_filename = root_folder + date_path + "\Observations\\" + "vic-" + graph_type + "-" + time_now + ".png"
try:
with open(graph_filename, 'r') as f:
print "Graph already created"
except IOError:
x = []
y = []
z = []
for i in range(0, len(self.obs_list)):
if self.obs_list[i].lcl == "-":
continue
y.append(self.obs_list[i].lat)
x.append(self.obs_list[i].lon)
z.append(self.obs_list[i].lcl)
resolution = 100
xi = linspace(141, 150, resolution)
yi = linspace(-39, -34, resolution)
zi = griddata((x, y),
z, (xi[None, :], yi[:, None]),
method='cubic')
w = 15
h = 12.5
fig = plt.figure(figsize=(w, h),
facecolor='w',
frameon=False,
edgecolor="Yellow")
ax = fig.add_subplot(111)
ax.contourf(xi,
yi,
zi,
15,
cmap=plt.cm.jet,
vmin=min_lcl,
vmax=max_lcl)
C = ax.contour(xi, yi, zi, 20, linewidths=0.5, colors='k')
ax.clabel(C, inline=1, fontsize=6, fmt='%1.0f')
ax.scatter(x, y, marker='o', c='b', s=.1)
ax.axis([141, 150, -39, -34])
ax.set_axis_off()
fig.subplots_adjust(left=0,
right=1,
bottom=0,
top=1,
hspace=0,
wspace=0)
fig.tight_layout(pad=0)
print "Start Save for", graph_type
print graph_filename
self.SaveFigureAsImage(graph_filename, fig=fig)
print "End Save"
# Wind Graph
if True:
graph_type = "wind_vel"
time_now = arrow_get(self.obs_list[36].time,
'HH:mmA').format('HH-mm')
graph_filename = root_folder + date_path + "\Observations\\" + "vic-" + graph_type + "-" + time_now + ".png"
try:
with open(graph_filename, 'r') as f:
print "Graph already created"
except IOError:
x = []
y = []
z = []
for i in range(0, len(self.obs_list)):
if self.obs_list[i].wind_vel == "-":
continue
y.append(self.obs_list[i].lat)
x.append(self.obs_list[i].lon)
z.append(self.obs_list[i].wind_vel)
resolution = 100
xi = linspace(141, 150, resolution)
yi = linspace(-39, -34, resolution)
zi = griddata((x, y),
z, (xi[None, :], yi[:, None]),
method='cubic')
w = 15
h = 12.5
fig = plt.figure(figsize=(w, h),
facecolor='w',
frameon=False,
edgecolor="Yellow")
ax = fig.add_subplot(111)
ax.contourf(xi,
yi,
zi,
15,
cmap=plt.cm.jet,
vmin=min_wind_vel,
vmax=max_wind_vel)
C = ax.contour(xi, yi, zi, 20, linewidths=0.5, colors='k')
ax.clabel(C, inline=1, fontsize=6, fmt='%1.0f')
ax.scatter(x, y, marker='o', c='b', s=.1)
ax.axis([141, 150, -39, -34])
ax.set_axis_off()
fig.subplots_adjust(left=0,
right=1,
bottom=0,
top=1,
hspace=0,
wspace=0)
fig.tight_layout(pad=0)
print "Start Save for", graph_type
print graph_filename
self.SaveFigureAsImage(graph_filename, fig=fig)
print "End Save"
def create_all_overlays(self):
give_me_full_obs()
root_folder = "C:\Users\Nathan\Documents\Storm Chasing\Chases\\"
date_path = arrow_now().format('YYYY-MM-DD')
config_filename = root_folder + date_path + "\Observations\\" + "range.cfg"
dir_list = listdir(root_folder + date_path + "\Observations")
csv_list = []
for fn in dir_list:
if fn[-3:] == "csv":
csv_list.append(fn)
len_loc = len(csv_list)
for fn in csv_list:
with open(root_folder + date_path + "\Observations\\" + fn,
'r') as f:
i = 0
for line in f:
i += 1
line_list = line.split(',')
len_details = len(line_list)
len_time = i
len_details += 1
all_obs = [[["" for k in xrange(len_details)]
for j in xrange(len_time)] for i in xrange(len_loc)]
for i in range(0, len(csv_list)):
fn = csv_list[i]
with open(root_folder + date_path + "\Observations\\" + fn,
'r') as f:
j = 0
last_time = 0
for line in f:
line_list = line.split(',')
this_time = int(line_list[0])
if j == 0:
for k in range(0, len(line_list)):
all_obs[i][j][k] = line_list[k]
all_obs[i][j][-1] = str(fn).split('.')[0]
last_time = this_time
elif this_time - last_time != 1800:
continue
else:
for k in range(0, len(line_list)):
all_obs[i][j][k] = line_list[k]
all_obs[i][j][-1] = str(fn).split('.')[0]
last_time = this_time
j += 1
all_obs_sorted = [[["" for k in xrange(len_loc)]
for j in xrange(len_time)]
for i in xrange(len_details)]
for i in range(0, len_loc):
for j in range(0, len_time):
for k in range(0, len_details):
all_obs_sorted[k][j][i] = all_obs[i][j][k]
for gt in range(1, 6):
if gt == 1:
graph_type = "temp"
min_graph = 0
max_graph = 40
elif gt == 2:
graph_type = "dew"
min_graph = 0
max_graph = 25
elif gt == 3:
graph_type = "rain"
min_graph = 0
max_graph = 80
elif gt == 4:
graph_type = "pressure"
min_graph = 990
max_graph = 1050
elif gt == 5:
graph_type = "lcl"
min_graph = 0
max_graph = 3000
elif gt == 6:
graph_type = "rel_hum"
min_graph = 0
max_graph = 100
elif gt == 7:
graph_type = "wind_vel"
min_graph = 0
max_graph = 120
w = 15
h = 12.5
fig = plt.figure(figsize=(w, h),
facecolor='w',
frameon=False,
edgecolor="Yellow")
for i in range(0, len(all_obs_sorted[9])):
x = []
y = []
z = []
for obs in self.obs_list:
for j in range(0, len(all_obs_sorted[9][i])):
if obs.code == all_obs_sorted[9][i][j]:
try:
z.append(float(all_obs_sorted[gt][i][j]))
x.append(float(obs.lon))
y.append(float(obs.lat))
except ValueError:
continue
break
if True:
try:
time_now = arrow_get(all_obs_sorted[0][i][36]).to(
'Australia/Melbourne').format('HH-mm')
except parser.ParserError:
continue
graph_filename = root_folder + date_path + "\Observations\\" + "vic-" + graph_type + "-" + time_now + ".png"
try:
with open(graph_filename, 'r'):
print "Graph already exists:"
print graph_filename
continue
except IOError:
pass
resolution = 100
xi = linspace(141, 150, resolution)
yi = linspace(-39, -34, resolution)
zi = griddata((x, y),
z, (xi[None, :], yi[:, None]),
method='cubic')
fig.clf()
ax = fig.add_subplot(111)
ax.contourf(xi,
yi,
zi,
120,
cmap=plt.cm.jet,
vmin=min_graph,
vmax=max_graph)
try:
C = ax.contour(xi,
yi,
zi,
20,
linewidths=0.5,
colors='k')
ax.clabel(C, inline=1, fontsize=6, fmt='%1.0f')
except ValueError:
pass
ax.scatter(x, y, marker='o', c='b', s=.1)
ax.axis([141, 150, -39, -34])
ax.set_axis_off()
fig.subplots_adjust(left=0,
right=1,
bottom=0,
top=1,
hspace=0,
wspace=0)
fig.tight_layout(pad=0)
print "Start Save for", graph_type, time_now
print graph_filename
self.SaveFigureAsImage(graph_filename, fig=fig)
print "End Save"
def _get_last_notified(self, key):
with self.lock():
value = self.kvdb.conn.get(key)
return arrow_get(value) if value else None
def give_me_full_obs():
string_of_observation_locations = """aireys-inlet,Aireys Inlet,-38.4583,144.0883,105
avalon,Avalon Airport,-38.0287,144.4783,10.6
bairnsdale,Bairnsdale Airport,-37.8817,147.5669,49.4
ballarat,Ballarat Aerodrome,-37.5127,143.7911,435.2
bendigo,Bendigo Airport,-36.7395,144.3266,208
geelong-racecourse,Breakwater (Geelong Racecourse),-38.1737,144.3765,12.9
cape-nelson,Cape Nelson Lighthouse,-38.4306,141.5437,45.4
cape-otway,Cape Otway Lighthouse,-38.8556,143.5128,82
casterton,Casterton,-37.583,141.3339,130.6
cerberus,Cerberus,-38.3646,145.1785,12.69
charlton,Charlton,-36.2847,143.3341,131.7
mount-gellibrand,Colac (Mount Gellibrand),-38.2333,143.7925,261
coldstream,Coldstream,-37.7239,145.4092,83
combienbar,Combienbar AWS,-37.3417,149.0228,640
yanakie,Corner Inlet (Yanakie),-38.8051,146.1939,13.3
dartmoor,Dartmoor,-37.9222,141.2614,51
mount-hotham-airport,Dinner Plain (Mount Hotham Airport),-37.0491,147.3347,1295.4
east-sale,East Sale Airport,-38.1156,147.1322,4.6
edenhope,Edenhope Airport,-37.0222,141.2657,155
eildon-fire-tower,Eildon Fire Tower,-37.2091,145.8423,637
essendon-airport,Essendon Airport,-37.7276,144.9066,78.4
falls-creek,Falls Creek,-36.8708,147.2755,1765
ferny-creek,Ferny Creek,-37.8748,145.3496,512.9
frankston,Frankston AWS,-38.1481,145.1156,6
gabo-island,Gabo Island Lighthouse,-37.5679,149.9158,15.2
gelantipy,Gelantipy,-37.22,148.2625,755
mount-william,Grampians (Mount William),-37.295,142.6039,1150
hamilton,Hamilton Airport,-37.6486,142.0636,241.1
hopetoun-airport,Hopetoun Airport,-35.7151,142.3569,77.3
hunters-hill,Hunters Hill,-36.2136,147.5394,981
kanagulk,Kanagulk,-37.1169,141.8031,188.8
warracknabeal-airport,Kellalac (Warracknabeal Airport),-36.3204,142.4161,118.3
kyabram,Kyabram,-36.335,145.0638,105
laverton,Laverton RAAF,-37.8565,144.7566,20.1
longerenong,Longerenong,-36.6722,142.2991,133
mallacoota,Mallacoota,-37.5976,149.7289,22
mangalore,Mangalore Airport,-36.8886,145.1859,140.8
melbourne-olympic-park,Melbourne (Olympic Park),-37.8255,144.9816,7.53
melbourne-airport,Melbourne Airport,-37.6655,144.8321,113.4
mildura,Mildura Airport,-34.2358,142.0867,50
moorabbin-airport,Moorabbin Airport,-37.98,145.0962,12.1
mortlake,Mortlake Racecourse,-38.0737,142.7744,130
latrobe-valley,Morwell (Latrobe Valley Airport),-38.2094,146.4747,55.7
mount-baw-baw,Mount Baw Baw,-37.8383,146.2747,1561
mount-buller,Mount Buller,-37.145,146.4394,1707
mount-moornapa,Mount Moornapa,-37.7481,147.1428,480
mount-nowa-nowa,Mount Nowa Nowa,-37.6924,148.0908,350
nhill-aerodrome,Nhill Aerodrome,-36.3092,141.6486,138.9
omeo,Omeo,-37.1017,147.6008,689.8
orbost,Orbost,-37.6922,148.4667,62.65
port-fairy,Port Fairy AWS,-38.3906,142.2347,10
portland-airport,Portland (Cashmore Airport),-38.3148,141.4705,80.9
portland-harbour,Portland NTC AWS,-38.3439,141.6136,0
pound-creek,Pound Creek,-38.6297,145.8107,3
redesdale,Redesdale,-37.0194,144.5203,290
rhyll,Rhyll,-38.4612,145.3101,13.4
rutherglen,Rutherglen Research,-36.1047,146.5094,175
scoresby,Scoresby Research Institute,-37.871,145.2561,80
sheoaks,Sheoaks,-37.9075,144.1303,236.7
shepparton,Shepparton Airport,-36.4289,145.3947,113.9
stawell,Stawell Aerodrome,-37.072,142.7402,235.36
swan-hill,Swan Hill Aerodrome,-35.3766,143.5416,71
tatura,Tatura Inst Sustainable Ag,-36.4378,145.2672,114
viewbank,Viewbank,-37.7408,145.0972,66.1
kilmore-gap,Wallan (Kilmore Gap),-37.3807,144.9654,527.8
walpeup,Walpeup Research,-35.1201,142.004,105
wangaratta,Wangaratta Aero,-36.4206,146.3056,152.6
warrnambool,Warrnambool Airport NDB,-38.2867,142.4522,70.8
westmere,Westmere,-37.7067,142.9378,226
wilsons-promontory,Wilsons Promontory Lighthouse,-39.1297,146.4244,95
yarram-airport,Yarram Airport,-38.5647,146.7479,17.9
yarrawonga,Yarrawonga,-36.0294,146.0306,128.9"""
buf = StringIO(string_of_observation_locations)
loc_names = []
list_of_csvs = []
for i in buf:
loc = str(i).replace('\n', '').split(',')
loc_names.append(loc[0].lstrip())
urladdy = "http://www.bom.gov.au/vic/observations/vicall.shtml"
reader = urlopen(urladdy)
full_page = StringIO(str(reader.read()))
print "Scraping URL"
for i in full_page:
line = str(i).replace('\n', '')
for j in range(0, len(loc_names)):
if loc_names[j] in line:
if 'a href=' in line:
code = str(line).split('<')[2].split('>')[0].split(
'=')[1].split('.')[1]
list_of_csvs.append(
("http://www.bom.gov.au/fwo/IDV60801/IDV60801." +
str(code) + ".axf", loc_names[j]))
reader.close()
for sites in list_of_csvs:
print
print "Starting for", sites[1]
print "Sleeping, don't wake the baby"
sleep(0.5)
print "Wah wah wah"
url_reader = urlopen(sites[0])
line = url_reader.readline()
while '[data]' not in line:
line = url_reader.readline()
csv_reader = csv.DictReader(url_reader)
loc_obs = []
for row in csv_reader:
if row['name[80]'] == None:
continue
code = sites[1]
try:
obs_time = arrow_get(row['local_date_time_full[80]'],
'YYYYMMDDHHmmss',
tzinfo='Australia/Melbourne')
except ValueError:
continue
day_start = arrow_get(str(arrow_now().format('YYYY-MM-DD')) +
" 00:00:00",
'YYYY-MM-DD HH:mm:ss',
tzinfo='Australia/Melbourne').timestamp
iter_time = arrow_get(obs_time).timestamp
if iter_time < day_start:
continue
try:
temp = float(row['air_temp'])
if -50 < temp < 70:
pass
else:
temp = "-"
except ValueError:
temp = "-"
try:
dew = float(row['dewpt'])
if -50 < dew < 50:
pass
else:
dew = "-"
except ValueError:
dew = "-"
try:
rain = float(row['rain_trace[80]'].replace('"', ''))
if 0 < rain < 400:
pass
else:
rain = "0.0"
except ValueError:
rain = "0.0"
try:
pressure = float(row['press'])
if 900 < pressure < 1100:
pass
else:
pressure = "-"
except ValueError:
pressure = "-"
try:
rel_hum = int(row['rel_hum'])
if -100 < rel_hum < 100:
pass
else:
rel_hum = "-"
except ValueError:
rel_hum = "-"
try:
wind_vel = int(row['wind_spd_kmh'])
if -1 < wind_vel < 300:
pass
else:
wind_vel = "-"
except ValueError:
wind_vel = "-"
try:
wind_dir = row['wind_dir[80]']
except ValueError:
wind_dir = "-"
try:
lcl = 125 * (temp - dew)
except TypeError:
lcl = "-"
loc_obs.insert(
0,
str(obs_time.timestamp) + ',' + str(temp) + ',' + str(dew) +
',' + str(rain) + ',' + str(pressure) + ',' + str(lcl) + ',' +
str(rel_hum) + ',' + str(wind_vel) + ',' + wind_dir)
for loc_line in loc_obs:
file_time = arrow_get(loc_line.split(',')[0]).to(
'Australia/Melbourne').format('HH-mm')
root_folder = "C:\Users\Nathan\Documents\Storm Chasing\Chases\\"
date_path = arrow_now().format('YYYY-MM-DD')
try:
chdir(root_folder + date_path)
except WindowsError:
mkdir(root_folder + date_path)
try:
chdir(root_folder + date_path + "\Observations")
except WindowsError:
mkdir(root_folder + date_path + "\Observations")
filename = root_folder + date_path + "\Observations\\" + code + '.csv'
with open(filename, 'w') as f:
for loc_line in loc_obs:
f.writelines(loc_line + '\n')
def give_me_vic_obs():
string_of_observation_locations = """aireys-inlet,Aireys Inlet,-38.4583,144.0883,105
avalon,Avalon Airport,-38.0287,144.4783,10.6
bairnsdale,Bairnsdale Airport,-37.8817,147.5669,49.4
ballarat,Ballarat Aerodrome,-37.5127,143.7911,435.2
bendigo,Bendigo Airport,-36.7395,144.3266,208
geelong-racecourse,Breakwater (Geelong Racecourse),-38.1737,144.3765,12.9
cape-nelson,Cape Nelson Lighthouse,-38.4306,141.5437,45.4
cape-otway,Cape Otway Lighthouse,-38.8556,143.5128,82
casterton,Casterton,-37.583,141.3339,130.6
cerberus,Cerberus,-38.3646,145.1785,12.69
charlton,Charlton,-36.2847,143.3341,131.7
mount-gellibrand,Colac (Mount Gellibrand),-38.2333,143.7925,261
coldstream,Coldstream,-37.7239,145.4092,83
combienbar,Combienbar AWS,-37.3417,149.0228,640
yanakie,Corner Inlet (Yanakie),-38.8051,146.1939,13.3
dartmoor,Dartmoor,-37.9222,141.2614,51
mount-hotham-airport,Dinner Plain (Mount Hotham Airport),-37.0491,147.3347,1295.4
east-sale,East Sale Airport,-38.1156,147.1322,4.6
edenhope,Edenhope Airport,-37.0222,141.2657,155
eildon-fire-tower,Eildon Fire Tower,-37.2091,145.8423,637
essendon-airport,Essendon Airport,-37.7276,144.9066,78.4
falls-creek,Falls Creek,-36.8708,147.2755,1765
ferny-creek,Ferny Creek,-37.8748,145.3496,512.9
frankston,Frankston AWS,-38.1481,145.1156,6
gabo-island,Gabo Island Lighthouse,-37.5679,149.9158,15.2
gelantipy,Gelantipy,-37.22,148.2625,755
mount-william,Grampians (Mount William),-37.295,142.6039,1150
hamilton,Hamilton Airport,-37.6486,142.0636,241.1
hopetoun-airport,Hopetoun Airport,-35.7151,142.3569,77.3
hunters-hill,Hunters Hill,-36.2136,147.5394,981
kanagulk,Kanagulk,-37.1169,141.8031,188.8
warracknabeal-airport,Kellalac (Warracknabeal Airport),-36.3204,142.4161,118.3
kyabram,Kyabram,-36.335,145.0638,105
laverton,Laverton RAAF,-37.8565,144.7566,20.1
longerenong,Longerenong,-36.6722,142.2991,133
mallacoota,Mallacoota,-37.5976,149.7289,22
mangalore,Mangalore Airport,-36.8886,145.1859,140.8
melbourne-olympic-park,Melbourne (Olympic Park),-37.8255,144.9816,7.53
melbourne-airport,Melbourne Airport,-37.6655,144.8321,113.4
mildura,Mildura Airport,-34.2358,142.0867,50
moorabbin-airport,Moorabbin Airport,-37.98,145.0962,12.1
mortlake,Mortlake Racecourse,-38.0737,142.7744,130
latrobe-valley,Morwell (Latrobe Valley Airport),-38.2094,146.4747,55.7
mount-baw-baw,Mount Baw Baw,-37.8383,146.2747,1561
mount-buller,Mount Buller,-37.145,146.4394,1707
mount-moornapa,Mount Moornapa,-37.7481,147.1428,480
mount-nowa-nowa,Mount Nowa Nowa,-37.6924,148.0908,350
nhill-aerodrome,Nhill Aerodrome,-36.3092,141.6486,138.9
omeo,Omeo,-37.1017,147.6008,689.8
orbost,Orbost,-37.6922,148.4667,62.65
port-fairy,Port Fairy AWS,-38.3906,142.2347,10
portland-airport,Portland (Cashmore Airport),-38.3148,141.4705,80.9
portland-harbour,Portland NTC AWS,-38.3439,141.6136,0
pound-creek,Pound Creek,-38.6297,145.8107,3
redesdale,Redesdale,-37.0194,144.5203,290
rhyll,Rhyll,-38.4612,145.3101,13.4
rutherglen,Rutherglen Research,-36.1047,146.5094,175
scoresby,Scoresby Research Institute,-37.871,145.2561,80
sheoaks,Sheoaks,-37.9075,144.1303,236.7
shepparton,Shepparton Airport,-36.4289,145.3947,113.9
stawell,Stawell Aerodrome,-37.072,142.7402,235.36
swan-hill,Swan Hill Aerodrome,-35.3766,143.5416,71
tatura,Tatura Inst Sustainable Ag,-36.4378,145.2672,114
viewbank,Viewbank,-37.7408,145.0972,66.1
kilmore-gap,Wallan (Kilmore Gap),-37.3807,144.9654,527.8
walpeup,Walpeup Research,-35.1201,142.004,105
wangaratta,Wangaratta Aero,-36.4206,146.3056,152.6
warrnambool,Warrnambool Airport NDB,-38.2867,142.4522,70.8
westmere,Westmere,-37.7067,142.9378,226
wilsons-promontory,Wilsons Promontory Lighthouse,-39.1297,146.4244,95
yarram-airport,Yarram Airport,-38.5647,146.7479,17.9
yarrawonga,Yarrawonga,-36.0294,146.0306,128.9"""
print "Beginning URL Load"
buf = StringIO(string_of_observation_locations)
loc_names = []
obs_list = []
for i in buf:
loc = str(i).replace('\n', '').split(',')
obs_list.append(
ObservationLocation(loc[0].lstrip(), loc[1], float(loc[2]),
float(loc[3]), float(loc[4])))
loc_names.append(loc[0].lstrip())
urladdy = "http://www.bom.gov.au/vic/observations/vicall.shtml"
print "Opening URL"
reader = urlopen(urladdy)
full_page = StringIO(str(reader.read()))
print "Scraping URL"
for i in full_page:
line = str(i).replace('\n', '')
for j in range(0, len(obs_list)):
if obs_list[j].code in line:
if '-datetime ' in line:
obs_list[j].time = line.split('>')[1].split('<')[0].split(
'/')[1]
if '-tmp ' in line:
obs_list[j].temp = line.split('>')[1].split('<')[0]
if '-dewpoint ' in line:
obs_list[j].dew = line.split('>')[1].split('<')[0]
if '-rainsince9am ' in line:
obs_list[j].rain = line.split('>')[1].split('<')[0]
if '-press ' in line:
obs_list[j].pressure = line.split('>')[1].split('<')[0]
if '-relhum ' in line:
obs_list[j].rel_hum = line.split('>')[1].split('<')[0]
if '-wind-spd-kmh ' in line:
obs_list[j].wind_vel = line.split('>')[1].split('<')[0]
if '-wind-dir ' in line:
obs_list[j].wind_dir = line.split('>')[1].split('<')[0]
for i in range(0, len(obs_list)):
temp_ = obs_list[i].temp
dp_ = obs_list[i].dew
if temp_ == "-" or dp_ == "-":
obs_list[i].lcl = "-"
continue
lcl = 125 * (float(temp_) - float(dp_))
obs_list[i].lcl = lcl + obs_list[i].height
print "Saving CSV Files"
temp = []
dew = []
rain = []
pressure = []
rel_hum = []
lcl = []
wind_vel = []
for j in range(0, len(obs_list)):
root_folder = "C:\Users\Nathan\Documents\Storm Chasing\Chases\\"
date_path = arrow_now().format('YYYY-MM-DD')
try:
chdir(root_folder + date_path)
except WindowsError:
mkdir(root_folder + date_path)
try:
chdir(root_folder + date_path + "\Observations")
except WindowsError:
mkdir(root_folder + date_path + "\Observations")
try:
temp.append(float(obs_list[j].temp))
except ValueError:
pass
try:
dew.append(float(obs_list[j].dew))
except ValueError:
pass
try:
rain.append(float(obs_list[j].rain))
except ValueError:
pass
try:
pressure.append(float(obs_list[j].pressure))
except ValueError:
pass
try:
rel_hum.append(float(obs_list[j].rel_hum))
except ValueError:
pass
try:
lcl.append(float(obs_list[j].lcl))
except ValueError:
pass
try:
wind_vel.append(float(obs_list[j].wind_vel))
except ValueError:
pass
last_line = ""
csv_filename = root_folder + date_path + "\Observations\\" + obs_list[
j].code + ".csv"
print obs_list[j].time
local_time = str(arrow_now().format('YYYY-MM-DD')) + " " + str(
arrow_get(obs_list[j].time, 'HH:mmA').format('HH:mm'))
ts = arrow_get(local_time,
'YYYY-MM-DD HH:mm',
tzinfo='Australia/Melbourne').timestamp
write_line = str(ts) + "," + str(obs_list[j].temp) + "," + str(
obs_list[j].dew) + "," + str(obs_list[j].rain) + "," + str(
obs_list[j].pressure) + "," + str(obs_list[j].lcl) + "," + str(
obs_list[j].rel_hum) + "," + str(
obs_list[j].wind_vel) + "," + str(
obs_list[j].wind_dir) + "\n"
try:
with open(csv_filename, 'r') as f:
for line in f:
last_line = line
if write_line == last_line:
pass
else:
with open(csv_filename, 'a') as g:
g.write(write_line)
except IOError:
with open(csv_filename, 'a') as g:
g.write(write_line)
config_filename = root_folder + date_path + "\Observations\\" + "range.cfg"
min_temp = "-"
max_temp = "-"
min_dew = "-"
max_dew = "-"
min_pressure = "-"
max_pressure = "-"
min_rain = "-"
max_rain = "-"
min_rel_hum = "-"
max_rel_hum = "-"
min_lcl = "-"
max_lcl = "-"
min_wind_vel = "-"
max_wind_vel = "-"
try:
with open(config_filename, 'r') as f:
# Determine min / max temp
line = f.readline()
line_vals = line.split(',')
try:
if min(temp) < float(line_vals[0]):
min_temp = min(temp)
else:
min_temp = float(line_vals[0])
if max(temp) > float(line_vals[1]):
max_temp = max(temp)
else:
max_temp = float(line_vals[1])
except ValueError:
pass
# Determine min / max dew
line = f.readline()
line_vals = line.split(',')
try:
if min(dew) < float(line_vals[0]):
min_dew = min(dew)
else:
min_dew = float(line_vals[0])
if max(dew) > float(line_vals[1]):
max_dew = max(dew)
else:
max_dew = float(line_vals[1])
except ValueError:
pass
# Determine min / max rain
line = f.readline()
line_vals = line.split(',')
try:
if min(rain) < float(line_vals[0]):
min_rain = min(rain)
else:
min_rain = float(line_vals[0])
if max(rain) > float(line_vals[1]):
max_rain = max(rain)
else:
max_rain = float(line_vals[1])
except ValueError:
pass
# Determine min / max pressure
line = f.readline()
line_vals = line.split(',')
try:
if min(pressure) < float(line_vals[0]):
min_pressure = min(pressure)
else:
min_pressure = float(line_vals[0])
if max(pressure) > float(line_vals[1]):
max_pressure = max(pressure)
else:
max_pressure = float(line_vals[1])
except ValueError:
pass
# Determine min / max rel_hum
line = f.readline()
line_vals = line.split(',')
try:
if min(rel_hum) < float(line_vals[0]):
min_rel_hum = min(rel_hum)
else:
min_rel_hum = float(line_vals[0])
if max(rel_hum) > float(line_vals[1]):
max_rel_hum = max(rel_hum)
else:
max_rel_hum = float(line_vals[1])
except ValueError:
pass
# Determine min / max lcl
line = f.readline()
line_vals = line.split(',')
try:
if min(lcl) < float(line_vals[0]):
min_lcl = min(lcl)
else:
min_lcl = float(line_vals[0])
if max(lcl) > float(line_vals[1]):
max_lcl = max(lcl)
else:
max_lcl = float(line_vals[1])
except ValueError:
pass
# Determine min / max wind_vel
line = f.readline()
line_vals = line.split(',')
try:
if min(wind_vel) < float(line_vals[0]):
min_wind_vel = min(wind_vel)
else:
min_wind_vel = float(line_vals[0])
if max(wind_vel) > float(line_vals[1]):
max_wind_vel = max(wind_vel)
else:
max_wind_vel = float(line_vals[1])
except ValueError:
pass
except IOError:
min_temp = 0
max_temp = 45
min_dew = -10
max_dew = 25
min_pressure = 995
max_pressure = 1030
min_rain = 0
max_rain = 40
min_rel_hum = 0
max_rel_hum = 100
min_lcl = 0
max_lcl = 3000
min_wind_vel = 0
max_wind_vel = 5
with open(config_filename, 'w') as f:
line = str(min_temp) + "," + str(max_temp) + "\n"
f.writelines(line)
line = str(min_dew) + "," + str(max_dew) + "\n"
f.writelines(line)
line = str(min_rain) + "," + str(max_rain) + "\n"
f.writelines(line)
line = str(min_pressure) + "," + str(max_pressure) + "\n"
f.writelines(line)
line = str(min_rel_hum) + "," + str(max_rel_hum) + "\n"
f.writelines(line)
line = str(min_lcl) + "," + str(max_lcl) + "\n"
f.writelines(line)
line = str(min_wind_vel) + "," + str(max_wind_vel) + "\n"
f.writelines(line)
print "Finished Saving CSV Files"
return obs_list, loc_names
def _get_base_tweet_info_from_text_response(
tweet_id: str, response_text: str) -> Optional[_TweetByIdBaseInfo]:
parsed_json = json.loads(response_text)
return _TweetByIdBaseInfo(tweet_id, parsed_json['user']['screen_name'],
parsed_json['text'],
arrow_get(parsed_json['created_at']))
