def observation_update2(site_key):
site = Site.get_by_key_name(site_key)
if site is None:
return Response(status = 404)
url = "http://www.metoffice.gov.uk/public/data/PWSCache/BestForecast/Observation/%s?format=application/json" % site_key
result = urlfetch.fetch(url)
if result.status_code == 200:
observations = parse_observation(result.content)
# issued_date = parse_date(forecast["@dataDate"])
for date, data in timesteps(observations):
obs_timestep = ObservationTimestep.get_by_site_and_datetime(site, date)
if obs_timestep is None:
obs_timestep = ObservationTimestep(site = site, observation_datetime = date, observation_date = date.date())
for k,v in data.items():
prop_name = snake_case(k)
if hasattr(obs_timestep, prop_name):
if v == "missing":
v = None
elif prop_name == 'temperature':
v = float(v)
setattr(obs_timestep, prop_name, v)
obs_timestep.save()
#logging.info("%s, %s" % (str(date), str(ObservationTimestep)))
return Response(status = 204)
def get(self):
"""Handle GET requests."""
# build the feed's file name
feed_filename = snake_case(self.__class__.__name__)
self.fields = {}
self.params = flask.request.get_json(force=True, silent=True) or {}
self.limit = self.params.get('limit', DEFAULT_RESP_LIMIT)
trigger_identity = self.params.get('trigger_identity')
#Gets paramters based on the GET request to return the corresponding RSS
params = {"lang": request.args.get('lang'), "user": request.args.get('user'), \
"title": request.args.get('title')}
trigger_values = self.params.get("triggerFields", params)
for field, default_value in self.default_fields.items():
self.fields[field] = trigger_values.get(field)
if not self.fields[field] and default_value not in TEST_FIELDS:
self.fields[field] = default_value
if not self.fields[field]:
if field in self.optional_fields and self.fields[field] is not None:
self.fields[field] = ''
else:
flask.abort(400)
logging.info('%s: %s' % (self.__class__.__name__, trigger_identity))
data = self.get_data()
data = data[:self.limit]
feeds = render_template(feed_filename + '.xml', data=data)
response = make_response(feeds)
response.headers["Content-Type"] = "application/xml"
return response
def forecast_update2(site_key):
site = Site.get_by_key_name(site_key)
if site is None:
return Response(status=404)
forecast_url = "http://www.metoffice.gov.uk/public/data/PWSCache/BestForecast/Forecast/%s?format=application/json" % site_key
result = urlfetch.fetch(forecast_url)
if result.status_code == 200:
forecast = parse_forecast(result.content)
issued_date = parse_date(forecast["@dataDate"])
for date, data in timesteps(forecast):
forecast_timestep = ForecastTimestep.find_by_site_and_dates(
site, date, issued_date)
if forecast_timestep is None:
forecast_timestep = ForecastTimestep(
site=site,
forecast_datetime=date,
issued_datetime=issued_date,
forecast_date=date.date())
for k, v in data.items():
prop_name = snake_case(k)
if hasattr(forecast_timestep, prop_name):
if v == "missing":
v = None
setattr(forecast_timestep, prop_name, v)
forecast_timestep.save()
return Response(status=204)
def argument(self, name, description=None, variadic=False, parse=None):
"""Adds a new positional argument to this command."""
if any(a.variadic for a in self._arguments):
raise VariadicArgumentPositionError(name)
self._arguments.append(Argument(name, description, variadic, parse))
setattr(self, utils.snake_case(name), None)
def get(self):
"""Handle GET requests."""
# build the feed's file name
feed_filename = snake_case(self.__class__.__name__)
self.fields = {}
self.params = flask.request.get_json(force=True, silent=True) or {}
self.limit = self.params.get('limit', DEFAULT_RESP_LIMIT)
trigger_identity = self.params.get('trigger_identity')
#Gets paramters based on the GET request to return the corresponding RSS
params = {"lang": request.args.get('lang'), "user": request.args.get('user'), \
"title": request.args.get('title')}
trigger_values = self.params.get("triggerFields", params)
for field, default_value in self.default_fields.items():
self.fields[field] = trigger_values.get(field)
if not self.fields[field] and default_value not in TEST_FIELDS:
self.fields[field] = default_value
if not self.fields[field]:
if field in self.optional_fields and self.fields[
field] is not None:
self.fields[field] = ''
else:
flask.abort(400)
logging.info('%s: %s' % (self.__class__.__name__, trigger_identity))
data = self.get_data()
data = data[:self.limit]
feeds = render_template(feed_filename + '.xml', data=data)
response = make_response(feeds)
response.headers["Content-Type"] = "application/xml"
return response
def _parse_arguments(self, argv):
"""Parses all positional arguments for this command."""
arguments = self._arguments.copy()
while len(arguments):
if not len(argv):
break
argument = arguments.pop(0)
val = argv if argument.variadic else argv.pop(0)
if argument.parse:
try:
if argument.variadic:
val = list(map(argument.parse, val))
else:
val = argument.parse(val)
except Exception as e:
raise ParseError(e, val)
setattr(self, utils.snake_case(argument.name), val)
if len(arguments):
raise MissingArgumentsError(arguments)
return argv
def initialize_protocol(this_schema, out_dir):
protocol_name = snake_case(this_schema)
# TODO
# are we sure we want to change the case or the protocol
# or make it snake case?
protocol_name = protocol_name.lower()
protocol = Protocol()
protocol.set_defaults(protocol_name)
protocol.set_landing_page(get_landing_page(this_schema))
# create output directories
protocol_path = os.path.join(out_dir, "protocols")
protocol.set_directory = protocol_path
if not os.path.exists(protocol_path):
os.makedirs(protocol_path)
protocol.write(protocol_path)
print_info(
"protocol",
protocol_name,
os.path.join(protocol_path, protocol.get_filename()),
)
return protocol, protocol_path
def forecast_update(site_key):
site = Site.get_by_key_name(site_key)
if site is None:
return Response(status = 404)
forecast_url = "http://www.metoffice.gov.uk/public/data/PWSCache/BestForecast/Forecast/%s?format=application/json" % site_key
result = urlfetch.fetch(forecast_url)
if result.status_code == 200:
forecast = parse_forecast(result.content)
issued_date = parse_date(forecast["@dataDate"])
for date, day in days(forecast):
forecast_day = ForecastDay.get_by_key_name(make_key_name(site,date))
if forecast_day is None:
forecast_day = ForecastDay(key_name=make_key_name(site,date), forecast_date = date, site = site)
forecast_day.site = site
for timestep, data in day_timesteps(day):
w = Forecast()
w.issued = issued_date
for k,v in data.items():
prop_name = snake_case(k)
if hasattr(w, prop_name):
if v == "missing":
v = None
setattr(w, prop_name, v)
forecast_day.forecasts.add(timestep,w)
forecast_day.save()
site.save()
return Response(status = 204)
def get(self):
"""Handle GET requests."""
# build the feed's file name dynamically
feed_filename = snake_case(self.__class__.__name__)
self.fields = {}
self.params = dict((key, request.args.getlist(key)
if len(request.args.getlist(key)) > 1
else request.args.getlist(key)[0])
for key in request.args.keys())
self.params = dict(triggerFields=dict(self.params))
self.limit = self.params.get('limit', DEFAULT_RESP_LIMIT)
trigger_identity = self.params.get('trigger_identity')
trigger_values = self.params.get("triggerFields", {})
for field, default_value in self.default_fields.items():
self.fields[field] = trigger_values.get(field)
if self.fields[field] == '' and default_value not in TEST_FIELDS:
self.fields[field] = default_value
if not self.fields[field]:
if field in self.optional_fields and self.fields[field] is not None:
self.fields[field] = ''
else:
flask.abort(400)
logging.info('%s: %s' % (self.__class__.__name__, trigger_identity))
data = self.get_data()
data = data[:self.limit]
feeds = render_template(feed_filename + '.xml', data=data)
response = make_response(feeds)
response.headers["Content-Type"] = "application/xml"
return response
def forecast_update2(site_key):
site = Site.get_by_key_name(site_key)
if site is None:
return Response(status = 404)
forecast_url = "http://www.metoffice.gov.uk/public/data/PWSCache/BestForecast/Forecast/%s?format=application/json" % site_key
result = urlfetch.fetch(forecast_url)
if result.status_code == 200:
forecast = parse_forecast(result.content)
issued_date = parse_date(forecast["@dataDate"])
for date, data in timesteps(forecast):
forecast_timestep = ForecastTimestep.find_by_site_and_dates(site, date, issued_date)
if forecast_timestep is None:
forecast_timestep = ForecastTimestep(site = site, forecast_datetime = date, issued_datetime = issued_date, forecast_date = date.date())
for k,v in data.items():
prop_name = snake_case(k)
if hasattr(forecast_timestep, prop_name):
if v == "missing":
v = None
setattr(forecast_timestep, prop_name, v)
forecast_timestep.save()
return Response(status = 204)
def observation_update2(site_key):
site = Site.get_by_key_name(site_key)
if site is None:
return Response(status=404)
url = "http://www.metoffice.gov.uk/public/data/PWSCache/BestForecast/Observation/%s?format=application/json" % site_key
result = urlfetch.fetch(url)
if result.status_code == 200:
observations = parse_observation(result.content)
# issued_date = parse_date(forecast["@dataDate"])
for date, data in timesteps(observations):
obs_timestep = ObservationTimestep.get_by_site_and_datetime(
site, date)
if obs_timestep is None:
obs_timestep = ObservationTimestep(
site=site,
observation_datetime=date,
observation_date=date.date())
for k, v in data.items():
prop_name = snake_case(k)
if hasattr(obs_timestep, prop_name):
if v == "missing":
v = None
elif prop_name == 'temperature':
v = float(v)
setattr(obs_timestep, prop_name, v)
obs_timestep.save()
#logging.info("%s, %s" % (str(date), str(ObservationTimestep)))
return Response(status=204)
def forecast_update(site_key):
site = Site.get_by_key_name(site_key)
if site is None:
return Response(status=404)
forecast_url = "http://www.metoffice.gov.uk/public/data/PWSCache/BestForecast/Forecast/%s?format=application/json" % site_key
result = urlfetch.fetch(forecast_url)
if result.status_code == 200:
forecast = parse_forecast(result.content)
issued_date = parse_date(forecast["@dataDate"])
for date, day in days(forecast):
forecast_day = ForecastDay.get_by_key_name(
make_key_name(site, date))
if forecast_day is None:
forecast_day = ForecastDay(key_name=make_key_name(site, date),
forecast_date=date,
site=site)
forecast_day.site = site
for timestep, data in day_timesteps(day):
w = Forecast()
w.issued = issued_date
for k, v in data.items():
prop_name = snake_case(k)
if hasattr(w, prop_name):
if v == "missing":
v = None
setattr(w, prop_name, v)
forecast_day.forecasts.add(timestep, w)
forecast_day.save()
site.save()
return Response(status=204)
def _parse_options(self, argv):
"""Parses all options for this command."""
while len(argv):
if not argv[0].startswith('-'):
break
arg = argv.pop(0)
option = next((o for o in self._options if arg in o.aliases), None)
if option:
if option.consume == 0:
val = True
elif option.consume == 1:
unparsed_val = argv.pop(0)
else:
unparsed_val = argv[:option.consume]
argv = argv[option.consume:]
if option.parse and option.consume >= 1:
try:
val = option.parse(
unparsed_val) if option.consume >= 1 else list(
map(option.parse, unparsed_val))
except Exception as e:
raise ParseError(e, unparsed_val)
else:
val = unparsed_val
setattr(self, utils.snake_case(option.name), val)
else:
raise UnknownOptionError(arg)
return argv
def option(self,
long,
aliases=[],
arguments=[],
description=None,
parse=None):
"""Adds a new option or flag to this command."""
option = Option(long, aliases, arguments, description, parse)
self._options.append(option)
setattr(self, utils.snake_case(option.name), None)
def set_item_name(this_item):
if "item" not in this_item.keys():
item_name = convert_to_str(this_item["item_pref_label"])
elif isinstance(convert_to_str(this_item["item"]), float):
item_name = convert_to_str(this_item["item_pref_label"])
elif convert_to_str(this_item["item"]) == "":
item_name = convert_to_str(this_item["item_pref_label"])
else:
item_name = convert_to_str(this_item["item"])
item_name = snake_case(item_name)
return item_name
def observation_update(site_key):
site = Site.get_by_key_name(site_key)
if site is None:
return Response(status=404)
url = "http://www.metoffice.gov.uk/public/data/PWSCache/BestForecast/Observation/%s?format=application/json" % site_key
obs = {}
def get_db_observation(date):
key_name = make_key_name(site, date.date())
if key_name in obs:
return obs[key_name]
o = ObservationDay.get_by_key_name(key_name)
if o is None:
o = ObservationDay(key_name=key_name)
o.site = site
o.observation_date = date.date()
o.observations = Observations()
obs[key_name] = o
return o
result = urlfetch.fetch(url)
if result.status_code == 200:
observations = parse_observation(result.content)
issue_date = parse_date(observations['@issueDate'])
site.last_obs_issue_datetime = issue_date
site.last_obs_update_datetime = datetime.now()
for date, data in timesteps(observations):
o = get_db_observation(date)
o.lastdata_datetime = issue_date
w = Weather({})
for k, v in data.items():
prop_name = snake_case(k)
if hasattr(w, prop_name):
if v == "missing":
v = None
elif prop_name == 'temperature':
v = float(v)
setattr(w, prop_name, v)
o.observations.add(date, w)
for o in obs.values():
o.save()
site.save()
return Response(status=204)
def observation_update(site_key):
site = Site.get_by_key_name(site_key)
if site is None:
return Response(status = 404)
url = "http://www.metoffice.gov.uk/public/data/PWSCache/BestForecast/Observation/%s?format=application/json" % site_key
obs = {}
def get_db_observation(date):
key_name = make_key_name(site, date.date())
if key_name in obs:
return obs[key_name]
o = ObservationDay.get_by_key_name(key_name)
if o is None:
o = ObservationDay(key_name=key_name)
o.site = site
o.observation_date = date.date()
o.observations = Observations()
obs[key_name] = o
return o
result = urlfetch.fetch(url)
if result.status_code == 200:
observations = parse_observation(result.content)
issue_date = parse_date(observations['@issueDate'])
site.last_obs_issue_datetime = issue_date
site.last_obs_update_datetime = datetime.now()
for date, data in timesteps(observations):
o = get_db_observation(date)
o.lastdata_datetime = issue_date
w = Weather({})
for k,v in data.items():
prop_name = snake_case(k)
if hasattr(w, prop_name):
if v == "missing":
v = None
elif prop_name == 'temperature':
v = float(v)
setattr(w, prop_name, v)
o.observations.add(date, w)
for o in obs.values():
o.save()
site.save()
return Response(status = 204)
def scrape_specs_page(page_data):
main_dict = {}
if not page_data:
return main_dict
b = bs4.BeautifulSoup(page_data.text, 'html.parser')
main_dict['specs'] = []
# All spec information in platform specific tables
for table in b.find_all('table', class_='techInfo'):
tds = table.find_all('td')[1:] # First td is table header title
table_dict = dict([(snake_case(replace_xa0(h.text)),
[replace_xa0(a.text) for a in v.find_all('a')]) for h, v in pairwise(tds)])
table_dict['platform'] = replace_xa0(table.find('thead').text)
main_dict['specs'].append(table_dict)
return main_dict
def create_div_tuple(div):
if div.name == u'h2':
# ('header', platform name)
return 'header', replace_xa0(div.text)
elif div.name == u'b':
# ('patch', None)
return 'patch', None
elif 'class' in div.attrs and u'relInfo' in div['class']:
# ('patch_rel_info', {relInfoTitle: relInfoDetails, ...})
return 'patch_rel_info', dict([(replace_xa0(div.find(class_='relInfoTitle').text),
replace_xa0(div.find(class_='relInfoDetails').text))])
elif 'class' in div.attrs and u'floatholder' in div['class']:
# ('attr', {attr_name: [value, ...]})
return 'attr', {snake_case(replace_xa0(div.find(class_='fl').text)): [a.text for a in div.find_all('a')]}
elif div.find(class_='relInfo'):
# ('rel_info', {relInfoTitle: relInfoDetails, ...})
return 'rel_info', dict([(replace_xa0(r.find(class_='relInfoTitle').text),
replace_xa0(r.find(class_='relInfoDetails').text)) for r in div.find_all(class_='relInfo')])
else:
return None, None
def initialize_activity(protocol, items, out_dir):
activity = Activity()
activity_pref_label = items.activity_pref_label.unique()[0]
activity.set_pref_label(activity_pref_label)
activity_name = snake_case(activity_pref_label)
activity_name = activity_name.lower()
# TODO
# are we sure we want to change the case of the activity
# or make it snake case?
# try to get the name of the activity from the correct column in the TSV
activity.set_defaults(activity_name)
activity.set_filename(activity_name)
activity.set_pref_label(activity_pref_label)
activity.set_preamble(get_activity_preamble(items))
URI = ("../activities" + "/" +
activity.get_basename().replace("_schema", "") + "/" +
activity.get_filename())
activity.set_URI(URI)
activity_path = os.path.join(out_dir, "activities", activity.dir)
if not os.path.exists(activity_path):
os.makedirs(activity_path)
if not os.path.exists(os.path.join(activity_path, "items")):
os.makedirs(os.path.join(activity_path, "items"))
print_info(
"activity",
activity_pref_label,
os.path.join(activity_path, activity.get_filename()),
)
return protocol, activity, activity_path
def __setattr__(self, key, value):
if snake_case(key) in type(self).__annotations__:
object.__setattr__(self, snake_case(key), value)
else:
raise AttributeError()
def __getattribute__(self, item):
if snake_case(item) in type(self).__dict__.keys():
value = object.__getattribute__(self, snake_case(item))
return value
else:
return object.__getattribute__(self, item)
