def calibrate_data(params, raw_data, calib_data):
"""'Calibrate' raw data, using a user-supplied function."""
start = calib_data.before(datetime.max)
if start is None:
start = datetime.min
start = raw_data.after(start + SECOND)
if start is None:
return start
del calib_data[start:]
calibrator = Calib(params, raw_data)
count = 0
for data in raw_data[start:]:
idx = data['idx']
count += 1
if count % 10000 == 0:
logger.info("calib: %s", idx.isoformat(' '))
elif count % 500 == 0:
logger.debug("calib: %s", idx.isoformat(' '))
for key in ('rain', 'abs_pressure', 'temp_in'):
if data[key] is None:
logger.error('Ignoring invalid data at %s', idx.isoformat(' '))
break
else:
calib_data[idx] = calibrator.calib(data)
return start
def __init__(self, context):
self.context = context
self.params = context.params
self.status = context.status
self.raw_data = context.raw_data
self.calib_data = context.calib_data
self.hourly_data = context.hourly_data
self.daily_data = context.daily_data
self.monthly_data = context.monthly_data
self.flush = eval(self.params.get('config', 'frequent writes',
'False'))
# get directories
self.work_dir = self.params.get('paths', 'work', '/tmp/pywws')
if not os.path.isdir(self.work_dir):
os.makedirs(self.work_dir)
self.template_dir = self.params.get(
'paths', 'templates', os.path.expanduser('~/weather/templates/'))
self.graph_template_dir = self.params.get(
'paths', 'graph_templates',
os.path.expanduser('~/weather/graph_templates/'))
self.local_dir = self.params.get(
'paths', 'local_files', os.path.expanduser('~/weather/results/'))
self.module_dir = self.params.get(
'paths', 'modules', os.path.expanduser('~/weather/modules/'))
# create calibration object
self.calibrator = Calib(self.params, self.raw_data)
# create templater object
self.templater = pywws.template.Template(context)
# create plotter objects
self.plotter = pywws.plot.GraphPlotter(context, self.work_dir)
self.roseplotter = pywws.windrose.RosePlotter(context, self.work_dir)
# create FTP uploads directory
self.uploads_directory = os.path.join(self.work_dir, 'uploads')
if not os.path.isdir(self.uploads_directory):
os.makedirs(self.uploads_directory)
# delay creation of uploader object until we know it's needed
self.uploader = None
# delay creation of a Twitter object until we know it's needed
self.twitter = None
# get daytime end hour
self.day_end_hour, self.use_dst = pywws.process.get_day_end_hour(
self.params)
# parse "cron" sections
self.cron = {}
for section in self.params._config.sections():
if section.split()[0] != 'cron':
continue
import croniter
self.cron[section] = croniter.croniter(
self.params.get(section, 'format', ''))
self.cron[section].get_prev()
last_update = self.status.get_datetime('last update', section)
if last_update:
last_update += timezone.utcoffset(last_update)
while self.cron[section].get_current(datetime) <= last_update:
self.cron[section].get_next()
# create service uploader objects
self.services = {}
for section in list(self.cron.keys()) + [
'live', 'logged', 'hourly', '12 hourly', 'daily'
]:
for name in eval(self.params.get(section, 'services', '[]')):
if name in self.services:
continue
if os.path.exists(os.path.join(self.module_dir, name + '.py')):
sys.path.insert(0, self.module_dir)
mod = importlib.import_module(name)
del sys.path[0]
else:
mod = importlib.import_module('pywws.service.' + name)
self.services[name] = mod.ToService(context)
# check for obsolete entries
if self.params.get(section, 'twitter'):
logger.error('Obsolete twitter entry in weather.ini [%s]',
section)
if self.params.get(section, 'yowindow'):
logger.error('Obsolete yowindow entry in weather.ini [%s]',
section)
# check for 'local' template results
if os.path.isdir(self.local_dir):
return
has_local = False
for section in list(self.cron.keys()) + [
'live', 'logged', 'hourly', '12 hourly', 'daily'
]:
for template, flags in self._parse_templates(section, 'text'):
if 'L' in flags:
has_local = True
for template, flags in self._parse_templates(section, 'plot'):
if 'L' in flags:
has_local = True
if has_local:
os.makedirs(self.local_dir)
break
def __init__(self, params, status,
raw_data, calib_data, hourly_data, daily_data, monthly_data,
asynch=False):
self.logger = logging.getLogger('pywws.Tasks.RegularTasks')
self.params = params
self.status = status
self.raw_data = raw_data
self.calib_data = calib_data
self.hourly_data = hourly_data
self.daily_data = daily_data
self.monthly_data = monthly_data
self.asynch = asynch
self.flush = eval(self.params.get('config', 'frequent writes', 'False'))
# get directories
self.work_dir = self.params.get('paths', 'work', '/tmp/weather')
if not os.path.isdir(self.work_dir):
raise RuntimeError(
'Directory "' + self.work_dir + '" does not exist.')
self.template_dir = self.params.get(
'paths', 'templates', os.path.expanduser('~/weather/templates/'))
self.graph_template_dir = self.params.get(
'paths', 'graph_templates', os.path.expanduser('~/weather/graph_templates/'))
self.local_dir = self.params.get(
'paths', 'local_files', os.path.expanduser('~/weather/results/'))
# create calibration object
self.calibrator = Calib(self.params, self.raw_data)
# create templater object
self.templater = Template.Template(
self.params, self.status, self.calib_data, self.hourly_data,
self.daily_data, self.monthly_data)
# create plotter objects
self.plotter = Plot.GraphPlotter(
self.params, self.status, self.calib_data, self.hourly_data,
self.daily_data, self.monthly_data, self.work_dir)
self.roseplotter = WindRose.RosePlotter(
self.params, self.status, self.calib_data, self.hourly_data,
self.daily_data, self.monthly_data, self.work_dir)
# create FTP uploader object
self.uploader = Upload.Upload(self.params)
self.uploads_directory = os.path.join(self.work_dir, 'uploads')
if not os.path.isdir(self.uploads_directory):
os.mkdir(self.uploads_directory)
# delay creation of a Twitter object until we know it's needed
self.twitter = None
# create a YoWindow object
self.yowindow = YoWindow.YoWindow(self.calib_data)
# get daytime end hour, in UTC
self.day_end_hour = eval(params.get('config', 'day end hour', '21'))
self.day_end_hour = (self.day_end_hour - (STDOFFSET.seconds // 3600)) % 24
# parse "cron" sections
self.cron = {}
for section in self.params._config.sections():
if section.split()[0] != 'cron':
continue
import croniter
self.cron[section] = croniter.croniter(
self.params.get(section, 'format', ''))
self.cron[section].get_prev()
last_update = self.status.get_datetime('last update', section)
if last_update:
last_update = last_update + Local.utcoffset(last_update)
while self.cron[section].get_current(datetime) <= last_update:
self.cron[section].get_next()
# create service uploader objects
self.services = {}
for section in self.cron.keys() + [
'live', 'logged', 'hourly', '12 hourly', 'daily']:
for name in eval(self.params.get(section, 'services', '[]')):
if name not in self.services:
self.services[name] = ToService(
self.params, self.status, self.calib_data,
service_name=name)
# check for deprecated syntax
if self.params.get(section, 'twitter') not in (None, '[]'):
self.logger.warning(
'Deprecated twitter entry in [%s]', section)
if self.params.get(section, 'yowindow'):
self.logger.warning(
'Deprecated yowindow entry in [%s]', section)
# create queues for things to upload / send
self.tweet_queue = deque()
self.service_queue = {}
for name in self.services:
self.service_queue[name] = deque()
self.uploads_queue = deque()
# start asynchronous thread to do uploads
if self.asynch:
self.logger.info('Starting asynchronous thread')
self.shutdown_thread = threading.Event()
self.wake_thread = threading.Event()
self.thread = threading.Thread(target=self._asynch_thread)
self.thread.start()
def __init__(self, context):
self.context = context
self.params = context.params
self.status = context.status
self.raw_data = context.raw_data
self.calib_data = context.calib_data
self.hourly_data = context.hourly_data
self.daily_data = context.daily_data
self.monthly_data = context.monthly_data
self.flush = literal_eval(
self.params.get('config', 'frequent writes', 'False'))
# get directories
self.template_dir = self.params.get(
'paths', 'templates', os.path.expanduser('~/weather/templates/'))
self.graph_template_dir = self.params.get(
'paths', 'graph_templates',
os.path.expanduser('~/weather/graph_templates/'))
self.module_dir = self.params.get(
'paths', 'modules', os.path.expanduser('~/weather/modules/'))
# create calibration object
self.calibrator = Calib(self.params, self.raw_data)
# create templater object
self.templater = pywws.template.Template(context)
# create plotter objects
self.plotter = pywws.plot.GraphPlotter(context, self.context.work_dir)
self.roseplotter = pywws.windrose.RosePlotter(context,
self.context.work_dir)
# get daytime end hour
self.day_end_hour, self.use_dst = pywws.process.get_day_end_hour(
self.params)
# parse "cron" sections
self.cron = {}
for section in self.params._config.sections():
if section.split()[0] != 'cron':
continue
import croniter
last_update = self.status.get_datetime('last update', section)
last_update = last_update or datetime.utcnow()
self.cron[section] = croniter.croniter(
self.params.get(section, 'format', ''),
start_time=time_zone.utc_to_local(last_update))
self.cron[section].get_next()
# create service uploader objects
self.services = {}
for section in list(self.cron.keys()) + [
'live', 'logged', 'hourly', '12 hourly', 'daily'
]:
for name, options in self._parse_templates(section, 'services'):
if name in self.services:
continue
if os.path.exists(os.path.join(self.module_dir, name + '.py')):
sys.path.insert(0, self.module_dir)
mod = importlib.import_module(name)
del sys.path[0]
else:
mod = importlib.import_module('pywws.service.' + name)
self.services[name] = mod.ToService(context)
# check for obsolete entries
if self.params.get(section, 'twitter'):
logger.error('Obsolete twitter entry in weather.ini [%s]',
section)
if self.params.get(section, 'yowindow'):
logger.error('Obsolete yowindow entry in weather.ini [%s]',
section)
