def process_from_source():
logging.info('Processing from source')
url = _noaa['sourceUrl']
archive_name = 'Observation-{name}.{extension}'.format(name=_build_archive_identifier(), extension='zip')
observation_path = utilities.real_path(join(_system['dataPath'], _noaa_observation_key))
file_path = utilities.real_path(join(observation_path, archive_name))
if not os.path.exists(observation_path):
os.makedirs(observation_path)
logging.info('%s -> %s', url, file_path)
utilities.stream_url_to_file(_noaa['sourceUrl'], file_path)
logging.info('Processing from %s ...',file_path)
entry_generator = utilities.generate_files_from_zip(file_path, '.*')
tmp_file = None
with tempfile.NamedTemporaryFile(delete=False) as tmp:
tmp_file = tmp
for success, file_name, xml_document, json_document in _iterations_over_entries(entry_generator):
if success:
tmp.write(bytes(json.dumps(json_document, separators=(',',':')), "utf-8"))
tmp.write(bytes("\n", "utf-8"))
else:
xml_file = '{0}_{1}.xml'.format(file_name.replace('.xml',''), time.strftime('%Y%m%d@%H:%M'))
with open(join(observation_path, xml_file), 'w') as output:
output.write(str(xml_document))
import_file = join(observation_path, '{0}.import'.format(time.strftime('%Y%m%d_%H%M')))
os.rename(os.path.abspath(tmp_file.name), import_file)
import logging
import os
from model import ObservationSchemata
from os.path import join
from datetime import datetime
import predictor.utilities as utilities
import predictor.model as model
import tempfile
import time
from model import RequiredPropertyAbsentException
_noaa_observation_key = 'noaa.observation'
_global_observation_key = 'global'
with open(utilities.real_path('~/.prediction/Settings.json')) as settings_file:
_settings = json.load(settings_file)
_system = _settings[_global_observation_key]
_noaa = _settings[_noaa_observation_key]
with open(utilities.real_path(os.path.join('~/.prediction','ObservationSchemata.json'))) as schemata_file:
_schematas = json.load(schemata_file)
_observation_schemata = ObservationSchemata(_schematas[_noaa_observation_key])
def process_from_source():
logging.info('Processing from source')
url = _noaa['sourceUrl']
archive_name = 'Observation-{name}.{extension}'.format(name=_build_archive_identifier(), extension='zip')
observation_path = utilities.real_path(join(_system['dataPath'], _noaa_observation_key))