コード例 #1
0
def process_from_source():
    logging.info('Processing from source')
    url = _noaa['sourceUrl']
    archive_name = 'Observation-{name}.{extension}'.format(name=_build_archive_identifier(), extension='zip')
    observation_path = utilities.real_path(join(_system['dataPath'], _noaa_observation_key))
    file_path = utilities.real_path(join(observation_path, archive_name))

    if not os.path.exists(observation_path):
        os.makedirs(observation_path)

    logging.info('%s -> %s', url, file_path)
    utilities.stream_url_to_file(_noaa['sourceUrl'], file_path)
    logging.info('Processing from %s ...',file_path)

    entry_generator = utilities.generate_files_from_zip(file_path, '.*')


    tmp_file = None
    with tempfile.NamedTemporaryFile(delete=False) as tmp:
        tmp_file = tmp
        for success, file_name, xml_document, json_document in _iterations_over_entries(entry_generator):
            if success:
                tmp.write(bytes(json.dumps(json_document, separators=(',',':')), "utf-8"))
                tmp.write(bytes("\n", "utf-8"))
            else:
                xml_file = '{0}_{1}.xml'.format(file_name.replace('.xml',''), time.strftime('%Y%m%d@%H:%M'))
                with open(join(observation_path, xml_file), 'w') as output:
                    output.write(str(xml_document))

    import_file = join(observation_path, '{0}.import'.format(time.strftime('%Y%m%d_%H%M')))
    os.rename(os.path.abspath(tmp_file.name), import_file)
コード例 #2
0
import logging
import os
from model import ObservationSchemata
from os.path import join
from datetime import datetime
import predictor.utilities  as utilities
import predictor.model      as model
import tempfile
import time

from model import RequiredPropertyAbsentException

_noaa_observation_key = 'noaa.observation'
_global_observation_key = 'global'

with open(utilities.real_path('~/.prediction/Settings.json')) as settings_file:
    _settings = json.load(settings_file)

_system  = _settings[_global_observation_key]
_noaa = _settings[_noaa_observation_key]
with open(utilities.real_path(os.path.join('~/.prediction','ObservationSchemata.json'))) as schemata_file:
    _schematas = json.load(schemata_file)

_observation_schemata = ObservationSchemata(_schematas[_noaa_observation_key])


def process_from_source():
    logging.info('Processing from source')
    url = _noaa['sourceUrl']
    archive_name = 'Observation-{name}.{extension}'.format(name=_build_archive_identifier(), extension='zip')
    observation_path = utilities.real_path(join(_system['dataPath'], _noaa_observation_key))