def test_compose(self, allow_partial): """Test the compose method's custom conversion options.""" key_vals = {"a": "this Is A-Test b_test c test"} new_str = compose("{a!c}", key_vals, allow_partial=allow_partial) assert new_str == "This is a-test b_test c test" new_str = compose("{a!h}", key_vals, allow_partial=allow_partial) assert new_str == "thisisatestbtestctest" new_str = compose("{a!H}", key_vals, allow_partial=allow_partial) assert new_str == "THISISATESTBTESTCTEST" new_str = compose("{a!l}", key_vals, allow_partial=allow_partial) assert new_str == "this is a-test b_test c test" new_str = compose("{a!R}", key_vals, allow_partial=allow_partial) assert new_str == "thisIsATestbtestctest" new_str = compose("{a!t}", key_vals, allow_partial=allow_partial) assert new_str == "This Is A-Test B_Test C Test" new_str = compose("{a!u}", key_vals, allow_partial=allow_partial) assert new_str == "THIS IS A-TEST B_TEST C TEST" # builtin repr new_str = compose("{a!r}", key_vals, allow_partial=allow_partial) assert new_str == "'this Is A-Test b_test c test'" # no formatting new_str = compose("{a}", key_vals, allow_partial=allow_partial) assert new_str == "this Is A-Test b_test c test" # bad formatter with pytest.raises(ValueError): new_str = compose("{a!X}", key_vals, allow_partial=allow_partial) assert new_str == "this Is A-Test b_test c test"
def process_message(msg, config): """Process the message.""" pattern = config["output_file_pattern"] input_files = [ urlsplit(item["uri"]).path for item in msg.data["collection"] ] data = msg.data.copy() data["proc_time"] = datetime.utcnow() try: aliases = get_aliases(config["aliases"]) except KeyError: aliases = {} for key in aliases: if key in data: data[key] = aliases[key].get(data[key], data[key]) try: min_length = int(config.get('min_length')) except NoOptionError: min_length = 0 if data["end_time"] - data["start_time"] < timedelta(minutes=min_length): logger.info('Pass too short, skipping: %s to %s', str(data["start_time"]), str(data["end_time"])) return output_file = compose(pattern, data) with bunzipped(input_files) as files_to_read: keyvals = { "input_files": " ".join(files_to_read), "output_file": output_file } cmd_pattern = config["command"] cmd = compose(cmd_pattern, keyvals) logger.info("Running %s", cmd) if "stdout" in config: stdout_file = compose(config["stdout"], keyvals) with open(stdout_file, "w") as output: p = Popen(cmd.split(), stderr=PIPE, stdout=output) err_reader = threading.Thread(target=reader, args=(p.stderr, logger.error)) err_reader.start() err_reader.join() else: popen(cmd) msg.type = "file" new_data = msg.data.copy() del new_data["collection"] new_data["filename"] = os.path.basename(output_file) new_data["uri"] = output_file if "publish_topic" in config: msg.subject = config.get("publish_topic") msg2 = Message(msg.subject, "file", new_data) return msg2
def test_compose(self): """Test the compose method's custom conversion options.""" key_vals = {'a': 'this Is A-Test b_test c test'} new_str = compose("{a!c}", key_vals) self.assertEqual(new_str, 'This is a-test b_test c test') new_str = compose("{a!h}", key_vals) self.assertEqual(new_str, 'thisisatestbtestctest') new_str = compose("{a!H}", key_vals) self.assertEqual(new_str, 'THISISATESTBTESTCTEST') new_str = compose("{a!l}", key_vals) self.assertEqual(new_str, 'this is a-test b_test c test') new_str = compose("{a!R}", key_vals) self.assertEqual(new_str, 'thisIsATestbtestctest') new_str = compose("{a!t}", key_vals) self.assertEqual(new_str, 'This Is A-Test B_Test C Test') new_str = compose("{a!u}", key_vals) self.assertEqual(new_str, 'THIS IS A-TEST B_TEST C TEST') # builtin repr new_str = compose("{a!r}", key_vals) self.assertEqual(new_str, '\'this Is A-Test b_test c test\'') # no formatting new_str = compose("{a}", key_vals) self.assertEqual(new_str, 'this Is A-Test b_test c test') # bad formatter self.assertRaises(ValueError, compose, "{a!X}", key_vals) self.assertEqual(new_str, 'this Is A-Test b_test c test')
def process_message(msg, config): pattern = config["output_file_pattern"] input_files = [item["uri"] for item in msg.data["collection"]] data = msg.data.copy() data["proc_time"] = datetime.utcnow() try: aliases = get_aliases(config["aliases"]) except KeyError: aliases = {} for key in aliases: if key in data: data[key] = aliases[key].get(data[key], data[key]) try: min_length = int(config.get('min_length')) except NoOptionError: min_length = 0 if data["end_time"] - data["start_time"] < timedelta(minutes=min_length): LOG.info('Pass too short, skipping: %s to %s', str(data["start_time"]), str(data["end_time"])) return output_file = compose(pattern, data) with bunzipped(input_files) as files_to_read: keyvals = {"input_files": " ".join(files_to_read), "output_file": output_file} cmd_pattern = config["command"] cmd = compose(cmd_pattern, keyvals) LOG.info("Running %s", cmd) if "stdout" in config: stdout_file = compose(config["stdout"], keyvals) with open(stdout_file, "w") as output: p = Popen(cmd.split(), stderr=PIPE, stdout=output) err_reader = threading.Thread(target=reader, args=(p.stderr, LOG.error)) err_reader.start() err_reader.join() else: popen(cmd) msg.type = "file" new_data = msg.data.copy() del new_data["collection"] new_data["filename"] = os.path.basename(output_file) new_data["uri"] = output_file msg2 = Message(msg.subject, "file", new_data) return msg2
def move_aapp_log_files(config): """ Move AAPP processing log files from AAPP working directory in to sub-directory. The directory path is defined in config file (aapp_log_files) """ try: filelist = glob( '%s/*.log' % config['aapp_processes'][config.process_name]['working_dir']) try: tmp_config = config.config.copy() tmp_config.update( tmp_config['aapp_processes'][config.process_name]) _outdir = compose(tmp_config['aapp_outdir_format'], tmp_config) destination = os.path.join( tmp_config['aapp_log_files_archive_dir'], _outdir) except KeyError as err: LOG.error( "Failed to compose log files dir: {}. Missing key:{}".format( config['aapp_processes'][config.process_name] ['aapp_outdir_format'], err)) return False except Exception as err: LOG.error("Failed to compose log files dir: {}. Error:{}".format( config['aapp_processes'][config.process_name] ['aapp_outdir_format'], err)) return False LOG.debug("move_aapp_log_files destination: " + destination) if not os.path.exists(destination): try: os.makedirs(destination) except OSError as err: LOG.error("Can't create directory: {} because: {}".format( destination, err)) return False # FIXME: Check! else: LOG.debug("Created new directory for AAPP log files:" + destination) for file_name in filelist: try: base_filename = os.path.basename(file_name) dst = os.path.join(destination, base_filename) shutil.move(file_name, dst) except Exception as err: LOG.warning("Failed to move log file: {} to: {}".format( file_name, dst)) else: LOG.debug("Moved {} to {}".format(file_name, dst)) except OSError as err: LOG.error("Moving AAPP log files to " + destination + " failed ", err) LOG.info("AAPP log files saved in to " + destination) return True
def publish_level1(publisher, config, msg, filelist, station_name, environment): """Send a publish message, one message per file in the filelist.""" for file in filelist: LOG.debug("Handeling file for sending: {}".format(file)) msg_to_send = {} try: msg_to_send = msg.data.copy() if 'dataset' in msg_to_send: del msg_to_send['dataset'] msg_to_send['uri'] = "file://{}{}".format( config['aapp_processes'][config.process_name] ['message_providing_server'], file['file']) msg_to_send['filename'] = os.path.basename(file['file']) msg_to_send['uid'] = os.path.basename(file['file']) msg_to_send['sensor'] = config['aapp_static_configuration'][ 'sensor_name_converter'].get(file['sensor'], file['sensor']) msg_to_send['orbit_number'] = config['orbit_number'] msg_to_send['format'] = "AAPP" msg_to_send['type'] = 'Binary' msg_to_send['data_processing_level'] = file['level'].upper( ).replace("L", "") LOG.debug('level in message: ' + str(msg_to_send['data_processing_level'])) msg_to_send['start_time'] = config['starttime'] msg_to_send['end_time'] = config['endtime'] msg_to_send['station'] = station_name msg_to_send['env'] = environment except KeyError as ke: LOG.error("KeyError, missing key: {}".format(ke)) except Exception as err: LOG.error( "Failed to build publish message with error: {}".format(err)) continue try: publish_to = compose( config['aapp_processes'][config.process_name] ['publish_sift_format'], msg_to_send) except KeyError: LOG.warning( "Unknown Key used in format: {}. Check spelling and/or availability." .format(config['aapp_processes'][config.process_name] ['publish_sift_format'])) LOG.warning("Available keys are:") for key in msg_to_send: LOG.warning("{} = {}".format(key, msg_to_send[key])) LOG.error("Can not publish these data!") return False except ValueError as ve: LOG.error("Value Error: {}".format(ve)) return LOG.debug("Publish to:{}".format(publish_to)) message = Message(publish_to, "file", msg_to_send).encode() LOG.debug("sending: " + str(message)) publisher.send(message)
def test_partial_compose_simple(self): """Test partial compose with a simple use case.""" fmt = "{variant:s}/{platform_name}_{start_time:%Y%m%d_%H%M}_{product}.{format}" composed = compose( fmt=fmt, keyvals={"platform_name": "foo", "format": "bar"}, allow_partial=True ) assert composed == "{variant:s}/foo_{start_time:%Y%m%d_%H%M}_{product}.bar"
def get_files(delta=30): """Get the tlefiles from the last delta days """ now = datetime.utcnow() files = [] for day in range(delta): pattern = compose(REF_PATTERN, {"date": now - timedelta(days=day)}) files.extend(glob.glob(pattern)) return files
def test_match(self, fmt, string, expected): """Test cases expected to be matched.""" # Test parsed value parsed = parse(fmt, string) assert parsed['foo'] == expected # Test round trip composed = compose(fmt, {'foo': expected}) parsed = parse(fmt, composed) assert parsed['foo'] == expected
def _get_bandfilenames(self, **options): """Get filename for each band""" conf = options[self.platform_name + '-viirs'] rootdir = conf['rootdir'] for section in conf: if not section.startswith('section'): continue bandnames = conf[section]['bands'] for band in bandnames: filename = os.path.join(rootdir, conf[section]['filename']) self.bandfilenames[band] = compose( filename, {'bandname': band})
def _get_bandfilenames(self, **options): """Get filename for each band""" conf = options[self.platform_name + '-viirs'] rootdir = conf['rootdir'] for section in conf: if not section.startswith('section'): continue bandnames = conf[section]['bands'] for band in bandnames: filename = os.path.join(rootdir, conf[section]['filename']) self.bandfilenames[band] = compose(filename, {'bandname': band})
def _compose_destination(destination, msg): try: _destination = compose(destination, msg.data) except KeyError as ke: LOGGER.error("Format identifier is missing from the msg.data: %s", str(ke)) raise except ValueError as ve: LOGGER.error( "Type of format identifier doesn't match the type in m msg.data: %s", str(ve)) raise except AttributeError as ae: LOGGER.error("msg or msg.data is None: %s", str(ae)) raise return _destination
def rename_file(process_config, values): """ Do the actual renaming and checking """ sensor = next(iter(values)) process_file = "process_{}".format(sensor) try: process_config[process_file] except KeyError as err: LOG.error("No such key: {}".format(err)) return False #Need to build a dict to use in trollsift compose tmp_process_config = process_config.config.copy() tmp_process_config.update( tmp_process_config['aapp_processes'][process_config.process_name]) #for key, value in tmp_process_config['aapp_processes'][process_config.process_name].iteritems(): #print key,value # tmp_process_config[key] = value #tmp_process_config.update(tmp_process_config['aapp_processes'][process_config.process_name]['rename_aapp_files']) tmp_process_config['data_type'] = values[sensor]['data_type'] tmp_process_config['data_level'] = values[sensor]['data_level'] #for key, value in tmp_process_config['aapp_processes'][process_config.process_name]['rename_aapp_files'].iteritems(): # #print key,value new_name = "" if process_config[process_file]: if os.path.exists(values[sensor]['aapp_file']): try: _outdir = compose(tmp_process_config['aapp_outdir_format'], tmp_process_config) dir = os.path.join(tmp_process_config['aapp_outdir_base'], _outdir) _new_name = compose( process_config['aapp_processes'][ process_config.process_name]['rename_aapp_compose'], tmp_process_config) new_name = os.path.join(dir, _new_name) except KeyError as err: LOG.error( "Failed to compose new filename: {}. Missing key:{}". format( process_config['aapp_processes'] [process_config.process_name]['rename_aapp_compose'], err)) return False except Exception as err: LOG.error( "Failed to compose new filename: {}. Error:{}".format( process_config['aapp_processes'] [process_config.process_name]['rename_aapp_compose'], err)) return False try: if not os.path.exists(os.path.dirname(new_name)): LOG.debug("Need to create directory: {}".format( os.path.dirname(new_name))) os.makedirs(os.path.dirname(new_name)) except OSError as oe: LOG.error("Could not create directory: {} with {}".format( os.path.dirname(new_name), oe)) return False try: #shutil.move(process_config['aapp_static_configuration']['decommutation_files'][inputfile],new_name) shutil.move(values[sensor]['aapp_file'], new_name) #LOG.debug("Renamed: {} to {}".format(process_config['aapp_static_configuration']['decommutation_files'][inputfile], new_name)) LOG.debug("Renamed: {} to {}".format( values[sensor]['aapp_file'], new_name)) except OSError as e: LOG.error("Failed to rename {} to {}. {}".format( process_config[inputfile], new_name, e)) LOG.error("Please check previous processing") return False else: LOG.error( "Excpected file {} does not exists. Please check previous processing." .format(values[sensor]['aapp_file'])) return False else: return False _tmp = {} _tmp['file'] = new_name _tmp['sensor'] = sensor _tmp['level'] = tmp_process_config['data_level'] return _tmp
def request_push(msg, destination, login, publisher=None, **kwargs): """Request a push for data.""" huid = add_to_ongoing(msg) if huid is None: return if already_received(msg): timeout = float(kwargs["req_timeout"]) send_ack(msg, timeout) return for msg in iterate_messages(huid): try: _destination = compose(destination, msg.data) except KeyError as ke: LOGGER.error("Format identifier is missing from the msg.data: %s", str(ke)) raise except ValueError as ve: LOGGER.error( "Type of format identifier doesn't match the type in m msg.data: %s", str(ve)) raise except AttributeError as ae: LOGGER.error("msg or msg.data is None: %s", str(ae)) raise req, fake_req = create_push_req_message(msg, _destination, login) LOGGER.info("Requesting: %s", str(fake_req)) timeout = float(kwargs["transfer_req_timeout"]) local_dir = create_local_dir(_destination, kwargs.get('ftp_root', '/')) if publisher: publisher.send(str(fake_req)) response, hostname = send_request(msg, req, timeout) if response and response.type in ['file', 'collection', 'dataset']: LOGGER.debug("Server done sending file") add_to_file_cache(msg) if publisher: # Send an 'ack' message so that possible hot spares know # the primary has completed the request msg = Message(msg.subject, 'ack', msg.data) LOGGER.debug( "Sending a public 'ack' of completed transfer: %s", str(msg)) publisher.send(str(msg)) try: lmsg = unpack_and_create_local_message(response, local_dir, **kwargs) except IOError: LOGGER.exception("Couldn't unpack %s", str(response)) continue if publisher: lmsg = make_uris(lmsg, _destination, login) lmsg.data['origin'] = response.data['request_address'] lmsg.data.pop('request_address', None) lmsg = replace_mda(lmsg, kwargs) lmsg.data.pop('destination', None) LOGGER.debug("publishing %s", str(lmsg)) publisher.send(str(lmsg)) terminate_transfers(huid, float(kwargs["req_timeout"])) break else: LOGGER.error("Failed to get valid response from server %s: %s", str(hostname), str(response)) else: LOGGER.warning('Could not get a working source for requesting %s', str(msg)) terminate_transfers(huid, float(kwargs["req_timeout"]))
def start_zipcollector(registry, message, options, **kwargs): """From a posttroll (gatherer) message start the pytroll zip collector.""" del kwargs outdir_destination = options['destination_output_dir'] outdir_local = options['local_output_dir'] requested_tslots = options['requested_timeslots'] logger.info("") logger.info("registry dict: " + str(registry)) logger.info("\tMessage:") logger.info(message) if message is None: return registry elif (message.type != 'dataset'): logger.warning("Message type is not a collection! Type=%s", str(message.type)) return registry if 'start_time' in message.data: start_time = message.data['start_time'] scene_id = start_time.strftime('%Y%m%d%H%M') else: logger.error("No start time in message!") start_time = None return registry if 'end_time' in message.data: end_time = message.data['end_time'] else: logger.warning("No end time in message!") end_time = start_time + timedelta(seconds=60 * 12) # noqa if 'seviri' not in message.data['sensor']: logger.debug("Scene is not supported") logger.warning("Sensor {0} is not SEVIRI! Continue".format( str(message.data['sensor']))) return registry else: registry[scene_id] = len(message.data['dataset']) # Now check that the time slot is among those requested logger.debug("Wanted time slots: %s", str(requested_tslots)) wanted_timeslot = False if '%.2d' % start_time.minute in requested_tslots: wanted_timeslot = True if wanted_timeslot: logger.info("Time slot {0} is requested".format(start_time)) # Example filename: # (service=0deg-lvl1) # met10___hritglob1708171100.tgz satid = PLATFORM_NAME.get(message.data['platform_name'], 'met10') filename = compose(options['archive_filename'], { 'start_time': start_time, 'satid': satid }) local_filepath = os.path.join(outdir_local, filename) dest_filepath = os.path.join(outdir_destination, filename + '_original') # Create the tar archive: logger.debug("Create gzipped tar archive: %s", local_filepath) status = True try: with tarfile.open(local_filepath, "w|gz") as archive: for item in message.data['dataset']: filepath = urlparse(item['uri']).path archive.add(filepath, arcname=item['uid']) copy_file_to_destination(local_filepath, dest_filepath) monitor_msg = "File successfully created" except Exception as err: monitor_msg = "Failed generating tar file: " + str(err) status = False if 'monitoring_hook' in options: options['monitoring_hook'](status, monitor_msg) else: logger.error("Configuration lacking a monitoring_hook entry!") else: logger.info( "Time slot {0} NOT requested. Do nothing".format(start_time)) return registry
def test_partial_compose_repeated_vars_with_different_formatting(self): """Test partial compose with a fmt with repeated vars with different formatting.""" fmt = "/foo/{start_time:%Y%m}/bar/{baz}_{start_time:%Y%m%d_%H%M}.{format}" composed = compose(fmt=fmt, keyvals={"format": "qux"}, allow_partial=True) assert composed == "/foo/{start_time:%Y%m}/bar/{baz}_{start_time:%Y%m%d_%H%M}.qux"
def test_partial_compose_with_similarly_named_params(self): """Test that partial compose handles well vars with common substrings in name.""" original_fmt = "{foo}{afooo}{fooo}.{bar}/{baz:%Y}/{baz:%Y%m%d_%H}/{baz:%Y}/{bar:d}" composed = compose(fmt=original_fmt, keyvals={"afooo": "qux"}, allow_partial=True) assert composed == "{foo}qux{fooo}.{bar}/{baz:%Y}/{baz:%Y%m%d_%H}/{baz:%Y}/{bar:d}"
def test_partial_compose_is_identity_with_empty_keyvals(self, original_fmt): """Test that partial compose leaves the input untouched if no keyvals at all.""" assert compose(fmt=original_fmt, keyvals={}, allow_partial=True) == original_fmt
def test_that_some_invalid_fmt_can_confuse_partial_compose(self): """Test that a fmt with a weird char can confuse partial compose.""" fmt = "{foo?}_{bar}_{foo}.qux" with pytest.raises(ValueError): _ = compose(fmt=fmt, keyvals={}, allow_partial=True)
def download_tle(config, timestamp, dir_data_tle): user = os.getenv("PAR_NAVIGATION_TLE_USER", "xxxxxx") passwd = os.getenv("PAR_NAVIGATION_TLE_PASSWD", "xxxxxx") url = os.getenv("PAR_NAVIGATION_TLE_URL_DOWNLOAD") timeout = 60 catalogue = "25338,26536,27453,28654,33591,37849,29499,38771,27431,32958,37214,25994,27424" tle_infile = "" tle_dict = {} tle_dict['timestamp'] = timestamp tle_cnf = [] try: tle_cnf = config['aapp_processes'][config.process_name]['tle_download'] except KeyError as ke: LOG.info("No tle_download config found. Using the default.") pass finally: tle_cnf.append({'url': url, 'user': user, 'passwd': passwd, 'timeout': timeout, 'catalogue': catalogue}) try: tle_infile = compose(config['aapp_processes'][config.process_name]['tle_infile_format'], tle_dict) except KeyError as ke: if 'tle_infile_format' in ke: tle_infile = compose('tle_{timestamp:%Y%m%d_%H%M}.txt', tle_dict) LOG.warning("Using default TLE file name format: %s", tle_infile) pass else: LOG.error("Key error: {}".format(ke)) LOG.error("Valid keys :") for key in tle_dict.keys(): LOG.error("{}".format(key)) raise except: raise tle_file_list = [] for cnf in tle_cnf: for cnf_url in cnf['url'].split(): LOG.debug("Will try to download TLE from {}.".format(cnf_url)) if "space-track" in cnf_url: # Do the special space-track login status = False returncode = 0 stdout = "" stderr = "" cmd = "wget -T {} --post-data=\"identity={}&password={}\" --cookies=on ", "--keep-session-cookies --save-cookies=cookies_spacetrack \"{}/ajaxauth/login\" -olog".format( cnf['timeout'], cnf['user'], cnf['passwd'], cnf_url) try: status, returncode, stdout, stderr = run_shell_command(cmd) except: LOG.error("Failed running command: {} with return code: {}".format(cmd, returncode)) LOG.error("stdout: {}".format(stdout)) LOG.error("stderr: {}".format(stderr)) return_status = False else: if returncode != 0: LOG.debug("Running command: {} with return code: {}".format(cmd, returncode)) LOG.debug("stdout: {}".format(stdout)) LOG.debug("stderr: {}".format(stderr)) else: cmd = "wget -T {} --keep-session-cookies --load-cookies=cookies_spacetrack -O weather.txt ", "\"{}/basicspacedata/query/class/tle_latest/ORDINAL/1/NORAD_CAT_ID/{}/orderby/TLE_LINE1\"".format( cnf['timeout'], cnf_url, cnf['catalogue']) try: status, returncode, stdout, stderr = run_shell_command(cmd) except: LOG.error("Failed running command: {} with return code: {}".format(cmd, returncode)) LOG.error("stdout: {}".format(stdout)) LOG.error("stderr: {}".format(stderr)) else: if returncode != 0: LOG.debug("Running command: {} with return code: {}".format(cmd, returncode)) LOG.debug("stdout: {}".format(stdout)) LOG.debug("stderr: {}".format(stderr)) else: LOG.debug("TLE download ok") if os.path.exists("weather.txt"): try: tle_file = open("weather.txt", 'r') tle_string = tle_file.read() tle_file.close() tle_file_out = os.path.join(dir_data_tle, tle_infile) with open(tle_file_out, "a") as tle_file: tle_file.write(tle_string) if tle_file_out not in tle_file_list: tle_file_list.append(tle_file_out) except Exception as ex: LOG.debug("Failed rename tle download file: {}".format(ex)) raise else: import urllib2 f = urllib2.urlopen(cnf_url) tle_string = f.read() f.close() tle_file_out = os.path.join(dir_data_tle, tle_infile) with open(tle_file_out, "a") as tle_file: tle_file.write(tle_string) if tle_file_out not in tle_file_list: tle_file_list.append(tle_file_out) return tle_file_list
def do_tleing(config, timestamp, satellite): """Get the tle-file and copy them to the AAPP data structure and run the AAPP tleing script and executable""" return_status = True # This function relays on beeing in a working directory try: current_dir = os.getcwd() # Store the dir to change back to after function complete except OSError as ose: LOG.error("Failed to get current working dir: {}".format(ose)) raise os.chdir(config['aapp_processes'][config.process_name]['working_dir']) tle_match_tests = (('.*(\d{4})(\d{2})(\d{2})_?-?T?(\d{2})(\d{2})(\d{2}).*', _do_6_matches), ('.*(\d{4})(\d{2})(\d{2})_?-?T?(\d{2})(\d{2}).*', _do_5_matches), ('.*(\d{4})(\d{2})(\d{2})_?-?T?(\d{2}).*', _do_4_matches), ('.*(\d{4})(\d{2})(\d{2}).*', _do_3_matches), ('.*(\d{2})(\d{2})(\d{2}).*', _do_3_matchesYY)) if '7' in os.getenv('AAPP_PREFIX'): aapp_env = 'AAPP_ENV7' elif '8' in os.getenv('AAPP_PREFIX'): aapp_env = 'AAPP_ENV8' else: aapp_env = 'AAPP_ENV' if 'dir_navigation' in config['aapp_processes'][config.process_name]: LOG.warning("Override the env variable set in {} DIR_NAVIGATION from {} to {}.".format( aapp_env, os.environ['DIR_NAVIGATION'], config['aapp_processes'][config.process_name]['dir_navigation'])) os.environ['DIR_NAVIGATION'] = config['aapp_processes'][config.process_name]['dir_navigation'] # Need to update DIR_DATA_TLE dir to be sure this is corect if not tle_indir is given os.environ['DIR_DATA_TLE'] = os.path.join(os.getenv('DIR_NAVIGATION'), 'tle_db') if 'tle_indir' in config['aapp_processes'][config.process_name]: tle_indir = config['aapp_processes'][config.process_name]['tle_indir'] LOG.warning("Override the env variable set in {} DIR_DATA_TLE from {} to {}.".format( aapp_env, os.environ['DIR_DATA_TLE'], tle_indir)) os.environ['DIR_DATA_TLE'] = tle_indir # variables for the TLE HOME directory DIR_DATA_TLE = os.getenv('DIR_DATA_TLE', os.path.join(os.getenv('DIR_NAVIGATION'), 'orb_elem')) # This is needed by AAPP tleing. Try other if not existing if not os.path.exists(DIR_DATA_TLE): LOG.warning("Dir " + DIR_DATA_TLE + " does not exist. Create ") try: os.makedirs(DIR_DATA_TLE) except: LOG.error("Failed to create %s. Can not handle TLEs without this", DIR_DATA_TLE) raise if 'tle_file_to_data_diff_limit_days' in config['aapp_processes'][config.process_name]: select_closest_tle_file_to_data = True min_closest_tle_file = int( config['aapp_processes'][config.process_name]['tle_file_to_data_diff_limit_days']) * 24 * 60 * 60 else: select_closest_tle_file_to_data = False TLE_INDEX = os.path.join(DIR_DATA_TLE, "tle_{}.index".format(satellite)) tle_search_dir = DIR_DATA_TLE tle_file_list = [] # dict to hold needed tle keys tle_dict = {} if not select_closest_tle_file_to_data: if os.path.exists(TLE_INDEX): # Loop over all tle files, and only do tle tle_files = [s for s in glob(os.path.join(DIR_DATA_TLE, 'tle*txt')) if os.path.isfile(os.path.join(DIR_DATA_TLE, s))] tle_files.sort(key=lambda s: os.path.getctime(os.path.join(DIR_DATA_TLE, s))) tle_index_mtime = os.path.getmtime(TLE_INDEX) for s in tle_files: if os.path.getmtime(os.path.join(DIR_DATA_TLE, s)) > tle_index_mtime: tle_file_list.append(s) if len(tle_file_list) == 0: import time LOG.warning(("No newer tle files than last update of the index file. " + "Last update of index file is {:d}s. If more than a few days you should check.".format( int(time.time() - tle_index_mtime)))) else: LOG.info("Will use tle files {}".format(tle_file_list)) else: LOG.warning("index file does not exist. If this is the first run of AAPP tleing.exe it is ok," " otherwise it is a bit suspisiuos.") try: tle_files = [s for s in os.listdir(DIR_DATA_TLE) if os.path.isfile(os.path.join(DIR_DATA_TLE, s))] tle_files.sort(key=lambda s: os.path.getctime(os.path.join(DIR_DATA_TLE, s))) tle_file_list = tle_files except OSError: LOG.warning("Found to tle files .... ") else: # dict to hold needed tle keys tle_dict['timestamp'] = timestamp try: infile = compose(config['aapp_processes'][config.process_name]['tle_infile_format'], tle_dict) except KeyError as ke: LOG.error("Key error: {}".format(ke)) LOG.error("Valid keys :") for key in tle_dict.keys(): LOG.error("{}".format(key)) raise except: raise LOG.debug("tle file name: {}".format(infile)) # Check if I can read the tle file. first_search = True # FIXME: In AAPP default get_tle script direcory timestamp is TLE_MONTH=`date +%Y-\%m` for tle_search_dir in [compose(os.path.join(DIR_DATA_TLE, "{timestamp:%Y_%m}"), tle_dict), DIR_DATA_TLE]: if not os.path.exists(tle_search_dir): LOG.debug("tle_search_dir {} does not exists.".format(tle_search_dir)) continue LOG.debug("tle_search_dir {}".format(tle_search_dir)) try: with open(os.path.join(tle_search_dir, infile)) as tle_file: del tle_file_list[:] tle_file_list.append(os.path.join(tle_search_dir, infile)) min_closest_tle_file = 0 pass except IOError as e: LOG.warning("Could not find tle file: {}. Try find closest ... ".format(infile)) tle_file_list = glob(os.path.join(tle_search_dir, '*')) LOG.debug("tle file list: {}".format(tle_file_list)) LOG.debug(tle_file_list) infile_closest = "" for tle_file_name in tle_file_list: for regex, test in tle_match_tests: m = re.match(regex, tle_file_name) if m: try: LOG.debug("{} {}".format(tle_file_name, test(m))) delta = timestamp - test(m) if (abs(delta.total_seconds()) < min_closest_tle_file): min_closest_tle_file = abs(delta.total_seconds()) # infile_closest = os.path.basename(tle_file_name) infile_closest = tle_file_name LOG.debug("Closest tle infile so far: {}".format(infile_closest)) except ValueError: pass if infile_closest: del tle_file_list[:] tle_file_list.append(infile_closest) break else: if not first_search: LOG.error("Could not find tle file close enough to timestamp {} with limit {}".format( timestamp, min_closest_tle_file)) LOG.error("Update your TLE files or adjust the limit(Not recomended!).") first_search = False else: break # DIR_DATA_TLE = tle_search_dir if tle_file_list: LOG.debug("Use this: {} offset {}s".format(tle_file_list, min_closest_tle_file)) if not tle_file_list and config['aapp_processes'][config.process_name]['download_tle_files']: LOG.warning("Found no tle files. Try to download ... ") tle_file_list = download_tle(config, timestamp, DIR_DATA_TLE) for tle_file in tle_file_list: archive = False # SATellite IDentification mandatory # so take care of default values os.environ['SATID_FILE'] = os.getenv('SATID_FILE', 'satid.txt') """Don't use the tle_indir because this is handeled by the tleing script""" if (DIR_DATA_TLE != tle_search_dir): tle_filename = compose(os.path.join("{timestamp:%Y_%m}", os.path.basename(tle_file)), tle_dict) else: tle_filename = os.path.basename(tle_file) status = False returncode = 0 stdout = "" stderr = "" cmd = "tleing.exe" stdin = "{}\n{}\n{}\n{}\n".format(DIR_DATA_TLE, tle_filename, satellite, TLE_INDEX) LOG.debug('stdin arguments to command: ' + str(stdin)) try: status, returncode, stdout, stderr = run_shell_command(cmd, stdin=stdin) except: LOG.error("Failed running command: {} with return code: {}".format(cmd, returncode)) LOG.error("stdout: {}".format(stdout)) LOG.error("stderr: {}".format(stderr)) return_status = False else: if returncode != 0: LOG.debug("Running command: {} with return code: {}".format(cmd, returncode)) LOG.debug("stdout: {}".format(stdout)) LOG.debug("stderr: {}".format(stderr)) elif not os.path.exists(TLE_INDEX): LOG.error("index file: {} does not exist after tleing. Something is wrong.".format(TLE_INDEX)) LOG.debug("Running command: {} with return code: {}".format(cmd, returncode)) LOG.debug("stdout: {}".format(stdout)) LOG.debug("stderr: {}".format(stderr)) else: LOG.debug("Running command: {} with return code: {}".format(cmd, returncode)) LOG.debug("stdout: {}".format(stdout)) LOG.debug("stderr: {}".format(stderr)) LOG.debug("DIR_DATA_TLE : {}".format(DIR_DATA_TLE)) LOG.debug("tle_file : {}".format(os.path.basename(tle_file))) LOG.debug("satellite : {}".format(satellite)) LOG.debug("TLE_INDEX : {}".format(TLE_INDEX)) # When a index file is generated above one line is added for each tle file. # If several tle files contains equal TLEs each of these TLEs generate one line in the index file # To avoid this, sort the index file keeping only unique lines(skipping the tle filename at the end # The sort options +0b -3b is guessed to be sort from column 0 to 3, but this is not documented # Could cause problems with future version of sort. # See eg. http://search.cpan.org/~sdague/ppt-0.12/bin/sort # cmd="sort -u -o {} +0b -3b {}".format(os.path.join(DIR_DATA_TLE, "{}.sort".format(TLE_INDEX)), # os.path.join(DIR_DATA_TLE, TLE_INDEX)) if os.path.exists(TLE_INDEX): cmd = "sort -u +0b -3b {}".format(TLE_INDEX) try: status, returncode, stdout, stderr = run_shell_command( cmd, stdout_logfile="{}.sort1".format(TLE_INDEX)) except: LOG.error("Failed running command: {} with return code: {}".format(cmd, returncode)) LOG.error("stdout: {}".format(stdout)) LOG.error("stderr: {}".format(stderr)) return_status = False else: if returncode == 0 and os.path.exists("{}.sort1".format(TLE_INDEX)): cmd = "grep -v NaN {}.sort1".format(TLE_INDEX) try: status, returncode, stdout, stderr = run_shell_command( cmd, stdout_logfile="{}.sort".format(TLE_INDEX)) except: LOG.error("Failed running command: {} with return code: {}".format(cmd, returncode)) LOG.error("stdout: {}".format(stdout)) LOG.error("stderr: {}".format(stderr)) return_status = False else: try: os.remove(TLE_INDEX) os.remove("{}.sort1".format(TLE_INDEX)) except OSError as e: LOG.error("Failed to remove unsorted and duplicated index file: {} with {}" .format(TLE_INDEX, e)) else: try: os.rename("{}.sort".format(TLE_INDEX), TLE_INDEX) archive = True except: LOG.error("Failed to rename sorted index file to original name.") else: LOG.error("Returncode other than 0: {} or tle index sort file does exists.".format( returncode, "{}.sort".format(TLE_INDEX))) else: LOG.error("tle index file: {} does not exists after tleing before sort. This can not happen.") # If a new tle is used and archive dir is given in config, copy TLEs to archive if archive and ('tle_archive_dir' in config['aapp_processes'][config.process_name]): archive_dict = {} archive_dict['tle_indir'] = config['aapp_processes'][config.process_name]['tle_indir'] for tle_file_name in tle_file_list: for regex, test in tle_match_tests: m = re.match(regex, tle_file_name) if m: try: archive_dict['timestamp'] = test(m) tle_archive_dir = compose( config['aapp_processes'][config.process_name]['tle_archive_dir'], archive_dict) if not os.path.exists(tle_archive_dir): try: os.makedirs(tle_archive_dir) except: LOG.error("Failed to make archive dir: {}".format(tle_archive_dir)) except ValueError: LOG.exception('Failed in archive step...') pass try: copy(tle_file_name, tle_archive_dir) LOG.debug("Copied {} to {}.".format(tle_file_name, tle_archive_dir)) archive = False except IOError as ioe: LOG.error("Failed to copy TLE file: {} to archive: {} because {}".format( tle_file_name, tle_archive_dir, ioe)) LOG.error("CWD: {}".format(os.getcwd())) # Change back after this is done os.chdir(current_dir) return return_status
def test_default_compose_is_strict(self): """Make sure the default compose call does not accept partial composition.""" fmt = "{foo}_{bar}.qux" with pytest.raises(KeyError): _ = compose(fmt, {"foo": "foo"})