Beispiel #1
0
    def test_modify_log_file(self):
        """Test writing a new log file then reverting that change."""
        test_log_file = "logs/test.log"

        # Get initial config
        config = lib.get_config()
        orig_log_file = config["logging"]["log_file"]

        # Modify config
        config["logging"]["log_file"] = test_log_file
        lib.write_config(config)

        # Get and check updated config
        updated_config = lib.get_config()
        assert updated_config == config
        assert updated_config["logging"]["log_file"] == test_log_file

        # Revert change
        config["logging"]["log_file"] = orig_log_file
        lib.write_config(config)

        # Test reverted change
        updated_config = lib.get_config()
        assert updated_config == config
        assert updated_config["logging"]["log_file"] == orig_log_file
Beispiel #2
0
    def __init__(self,
                 ctrl_addr="tcp://127.0.0.1:60000",
                 sub_addr="tcp://127.0.0.1:60001"):

        # Get config, build logger
        self.config = lib.get_config()
        self.logger = lib.get_logger()

        # Build control client
        try:
            self.ctrl_client = ctrl_client_mod.CtrlClient(ctrl_addr)
        except Exception, e:
            self.logger.error("Couldn't build CtrlClient; ctrl_addr: {},"
                              " error: {}".format(ctrl_addr, e))
            sys.exit(-1)
Beispiel #3
0
    def __init__(self,
                 ctrl_addr="tcp://127.0.0.1:60000",
                 sub_addr="tcp://127.0.0.1:60001"):

        # Get config, build logger
        self.config = lib.get_config()
        self.logger = lib.get_logger()

        # Build control client
        try:
            self.ctrl_client = ctrl_client_mod.CtrlClient(ctrl_addr)
        except Exception, e:
            self.logger.error("Couldn't build CtrlClient; ctrl_addr: {},"
                              " error: {}".format(ctrl_addr, e))
            sys.exit(-1)
from pathlib import Path
import platform
import os
import sys
import time

from lib.lib import read_ids, get_config, linux2win
from lib.logging_utils import create_logger, create_logfile_path
from lib.order import submit_order, poll_for_success
# from submit_order import submit_order
from lib.order import download_parallel
import lib.constants as constants

logger = create_logger(__name__, 'sh', 'INFO')

default_dst_parent = get_config(constants.DOWNLOAD_LOC)
if platform.system() == constants.WINDOWS:
    default_dst_parent = linux2win(default_dst_parent)

# DELIVERY
DELIVERY_OPTIONS = [constants.AWS, constants.ZIP]


def order_and_download(order_name,
                       order_ids_path,
                       order_selection_path,
                       out_orders_list,
                       order_product_bundle,
                       remove_onhand=True,
                       order_only=False,
                       delivery=constants.ZIP,
Beispiel #5
0
    # Need to have process start from root of repo for imports to work
    cwd = str(os.path.join(os.path.abspath(os.path.dirname(__file__)), ".."))
    process_description = ["./bot/server/ctrl_server.py", str(args.test_mode)]
    server = Popen(process_description, preexec_fn=preexec_fn, cwd=cwd)
    # Give server a chance to get up and running
    sleep(.3)

if args.pilot:
    import pilot as pilot_mod
    print "Starting pilot"
    pilot = pilot_mod.Pilot()
    pilot.run()

if args.cli:
    # Build addresses of remote resources
    config = lib.get_config("config.yaml")
    ctrl_addr = "{protocol}://{host}:{port}".format(
        protocol=config["server_protocol"],
        host=config["server_host"],
        port=config["ctrl_server_port"])
    sub_addr = "{protocol}://{host}:{port}".format(
        protocol=config["server_protocol"],
        host=config["server_host"],
        port=config["pub_server_port"])

    print "Starting CLI"
    try:
        cli_mod.CLI(ctrl_addr, sub_addr).cmdloop()
    except KeyboardInterrupt:
        print "Exiting CLI"
Beispiel #6
0
# External modules
sys.path.append(str(Path(__file__).parent / '..'))
try:
    from db_utils.db import Postgres, ids2sql
except ImportError as e:
    logger.error('db_utils module not found. It should be adjacent to '
                 'the planet_tools directory. Path: \n{}'.format(sys.path))
    sys.exit()

# Params
PLATFORM_LOCATION = 'platform_location'
REQUIRED_FIELDS = [constants.ID, constants.SHELVED_LOC]
# Transfer methods
TM_LINK = 'link'
TM_COPY = 'copy'
shelved_base = get_config(constants.SHELVED_LOC)
if platform.system() == constants.WINDOWS:
    shelved_base = Path(linux2win(shelved_base))
else:
    shelved_base = Path(shelved_base)


def load_selection(scene_ids_path=None, footprint_path=None):
    # If path to file of scene ids is provided, look up locations in onhand table
    # Get scene ids
    if scene_ids_path:
        scene_ids = read_ids(scene_ids_path)
        logger.info('Total IDs found: {:,}'.format(len(scene_ids)))
        scene_ids = set(scene_ids)
        logger.info('Unique IDs found: {:,}'.format(len(scene_ids)))
Beispiel #7
0
import time

from tqdm import tqdm

try:
    import boto3
except ImportError:
    print('Warning: boto3 import failed, delivery via AWS will not work.')

from lib.lib import get_config
from lib.logging_utils import create_logger

logger = create_logger(__name__, 'sh', 'INFO')

# AWS
AWS_PARAMS = get_config("aws")
AWS_ACCESS_KEY_ID = AWS_PARAMS["aws_access_key_id"]
AWS_SECRET_ACCESS_KEY = AWS_PARAMS["aws_secret_access_key"]
AWS_BUCKET = AWS_PARAMS["aws_bucket"]
AWS_REGION = AWS_PARAMS["aws_region"]
AWS_PATH_PREFIX = AWS_PARAMS["aws_path_prefix"]
BUCKET_NAME = 'pgc-data'

# Constants
S3 = 's3'


def connect_aws_bucket(bucket_name=BUCKET_NAME,
                       aws_access_key_id=AWS_ACCESS_KEY_ID,
                       aws_secret_access_key=AWS_SECRET_ACCESS_KEY):
    s3 = boto3.resource(
Beispiel #8
0
    # Need to have process start from root of repo for imports to work
    cwd = str(os.path.join(os.path.abspath(os.path.dirname(__file__)), ".."))
    process_description = ["./bot/server/ctrl_server.py", str(args.test_mode)]
    server = Popen(process_description, preexec_fn=preexec_fn, cwd=cwd)
    # Give server a chance to get up and running
    sleep(.3)

if args.pilot:
    import pilot as pilot_mod
    print "Starting pilot"
    pilot = pilot_mod.Pilot()
    pilot.run()

if args.cli:
    # Build addresses of remote resources
    config = lib.get_config("config.yaml")
    ctrl_addr = "{protocol}://{host}:{port}".format(
        protocol=config["server_protocol"],
        host=config["server_host"],
        port=config["ctrl_server_port"])
    sub_addr = "{protocol}://{host}:{port}".format(
        protocol=config["server_protocol"],
        host=config["server_host"],
        port=config["pub_server_port"])

    print "Starting CLI"
    try:
        cli_mod.CLI(ctrl_addr, sub_addr).cmdloop()
    except KeyboardInterrupt:
        print "Exiting CLI"
Beispiel #9
0
 def test_invalid_state(self):
     """Test passing a non-boolean value for state param."""
     with self.assertRaises(TypeError):
         lib.set_testing("not_bool")
     new_config = lib.get_config()
     assert self.orig_config == new_config
Beispiel #10
0
 def test_same(self):
     """Test writing the current value of testing flag."""
     lib.set_testing(self.orig_config["testing"])
     new_config = lib.get_config()
     assert new_config == self.orig_config
Beispiel #11
0
 def setUp(self):
     """Get and store original config."""
     self.orig_config = lib.get_config()
Beispiel #12
0
 def test_same(self):
     """Confirm that writing without changes produces no change."""
     config = lib.get_config()
     lib.write_config(config)
     result_config = lib.get_config()
     assert config == result_config
Beispiel #13
0
 def test_invalid(self):
     """Test proper failure for fake config file."""
     with self.assertRaises(IOError):
         config = lib.get_config("fake.yaml")
Beispiel #14
0
 def test_type(self):
     """Confirm that type of config is a dict."""
     config = lib.get_config()
     assert type(config) is dict
Beispiel #15
0
    sys.exit()

# Constants
WINDOWS = 'Windows'
LINUX = 'Linux'
COPY = 'copy'
LINK = 'link'
# Databse
SANDWICH = 'sandwich'
PLANET = 'planet'

# config.json keys
SHELVED_LOC = 'shelved_loc'

# Destination directory for shelving
PLANET_DATA_DIR = get_config(SHELVED_LOC)
if platform.system() == WINDOWS:
    PLANET_DATA_DIR = linux2win(PLANET_DATA_DIR)

# Filename of order-level manifests, used for locating
MANIFEST_JSON = 'manifest.json'

# Index table name and unique constraint
INDEX_TBL = 'scenes_onhand'
INDEX_UNIQUE_CONSTRAINT = get_config('db')['tables'][INDEX_TBL]['unique_id']
INDEX_GEOM = 'geometry'
INDEX_CRS = 'epsg:4326'


def determine_copy_fxn(transfer_method: str) -> Callable:
    """
Beispiel #16
0
    # Need to have process start from root of repo for imports to work
    cwd = str(os.path.join(os.path.abspath(os.path.dirname(__file__)), ".."))
    process_description = ["./bot/server/ctrl_server.py", str(args.test_mode)]
    server = Popen(process_description, preexec_fn=preexec_fn, cwd=cwd)
    # Give server a chance to get up and running
    sleep(.3)

if args.pilot:
    import pilot as pilot_mod
    print "Starting pilot"
    pilot = pilot_mod.Pilot()
    pilot.run()

if args.cli:
    # Build addresses of remote resources
    config = lib.get_config()
    ctrl_addr = "{protocol}://{host}:{port}".format(
        protocol=config["server_protocol"],
        host=config["server_host"],
        port=config["ctrl_server_port"])
    sub_addr = "{protocol}://{host}:{port}".format(
        protocol=config["server_protocol"],
        host=config["server_host"],
        port=config["pub_server_port"])

    print "Starting CLI"
    try:
        cli_mod.CLI(ctrl_addr, sub_addr).cmdloop()
    except KeyboardInterrupt:
        print "Exiting CLI"