def __init__(self):
        Extractor.__init__(self)

        self.setup()

        logging.getLogger('pyclowder').setLevel(logging.DEBUG)
        logging.getLogger('__main__').setLevel(logging.DEBUG)
Beispiel #2
0
    def __init__(self):
        Extractor.__init__(self)

        image_binary = os.getenv('IMAGE_BINARY', '')
        image_type = os.getenv('IMAGE_TYPE', 'png')
        image_thumbnail_command = os.getenv('IMAGE_THUMBNAIL_COMMAND', '@BINARY@ @INPUT@ @OUTPUT@')
        image_preview_command = os.getenv('IMAGE_PREVIEW_COMMAND', '@BINARY@ @INPUT@ @OUTPUT@')
        preview_binary = os.getenv('PREVIEW_BINARY', '')
        preview_type = os.getenv('PREVIEW_TYPE', '')
        preview_command = os.getenv('PREVIEW_COMMAND', '@BINARY@ @INPUT@ @OUTPUT@')

        # add any additional arguments to parser
        self.parser.add_argument('--image-binary', nargs='?', dest='image_binary', default=image_binary,
                                 help='Image Binary used to for image thumbnail/preview (default=%s)' % image_binary)
        self.parser.add_argument('--image-type', nargs='?', dest='image_type', default=image_type,
                                 help='Image type of thumbnail/preview (default=%s)' % image_type)
        self.parser.add_argument('--image-thumbnail-command', nargs='?', dest='image_thumbnail_command',
                                 default=image_thumbnail_command,
                                 help='Image command line for thumbnail (default=%s)' % image_thumbnail_command)
        self.parser.add_argument('--image-preview-command', nargs='?', dest='image_preview_command',
                                 default=image_preview_command,
                                 help='Image command line for preview (default=%s)' % image_preview_command)
        self.parser.add_argument('--preview-binary', nargs='?', dest='preview_binary', default=preview_binary,
                                 help='Binary used to generate preview (default=%s)' % preview_binary)
        self.parser.add_argument('--preview-type', nargs='?', dest='preview_type', default=preview_type,
                                 help='Preview type (default=%s)' % preview_type)
        self.parser.add_argument('--preview-command', nargs='?', dest='preview_command', default=preview_command,
                                 help='Command line for preview (default=%s)' % preview_command)

        # parse command line and load default logging configuration
        self.setup()

        # setup logging for the exctractor
        logging.getLogger('pyclowder').setLevel(logging.DEBUG)
        logging.getLogger('__main__').setLevel(logging.DEBUG)
Beispiel #3
0
    def __init__(self):
        Extractor.__init__(self)

        # parse command line and load default logging configuration
        self.setup()

        # setup logging for the exctractor
        logging.getLogger('pyclowder').setLevel(logging.DEBUG)
        logging.getLogger('__main__').setLevel(logging.DEBUG)
Beispiel #4
0
    def __init__(self):
        Extractor.__init__(self)

        # parse command line and load default logging configuration
        self.setup()
        logging.basicConfig(level=logging.INFO)
        # setup logging for the exctractor
        logging.getLogger("pika").setLevel(logging.WARNING)
        logging.getLogger("urllib3").setLevel(logging.WARNING)

        logging.getLogger('pyclowder').setLevel(logging.INFO)
        logging.getLogger('__main__').setLevel(logging.INFO)
    def __init__(self):
        Extractor.__init__(self)

        # add any additional arguments to parser
        # self.parser.add_argument('--max', '-m', type=int, nargs='?', default=-1,
        #                          help='maximum number (default=-1)')

        # parse command line and load default logging configuration
        self.setup()

        # setup logging for the exctractor
        logging.getLogger('pyclowder').setLevel(logging.DEBUG)
        logging.getLogger('__main__').setLevel(logging.DEBUG)
Beispiel #6
0
    def __init__(self):
        Extractor.__init__(self)

        # parse command line and load default logging configuration
        self.setup()

        # setup logging for the exctractor
        logging.getLogger('pyclowder').setLevel(logging.DEBUG)
        logging.getLogger('__main__').setLevel(logging.DEBUG)
        self.logger = logging.getLogger(__name__)

        self.selenium = os.getenv('SELENIUM_URI', 'http://localhost:4444/wd/hub')
        self.window_size = (1366, 768)  # the default
        self.read_settings()
    def __init__(self):
        Extractor.__init__(self)

        # add any additional arguments to parser
        # self.parser.add_argument('--max', '-m', type=int, nargs='?', default=-1,
        #                          help='maximum number (default=-1)')
        self.parser.add_argument('--influxHost',
                                 dest="influx_host",
                                 type=str,
                                 nargs='?',
                                 default="terra-logging.ncsa.illinois.edu",
                                 help="InfluxDB URL for logging")
        self.parser.add_argument('--influxPort',
                                 dest="influx_port",
                                 type=int,
                                 nargs='?',
                                 default=8086,
                                 help="InfluxDB port")
        self.parser.add_argument('--influxUser',
                                 dest="influx_user",
                                 type=str,
                                 nargs='?',
                                 default="terra",
                                 help="InfluxDB username")
        self.parser.add_argument('--influxPass',
                                 dest="influx_pass",
                                 type=str,
                                 nargs='?',
                                 default="",
                                 help="InfluxDB password")
        self.parser.add_argument('--influxDB',
                                 dest="influx_db",
                                 type=str,
                                 nargs='?',
                                 default="extractor_db",
                                 help="InfluxDB databast")

        # parse command line and load default logging configuration
        self.setup()

        # setup logging for the exctractor
        logging.getLogger('pyclowder').setLevel(logging.DEBUG)
        logging.getLogger('__main__').setLevel(logging.DEBUG)

        # assign other arguments
        self.influx_host = self.args.influx_host
        self.influx_port = self.args.influx_port
        self.influx_user = self.args.influx_user
        self.influx_pass = self.args.influx_pass
        self.influx_db = self.args.influx_db
Beispiel #8
0
    def __init__(self):
        Extractor.__init__(self)

        self.context = {
            "scan":
            "https://clowder.ncsa.illinois.edu/contexts/clamav#scan",
            "database":
            "https://clowder.ncsa.illinois.edu/contexts/clamav#database",
        }
        print(self.database_info())

        # parse command line and load default logging configuration
        self.setup()

        # setup logging for the exctractor
        logging.getLogger('pyclowder').setLevel(logging.DEBUG)
        logging.getLogger('__main__').setLevel(logging.DEBUG)
	def __init__(self):
		Extractor.__init__(self)

		# add any additional arguments to parser
		# self.parser.add_argument('--max', '-m', type=int, nargs='?', default=-1,
		#                          help='maximum number (default=-1)')
		self.parser.add_argument('--output', '-o', dest="output_dir", type=str, nargs='?',
								 default="/home/extractor/sites/ua-mac/Level_1/netcdf",
								 help="root directory where timestamp & output directories will be created")

		# parse command line and load default logging configuration
		self.setup()

		# setup logging for the exctractor
		logging.getLogger('pyclowder').setLevel(logging.DEBUG)
		logging.getLogger('__main__').setLevel(logging.DEBUG)

		# assign other arguments
		self.output_dir = self.args.output_dir
    def __init__(self):
        Extractor.__init__(self)

        # parse command line and load default logging configuration
        self.setup()

        self.extractorName = os.getenv('RABBITMQ_QUEUE', "ncsa.geoshp.preview")
        self.geoServer = os.getenv('GEOSERVER_URL')
        self.gs_username = os.getenv('GEOSERVER_USERNAME', 'admin')
        self.gs_password = os.getenv('GEOSERVER_PASSWORD', 'geoserver')
        self.proxy_url = os.getenv('PROXY_URL',
                                   'http://localhost:9000/api/proxy/')
        self.proxy_on = os.getenv('PROXY_ON', 'false')

        self.datasetid = None
        self.logger = logging.getLogger('geoshp preview')
        self.logger.setLevel(logging.DEBUG)
        # setup logging for the exctractor
        logging.getLogger('pyclowder').setLevel(logging.DEBUG)
        logging.getLogger('__main__').setLevel(logging.DEBUG)
Beispiel #11
0
    def __init__(self):
        Extractor.__init__(self)

        image_binary = os.getenv('IMAGE_BINARY', '/usr/bin/identify')

        # add any additional arguments to parser
        self.parser.add_argument(
            '--image-binary',
            nargs='?',
            dest='image_binary',
            default=image_binary,
            help='Image Binary used to for image thumbnail/preview (default=%s)'
            % image_binary)

        # parse command line and load default logging configuration
        self.setup()

        # setup logging for the exctractor
        logging.getLogger('pyclowder').setLevel(logging.DEBUG)
        logging.getLogger('__main__').setLevel(logging.DEBUG)
Beispiel #12
0
    def __init__(self):
        Extractor.__init__(self)

        self.parser.add_argument(
            '--plots',
            dest="plots_shp",
            type=str,
            nargs='?',
            default=
            "/home/extractor/extractors-metadata/sensorposition/shp/sorghumexpfall2016v5/sorghumexpfall2016v5_lblentry_1to7.shp",
            help=".shp file containing plots")

        # parse command line and load default logging configuration
        self.setup()

        # setup logging for the exctractor
        logging.getLogger('pyclowder').setLevel(logging.DEBUG)
        logging.getLogger('__main__').setLevel(logging.DEBUG)

        self.plots_shp = self.args.plots_shp
Beispiel #13
0
    def __init__(self):
        Extractor.__init__(self)

        # add any additional arguments to parser
        # self.parser.add_argument('--max', '-m', type=int, nargs='?', default=-1,
        #                          help='maximum number (default=-1)')
        self.parser.add_argument(
            '--sensor',
            dest="sensor_name",
            type=str,
            nargs='?',
            default=('AZMET Maricopa Weather Station'),
            help="sensor name where streams and datapoints should be posted")
        self.parser.add_argument(
            '--aggregation',
            dest="agg_cutoff",
            type=int,
            nargs='?',
            default=(300),
            help="minute chunks to aggregate records into (default is 5 mins)")
        self.parser.add_argument('--influxHost',
                                 dest="influx_host",
                                 type=str,
                                 nargs='?',
                                 default="terra-logging.ncsa.illinois.edu",
                                 help="InfluxDB URL for logging")
        self.parser.add_argument('--influxPort',
                                 dest="influx_port",
                                 type=int,
                                 nargs='?',
                                 default=8086,
                                 help="InfluxDB port")
        self.parser.add_argument('--influxUser',
                                 dest="influx_user",
                                 type=str,
                                 nargs='?',
                                 default="terra",
                                 help="InfluxDB username")
        self.parser.add_argument('--influxPass',
                                 dest="influx_pass",
                                 type=str,
                                 nargs='?',
                                 default="",
                                 help="InfluxDB password")
        self.parser.add_argument('--influxDB',
                                 dest="influx_db",
                                 type=str,
                                 nargs='?',
                                 default="extractor_db",
                                 help="InfluxDB databast")

        # parse command line and load default logging configuration
        self.setup()

        # setup logging for the exctractor
        logging.getLogger('pyclowder').setLevel(logging.DEBUG)
        logging.getLogger('__main__').setLevel(logging.DEBUG)

        # assign other arguments
        self.sensor_name = self.args.sensor_name
        self.agg_cutoff = self.args.agg_cutoff
        self.influx_host = self.args.influx_host
        self.influx_port = self.args.influx_port
        self.influx_user = self.args.influx_user
        self.influx_pass = self.args.influx_pass
        self.influx_db = self.args.influx_db
    def __init__(self):
        Extractor.__init__(self)

        # add any additional arguments to parser
        # self.parser.add_argument('--max', '-m', type=int, nargs='?', default=-1,
        #                          help='maximum number (default=-1)')
        self.parser.add_argument(
            '--output',
            '-o',
            dest="output_dir",
            type=str,
            nargs='?',
            default=
            "/home/extractor/sites/ua-mac/Level_1/stereoTop_canopyCover",
            help=
            "root directory where timestamp & output directories will be created"
        )
        self.parser.add_argument(
            '--overwrite',
            dest="force_overwrite",
            type=bool,
            nargs='?',
            default=False,
            help=
            "whether to overwrite output file if it already exists in output directory"
        )
        self.parser.add_argument(
            '--betyURL',
            dest="bety_url",
            type=str,
            nargs='?',
            default=
            "https://terraref.ncsa.illinois.edu/bety/api/beta/traits.csv",
            help=
            "traits API endpoint of BETY instance that outputs should be posted to"
        )
        self.parser.add_argument(
            '--betyKey',
            dest="bety_key",
            type=str,
            nargs='?',
            default=False,
            help="API key for BETY instance specified by betyURL")
        self.parser.add_argument(
            '--plots',
            dest="plots_shp",
            type=str,
            nargs='?',
            default=
            "/home/extractor/extractors-metadata/sensorposition/shp/sorghumexpfall2016v5/sorghumexpfall2016v5_lblentry_1to7.shp",
            help=".shp file containing plots")
        self.parser.add_argument('--influxHost',
                                 dest="influx_host",
                                 type=str,
                                 nargs='?',
                                 default="terra-logging.ncsa.illinois.edu",
                                 help="InfluxDB URL for logging")
        self.parser.add_argument('--influxPort',
                                 dest="influx_port",
                                 type=int,
                                 nargs='?',
                                 default=8086,
                                 help="InfluxDB port")
        self.parser.add_argument('--influxUser',
                                 dest="influx_user",
                                 type=str,
                                 nargs='?',
                                 default="terra",
                                 help="InfluxDB username")
        self.parser.add_argument('--influxPass',
                                 dest="influx_pass",
                                 type=str,
                                 nargs='?',
                                 default="",
                                 help="InfluxDB password")
        self.parser.add_argument('--influxDB',
                                 dest="influx_db",
                                 type=str,
                                 nargs='?',
                                 default="extractor_db",
                                 help="InfluxDB databast")

        # parse command line and load default logging configuration
        self.setup()

        # setup logging for the exctractor
        logging.getLogger('pyclowder').setLevel(logging.DEBUG)
        logging.getLogger('__main__').setLevel(logging.DEBUG)

        # assign other arguments
        self.output_dir = self.args.output_dir
        self.force_overwrite = self.args.force_overwrite
        self.bety_url = self.args.bety_url
        self.bety_key = self.args.bety_key
        self.plots_shp = self.args.plots_shp
        self.influx_host = self.args.influx_host
        self.influx_port = self.args.influx_port
        self.influx_user = self.args.influx_user
        self.influx_pass = self.args.influx_pass
        self.influx_db = self.args.influx_db
    def __init__(self):
        Extractor.__init__(self)

        influx_host = os.getenv("INFLUXDB_HOST",
                                "terra-logging.ncsa.illinois.edu")
        influx_port = os.getenv("INFLUXDB_PORT", 8086)
        influx_db = os.getenv("INFLUXDB_DB", "extractor_db")
        influx_user = os.getenv("INFLUXDB_USER", "terra")
        influx_pass = os.getenv("INFLUXDB_PASSWORD", "")

        # add any additional arguments to parser
        self.parser.add_argument(
            '--output',
            '-o',
            dest="output_dir",
            type=str,
            nargs='?',
            default="/home/extractor/sites/ua-mac/Level_1/stereoTop_geotiff",
            help=
            "root directory where timestamp & output directories will be created"
        )
        self.parser.add_argument(
            '--overwrite',
            dest="force_overwrite",
            type=bool,
            nargs='?',
            default=False,
            help=
            "whether to overwrite output file if it already exists in output directory"
        )
        self.parser.add_argument('--influxHost',
                                 dest="influx_host",
                                 type=str,
                                 nargs='?',
                                 default="terra-logging.ncsa.illinois.edu",
                                 help="InfluxDB URL for logging")
        self.parser.add_argument('--influxPort',
                                 dest="influx_port",
                                 type=int,
                                 nargs='?',
                                 default=8086,
                                 help="InfluxDB port")
        self.parser.add_argument('--influxUser',
                                 dest="influx_user",
                                 type=str,
                                 nargs='?',
                                 default="terra",
                                 help="InfluxDB username")
        self.parser.add_argument('--influxPass',
                                 dest="influx_pass",
                                 type=str,
                                 nargs='?',
                                 default=influx_pass,
                                 help="InfluxDB password")
        self.parser.add_argument('--influxDB',
                                 dest="influx_db",
                                 type=str,
                                 nargs='?',
                                 default="extractor_db",
                                 help="InfluxDB database")

        # parse command line and load default logging configuration
        self.setup()

        # setup logging for the exctractor
        logging.getLogger('pyclowder').setLevel(logging.DEBUG)
        logging.getLogger('__main__').setLevel(logging.DEBUG)

        # assign other arguments
        self.output_dir = self.args.output_dir
        self.force_overwrite = self.args.force_overwrite
        self.influx_params = {
            "host": influx_host,
            "port": influx_port,
            "db": influx_db,
            "user": influx_user,
            "pass": influx_pass
        }
    def __init__(self):
        Extractor.__init__(self)

        # add any additional arguments to parser
        self.parser.add_argument(
            '--output',
            '-o',
            dest="output_dir",
            type=str,
            nargs='?',
            default="/home/extractor/sites/ua-mac/Level_1/fullfield",
            help=
            "root directory where timestamp & output directories will be created"
        )
        self.parser.add_argument(
            '--overwrite',
            dest="force_overwrite",
            type=bool,
            nargs='?',
            default=False,
            help=
            "whether to overwrite output file if it already exists in output directory"
        )
        self.parser.add_argument('--mainspace',
                                 dest="mainspace",
                                 type=str,
                                 nargs='?',
                                 default="58da6b924f0c430e2baa823f",
                                 help="Space UUID in Clowder to store results")
        self.parser.add_argument('--influxHost',
                                 dest="influx_host",
                                 type=str,
                                 nargs='?',
                                 default="terra-logging.ncsa.illinois.edu",
                                 help="InfluxDB URL for logging")
        self.parser.add_argument('--influxPort',
                                 dest="influx_port",
                                 type=int,
                                 nargs='?',
                                 default=8086,
                                 help="InfluxDB port")
        self.parser.add_argument('--influxUser',
                                 dest="influx_user",
                                 type=str,
                                 nargs='?',
                                 default="terra",
                                 help="InfluxDB username")
        self.parser.add_argument('--influxPass',
                                 dest="influx_pass",
                                 type=str,
                                 nargs='?',
                                 default="",
                                 help="InfluxDB password")
        self.parser.add_argument('--influxDB',
                                 dest="influx_db",
                                 type=str,
                                 nargs='?',
                                 default="extractor_db",
                                 help="InfluxDB databast")

        # parse command line and load default logging configuration
        self.setup()

        # setup logging for the exctractor
        logging.getLogger('pyclowder').setLevel(logging.DEBUG)
        logging.getLogger('__main__').setLevel(logging.DEBUG)

        # assign other arguments
        self.output_dir = self.args.output_dir
        self.force_overwrite = self.args.force_overwrite
        self.mainspace = self.args.mainspace
        self.influx_host = self.args.influx_host
        self.influx_port = self.args.influx_port
        self.influx_user = self.args.influx_user
        self.influx_pass = self.args.influx_pass
        self.influx_db = self.args.influx_db