Python add_dataset_to_sciwms Examples

Example #1

File: get_ncml_daily_iso.py Project: alexisbeudin/usgs-cmg-portal

logger_name = 'thredds_crawler'
logger = logging.getLogger(logger_name)
fh = logging.handlers.RotatingFileHandler(
    '/usgs/data0/iso/logs/iso_harvest.log',
    maxBytes=1024 * 1024 * 10,
    backupCount=5)
fh.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
    '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
logger.addHandler(fh)
logger.addHandler(ch)
logger.setLevel(logging.DEBUG)

skips = Crawl.SKIPS + ['.*MATLAB.*', '\..*']
select = ['.*00_dir.*.ncml']

metadata_files = get_metadata(thredds_servers=THREDDS_SERVERS,
                              save_dir=SAVE_DIR,
                              skips=skips,
                              select=select,
                              logger_name=None)
add_dataset_to_sciwms(rest_url=SCIWMS_REST_URL,
                      user=USER,
                      password=PASSWORD,
                      metadata_files=metadata_files,
                      logger_name=None)

Example #2

    '/opt/docker/harvest/logs/iso_harvest.log',
    maxBytes=1024 * 1024 * 10,
    backupCount=5)
fh.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
    '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
logger.addHandler(fh)
logger.addHandler(ch)
logger.setLevel(logging.DEBUG)

skips = Crawl.SKIPS + ['.*MATLAB.*', '\..*']
select = ['.*00_dir.*.ncml']

metadata_files = get_metadata(thredds_servers=THREDDS_SERVERS,
                              save_dir=SAVE_DIR,
                              skips=skips,
                              select=select,
                              logger_name=None)
'''
add_dataset_to_sciwms(rest_url=SCIWMS_REST_URL,
                      user=USER,
                      password=PASSWORD,
                      metadata_files=metadata_files,
                      logger_name=None
                      )
'''

Example #3

File: get_ts_iso.py Project: dkralston/usgs-cmg-portal

logger_name = 'thredds_crawler'
logger = logging.getLogger('thredds_crawler')
fh = logging.handlers.RotatingFileHandler('/usgs/data0/iso/logs/iso_harvest.log', maxBytes=1024*1024*10, backupCount=5)
fh.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
logger.addHandler(fh)
logger.addHandler(ch)
logger.setLevel(logging.DEBUG)

SAVE_DIR="/usgs/data0/iso/iso_records"

THREDDS_SERVERS = {
   "usgs-ts":   "http://geoport.whoi.edu/thredds/catalog/usgs/data2/emontgomery/stellwagen/CF-1.6/catalog.html"
}

metadata_files = get_metadata(thredds_servers=THREDDS_SERVERS,
                              save_dir=SAVE_DIR,
                              logger_name=logger_name
                              )
add_dataset_to_sciwms(rest_url=SCIWMS_REST_URL,
                      user=USER,
                      password=PASSWORD,
                      metadata_files=metadata_files,
                      logger_name=logger_name
                      )

Example #4

File: get_daily_iso.py Project: USGS-CMG/usgs-cmg-portal

ch.setFormatter(formatter)
logger.addHandler(fh)
logger.addHandler(ch)
logger.setLevel(logging.DEBUG)

SAVE_DIR="/opt/docker/pycsw/force/iso_records"

'''
THREDDS_SERVERS = {
    "coawst":   "https://gamone.whoi.edu/thredds/catalog/coawst_4/use/fmrc/catalog.html",
    "estofs_wh": "http://geoport-dev.whoi.edu/thredds/estofs_agg.html",
    "useast":  "http://omgsrv1.meas.ncsu.edu:8080/thredds/catalog/fmrc/us_east/catalog.html",
    "smast_archives": "http://www.smast.umassd.edu:8080/thredds/archives.html"}
'''

THREDDS_SERVERS = {
    "coawst":   "https://gamone.whoi.edu/thredds/catalog/coawst_4/use/fmrc/catalog.html"}

metadata_files = get_metadata(thredds_servers=THREDDS_SERVERS,
                              save_dir=SAVE_DIR,
                              logger_name=logger_name
                              )
'''
add_dataset_to_sciwms(rest_url=SCIWMS_REST_URL,
                      user=USER,
                      password=PASSWORD,
                      metadata_files=metadata_files,
                      logger_name=logger_name
                      )
'''