def get(self):
index = Index(INDEX_DB_URL)
collection_name = request.args.get('collection_name')
start_time = request.args.get('start_time')
end_time = request.args.get('end_time')
start_time = timestamp_parser.parse_datetime(
start_time, default=timestamp_parser.min_datetime)
end_time = timestamp_parser.parse_datetime(
end_time, default=timestamp_parser.max_datetime)
# refresh if needed (might be time consuming)
scraper = CollectionRefreshScraper(index)
scraper.set_refresh_scope(collection_name, start_time, end_time)
driver = ScraperDriver(scraper, 4, 0.5)
driver.harvest()
scraper.sync_index()
# retrieve from index DB
result = index.get_granules(collection_name, start_time, end_time)
print("read index for %s" % collection_name, flush=True)
return result
def _setup_tag_invalid_meta_data_file(confirm_nuke):
exif = ExifEditor(Configuration())
index = Index(TEST_INDEX_LOCATION, exif)
core = TieCoreImpl(exif, index)
cli.run_cmd(["cp", READ_FILE, WRITE_FILE])
frontend = FrontendTest(confirm_nuke, [])
tie_main.run(core, RunOptions(["tag", "foo", WRITE_FILE]), frontend)
def post(self):
print("1 TRAP SIGURG on " + str(os.getpid()))
debug_thread = threading.Thread(target=debug_dump_watch)
debug_thread.start()
# get params
# collection_name = request.form.get('collection_name')
catalog_url = request.form.get('catalog_url')
if 'thredds.ucar.edu' in catalog_url:
Dataset.default_authority = 'edu.ucar.unidata'
if 'rda.ucar.edu' in catalog_url:
Dataset.default_authority = 'edu.ucar.rda'
job_id = str(uuid.uuid4())[:8] # short id for uniqueness
index = Index(INDEX_DB_URL)
output_dir = RECORDS_DIR + '/result.' + job_id
scraper = CollectionImportScraper(output_dir, index)
scraper.add_catalog(catalog_url=catalog_url)
driver = ScraperDriver(scraper, 40, 1)
driver.harvest()
scraper.sync_index()
# complete
print("indexing harvest complete", flush=True)
print("importing %s" % output_dir, flush=True)
print("")
PycswHelper().load_records(output_dir)
return output_dir
def main(*args):
try:
setup_sys_path()
configuration = config.load_user_config()
run_options = RunOptions(list(args[1:]))
frontend_type = run_options.frontend
front_end = ff.from_type(frontend_type)
index_root_dir = configuration.index_path
exif = ExifEditor(configuration)
index = Index(index_root_dir, exif)
core = TieCoreImpl(exif, index)
if run_options.action == Action.help:
print_usage()
else:
tie_main.run(core, run_options, front_end)
except ParseError as parse_error:
printerr("Error: " + parse_error.msg)
sys.exit(EXIT_CODE_PARSE_ERROR)
except InvalidMetaDataError as meta_data_error:
printerr("Error: " + meta_data_error.msg)
sys.exit(EXIT_CODE_INVALID_META_DATA)
except KeyboardInterrupt:
printerr("Application aborted by user")
sys.exit(EXIT_CODE_INVALID_META_DATA)
except FileNotFoundError:
# No need to print it. this is already done by subprocess
sys.exit(EXIT_CODE_FILE_NOT_FOUND)
except CalledProcessError:
# No need to print it. this is already done by subprocess
sys.exit(EXIT_CODE_UNKNOWN_SUBPROCESS_ERROR)
def setUp(self):
_remove_index()
self.exif = ee.ExifEditor(Configuration())
self.files_base_path = _path_to_linkname(os.path.abspath("../res"))
self.index = Index(TEST_INDEX_LOCATION, self.exif)
self.tie_core = TieCoreImpl(self.exif, self.index)
def setUp(self):
_remove_index()
self.index = Index(TEST_INDEX_LOCATION, ee.ExifEditor(Configuration()))
self.files_base_path = _path_to_linkname(os.path.abspath("../res"))
def index(index_path):
return Index(index_path)