def main() -> bool:
try:
RedVoxConfig.from_env()
print("Found RedVox configuration in environment")
return True
except:
pass
try:
RedVoxConfig.from_home()
print("Found RedVox configuration in user's home directory")
return True
except:
pass
return False
def cloud_client(
redvox_config: Optional[RedVoxConfig] = RedVoxConfig.find(),
refresh_token_interval: float = 600.0,
timeout: float = 10.0,
):
"""
Function that can be used within a "with" block to automatically handle the closing of open resources.
Creates and returns a CloudClient that will automatically be closed when exiting the with block or if an error
occurs.
See https://docs.python.org/3/library/contextlib.html for more info.
:param redvox_config: The Redvox endpoint configuration.
:param refresh_token_interval: An optional token refresh interval
:param timeout: An optional timeout.
:return: A CloudClient.
"""
if redvox_config is None:
raise cloud_errors.CloudApiError(
"A RedVoxConfig was not found in the environment and one wasn't provided"
)
client: CloudClient = CloudClient(redvox_config, refresh_token_interval,
timeout)
try:
yield client
finally:
if client is not None:
client.close()
def __init__(self, parent=None):
super(DataDownloadGui, self).__init__(parent)
self.setWindowTitle("RedVox Cloud Data Downloader")
self.setMinimumWidth(1600)
self.setMinimumHeight(900)
redvox_config: Optional[RedVoxConfig] = RedVoxConfig.find()
self.authentication_widget: AuthenticationWidget = AuthenticationWidget(
redvox_config)
self.server_selection_widget: ServerSelectionWidget = ServerSelectionWidget(
redvox_config)
self.window_selection_widget: WindowSelectionWidget = (
WindowSelectionWidget())
self.station_ids_widget: QTextEdit = QTextEdit()
self.api_selection_widget: ApiSelectionWidget = ApiSelectionWidget(
)
self.output_directory_widget: OutputDirectoryWidget = (
OutputDirectoryWidget())
self.download_data_btn: QPushButton = QPushButton("Download Data")
# noinspection PyUnresolvedReferences
self.download_data_btn.clicked.connect(self.download_data)
self.log: QTextEdit = QTextEdit()
self.log.setReadOnly(True)
self.info = lambda msg: self.log.append(
f"{datetime.datetime.utcnow()}: {msg}")
layout: QVBoxLayout = QVBoxLayout()
layout.addWidget(QLabel("RedVox Authentication"))
layout.addWidget(self.authentication_widget)
layout.addWidget(QLabel("Server Selection"))
layout.addWidget(self.server_selection_widget)
layout.addWidget(QLabel("Data Window Selection"))
layout.addWidget(self.window_selection_widget)
layout.addWidget(QLabel("Station IDs (one per line)"))
layout.addWidget(self.station_ids_widget)
layout.addWidget(QLabel("API Selection"))
layout.addWidget(self.api_selection_widget)
layout.addWidget(QLabel("Output Directory"))
layout.addWidget(self.output_directory_widget)
layout.addWidget(self.download_data_btn)
layout.addWidget(QLabel("Log"))
layout.addWidget(self.log)
self.setLayout(layout)
def health_check(
redvox_config: RedVoxConfig,
session: Optional[requests.Session] = None,
timeout: Optional[float] = 10.0,
) -> bool:
"""
Check that the Cloud API endpoint is up.
:param redvox_config: The API config.
:param session: An (optional) session for re-using an HTTP client.
:param timeout: An optional timeout.
:return: True if the endpoint is up, False otherwise.
"""
url: str = redvox_config.url(RoutesV1.HEALTH_CHECK)
if session:
resp: requests.Response = session.get(url, timeout=timeout)
else:
resp = requests.get(url, timeout=timeout)
if resp.status_code == 200:
return True
return False
def post_req(
redvox_config: RedVoxConfig,
route: str,
req: Any,
resp_transform: Callable[[requests.Response], Any],
session: Optional[requests.Session] = None,
timeout: Optional[float] = 10.0,
) -> Optional[Any]:
"""
Performs an HTTP POST request.
:param redvox_config: API endpoint configuration.
:param route: Route to POST to.
:param req: Request to send in POST.
:param resp_transform: Function to transform the response into something we can use.
:param session: The HTTP session.
:param timeout: An (optional) timeout.
:return: The optional response.
"""
url: str = redvox_config.url(route)
# noinspection Mypy
req_dict: Dict = req.to_dict()
try:
if session:
resp: requests.Response = session.post(url,
json=req_dict,
timeout=timeout)
else:
resp = requests.post(url, json=req_dict, timeout=timeout)
if resp.status_code == 200:
# noinspection Mypy
return resp_transform(resp)
else:
return None
except requests.RequestException as ex:
raise cloud_errors.ApiConnectionError(
f"Error making POST request to {url}: with body: {req_dict}: {ex}")
def data_req_report_args(args) -> None:
"""
Wrapper function that calls the data_req_report.
:param args: Args from argparse.
"""
if not check_out_dir(args.out_dir):
determine_exit(False)
# Rebuild RedVox config from potentially optional passed in args
if args.email is None:
log.error(
f"The argument 'email' is required, but was not found in the environment or provided."
)
determine_exit(False)
if args.password is None:
log.error(
f"The argument 'password' is required, but was not found in the environment or provided."
)
determine_exit(False)
redvox_config: RedVoxConfig = RedVoxConfig(
args.email,
args.password,
args.protocol,
args.host,
args.port,
args.secret_token,
)
determine_exit(
data_req_report(
redvox_config,
args.report_id,
args.out_dir,
args.retries,
))
def data_req_args(args) -> None:
"""
Wrapper function that calls the data_req.
:param args: Args from argparse.
"""
if not check_out_dir(args.out_dir):
determine_exit(False)
api_type: DataRangeReqType = DataRangeReqType[args.api_type]
# Rebuild RedVox config from potentially optional passed in args
if args.email is None:
log.error(
f"The argument 'email' is required, but was not found in the environment or provided."
)
determine_exit(False)
if args.password is None:
log.error(
f"The argument 'password' is required, but was not found in the environment or provided."
)
determine_exit(False)
redvox_config: RedVoxConfig = RedVoxConfig(
args.email,
args.password,
args.protocol,
args.host,
args.port,
args.secret_token,
)
determine_exit(
data_req.make_data_req(args.out_dir, redvox_config, args.req_start_s,
args.req_end_s, args.station_ids, api_type,
args.retries, args.timeout,
not args.disable_timing_correction))
def __init__(
self,
redvox_config: Optional[RedVoxConfig] = RedVoxConfig.find(),
refresh_token_interval: float = 600.0,
timeout: Optional[float] = 10.0,
):
"""
Instantiates this client.
:param redvox_config: The Redvox endpoint configuration.
:param refresh_token_interval: An optional interval in seconds that the auth token should be refreshed.
:param timeout: An optional timeout
"""
if redvox_config is None:
raise cloud_errors.CloudApiError(
"A RedVoxConfig was not found in the environment and one wasn't provided"
)
if refresh_token_interval <= 0:
raise cloud_errors.CloudApiError(
"refresh_token_interval must be strictly > 0")
if timeout is not None and (timeout <= 0):
raise cloud_errors.CloudApiError("timeout must be strictly > 0")
self.redvox_config: RedVoxConfig = redvox_config
self.refresh_token_interval: float = refresh_token_interval
self.timeout: Optional[float] = timeout
self.__session = (requests.Session()
) # This must be initialized before the auth req!
self.__refresh_timer = None
self.__authenticate()
self.__refresh_timer = threading.Timer(self.refresh_token_interval,
self.__refresh_token)
self.__refresh_timer.start()
def main():
"""
Entry point into the CLI.
"""
redvox_config: Optional[RedVoxConfig] = RedVoxConfig.find()
parser: argparse.ArgumentParser = argparse.ArgumentParser(
"redvox-cli",
description="Command line tools for viewing, converting,"
" and downloading RedVox data.",
)
parser.add_argument("--verbose",
"-v",
help="Enable verbose logging",
action="count",
default=0)
sub_parser = parser.add_subparsers()
sub_parser.required = True
sub_parser.dest = "command"
# Cloud data retrieval
cloud_download_parser = sub_parser.add_parser("cloud-download")
cloud_download_parser.set_defaults(
func=lambda _: cloud_data_retrieval.run_gui())
# Gallery
gallery_parser = sub_parser.add_parser("gallery")
gallery_parser.add_argument("rdvxm_paths",
help="One or more rdvxm files",
nargs="+")
gallery_parser.set_defaults(func=gallery_args)
# rdvxz -> rdvxm
rdvxz_to_rdvxm_parser = sub_parser.add_parser(
"rdvxz-to-rdvxm",
help="Convert rdvxz (API 900) to rdvxm (API 1000/M) files")
rdvxz_to_rdvxm_parser.add_argument(
"rdvxz_paths",
help="One or more rdvxz files to convert to json files",
nargs="+",
)
rdvxz_to_rdvxm_parser.add_argument(
"--out-dir",
"-o",
help=
"Optional output directory (will use same directory as source files by "
"default)",
)
rdvxz_to_rdvxm_parser.set_defaults(func=rdvxz_to_rdvxm)
# rdvxm -> rdvxz
rdvxm_to_rdvxz_parser = sub_parser.add_parser(
"rdvxm-to-rdvxz",
help="Convert rdvxm (API 1000/M) to rdvxx (API 900) files")
rdvxm_to_rdvxz_parser.add_argument(
"rdvxm_paths",
help="One or more rdvxm files to convert to json files",
nargs="+",
)
rdvxm_to_rdvxz_parser.add_argument(
"--out-dir",
"-o",
help=
"Optional output directory (will use same directory as source files by "
"default)",
)
rdvxm_to_rdvxz_parser.set_defaults(func=rdvxm_to_rdvxz)
# rdvxz -> json
rdvxz_to_json_parser = sub_parser.add_parser(
"rdvxz-to-json", help="Convert rdvxz files to json files")
rdvxz_to_json_parser.add_argument(
"rdvxz_paths",
help="One or more rdvxz files to convert to json files",
nargs="+",
)
rdvxz_to_json_parser.add_argument(
"--out-dir",
"-o",
help=
"Optional output directory (will use same directory as source files by "
"default)",
)
rdvxz_to_json_parser.set_defaults(func=rdvxz_to_json_args)
# rdvxm -> json
rdvxm_to_json_parser = sub_parser.add_parser(
"rdvxm-to-json", help="Convert rdvxm files to json files")
rdvxm_to_json_parser.add_argument(
"rdvxm_paths",
help="One or more rdvxm files to convert to json files",
nargs="+",
)
rdvxm_to_json_parser.add_argument(
"--out-dir",
"-o",
help=
"Optional output directory (will use same directory as source files by "
"default)",
)
rdvxm_to_json_parser.set_defaults(func=rdvxm_to_json_args)
# json -> rdvxz
json_to_rdvxz_parser = sub_parser.add_parser(
"json-to-rdvxz", help="Convert json files to rdvxz files")
json_to_rdvxz_parser.add_argument(
"json_paths",
help="One or more json files to convert to rdvxz files",
nargs="+")
json_to_rdvxz_parser.add_argument(
"--out-dir",
"-o",
help=
"Optional output directory (will use same directory as source files by "
"default)",
)
json_to_rdvxz_parser.set_defaults(func=json_to_rdvxz_args)
# json -> rdvxm
json_to_rdvxm_parser = sub_parser.add_parser(
"json-to-rdvxm", help="Convert json files to rdvxm files")
json_to_rdvxm_parser.add_argument(
"json_paths",
help="One or more json files to convert to rdvxm files",
nargs="+")
json_to_rdvxm_parser.add_argument(
"--out-dir",
"-o",
help=
"Optional output directory (will use same directory as source files by "
"default)",
)
json_to_rdvxm_parser.set_defaults(func=json_to_rdvxm_args)
# sort unstructured data into structured data
sort_unstructured_parser = sub_parser.add_parser(
"sort-unstructured",
help=
"Sorts unstructured RedVox files into their structured counterpart",
)
sort_unstructured_parser.add_argument(
"input_dir",
help=
"Directory containing RedVox files to sort into a structured layout",
)
sort_unstructured_parser.add_argument(
"--out-dir",
"-o",
help="Optional output directory (current working directory by default)",
)
sort_unstructured_parser.add_argument(
"--mv",
help=
"When set, file contents will be moved to the structured layout rather than copied.",
action="store_true",
)
sort_unstructured_parser.set_defaults(func=sort_unstructured_args)
# print rdvxz
rdvxz_print_parser = sub_parser.add_parser(
"print-z", help="Print contents of rdvxz files to stdout")
rdvxz_print_parser.add_argument("rdvxz_paths",
help="One or more rdvxz files to print",
nargs="+")
rdvxz_print_parser.set_defaults(func=rdvxz_print_stdout_args)
# print rdvxm
rdvxm_print_parser = sub_parser.add_parser(
"print-m", help="Print contents of rdvxm files to stdout")
rdvxm_print_parser.add_argument("rdvxm_paths",
help="One or more rdvxm files to print",
nargs="+")
rdvxm_print_parser.set_defaults(func=rdvxm_print_stdout_args)
# validation
rdvxm_validation_parser = sub_parser.add_parser(
"validate-m", help="Validate the structure of API M files")
rdvxm_validation_parser.add_argument(
"rdvxm_paths", help="One or more rdvxm files to print", nargs="+")
rdvxm_validation_parser.set_defaults(func=validate_rdvxm_args)
# data_req
data_req_parser = sub_parser.add_parser(
"data-req", help="Request bulk RedVox data from RedVox servers")
data_req_parser.add_argument(
"--email",
help="redvox.io account email",
default=map_or_default(redvox_config, lambda config: config.username,
None),
)
data_req_parser.add_argument(
"--password",
help="redvox.io account password",
default=map_or_default(redvox_config, lambda config: config.password,
None),
)
data_req_parser.add_argument(
"--out-dir",
help=
"The output directory that RedVox files will be written to (default=.)",
default=".",
)
data_req_parser.add_argument("--disable-timing-correction",
help="Disables query timing correction",
default=False,
action="store_true")
data_req_parser.add_argument(
"--retries",
help=
"The number of times the client should retry getting a file on failure "
"(default=1)",
default=1,
choices=set(range(0, 6)),
type=int,
)
data_req_parser.add_argument(
"--host",
help="Data server host",
default=map_or_default(redvox_config, lambda config: config.host,
"redvox.io"),
)
data_req_parser.add_argument(
"--port",
type=int,
help="Data server port",
default=map_or_default(redvox_config, lambda config: config.port,
8080),
)
data_req_parser.add_argument(
"--protocol",
help="One of either http or https",
choices=["https", "http"],
default=map_or_default(redvox_config, lambda config: config.protocol,
"https"),
)
data_req_parser.add_argument(
"--secret-token",
help=
"A shared secret token provided by RedVox required for accessing the data "
"request service",
default=map_or_default(redvox_config,
lambda config: config.secret_token, None),
)
data_req_parser.add_argument(
"--api-type",
help="Data API to be retrieved",
choices=["API_900", "API_1000", "API_900_1000"],
default="API_900_1000",
)
data_req_parser.add_argument(
"--timeout",
help="Read timeout in seconds (default=10 seconds)",
type=int,
default=10)
data_req_parser.add_argument(
"req_start_s",
type=int,
help="Data request start as number of seconds since the epoch UTC",
)
data_req_parser.add_argument(
"req_end_s",
type=int,
help="Data request end as number of seconds since the epoch UTC",
)
data_req_parser.add_argument(
"station_ids",
nargs="+",
help="A list of RedVox ids delimited by a space")
data_req_parser.set_defaults(func=data_req_args)
# data req report
data_req_report_parser = sub_parser.add_parser(
"data-req-report",
help="Request bulk RedVox data from the RedVox servers")
data_req_report_parser.add_argument(
"--out-dir",
help=
"The output directory that RedVox files will be written to (default=.)",
default=".",
)
data_req_report_parser.add_argument(
"--email",
help="redvox.io account email",
default=map_or_default(redvox_config, lambda config: config.username,
None),
)
data_req_report_parser.add_argument(
"--password",
help="redvox.io account password",
default=map_or_default(redvox_config, lambda config: config.password,
None),
)
data_req_report_parser.add_argument(
"--retries",
help=
"The number of times the client should retry getting a file on failure "
"(default=1)",
default=1,
choices=set(range(0, 6)),
type=int,
)
data_req_report_parser.add_argument(
"--host",
help="Data server host",
default=map_or_default(redvox_config, lambda config: config.host,
"redvox.io"),
)
data_req_report_parser.add_argument(
"--port",
type=int,
help="Data server port",
default=map_or_default(redvox_config, lambda config: config.port,
8080),
)
data_req_report_parser.add_argument(
"--protocol",
help="One of either http or https (default https)",
choices=["https", "http"],
default=map_or_default(redvox_config, lambda config: config.protocol,
"https"),
)
data_req_report_parser.add_argument(
"--secret-token",
help=
"A shared secret token provided by RedVox required for accessing the data "
"request service",
default=map_or_default(redvox_config,
lambda config: config.secret_token, None),
)
data_req_report_parser.add_argument(
"report_id",
type=str,
help="The full report id that data is being requested for",
)
data_req_report_parser.set_defaults(func=data_req_report_args)
# Parse the args
args = parser.parse_args()
# Setup logging
log_levels: Dict[int, str] = {0: "WARN", 1: "INFO", 2: "DEBUG"}
log_level: str = (log_levels[args.verbose]
if args.verbose in log_levels else log_levels[0])
logging.basicConfig(
level=log_level,
format=
"[%(levelname)s:%(process)d:%(filename)s:%(module)s:%(funcName)s:%(lineno)d:%(asctime)s]"
" %(message)s",
)
log.info("Running with args=%s and log_level=%s", str(args), log_level)
# Try calling the appropriate handler
# pylint: disable=W0703
try:
args.func(args)
except Exception as e:
log.error("Encountered an error: %s", str(e))
sys.exit(1)
def download_data(self):
validation_errors: List[str] = self.validate()
if len(validation_errors) > 0:
self.info("Errors encountered")
for error in validation_errors:
self.info(error)
return
username: str = self.authentication_widget.username.text()
password: str = self.authentication_widget.password.text()
protocol: str
if self.server_selection_widget.https.isChecked():
protocol = "https"
else:
protocol = "http"
host: str = self.server_selection_widget.host.text()
port: int = int(self.server_selection_widget.port.text())
out_dir: str = self.output_directory_widget.output_dir.text()
redvox_config: RedVoxConfig = RedVoxConfig(username, password,
protocol, host, port)
req_start: int = int(self.window_selection_widget.start_dt.
dateTime().toSecsSinceEpoch())
req_end: int = int(self.window_selection_widget.end_dt.dateTime().
toSecsSinceEpoch())
station_ids: List[str] = list(
map(str.strip,
self.station_ids_widget.toPlainText().splitlines(False)))
api_type: DataRangeReqType
if self.api_selection_widget.api_900_1000.isChecked():
# noinspection PyTypeChecker
api_type = DataRangeReqType.API_900_1000
elif self.api_selection_widget.api_1000.isChecked():
# noinspection PyTypeChecker
api_type = DataRangeReqType.API_1000
else:
# noinspection PyTypeChecker
api_type = DataRangeReqType.API_900
retries: int = int(self.server_selection_widget.retries.text())
timeout: int = int(self.server_selection_widget.timeout.text())
correct_query_timing: bool = (
not self.server_selection_widget.disable_timing_correction.
isChecked())
manager = Manager()
out_queue = manager.Queue(1024)
self.info("Starting data query")
process = Process(
target=download_process,
args=(
redvox_config,
timeout,
req_start,
req_end,
station_ids,
api_type,
correct_query_timing,
out_dir,
retries,
out_queue,
),
)
process.start()
result: str = ""
while result != "done":
# noinspection PyBroadException
try:
result = out_queue.get(block=True, timeout=0.1)
self.info(result)
QApplication.processEvents()
except:
pass