def main() -> bool: try: RedVoxConfig.from_env() print("Found RedVox configuration in environment") return True except: pass try: RedVoxConfig.from_home() print("Found RedVox configuration in user's home directory") return True except: pass return False
def cloud_client( redvox_config: Optional[RedVoxConfig] = RedVoxConfig.find(), refresh_token_interval: float = 600.0, timeout: float = 10.0, ): """ Function that can be used within a "with" block to automatically handle the closing of open resources. Creates and returns a CloudClient that will automatically be closed when exiting the with block or if an error occurs. See https://docs.python.org/3/library/contextlib.html for more info. :param redvox_config: The Redvox endpoint configuration. :param refresh_token_interval: An optional token refresh interval :param timeout: An optional timeout. :return: A CloudClient. """ if redvox_config is None: raise cloud_errors.CloudApiError( "A RedVoxConfig was not found in the environment and one wasn't provided" ) client: CloudClient = CloudClient(redvox_config, refresh_token_interval, timeout) try: yield client finally: if client is not None: client.close()
def __init__(self, parent=None): super(DataDownloadGui, self).__init__(parent) self.setWindowTitle("RedVox Cloud Data Downloader") self.setMinimumWidth(1600) self.setMinimumHeight(900) redvox_config: Optional[RedVoxConfig] = RedVoxConfig.find() self.authentication_widget: AuthenticationWidget = AuthenticationWidget( redvox_config) self.server_selection_widget: ServerSelectionWidget = ServerSelectionWidget( redvox_config) self.window_selection_widget: WindowSelectionWidget = ( WindowSelectionWidget()) self.station_ids_widget: QTextEdit = QTextEdit() self.api_selection_widget: ApiSelectionWidget = ApiSelectionWidget( ) self.output_directory_widget: OutputDirectoryWidget = ( OutputDirectoryWidget()) self.download_data_btn: QPushButton = QPushButton("Download Data") # noinspection PyUnresolvedReferences self.download_data_btn.clicked.connect(self.download_data) self.log: QTextEdit = QTextEdit() self.log.setReadOnly(True) self.info = lambda msg: self.log.append( f"{datetime.datetime.utcnow()}: {msg}") layout: QVBoxLayout = QVBoxLayout() layout.addWidget(QLabel("RedVox Authentication")) layout.addWidget(self.authentication_widget) layout.addWidget(QLabel("Server Selection")) layout.addWidget(self.server_selection_widget) layout.addWidget(QLabel("Data Window Selection")) layout.addWidget(self.window_selection_widget) layout.addWidget(QLabel("Station IDs (one per line)")) layout.addWidget(self.station_ids_widget) layout.addWidget(QLabel("API Selection")) layout.addWidget(self.api_selection_widget) layout.addWidget(QLabel("Output Directory")) layout.addWidget(self.output_directory_widget) layout.addWidget(self.download_data_btn) layout.addWidget(QLabel("Log")) layout.addWidget(self.log) self.setLayout(layout)
def health_check( redvox_config: RedVoxConfig, session: Optional[requests.Session] = None, timeout: Optional[float] = 10.0, ) -> bool: """ Check that the Cloud API endpoint is up. :param redvox_config: The API config. :param session: An (optional) session for re-using an HTTP client. :param timeout: An optional timeout. :return: True if the endpoint is up, False otherwise. """ url: str = redvox_config.url(RoutesV1.HEALTH_CHECK) if session: resp: requests.Response = session.get(url, timeout=timeout) else: resp = requests.get(url, timeout=timeout) if resp.status_code == 200: return True return False
def post_req( redvox_config: RedVoxConfig, route: str, req: Any, resp_transform: Callable[[requests.Response], Any], session: Optional[requests.Session] = None, timeout: Optional[float] = 10.0, ) -> Optional[Any]: """ Performs an HTTP POST request. :param redvox_config: API endpoint configuration. :param route: Route to POST to. :param req: Request to send in POST. :param resp_transform: Function to transform the response into something we can use. :param session: The HTTP session. :param timeout: An (optional) timeout. :return: The optional response. """ url: str = redvox_config.url(route) # noinspection Mypy req_dict: Dict = req.to_dict() try: if session: resp: requests.Response = session.post(url, json=req_dict, timeout=timeout) else: resp = requests.post(url, json=req_dict, timeout=timeout) if resp.status_code == 200: # noinspection Mypy return resp_transform(resp) else: return None except requests.RequestException as ex: raise cloud_errors.ApiConnectionError( f"Error making POST request to {url}: with body: {req_dict}: {ex}")
def data_req_report_args(args) -> None: """ Wrapper function that calls the data_req_report. :param args: Args from argparse. """ if not check_out_dir(args.out_dir): determine_exit(False) # Rebuild RedVox config from potentially optional passed in args if args.email is None: log.error( f"The argument 'email' is required, but was not found in the environment or provided." ) determine_exit(False) if args.password is None: log.error( f"The argument 'password' is required, but was not found in the environment or provided." ) determine_exit(False) redvox_config: RedVoxConfig = RedVoxConfig( args.email, args.password, args.protocol, args.host, args.port, args.secret_token, ) determine_exit( data_req_report( redvox_config, args.report_id, args.out_dir, args.retries, ))
def data_req_args(args) -> None: """ Wrapper function that calls the data_req. :param args: Args from argparse. """ if not check_out_dir(args.out_dir): determine_exit(False) api_type: DataRangeReqType = DataRangeReqType[args.api_type] # Rebuild RedVox config from potentially optional passed in args if args.email is None: log.error( f"The argument 'email' is required, but was not found in the environment or provided." ) determine_exit(False) if args.password is None: log.error( f"The argument 'password' is required, but was not found in the environment or provided." ) determine_exit(False) redvox_config: RedVoxConfig = RedVoxConfig( args.email, args.password, args.protocol, args.host, args.port, args.secret_token, ) determine_exit( data_req.make_data_req(args.out_dir, redvox_config, args.req_start_s, args.req_end_s, args.station_ids, api_type, args.retries, args.timeout, not args.disable_timing_correction))
def __init__( self, redvox_config: Optional[RedVoxConfig] = RedVoxConfig.find(), refresh_token_interval: float = 600.0, timeout: Optional[float] = 10.0, ): """ Instantiates this client. :param redvox_config: The Redvox endpoint configuration. :param refresh_token_interval: An optional interval in seconds that the auth token should be refreshed. :param timeout: An optional timeout """ if redvox_config is None: raise cloud_errors.CloudApiError( "A RedVoxConfig was not found in the environment and one wasn't provided" ) if refresh_token_interval <= 0: raise cloud_errors.CloudApiError( "refresh_token_interval must be strictly > 0") if timeout is not None and (timeout <= 0): raise cloud_errors.CloudApiError("timeout must be strictly > 0") self.redvox_config: RedVoxConfig = redvox_config self.refresh_token_interval: float = refresh_token_interval self.timeout: Optional[float] = timeout self.__session = (requests.Session() ) # This must be initialized before the auth req! self.__refresh_timer = None self.__authenticate() self.__refresh_timer = threading.Timer(self.refresh_token_interval, self.__refresh_token) self.__refresh_timer.start()
def main(): """ Entry point into the CLI. """ redvox_config: Optional[RedVoxConfig] = RedVoxConfig.find() parser: argparse.ArgumentParser = argparse.ArgumentParser( "redvox-cli", description="Command line tools for viewing, converting," " and downloading RedVox data.", ) parser.add_argument("--verbose", "-v", help="Enable verbose logging", action="count", default=0) sub_parser = parser.add_subparsers() sub_parser.required = True sub_parser.dest = "command" # Cloud data retrieval cloud_download_parser = sub_parser.add_parser("cloud-download") cloud_download_parser.set_defaults( func=lambda _: cloud_data_retrieval.run_gui()) # Gallery gallery_parser = sub_parser.add_parser("gallery") gallery_parser.add_argument("rdvxm_paths", help="One or more rdvxm files", nargs="+") gallery_parser.set_defaults(func=gallery_args) # rdvxz -> rdvxm rdvxz_to_rdvxm_parser = sub_parser.add_parser( "rdvxz-to-rdvxm", help="Convert rdvxz (API 900) to rdvxm (API 1000/M) files") rdvxz_to_rdvxm_parser.add_argument( "rdvxz_paths", help="One or more rdvxz files to convert to json files", nargs="+", ) rdvxz_to_rdvxm_parser.add_argument( "--out-dir", "-o", help= "Optional output directory (will use same directory as source files by " "default)", ) rdvxz_to_rdvxm_parser.set_defaults(func=rdvxz_to_rdvxm) # rdvxm -> rdvxz rdvxm_to_rdvxz_parser = sub_parser.add_parser( "rdvxm-to-rdvxz", help="Convert rdvxm (API 1000/M) to rdvxx (API 900) files") rdvxm_to_rdvxz_parser.add_argument( "rdvxm_paths", help="One or more rdvxm files to convert to json files", nargs="+", ) rdvxm_to_rdvxz_parser.add_argument( "--out-dir", "-o", help= "Optional output directory (will use same directory as source files by " "default)", ) rdvxm_to_rdvxz_parser.set_defaults(func=rdvxm_to_rdvxz) # rdvxz -> json rdvxz_to_json_parser = sub_parser.add_parser( "rdvxz-to-json", help="Convert rdvxz files to json files") rdvxz_to_json_parser.add_argument( "rdvxz_paths", help="One or more rdvxz files to convert to json files", nargs="+", ) rdvxz_to_json_parser.add_argument( "--out-dir", "-o", help= "Optional output directory (will use same directory as source files by " "default)", ) rdvxz_to_json_parser.set_defaults(func=rdvxz_to_json_args) # rdvxm -> json rdvxm_to_json_parser = sub_parser.add_parser( "rdvxm-to-json", help="Convert rdvxm files to json files") rdvxm_to_json_parser.add_argument( "rdvxm_paths", help="One or more rdvxm files to convert to json files", nargs="+", ) rdvxm_to_json_parser.add_argument( "--out-dir", "-o", help= "Optional output directory (will use same directory as source files by " "default)", ) rdvxm_to_json_parser.set_defaults(func=rdvxm_to_json_args) # json -> rdvxz json_to_rdvxz_parser = sub_parser.add_parser( "json-to-rdvxz", help="Convert json files to rdvxz files") json_to_rdvxz_parser.add_argument( "json_paths", help="One or more json files to convert to rdvxz files", nargs="+") json_to_rdvxz_parser.add_argument( "--out-dir", "-o", help= "Optional output directory (will use same directory as source files by " "default)", ) json_to_rdvxz_parser.set_defaults(func=json_to_rdvxz_args) # json -> rdvxm json_to_rdvxm_parser = sub_parser.add_parser( "json-to-rdvxm", help="Convert json files to rdvxm files") json_to_rdvxm_parser.add_argument( "json_paths", help="One or more json files to convert to rdvxm files", nargs="+") json_to_rdvxm_parser.add_argument( "--out-dir", "-o", help= "Optional output directory (will use same directory as source files by " "default)", ) json_to_rdvxm_parser.set_defaults(func=json_to_rdvxm_args) # sort unstructured data into structured data sort_unstructured_parser = sub_parser.add_parser( "sort-unstructured", help= "Sorts unstructured RedVox files into their structured counterpart", ) sort_unstructured_parser.add_argument( "input_dir", help= "Directory containing RedVox files to sort into a structured layout", ) sort_unstructured_parser.add_argument( "--out-dir", "-o", help="Optional output directory (current working directory by default)", ) sort_unstructured_parser.add_argument( "--mv", help= "When set, file contents will be moved to the structured layout rather than copied.", action="store_true", ) sort_unstructured_parser.set_defaults(func=sort_unstructured_args) # print rdvxz rdvxz_print_parser = sub_parser.add_parser( "print-z", help="Print contents of rdvxz files to stdout") rdvxz_print_parser.add_argument("rdvxz_paths", help="One or more rdvxz files to print", nargs="+") rdvxz_print_parser.set_defaults(func=rdvxz_print_stdout_args) # print rdvxm rdvxm_print_parser = sub_parser.add_parser( "print-m", help="Print contents of rdvxm files to stdout") rdvxm_print_parser.add_argument("rdvxm_paths", help="One or more rdvxm files to print", nargs="+") rdvxm_print_parser.set_defaults(func=rdvxm_print_stdout_args) # validation rdvxm_validation_parser = sub_parser.add_parser( "validate-m", help="Validate the structure of API M files") rdvxm_validation_parser.add_argument( "rdvxm_paths", help="One or more rdvxm files to print", nargs="+") rdvxm_validation_parser.set_defaults(func=validate_rdvxm_args) # data_req data_req_parser = sub_parser.add_parser( "data-req", help="Request bulk RedVox data from RedVox servers") data_req_parser.add_argument( "--email", help="redvox.io account email", default=map_or_default(redvox_config, lambda config: config.username, None), ) data_req_parser.add_argument( "--password", help="redvox.io account password", default=map_or_default(redvox_config, lambda config: config.password, None), ) data_req_parser.add_argument( "--out-dir", help= "The output directory that RedVox files will be written to (default=.)", default=".", ) data_req_parser.add_argument("--disable-timing-correction", help="Disables query timing correction", default=False, action="store_true") data_req_parser.add_argument( "--retries", help= "The number of times the client should retry getting a file on failure " "(default=1)", default=1, choices=set(range(0, 6)), type=int, ) data_req_parser.add_argument( "--host", help="Data server host", default=map_or_default(redvox_config, lambda config: config.host, "redvox.io"), ) data_req_parser.add_argument( "--port", type=int, help="Data server port", default=map_or_default(redvox_config, lambda config: config.port, 8080), ) data_req_parser.add_argument( "--protocol", help="One of either http or https", choices=["https", "http"], default=map_or_default(redvox_config, lambda config: config.protocol, "https"), ) data_req_parser.add_argument( "--secret-token", help= "A shared secret token provided by RedVox required for accessing the data " "request service", default=map_or_default(redvox_config, lambda config: config.secret_token, None), ) data_req_parser.add_argument( "--api-type", help="Data API to be retrieved", choices=["API_900", "API_1000", "API_900_1000"], default="API_900_1000", ) data_req_parser.add_argument( "--timeout", help="Read timeout in seconds (default=10 seconds)", type=int, default=10) data_req_parser.add_argument( "req_start_s", type=int, help="Data request start as number of seconds since the epoch UTC", ) data_req_parser.add_argument( "req_end_s", type=int, help="Data request end as number of seconds since the epoch UTC", ) data_req_parser.add_argument( "station_ids", nargs="+", help="A list of RedVox ids delimited by a space") data_req_parser.set_defaults(func=data_req_args) # data req report data_req_report_parser = sub_parser.add_parser( "data-req-report", help="Request bulk RedVox data from the RedVox servers") data_req_report_parser.add_argument( "--out-dir", help= "The output directory that RedVox files will be written to (default=.)", default=".", ) data_req_report_parser.add_argument( "--email", help="redvox.io account email", default=map_or_default(redvox_config, lambda config: config.username, None), ) data_req_report_parser.add_argument( "--password", help="redvox.io account password", default=map_or_default(redvox_config, lambda config: config.password, None), ) data_req_report_parser.add_argument( "--retries", help= "The number of times the client should retry getting a file on failure " "(default=1)", default=1, choices=set(range(0, 6)), type=int, ) data_req_report_parser.add_argument( "--host", help="Data server host", default=map_or_default(redvox_config, lambda config: config.host, "redvox.io"), ) data_req_report_parser.add_argument( "--port", type=int, help="Data server port", default=map_or_default(redvox_config, lambda config: config.port, 8080), ) data_req_report_parser.add_argument( "--protocol", help="One of either http or https (default https)", choices=["https", "http"], default=map_or_default(redvox_config, lambda config: config.protocol, "https"), ) data_req_report_parser.add_argument( "--secret-token", help= "A shared secret token provided by RedVox required for accessing the data " "request service", default=map_or_default(redvox_config, lambda config: config.secret_token, None), ) data_req_report_parser.add_argument( "report_id", type=str, help="The full report id that data is being requested for", ) data_req_report_parser.set_defaults(func=data_req_report_args) # Parse the args args = parser.parse_args() # Setup logging log_levels: Dict[int, str] = {0: "WARN", 1: "INFO", 2: "DEBUG"} log_level: str = (log_levels[args.verbose] if args.verbose in log_levels else log_levels[0]) logging.basicConfig( level=log_level, format= "[%(levelname)s:%(process)d:%(filename)s:%(module)s:%(funcName)s:%(lineno)d:%(asctime)s]" " %(message)s", ) log.info("Running with args=%s and log_level=%s", str(args), log_level) # Try calling the appropriate handler # pylint: disable=W0703 try: args.func(args) except Exception as e: log.error("Encountered an error: %s", str(e)) sys.exit(1)
def download_data(self): validation_errors: List[str] = self.validate() if len(validation_errors) > 0: self.info("Errors encountered") for error in validation_errors: self.info(error) return username: str = self.authentication_widget.username.text() password: str = self.authentication_widget.password.text() protocol: str if self.server_selection_widget.https.isChecked(): protocol = "https" else: protocol = "http" host: str = self.server_selection_widget.host.text() port: int = int(self.server_selection_widget.port.text()) out_dir: str = self.output_directory_widget.output_dir.text() redvox_config: RedVoxConfig = RedVoxConfig(username, password, protocol, host, port) req_start: int = int(self.window_selection_widget.start_dt. dateTime().toSecsSinceEpoch()) req_end: int = int(self.window_selection_widget.end_dt.dateTime(). toSecsSinceEpoch()) station_ids: List[str] = list( map(str.strip, self.station_ids_widget.toPlainText().splitlines(False))) api_type: DataRangeReqType if self.api_selection_widget.api_900_1000.isChecked(): # noinspection PyTypeChecker api_type = DataRangeReqType.API_900_1000 elif self.api_selection_widget.api_1000.isChecked(): # noinspection PyTypeChecker api_type = DataRangeReqType.API_1000 else: # noinspection PyTypeChecker api_type = DataRangeReqType.API_900 retries: int = int(self.server_selection_widget.retries.text()) timeout: int = int(self.server_selection_widget.timeout.text()) correct_query_timing: bool = ( not self.server_selection_widget.disable_timing_correction. isChecked()) manager = Manager() out_queue = manager.Queue(1024) self.info("Starting data query") process = Process( target=download_process, args=( redvox_config, timeout, req_start, req_end, station_ids, api_type, correct_query_timing, out_dir, retries, out_queue, ), ) process.start() result: str = "" while result != "done": # noinspection PyBroadException try: result = out_queue.get(block=True, timeout=0.1) self.info(result) QApplication.processEvents() except: pass