def do(self): usagestring = "DECam_clusters_upload.py date <options>" tstart = time.time() # read in the options from the param file and the command line # some convoluted syntax here, making it so param file is not required parser = self.add_options(usage=usagestring) options, args = parser.parse_known_args() config = configparser.ConfigParser() config.read("%s/settings.ini" % djangoSettings.PROJECT_DIR) parser = self.add_options(usage=usagestring, config=config) options, args = parser.parse_known_args() self.options = options try: nsn = self.main() except Exception as e: print(e) nsn = 0 smtpserver = "%s:%s" % (options.SMTP_HOST, options.SMTP_PORT) from_addr = "*****@*****.**" % options.SMTP_LOGIN subject = "QUB Transient Upload Failure" print("Sending error email") html_msg = "Alert : YSE_PZ Failed to upload transients from PSST in QUB_data.py\n" html_msg += "Error : %s" sendemail(from_addr, options.dbemail, subject, html_msg % (e), options.SMTP_LOGIN, options.dbemailpassword, smtpserver) print('QUB -> YSE_PZ took %.1f seconds for %i transients' % (time.time() - tstart, nsn))
def main(argv=None): """ The main function which creates the pipeline and runs it. """ parser = argparse.ArgumentParser() # Here we add some specific command line arguments we expect. # This defaults the output table in your BigQuery you'll have # to create the example_data dataset yourself using bq mk temp parser.add_argument('--output', dest='output', required=False) # Parse arguments from the command line. known_args, pipeline_args = parser.parse_known_args(argv) table_spec = bigquery.TableReference( projectId=os.getenv('GOOGLE_PROJECT_ID'), datasetId=os.getenv('GOOGLE_DATASET_ID'), tableId=os.getenv('GOOGLE_TABLE_ID')) pipeline_options = PipelineOptions(flags=pipeline_args, ) pipeline = beam.Pipeline(options=pipeline_options) (pipeline | 'Get Data From Business Central ' >> beam.Create(get_data_from_bc365()) | 'Format Dates' >> beam.ParDo(FormatElements()) | 'Write To BigQuery' >> beam.io.WriteToBigQuery( table_spec, write_disposition=beam.io.BigQueryDisposition.WRITE_TRUNCATE, create_disposition=beam.io.BigQueryDisposition.CREATE_NEVER, custom_gcs_temp_location=SYNC_BUCKET)) pipeline.run().wait_until_finish()
def main(): parser = argparse.ArgumentParser( description="craigslist car finder", parents=()) parser.add_argument("-c", "--city", default='austin', help='which city to search for') parser.add_argument("-b", "--brand", default='honda', help='car brand') parser.add_argument("-m", "--model", default='civic', help='car model') parser.add_argument("-p", "--minimum_price", default='4000') parser.add_argument("-y", "--minimum_year", default='2007') parser.add_argument("-d", "--maximum_odometer", default='100000', help='maximum miles travelled by car before purchase') parser.add_argument("-t", "--blacklist_titles", nargs='+', default=['salvage', 'rebuilt'], help='List unacceptable states for car, e.g. You may want to filter out cars that \ have been totalled or salvaged') parser.add_argument("-w", "--week_range", default=2, help='number of weeks to search car listings for starting from now') parser.add_argument("-l", "--max_results", default=20, help='limit to this number of results for cars returned') parser.add_argument("-o", "--output", help='write matching cars to file') parser.add_argument("-v", "--verbose", action='store_true', help='print debug output') try: args, extra_args = parser.parse_known_args() except Exception, e: print e sys.exit(1)
def main(): # from .state import * from . import utils import argparse global options parser = argparse.ArgumentParser() group = parser.add_mutually_exclusive_group() group.add_argument("-v", "--verbose", action="count", default=0, help="verbose logging") group.add_argument("-q", "--quiet", action="count", default=0, help="quiet logging") options, args = parser.parse_known_args() utils.setup_logging(options.verbose - options.quiet) # state.session = MLBStreamSession.new() # raise Exception(state.session.token) raise Exception(PROVIDERS)
def main(): import argparse parser = argparse.ArgumentParser(description='Warden IMAPReceiver configuration file parser') parser.add_argument('--config', help="Path to the Warden IMAPReceiver configuration file.", dest='config') parser.add_argument('--server', help='Run continually in server mode (default is to run once)', action='store_true', default=False) args, unknown = parser.parse_known_args(sys.argv) imapreceiver_configuration_file = os.path.abspath(os.path.expanduser(args.config)) if not os.path.exists(imapreceiver_configuration_file): log.error('The IMAPReceiver config file specified ("%s") does not exist!' % imapreceiver_configuration_file) sys.exit(1) config = ConfigParser.SafeConfigParser() config.read(imapreceiver_configuration_file) imapreceiver = IMAPReceiver(config) if args.server: imapreceiver.start() else: imapreceiver.connect_to_mailbox() imapreceiver.check_mails()
def main(): # Spawn threads, one for each feed. # gsb-host and mongod-host should be passed on command line parser = argparse.ArgumentParser() parser.add_argument('-g', '--gsb-url', help='GSB Agent URL', dest='gsb_url') parser.add_argument('-m', '--mongod-url', help='Mongo DB Server URL', dest='mongod_url') args = parser.parse_known_args() # Fetch API keys from apikeys.conf with open('apikeys.conf') as f: keys_raw = f.read().split('\n') keys_d = dict() for i in keys_raw: if i is not None and i is not '': keys_d[i.split(':')[0]] = i.split(':')[1] feedProcessor = FeedProcessor(args, keys_d) signal.signal(signal.SIGINT, feedProcessor.sigint_handler) connect("threatint", host=args[0].mongod_url) logging.basicConfig(filename="url_record.log", level=logging.INFO) for i in sorted(dir(feedProcessor)): if i.endswith('_worker'): # Worker methods that fetch a feed and add to DB end with '_worker'. Start them all. new_thread = threading.Thread(target=getattr(feedProcessor, i)) try: new_thread.start() logging.info("Started thread with target {}.".format(i)) except RuntimeError: logging.exception( "Could not start thread with target {}. Already started?". format(i)) # Start a thread that queries the DB one by one and updates DNS query values. while sigint_exit == False: # Hack to keep main thread active to catch SIGINT to kill the entire process. time.sleep(2) sys.exit(0)
def parse_arguments(): # parse input arguments parser = argparse.ArgumentParser() parser.add_argument( '--search-query', type=str, default=DEF_QUERY, help= 'query used for arxiv API. See http://arxiv.org/help/api/user-manual#detailed_examples' ) parser.add_argument('--start-index', type=int, default=0, help='0 = most recent API result') parser.add_argument('--max-index', type=int, default=3000, help='upper bound on the paper index we will fetch') parser.add_argument('--results-per-iteration', type=int, default=100, help='passed to arxiv API') parser.add_argument( '--wait-time', type=float, default=5.0, help='wait time allows being gentle on the arxiv API (in seconds)') parser.add_argument( '--break-on-no-added', type=int, default=1, help= 'break out early if all returned query papers are already in db? 1=yes, 0=no' ) args, unknown = parser.parse_known_args() return args
def init(): parser = argparse.ArgumentParser(description="Generates a dataset by compiling generated data properties using a certain dataset model") parser.add_argument('--model', type=str, default='matrix', help='The name of the dataset model to use. Defaults to matrix.') parser.add_argument('properties', type=str, default='openPrice,closePrice,gasPrice', help='A list of the names of the properties to use, separated by a comma.') parser.add_argument('targets', type=str, default='highPrice', help='A list of target property names, separated by a comma.') parser.add_argument('--start', type=str, default=None, help='The start date. YYYY-MM-DD-HH') parser.add_argument('--end', type=str, default=None, help='The end date. YYYY-MM-DD-HH') parser.add_argument('--filename', type=str, default=None, help='The target filename / dir to save the pickled dataset to. Defaults to "data/dataset.pickle"') parser.add_argument('--overwrite', dest='overwrite', action='store_true', help="If the filename already exists, overwrite it.") parser.add_argument('--ratio', type=str, default='1', help='On how many fragments to split the main dataset. For example, "1:2:3" will create three datasets with sizes proportional to what given.') parser.add_argument('--shuffle', dest='shuffle', action="store_true", help="Shuffle the generated dataset and labels.") parser.set_defaults(shuffle=False) parser.set_defaults(overwrite=False) args, _ = parser.parse_known_args() if len(_) != 0: raise ValueError("Provided flags %s cannot be understood." % str(_)) if args.filename == None: filename = "data/dataset_" + str(args.start) + "-" + str(args.end) + ".pickle" else: filename = args.filename start = args.start end = args.end start = dateutil.parser.parse(start) if start is not None else None end = dateutil.parser.parse(end) if end is not None else None try: ratio = [int(x) for x in args.ratio.split(':')] except ValueError: print("Error while reading the given ratio. Did you format it in the correct way?") return run(args.model, args.properties.split(','), args.targets.split(','), filename, start=start, end=end, ratio=ratio, shuffle=args.shuffle, overwrite=args.overwrite)
def main(): global options global logger today = datetime.now(pytz.timezone('US/Eastern')).date() init_parser = argparse.ArgumentParser() init_parser.add_argument("-p", "--profile", help="use alternate config profile") options, args = init_parser.parse_known_args() config.load(merge_default=True) if options.profile: config.settings.set_profile(options.profile) player.Player.load() parser = argparse.ArgumentParser() group = parser.add_mutually_exclusive_group() group.add_argument("-v", "--verbose", action="count", default=0, help="verbose logging") group.add_argument("-q", "--quiet", action="count", default=0, help="quiet logging") parser.add_argument("-d", "--debug-console", help="show logging console (disables task manager UI)", action="store_true") parser.add_argument("spec", metavar="SPECIFIER", help="media specifier", nargs="?") options, args = parser.parse_known_args(args) state.options = AttrDict(vars(options)) logger = logging.getLogger() providers.load() model.init() sh = logging.StreamHandler() state.logger = setup_logging(options.verbose - options.quiet, quiet_stdout=False) providers.load_config() with db_session(optimistic=False): model.MediaFeed.purge_all( min_items=config.settings.profile.cache.min_items, max_items=config.settings.profile.cache.max_items, max_age=config.settings.profile.cache.max_age) spec = None logger.debug(f"{PACKAGE_NAME} starting") state.asyncio_loop = asyncio.get_event_loop() state.task_manager = tasks.TaskManager() state.task_manager_task = state.asyncio_loop.create_task( state.task_manager.start()) log_file = os.path.join(config.CONFIG_DIR, f"{PACKAGE_NAME}.log") fh = logging.FileHandler(log_file) add_log_handler(fh) logging.getLogger("panwid").setLevel(logging.INFO) action, provider, selection, opts = providers.parse_spec(options.spec) if selection: run_cli(action, provider, selection, **opts) else: run_gui(action, provider, **opts)
except ImportError: pass # The warnings need to be before any of matplotlib imports, but after # dateutil.parser and setuptools (if present) which has syntax error with # the warnings enabled. Filtering by module does not work as this will be # raised by Python itself so `module=matplotlib.*` is out of question. import warnings # Python 3.6 deprecate invalid character-pairs \A, \* ... in non # raw-strings and other things. Let's not re-introduce them warnings.filterwarnings('error', '.*invalid escape sequence.*', category=DeprecationWarning) warnings.filterwarnings( 'default', r'.*inspect.getargspec\(\) is deprecated.*', category=DeprecationWarning) from matplotlib import test parser = argparse.ArgumentParser(add_help=False) parser.add_argument('--recursionlimit', type=int, default=0, help='Specify recursionlimit for test run') args, extra_args = parser.parse_known_args() print('Python byte-compilation optimization level:', sys.flags.optimize) retcode = test(argv=extra_args, recursionlimit=args.recursionlimit) sys.exit(retcode)
def main(): global options global logger today = datetime.now(pytz.timezone('US/Eastern')).date() init_parser = argparse.ArgumentParser() init_parser.add_argument("-c", "--config-file", help="use alternate config file") init_parser.add_argument("-p", "--profile", help="use alternate config profile") options, args = init_parser.parse_known_args() # -c used to refer to a config dir config_file = None if options.config_file: config_file = os.path.expanduser(options.config_file) if os.path.isdir(config_file): config_file = os.path.join(config_file, config.Config.DEFAULT_CONFIG_FILE) logger.warning("`-c` should refer to a file, not a directory. " f"using `{config_file}`") config.load(config_file, merge_default=True) if options.profile: for p in options.profile.split(","): config.settings.include_profile(p) programs.load() parser = argparse.ArgumentParser() group = parser.add_mutually_exclusive_group() group.add_argument("-v", "--verbose", action="count", default=0, help="verbose logging") group.add_argument("-q", "--quiet", action="count", default=0, help="quiet logging") parser.add_argument("uri", metavar="URI", help="media URI", nargs="?") options, args = parser.parse_known_args(args) if options.verbose: try: import ipdb pdb_on_exception("ipdb") except ImportError: pdb_on_exception() state.options = AttrDict(vars(options)) state.options.config_file = config_file logging.captureWarnings(True) logger = logging.getLogger() sh = logging.StreamHandler() state.logger = setup_logging(options.verbose - options.quiet, quiet_stdout=False) state.task_manager = tasks.TaskManager() providers.load() model.init() providers.load_config(default=state.app_data.selected_provider) providers.apply_settings() spec = None logger.debug(f"{PACKAGE_NAME} starting") if config.settings.profile.downloads.max_age > 0: with db_session(optimistic=False): model.MediaDownload.purge( age=config.settings.profile.downloads.max_age) state.start_task_manager() # state.task_manager_task = state.event_loop.create_task(state.task_manager.start()) log_file = os.path.join(config.settings.CONFIG_DIR, f"{PACKAGE_NAME}.log") fh = logging.FileHandler(log_file) add_log_handler(fh) logging.getLogger("panwid.dropdown").setLevel(logging.INFO) logging.getLogger("panwid.keymap").setLevel(logging.INFO) logging.getLogger("panwid.datatable").setLevel(logging.INFO) logging.getLogger("aio_mpv_jsonipc").setLevel(logging.INFO) action, provider, selection, opts = providers.parse_uri(options.uri) if selection: state.tui_enabled = False rc = run_cli(action, provider, selection, **opts) else: state.tui_enabled = True rc = run_gui(action, provider, **opts) state.stop_task_manager() return rc
def main(): global options global logger parser = argparse.ArgumentParser() parser.add_argument("-v", "--verbose", action="store_true") options, args = parser.parse_known_args() log_file = os.path.join(config.CONFIG_DIR, "mlbstreamer.log") formatter = logging.Formatter( "%(asctime)s [%(module)16s:%(lineno)-4d] [%(levelname)8s] %(message)s", datefmt="%Y-%m-%d %H:%M:%S" ) fh = logging.FileHandler(log_file) fh.setLevel(logging.DEBUG) fh.setFormatter(formatter) logger = logging.getLogger("mlbstreamer") logger.setLevel(logging.INFO) logger.addHandler(fh) ulh = UrwidLoggingHandler() ulh.setLevel(logging.DEBUG) ulh.setFormatter(formatter) logger.addHandler(ulh) logger.debug("mlbstreamer starting") config.settings.load() state.session = MLBSession.new() entries = Dropdown.get_palette_entries() entries.update(ScrollingListBox.get_palette_entries()) entries.update(DataTable.get_palette_entries()) # raise Exception(entries) palette = Palette("default", **entries) screen = urwid.raw_display.Screen() screen.set_terminal_properties(256) view = ScheduleView() log_console = widgets.ConsoleWindow() # log_box = urwid.BoxAdapter(urwid.LineBox(log_console), 10) pile = urwid.Pile([ ("weight", 1, urwid.LineBox(view)), (10, urwid.LineBox(log_console)) ]) def global_input(key): if key in ('q', 'Q'): raise urwid.ExitMainLoop() else: return False state.loop = urwid.MainLoop( pile, palette, screen=screen, unhandled_input=global_input, pop_ups=True ) ulh.connect(state.loop.watch_pipe(log_console.log_message)) logger.info("mlbstreamer starting") if options.verbose: logger.setLevel(logging.DEBUG) state.loop.run()
def main(): today = datetime.now().date() parser = argparse.ArgumentParser() parser.add_argument("-d", "--date", help="game date", type=valid_date, default=today) parser.add_argument("-b", "--beginning", help="play live streams from beginning", nargs="?", metavar="offset_from_game_start", type=int, const=-10) parser.add_argument("-r", "--resolution", help="stream resolution", default="720p") parser.add_argument("-s", "--save-stream", help="save stream to file", nargs="?", const=True) parser.add_argument("-v", "--verbose", action="store_true", help="verbose logging") parser.add_argument("--init-config", help="initialize configuration", action="store_true") parser.add_argument("game", metavar="game", nargs="?", help="team abbreviation or MLB game ID") options, args = parser.parse_known_args() logger = logging.getLogger("mlbstreamer") if options.verbose: logger.setLevel(logging.DEBUG) formatter = logging.Formatter( "%(asctime)s [%(levelname)8s] %(message)s", datefmt='%Y-%m-%d %H:%M:%S') handler = logging.StreamHandler(sys.stdout) handler.setFormatter(formatter) logger.addHandler(handler) else: logger.addHandler(logging.NullHandler()) if options.init_config: config.settings.init_config() sys.exit(0) config.settings.load() if not options.game: parser.error("option game") state.session = MLBSession.new() if options.game.isdigit(): game_id = int(options.game) else: season = today.year teams_url = ("http://statsapi.mlb.com/api/v1/teams" "?sportId={sport}&season={season}".format(sport=1, season=season)) teams = { team["fileCode"]: team["id"] for team in state.session.get(teams_url).json()["teams"] } if options.game not in teams: print("'%s' not a valid team code, must be one of:\n%s" % (options.game, " ".join(teams))) sys.exit(1) schedule = state.session.schedule(start=options.date, end=options.date, sport_id=1, team_id=teams[options.game]) game_id = schedule["dates"][0]["games"][0]["gamePk"] try: proc = play_stream(game_id, options.resolution, offset_from_beginning=options.beginning, output=options.save_stream) proc.wait() except MLBPlayException as e: logger.error(e)
def main(): parser = argparse.ArgumentParser() parser.add_argument("--pipeline-uri", type=str, help="A URI to JSON with instructions") parser.add_argument("--pipeline", type=str, help="JSON with instructions") parser.add_argument( "--sentinel-stac-id", type=str, help="STAC Item ID to process from the STAC collection") parser.add_argument( "--sentinel-collection-id", type=str, default=SENTINEL_ARCHIVE_COLLECTION_ID, ) parser.add_argument( "--stac-api-uri", type=str, default=os.environ.get("STAC_API_URI", "http://franklin:9090"), ) parser.add_argument( "--stac-api-uri-sentinel", type=str, default=os.environ.get("STAC_API_URI_SENTINEL", "https://earth-search.aws.element84.com/v0"), ) parser.add_argument("--s3-bucket", type=str, default=os.environ.get("S3_BUCKET", "sentinel-s2-data")) parser.add_argument( "--s3-prefix", type=str, default=os.environ.get("S3_PREFIX", "aviris-scene-cogs-l2"), ) parser.add_argument("--temp-dir", type=str, default=os.environ.get("TEMP_DIR", None)) parser.add_argument("--output-format", type=str, default=os.environ.get("GDAL_OUTPUT_FORMAT", "COG")) parser.add_argument( "--keep-temp-dir", action="store_true", help= "If provided, script does not delete temporary directory before script exits. Useful for debugging.", ) parser.add_argument( "--force", action="store_true", help= "If provided, force reingest StacItem even though this it is already present in the catalog.", ) try: warpMemoryLimit = int(os.environ.get("GDAL_WARP_MEMORY_LIMIT", None)) except TypeError: warpMemoryLimit = None # TODO: replace it with parser.parse_args() later cli_args, cli_unknown = parser.parse_known_args() # parse all cli arguments args = CliConfig(cli_args, cli_unknown) s3 = boto3.client("s3") stac_client_sentinel = STACClient(args.stac_api_uri_sentinel) stac_client = STACClient(args.stac_api_uri) collection = stac_client_sentinel.get_collection( args.sentinel_collection_id) SENTINEL_COG_COLLECTION = pystac.Collection( SENTINEL_COG_COLLECTION_ID, "Sentinel-2a and Sentinel-2b imagery, processed to Level 2A (Surface Reflectance) and converted to Cloud-Optimized GeoTIFFs", collection.extent, stac_extensions=COG_COLLECTION_EXTENSIONS) SENTINEL_COG_COLLECTION.links = [] SENTINEL_COG_COLLECTION.properties = {} SENTINEL_COG_COLLECTION.properties['eo:bands'] = SENTINEL_BANDS SENTINEL_COG_COLLECTION.properties[ 'hsi:wavelength_min'] = SENTINEL_WAVELENGTH_MIN SENTINEL_COG_COLLECTION.properties[ 'hsi:wavelength_max'] = SENTINEL_WAVELENGTH_MAX # GET STAC Item from SENTINEL Catalog item = stac_client_sentinel.get_collection_item( args.sentinel_collection_id, args.sentinel_stac_id) assets = item.assets bands_map = { 'B01': vsis3(strip_scheme(assets['B01'].href)), 'B02': vsis3(strip_scheme(assets['B02'].href)), 'B03': vsis3(strip_scheme(assets['B03'].href)), 'B04': vsis3(strip_scheme(assets['B04'].href)), 'B05': vsis3(strip_scheme(assets['B05'].href)), 'B06': vsis3(strip_scheme(assets['B06'].href)), 'B07': vsis3(strip_scheme(assets['B07'].href)), 'B08': vsis3(strip_scheme(assets['B08'].href)), 'B8A': vsis3(strip_scheme(assets['B8A'].href)), 'B09': vsis3(strip_scheme(assets['B09'].href)), 'B11': vsis3(strip_scheme(assets['B11'].href)), 'B12': vsis3(strip_scheme(assets['B12'].href)), 'AOT': vsis3(strip_scheme(assets['AOT'].href)), # 'WVP': vsis3(strip_scheme(assets['WVP'].href)), # 'SCL': vsis3(strip_scheme(assets['SCL'].href)) } # we don't need assets here, since the gather scripts knows what and how to download by the sentinel path properties = item.properties datetime = dateutil.parser.isoparse(properties['datetime']) # here "href": "s3://sentinel-s2-l2a/tiles/31/V/CE/2021/8/19/0/R60m/B01.jp2" # path is tiles/31/V/CE/2021/8/19/0 sentintel_path = 'tiles/{}/{}/{}/{}/{}/{}/{}'.format( properties['sentinel:utm_zone'], properties['sentinel:latitude_band'], properties['sentinel:grid_square'], str(datetime.year), str(datetime.month), str(datetime.day), properties['sentinel:sequence']) # Create new COG STAC Item cog_item_id = "{}_{}".format(SENTINEL_COG_COLLECTION.id, item.id) cog_item = pystac.Item( cog_item_id, item.geometry, item.bbox, item.datetime, item.properties, stac_extensions=COG_ITEM_EXTENSIONS, collection=SENTINEL_COG_COLLECTION.id, ) cog_item.properties['eo:bands'] = SENTINEL_COG_COLLECTION.properties[ 'eo:bands'] cog_item.properties[ 'hsi:wavelength_min'] = SENTINEL_COG_COLLECTION.properties[ 'hsi:wavelength_min'] cog_item.properties[ 'hsi:wavelength_max'] = SENTINEL_COG_COLLECTION.properties[ 'hsi:wavelength_max'] cog_item.properties['proj:epsg'] = '4326' # Create COG Collection if it doesn't exist if not stac_client.has_collection(SENTINEL_COG_COLLECTION.id): stac_client.post_collection(SENTINEL_COG_COLLECTION) if not args.force: # Exit early if COG STAC Item already exists try: stac_client.get_collection_item(SENTINEL_COG_COLLECTION.id, cog_item_id) logger.info(f'STAC Item {cog_item_id} already exists. Exiting.') activation_output(cog_item_id) return except requests.exceptions.HTTPError: pass _, s3_uri = gather_sentinel( f'{cog_item_id}.tiff', f's3://{args.s3_bucket}/{args.s3_prefix}/{sentintel_path}/', bands_map) # Add assets to COG STAC Item idx = 0 cog_item.add_asset( f'{args.output_asset_name}_{idx}', pystac.Asset(s3_uri, media_type=pystac.MediaType.COG, roles=["data"]), ) # Add COG Item to AVIRIS L2 STAC Collection logger.info(f"POST Item {cog_item.id} to {args.stac_api_uri}") item_data = stac_client.post_collection_item(SENTINEL_COG_COLLECTION.id, cog_item) if item_data.get('id', None): logger.info(f"Success: {item_data['id']}") activation_output(item_data['id']) else: logger.error(f"Failure: {item_data}") return -1
plt.savefig(filename, dpi = 1500) print("Saved accuracy graph at %s." % filename) if __name__ == "__main__": #if this is the main file, parse the command args parser = argparse.ArgumentParser(description="Module that loads given datasets and trains and evaluates one or more neural network models on that.") parser.add_argument('dataset', type=str, help="The filepath to the dataset/s.") parser.add_argument('--models', type=str, help="A list of the models that are going to be trained and evaluated. Default is all available.") parser.add_argument('--args', type=str, help="A list of arguments to be passed on to the models. In the format key1=value1,key2=value2.1;value2.2") parser.add_argument('--quiet', dest='quiet', action="store_true", help="Do not plot graphs, but save them as images.") parser.set_defaults(quiet=False) parser.add_argument('--shuffle', dest='shuffle', action="store_true", help="Shuffle the generated dataset and labels.") parser.set_defaults(shuffle=False) parser.add_argument('--trim-batch', dest='trim', action="store_true", help="Trim each dataset so that its length is divisible by the batch size.") parser.set_defaults(trim=False) args, _ = parser.parse_known_args() givenModels = args.models.split(',') if args.models else None modelArgs = {} pairs = args.args.split(',') for pair in pairs: key, value = pair.split('=') try: value = float(value) if value == int(value): value = int(value) except ValueError: if ':' in value: value = [int(i) for i in value.split(':')]
def main(): parser = argparse.ArgumentParser() subparsers = parser.add_subparsers(dest='test_type') # Host-side test args. host_cmd_parser = subparsers.add_parser( 'host-cmd', help='Runs a host-side test. Pass the host-side command to run after ' '"--". If --use-vm is passed, hostname and port for the device ' 'will be 127.0.0.1:9222.') host_cmd_parser.set_defaults(func=host_cmd) host_cmd_parser.add_argument( '--deploy-chrome', action='store_true', help='Will deploy a locally built Chrome binary to the device before ' 'running the host-cmd.') # GTest args. # TODO(bpastene): Rename 'vm-test' arg to 'gtest'. gtest_parser = subparsers.add_parser('vm-test', help='Runs a device-side gtest.') gtest_parser.set_defaults(func=device_test) gtest_parser.add_argument( '--test-exe', type=str, required=True, help='Path to test executable to run inside the device. If the value is ' '%s, the sanity test that ships with the device image runs instead. ' 'This test smokes-check the system browser (eg: loads a simple ' 'webpage, executes some javascript), so a fully-built Chrome binary ' 'that can get deployed to the device is expected to be available in ' 'the out-dir.' % SANITY_TEST_TARGET) # GTest args. Some are passed down to the test binary in the device. Others # are parsed here since they might need tweaking or special handling. gtest_parser.add_argument( '--test-launcher-summary-output', type=str, help='When set, will pass the same option down to the test and retrieve ' 'its result file at the specified location.') # Shard args are parsed here since we might also specify them via env vars. gtest_parser.add_argument('--test-launcher-shard-index', type=int, default=os.environ.get('GTEST_SHARD_INDEX', 0), help='Index of the external shard to run.') gtest_parser.add_argument('--test-launcher-total-shards', type=int, default=os.environ.get('GTEST_TOTAL_SHARDS', 1), help='Total number of external shards.') gtest_parser.add_argument( '--stop-ui', action='store_true', help='Will stop the UI service in the device before running the test.') gtest_parser.add_argument( '--trace-dir', type=str, help='When set, will pass down to the test to generate the trace and ' 'retrieve the trace files to the specified location.') gtest_parser.add_argument( '--env-var', nargs=2, action='append', default=[], help='Env var to set on the device for the duration of the test. ' 'Expected format is "--env-var SOME_VAR_NAME some_var_value". Specify ' 'multiple times for more than one var.') # Tast test args. # pylint: disable=line-too-long tast_test_parser = subparsers.add_parser( 'tast', help='Runs a device-side set of Tast tests. For more details, see: ' 'https://chromium.googlesource.com/chromiumos/platform/tast/+/master/docs/running_tests.md' ) tast_test_parser.set_defaults(func=device_test) tast_test_parser.add_argument( '--suite-name', type=str, required=True, help='Name to apply to the set of Tast tests to run. This has no effect ' 'on what is executed, but is used mainly for test results reporting ' 'and tracking (eg: flakiness dashboard).') tast_test_parser.add_argument( '--test-launcher-summary-output', type=str, help='Generates a simple GTest-style JSON result file for the test run.' ) # TODO(bpastene): Change all uses of "--conditional" to use "--attr-expr". tast_test_parser.add_argument( '--conditional', '--attr-expr', type=str, dest='conditional', help='A boolean expression whose matching tests will run ' '(eg: ("dep:chrome")).') tast_test_parser.add_argument( '--strip-chrome', action='store_true', help='Strips symbols from the browser before deploying to the device.') tast_test_parser.add_argument( '--test', '-t', action='append', dest='tests', help='A Tast test to run in the device (eg: "ui.ChromeLogin").') add_common_args(gtest_parser, tast_test_parser, host_cmd_parser) args, unknown_args = parser.parse_known_args() logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARN) if not args.use_vm and not args.device: # If we're not running on a VM, but haven't specified a hostname, assume # we're on a lab bot and are trying to run a test on a lab DUT. See if the # magic lab DUT hostname resolves to anything. (It will in the lab and will # not on dev machines.) try: socket.getaddrinfo(LAB_DUT_HOSTNAME, None) except socket.gaierror: logging.error('The default DUT hostname of %s is unreachable.', LAB_DUT_HOSTNAME) return 1 if args.use_vm: if not os.path.exists('/dev/kvm'): logging.error( '/dev/kvm is missing. Is KVM installed on this machine?') return 1 elif not os.access('/dev/kvm', os.W_OK): logging.error( '/dev/kvm is not writable as current user. Perhaps you should be ' 'root?') return 1 args.cros_cache = os.path.abspath(args.cros_cache) return args.func(args, unknown_args)
def init(): np.set_printoptions(precision=3, linewidth=180) parser = argparse.ArgumentParser( description= "Tool that can read historical data from the db or from a file and visualize it as a graph." ) parser.add_argument( 'data', type=str, nargs='*', help='A pickled file or file list, output from training.') parser.add_argument( '--keys', type=str, default=None, help='Filter out only one or multiple keys, separated by a comma.') parser.add_argument('--best', type=int, default=None, help='Only display the best N results.') parser.add_argument('--detail', dest='detail', action='store_true', help='Display the scores in detail.') parser.set_defaults(detail=False) parser.add_argument('--no-display', dest='display', action='store_false', help='Do not plot any graphs.') parser.set_defaults(display=True) args, _ = parser.parse_known_args() files = args.data for i in range(len(files)): if '\n' in files[i]: s = files[i].split('\n') subfiles = [f for f in s if f != '' and not f.isspace()] files[i] = None files.extend(subfiles) files = [f for f in files if f is not None] #print(files) histories = [] data = {} titles = files.copy() keys = args.keys.split(',') if args.keys is not None else None for f in files: with open(f, 'rb') as fi: element = pickle.load(fi) hist = element['history'] histories.append(hist) for i, hist in enumerate(histories): if keys is not None: hist2 = {} for key in keys: hist2[key] = hist[key] hist = hist2 for key in hist: if key not in data: data[key] = [] filename = basename(files[i]) #extract filename from path data[key].append({'file': filename, 'data': hist[key]}) for key in data: if key == 'val_loss' or key == 'loss' or key == 'rmse' or key == 'custom': func = min data[key] = sortResults(data[key], ascending=True, func=func) elif key == 'R2' or key == 'sign': func = max data[key] = sortResults(data[key], ascending=False, func=func) else: raise ValueError("Unexpected and unknown metric '%s'." % key) print("Best for %s: are as follows" % (key)) for i in range(min(5, len(data[key]))): val = data[key][i]['data'][-1] if func is None else func( data[key][i]['data']) print("#%d %s is %f at %s" % (i + 1, key, val, data[key][i]['file'])) if args.best is not None: data[key] = data[key][:args.best] if args.detail: resLines = "" joiner = ' & ' for i, hist in enumerate(histories): print("Detailed results for %s." % (basename(files[i]))) best = [] headers = [] for key in (hist if keys is None else keys): best.append(getCompareFuncBasedOnKey(key)(hist[key])) headers.append(key) print(headers) print(best) commLine = '%' + basename(files[i]) + '\n' resLine = str.join(joiner, ("%f" % val for val in best)) + '\\\\\n' resLines += commLine resLines += resLine print(str.join(joiner, headers)) print(resLine) print("All result lines for conveninece:") print(resLines) if args.display: drawAccuracyGraph(data)
def main(): # https://sfbay.craigslist.org/search/eby/cto? # sort=pricedsc& # hasPic=1& # search_distance=10& # postal=94610& # min_price=4000& # max_price=10000& # auto_make_model=toyota+corolla& # min_auto_year=2008& # min_auto_miles=100000& # max_auto_miles=120000& # condition=30& # condition=40& # auto_cylinders=2& # auto_title_status=1& # auto_transmission=2 parser = argparse.ArgumentParser(description="craigslist car finder", parents=()) parser.add_argument("-c", "--city", default='sfbay', help='which city to search for') parser.add_argument("-d", "--sort_results", default='pricedsc', help='how to sort results') parser.add_argument("-e", "--search_distance", default='40', help='maximum distance from search zip') parser.add_argument("-f", "--postal", default='94610', help='search zip') parser.add_argument("-p", "--min_price", default='2000') parser.add_argument("-o", "--max_price", default='16000') parser.add_argument("-b", "--make", default='kia', help='car make (brand)') parser.add_argument("-m", "--model", default='soul', help='car model') parser.add_argument("-y", "--min_auto_year", default='2008') parser.add_argument("-a", "--min_auto_miles", default='50000', help='minimum miles travelled by car before purchase') parser.add_argument("-i", "--max_auto_miles", default='150000', help='maximum miles travelled by car before purchase') parser.add_argument("-c1", "--condition_1", default='30') # 30 = excellent parser.add_argument("-c2", "--condition_2", default='40') # 40 = good parser.add_argument("-g1", "--auto_cylinders", default='2') # 2 = 4 cylinders parser.add_argument("-t1", "--auto_title_status", default='1') # 1 = clean parser.add_argument("-x1", "--auto_transmission", default='2') # 2 = automatic, 1 = manual parser.add_argument("-l", "--max_results", default=1000, help='limit to this number of results for cars returned') parser.add_argument("-w", "--week_range", default=2, help='number of weeks to search car listings for starting from now') parser.add_argument("-v", "--verbose", default='False', help='print debug output') parser.add_argument("-t", "--blacklist_titles", nargs='+', default=['salvage', 'rebuilt'], help='List unacceptable states for car, e.g. You may want to filter out cars that ' 'have been totalled or salvaged') # parser.add_argument("-o", "--output", help='write matching cars to file') try: args, extra_args = parser.parse_known_args() if args.verbose == 'True': verbose = True else: verbose = False if verbose: print('\n\nargs:', args) print('\n\nextra_args:', extra_args) except Exception as e: print(e) sys.exit(1) all_cars = get_craigslist_cars(args.city, args.sort_results, args.search_distance, args.postal, args.min_price, args.max_price, args.make, args.model, args.min_auto_year, args.min_auto_miles, args.max_auto_miles, args.condition_1, args.condition_2, args.auto_cylinders, args.auto_title_status, args.auto_transmission, args.max_results, verbose=verbose) if verbose: print('\n\nall_cars:') for i, car_attributes in enumerate(all_cars): print('\n\ncar #: {}'.format(i)) for key, value in car_attributes.items(): print('key: {}; value: {}'.format(key, value)) filtered_cars = filter_cars(all_cars, args.max_auto_miles, args.blacklist_titles, args.week_range, verbose=verbose) print('\n\nfiltered_cars:') final_car_attributes_list = [] for i, car_attributes in enumerate(filtered_cars): print('\n\ncar #: {}'.format(i)) final_car_attributes_list.extend(car_attributes.keys()) for key, value in car_attributes.items(): if key == 'posting_body': continue else: print('key: {}; value: {}'.format(key, value)) final_car_attributes_list = list(set(final_car_attributes_list)) print('\n\nfinal_car_attributes_list:\n', final_car_attributes_list) car_row_dict_list = [] for i, car_attributes in enumerate(filtered_cars): car_row_dict = collections.OrderedDict() for car_attribute in final_car_attributes_list: print('\n\n******************') print('car_attribute:', car_attribute) print('car_attributes[car_attribute]:', car_attributes[car_attribute]) try: if isinstance(car_attributes[car_attribute], str) or \ car_attributes[car_attribute] != car_attributes[car_attribute]: print('treat as string') car_row_dict[car_attribute] = car_attributes[car_attribute] else: try: print('treat as list') car_row_dict[car_attribute] = car_attributes[car_attribute][0] except IndexError as e: print('\ncar_attributes[car_attribute]:', car_attributes[car_attribute]) print('str(e):', str(e)) car_row_dict[car_attribute] = np.nan except KeyError: car_row_dict[car_attribute] = np.nan print('car_row_dict[car_attribute]:', car_row_dict[car_attribute]) car_row_dict_list.append(car_row_dict) df = pd.DataFrame.from_dict(car_row_dict_list) def do_it(element): element = element[21:25] return element df['model_make_year'] = df['posting_title'].apply(lambda element: do_it(element)) def do_it(element): element = element.replace('\'', '\'\'\'').replace('\n', ' ').replace('favorite this post', '').\ replace('hide this posting unhide', '').strip() return element df['posting_title'] = df['posting_title'].apply(lambda element: do_it(element)) def do_it(element): element = element.replace('\'', '\'\'\'').replace('\n', ' ').replace('QR Code Link to This Post', '').strip() return element df['posting_body'] = df['posting_body'].apply(lambda element: do_it(element)) print(df.columns) df = df[['model_make_year', 'odometer', 'price', 'time_posted', 'fuel', 'condition', 'transmission', 'title status', 'url', 'posting_title', 'drive', 'cylinders', 'VIN', 'type', 'size', 'paint color', 'attribute']] # , # 'posting_body']] df.sort_values(by=['model_make_year', 'odometer', 'price', 'time_posted'], inplace=True) with pd.option_context('display.max_rows', None, 'display.max_columns', None): print('\n\ndf.head():\n', df.head()) df.to_csv('results.csv')
def main(): parser = argparse.ArgumentParser() parser.add_argument("-f", "--fields", dest="inputfields", default=None, help="fields to emit (default all fields)") parser.add_argument("-t", "--types", dest="typenames", default=None, help="type converters for emitted fields") parser.add_argument("-i", "--insep", dest="insep", default=",", help="input field separator (default ',')") parser.add_argument("-a", "--array", dest="emit_arrays", default=False, action="store_true", help="emit arrays instead of objects") parser.add_argument("-H", "--header", dest="array_header", default=False, action="store_true", help="when emitting arrays, also emit a header array") parser.add_argument("--fullhelp", dest="full_help", action="store_true", default=False, help="display full usage") (options, args) = parser.parse_known_args() if options.full_help: usage() return 0 options.inputfields = (options.inputfields.split(",") if options.inputfields else []) options.typenames = (options.typenames.split(",") if options.typenames else []) if len(args) > 2: usage(sys.argv[0]) return 1 if len(args) >= 1: inf = open(args[0], "r") else: inf = sys.stdin if len(args) == 2: outf = open(args[1], "w") else: outf = sys.stdout reader = csv.DictReader(inf, delimiter=options.insep) if not options.inputfields: options.inputfields = reader.fieldnames try: generate_type_converters(options) except ValueError as exc: usage(exc.args) return 1 return translate(reader, options, outf)
def inference(l_args, s_ticker): import flair parser = argparse.ArgumentParser( prog='infer', description= """Print quick sentiment inference from last tweets that contain the ticker. This model splits the text into character-level tokens and uses the DistilBERT model to make predictions. DistilBERT is a distilled version of the powerful BERT transformer model. Not only time period of these, but also frequency. [Source: Twitter]""") parser.add_argument('-n', "--num", action="store", dest="n_num", type=int, default=100, choices=range(10, 101), help='num of latest tweets to infer from.') (ns_parser, l_unknown_args) = parser.parse_known_args(l_args) if l_unknown_args: print( f"The following args couldn't be interpreted: {l_unknown_args}\n") return # Get tweets using Twitter API params = { 'q': '$' + s_ticker, 'tweet_mode': 'extended', 'lang': 'en', 'count': str(ns_parser.n_num) } # Request Twitter API response = requests.get( 'https://api.twitter.com/1.1/search/tweets.json', params=params, headers={'authorization': 'Bearer ' + cfg.API_TWITTER_BEARER_TOKEN}) # Create dataframe df_tweets = pd.DataFrame() # Check that the API response was successful if response.status_code == 200: for tweet in response.json()['statuses']: row = get_data(tweet) df_tweets = df_tweets.append(row, ignore_index=True) # Load sentiment model sentiment_model = flair.models.TextClassifier.load('en-sentiment') print("") # We will append probability/sentiment preds later probs = [] sentiments = [] for s_tweet in df_tweets['text'].to_list(): tweet = clean_tweet(s_tweet, s_ticker) # Make sentiment prediction sentence = flair.data.Sentence(tweet) sentiment_model.predict(sentence) # Extract sentiment prediction (POSITIVE/NEGATIVE) and confidence (0-1) probs.append(sentence.labels[0].score) sentiments.append(sentence.labels[0].value) # Add probability and sentiment predictions to tweets dataframe df_tweets['probability'] = probs df_tweets['sentiment'] = sentiments # Add sentiment estimation (probability positive for POSITIVE sentiment, and negative for NEGATIVE sentiment) df_tweets['sentiment_estimation'] = df_tweets.apply( lambda row: row['probability'] * (-1, 1)[row['sentiment'] == 'POSITIVE'], axis=1).cumsum() # Cumulative sentiment_estimation df_tweets['prob_sen'] = df_tweets.apply( lambda row: row['probability'] * (-1, 1)[row['sentiment'] == 'POSITIVE'], axis=1) # Percentage of confidence if df_tweets['sentiment_estimation'].values[-1] > 0: n_pos = df_tweets[df_tweets['prob_sen'] > 0]['prob_sen'].sum() n_pct = round(100 * n_pos / df_tweets['probability'].sum()) else: n_neg = abs(df_tweets[df_tweets['prob_sen'] < 0]['prob_sen'].sum()) n_pct = round(100 * n_neg / df_tweets['probability'].sum()) # Parse tweets dt_from = dateutil.parser.parse(df_tweets['created_at'].values[-1]) dt_to = dateutil.parser.parse(df_tweets['created_at'].values[0]) print(f"From: {dt_from.strftime('%Y-%m-%d %H:%M:%S')}") print(f"To: {dt_to.strftime('%Y-%m-%d %H:%M:%S')}") print(f"{len(df_tweets)} tweets were analyzed.") dt_delta = (dt_to - dt_from) n_freq = dt_delta.total_seconds() / len(df_tweets) print(f"Frequency of approx 1 tweet every {round(n_freq)} seconds.") s_sen = f"{('NEGATIVE', 'POSITIVE')[int(df_tweets['sentiment_estimation'].values[-1] > 0)]}" s_conf = f"{round(100*df_tweets['sentiment_estimation'].values[-1]/df_tweets['probability'].sum())}%" print(f"The sentiment of {s_ticker} is: {s_sen} ({s_conf})") print("")
def sentiment(l_args, s_ticker): import flair parser = argparse.ArgumentParser( prog='sen', description= """Plot in-depth sentiment predicted from tweets from last days that contain pre-defined ticker. This model splits the text into character-level tokens and uses the DistilBERT model to make predictions. DistilBERT is a distilled version of the powerful BERT transformer model. Note that a big num of tweets extracted per hour in conjunction with a high number of days in the past, will make the algorithm take a long period of time to estimate sentiment. [Source: Twitter]""" ) # in reality this argument could be 100, but after testing it takes too long to compute which may not be acceptable parser.add_argument('-n', "--num", action="store", dest="n_tweets", type=int, default=10, choices=range(10, 61), help='num of tweets to extract per hour.') parser.add_argument('-d', "--days", action="store", dest="n_days_past", type=int, default=7, choices=range(1, 8), help='num of days in the past to extract tweets.') (ns_parser, l_unknown_args) = parser.parse_known_args(l_args) if l_unknown_args: print( f"The following args couldn't be interpreted: {l_unknown_args}\n") return # Setup API request params and headers headers = {'authorization': f'Bearer {cfg.API_TWITTER_BEARER_TOKEN}'} params = { 'query': f'({s_ticker}) (lang:en)', 'max_results': str(ns_parser.n_tweets), 'tweet.fields': 'created_at,lang' } # Date format string required by twitter dtformat = '%Y-%m-%dT%H:%M:%SZ' # Algorithm to extract dt_recent = datetime.now() - timedelta(seconds=20) dt_old = dt_recent - timedelta(days=ns_parser.n_days_past) print( f"From {dt_recent.date()} retrieving {ns_parser.n_tweets*24} tweets ({ns_parser.n_tweets} tweets/hour)" ) df_tweets = pd.DataFrame() while True: # Iterate until we haven't passed the old number of days if dt_recent < dt_old: break # Update past datetime dt_past = dt_recent - timedelta(minutes=60) if dt_past.day < dt_recent.day: print( f"From {dt_past.date()} retrieving {ns_parser.n_tweets*24} tweets ({ns_parser.n_tweets} tweets/hour)" ) # Assign from and to datetime parameters for the API params['start_time'] = dt_past.strftime(dtformat) params['end_time'] = dt_recent.strftime(dtformat) # Send API request response = requests.get( 'https://api.twitter.com/2/tweets/search/recent', params=params, headers=headers) # Update recent datetime dt_recent = dt_past # If response from API request is a success if response.status_code == 200: # Iteratively append our tweet data to our dataframe for tweet in response.json()['data']: row = get_data(tweet) df_tweets = df_tweets.append(row, ignore_index=True) # Load sentiment model print("") sentiment_model = flair.models.TextClassifier.load('en-sentiment') print("") # Append probability and sentiment preds later probs = [] sentiments = [] for s_tweet in df_tweets['text'].to_list(): tweet = clean_tweet(s_tweet, s_ticker) # Make sentiment prediction sentence = flair.data.Sentence(tweet) sentiment_model.predict(sentence) # Extract sentiment prediction (POSITIVE/NEGATIVE) and confidence (0-1) probs.append(sentence.labels[0].score) sentiments.append(sentence.labels[0].value) # Add probability and sentiment predictions to tweets dataframe df_tweets['probability'] = probs df_tweets['sentiment'] = sentiments # Sort tweets per date df_tweets.sort_index(ascending=False, inplace=True) # Add sentiment estimation (probability positive for POSITIVE sentiment, and negative for NEGATIVE sentiment) df_tweets['sentiment_estimation'] = df_tweets.apply( lambda row: row['probability'] * (-1, 1)[row['sentiment'] == 'POSITIVE'], axis=1).cumsum() # Cumulative sentiment_estimation df_tweets['prob_sen'] = df_tweets.apply( lambda row: row['probability'] * (-1, 1)[row['sentiment'] == 'POSITIVE'], axis=1) # Percentage of confidence if df_tweets['sentiment_estimation'].values[-1] > 0: n_pos = df_tweets[df_tweets['prob_sen'] > 0]['prob_sen'].sum() n_pct = round(100 * n_pos / df_tweets['probability'].sum()) else: n_neg = abs(df_tweets[df_tweets['prob_sen'] < 0]['prob_sen'].sum()) n_pct = round(100 * n_neg / df_tweets['probability'].sum()) s_sen = f"{('NEGATIVE', 'POSITIVE')[int(df_tweets['sentiment_estimation'].values[-1] > 0)]}" #df_tweets.to_csv(r'notebooks/tweets.csv', index=False) df_tweets.reset_index(inplace=True) # Plotting plt.subplot(211) plt.title( f"Twitter's {s_ticker} sentiment over time is {s_sen} ({n_pct} %)") plt.plot(df_tweets.index, df_tweets['sentiment_estimation'].values, lw=3, c='cyan') plt.xlim(df_tweets.index[0], df_tweets.index[-1]) plt.grid(b=True, which='major', color='#666666', linestyle='-', lw=1.5, alpha=0.5) plt.minorticks_on() plt.grid(b=True, which='minor', color='#999999', linestyle='-', alpha=0.2) plt.ylabel('Cumulative Sentiment') l_xticks = list() l_xlabels = list() l_xticks.append(0) l_xlabels.append(df_tweets['created_at'].values[0].split(' ')[0]) n_day = datetime.strptime(df_tweets['created_at'].values[0], "%Y-%m-%d %H:%M:%S").day n_idx = 0 n_next_idx = 0 for n_next_idx, dt_created in enumerate(df_tweets['created_at']): if datetime.strptime(dt_created, "%Y-%m-%d %H:%M:%S").day > n_day: l_xticks.append(n_next_idx) l_xlabels.append( df_tweets['created_at'].values[n_next_idx].split(' ')[0]) l_val_days = df_tweets['sentiment_estimation'].values[ n_idx:n_next_idx] - df_tweets['sentiment_estimation'].values[ n_idx] plt.plot(range(n_idx, n_next_idx), l_val_days, lw=3, c='tab:blue') n_day_avg = np.mean(l_val_days) if n_day_avg > 0: plt.hlines(n_day_avg, n_idx, n_next_idx, linewidth=2.5, linestyle='--', color='green', lw=3) else: plt.hlines(n_day_avg, n_idx, n_next_idx, linewidth=2.5, linestyle='--', color='red', lw=3) n_idx = n_next_idx n_day += 1 l_val_days = df_tweets['sentiment_estimation'].values[n_idx:] - df_tweets[ 'sentiment_estimation'].values[n_idx] plt.plot(range(n_idx, len(df_tweets)), l_val_days, lw=3, c='tab:blue') n_day_avg = np.mean(l_val_days) if n_day_avg > 0: plt.hlines(n_day_avg, n_idx, len(df_tweets), linewidth=2.5, linestyle='--', color='green', lw=3) else: plt.hlines(n_day_avg, n_idx, len(df_tweets), linewidth=2.5, linestyle='--', color='red', lw=3) l_xticks.append(len(df_tweets)) datetime.strptime(dt_created, "%Y-%m-%d %H:%M:%S") + timedelta(days=1) l_xlabels.append( datetime.strftime( datetime.strptime( df_tweets['created_at'].values[len(df_tweets) - 1], "%Y-%m-%d %H:%M:%S") + timedelta(days=1), "%Y-%m-%d")) plt.xticks(l_xticks, l_xlabels) plt.axhspan(plt.gca().get_ylim()[0], 0, facecolor='r', alpha=0.1) plt.axhspan(0, plt.gca().get_ylim()[1], facecolor='g', alpha=0.1) plt.subplot(212) plt.bar(df_tweets[df_tweets['prob_sen'] > 0].index, df_tweets[df_tweets['prob_sen'] > 0]['prob_sen'].values, color='green') plt.bar(df_tweets[df_tweets['prob_sen'] < 0].index, df_tweets[df_tweets['prob_sen'] < 0]['prob_sen'].values, color='red') for l_x in l_xticks[1:]: plt.vlines(l_x, -1, 1, linewidth=2, linestyle='--', color='k', lw=3) plt.xlim(df_tweets.index[0], df_tweets.index[-1]) plt.xticks(l_xticks, l_xlabels) plt.grid(b=True, which='major', color='#666666', linestyle='-') plt.ylabel('Sentiment') plt.xlabel("Time") plt.show()
def main(): parser = argparse.ArgumentParser() subparsers = parser.add_subparsers(dest='test_type') # Host-side test args. host_cmd_parser = subparsers.add_parser( 'host-cmd', help='Runs a host-side test. Pass the host-side command to run after ' '"--". If --use-vm is passed, hostname and port for the device ' 'will be 127.0.0.1:9222.') host_cmd_parser.set_defaults(func=host_cmd) host_cmd_parser.add_argument( '--deploy-chrome', action='store_true', help= 'Will deploy a locally built ash-chrome binary to the device before ' 'running the host-cmd.') host_cmd_parser.add_argument( '--deploy-lacros', action='store_true', help='Deploy a lacros-chrome instead of ash-chrome.') # GTest args. # TODO(bpastene): Rename 'vm-test' arg to 'gtest'. gtest_parser = subparsers.add_parser('vm-test', help='Runs a device-side gtest.') gtest_parser.set_defaults(func=device_test) gtest_parser.add_argument( '--test-exe', type=str, required=True, help='Path to test executable to run inside the device.') # GTest args. Some are passed down to the test binary in the device. Others # are parsed here since they might need tweaking or special handling. gtest_parser.add_argument( '--test-launcher-summary-output', type=str, help='When set, will pass the same option down to the test and retrieve ' 'its result file at the specified location.') gtest_parser.add_argument( '--stop-ui', action='store_true', help='Will stop the UI service in the device before running the test.') gtest_parser.add_argument( '--trace-dir', type=str, help='When set, will pass down to the test to generate the trace and ' 'retrieve the trace files to the specified location.') gtest_parser.add_argument( '--env-var', nargs=2, action='append', default=[], help='Env var to set on the device for the duration of the test. ' 'Expected format is "--env-var SOME_VAR_NAME some_var_value". Specify ' 'multiple times for more than one var.') # Tast test args. # pylint: disable=line-too-long tast_test_parser = subparsers.add_parser( 'tast', help='Runs a device-side set of Tast tests. For more details, see: ' 'https://chromium.googlesource.com/chromiumos/platform/tast/+/master/docs/running_tests.md' ) tast_test_parser.set_defaults(func=device_test) tast_test_parser.add_argument( '--suite-name', type=str, required=True, help='Name to apply to the set of Tast tests to run. This has no effect ' 'on what is executed, but is used mainly for test results reporting ' 'and tracking (eg: flakiness dashboard).') tast_test_parser.add_argument( '--test-launcher-summary-output', type=str, help='Generates a simple GTest-style JSON result file for the test run.' ) # TODO(bpastene): Change all uses of "--conditional" to use "--attr-expr". tast_test_parser.add_argument( '--conditional', '--attr-expr', type=str, dest='conditional', help='A boolean expression whose matching tests will run ' '(eg: ("dep:chrome")).') tast_test_parser.add_argument( '--strip-chrome', action='store_true', help='Strips symbols from ash-chrome before deploying to the device.') tast_test_parser.add_argument( '--deploy-lacros', action='store_true', help='Deploy a lacros-chrome instead of ash-chrome.') tast_test_parser.add_argument( '--tast-var', action='append', dest='tast_vars', help='Runtime variables for Tast tests, and the format are expected to ' 'be "key=value" pairs.') tast_test_parser.add_argument( '--test', '-t', action='append', dest='tests', help='A Tast test to run in the device (eg: "ui.ChromeLogin").') tast_test_parser.add_argument( '--gtest_filter', type=str, help="Similar to GTest's arg of the same name, this will filter out the " "specified tests from the Tast run. However, due to the nature of Tast's " 'cmd-line API, this will overwrite the value(s) of "--test" above.') add_common_args(gtest_parser, tast_test_parser, host_cmd_parser) args = sys.argv[1:] unknown_args = [] # If a '--' is present in the args, treat everything to the right of it as # args to the test and everything to the left as args to this test runner. # Otherwise treat all known args as args to this test runner and all unknown # args as test args. if '--' in args: unknown_args = args[args.index('--') + 1:] args = args[0:args.index('--')] if unknown_args: args = parser.parse_args(args=args) else: args, unknown_args = parser.parse_known_args() logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARN) if not args.use_vm and not args.device: # If we're not running on a VM, but haven't specified a hostname, assume # we're on a lab bot and are trying to run a test on a lab DUT. See if the # magic lab DUT hostname resolves to anything. (It will in the lab and will # not on dev machines.) try: socket.getaddrinfo(LAB_DUT_HOSTNAME, None) except socket.gaierror: logging.error('The default DUT hostname of %s is unreachable.', LAB_DUT_HOSTNAME) return 1 args.cros_cache = os.path.abspath(args.cros_cache) return args.func(args, unknown_args)
'Output directory, if not provided will output to current working directory', default=None) parser.add_argument('--verbosity', help='Verbosity (default=0)', default=0) parser.add_argument('--override', help='Overwrite NetCDF files (default=True)', default=True) parser.add_argument( '--acolite_path', help= 'Path to ACOLITE code (if not already in Python path) (default=None)', default=None) parser.add_argument('--satellite', help='Specifiy satellite (default=Sentinel-2A)', default='Sentinel-2A') args, unknown = parser.parse_known_args() ## acolite path if args.acolite_path is not None: sys.path.append(os.path.expanduser(args.acolite_path)) try: import acolite as ac except: print( 'Could not import acolite - make sure it is in your Python path, or provide it with --acolite_path' ) exit(1) ## parse output directory if args.output is not None: args.output = os.path.expanduser(args.output)
def main(): global options global logger today = datetime.now(pytz.timezone('US/Eastern')).date() init_parser = argparse.ArgumentParser() init_parser.add_argument("-p", "--profile", help="use alternate config profile") options, args = init_parser.parse_known_args() config.settings.load() if options.profile: config.settings.set_profile(options.profile) parser = argparse.ArgumentParser() # parser.add_argument("-d", "--date", help="game date", # type=utils.valid_date, # default=today) parser.add_argument("-r", "--resolution", help="stream resolution", default=config.settings.profile.default_resolution) group = parser.add_mutually_exclusive_group() group.add_argument("-v", "--verbose", action="count", default=0, help="verbose logging") group.add_argument("-q", "--quiet", action="count", default=0, help="quiet logging") parser.add_argument("game", metavar="game", help="game specifier", nargs="?") options, args = parser.parse_known_args() log_file = os.path.join(config.CONFIG_DIR, "mlbstreamer.log") # formatter = logging.Formatter( # "%(asctime)s [%(module)16s:%(lineno)-4d] [%(levelname)8s] %(message)s", # datefmt="%Y-%m-%d %H:%M:%S" # ) fh = logging.FileHandler(log_file) fh.setLevel(logging.DEBUG) # fh.setFormatter(formatter) logger = logging.getLogger("mlbstreamer") # logger.setLevel(logging.INFO) # logger.addHandler(fh) ulh = UrwidLoggingHandler() # ulh.setLevel(logging.DEBUG) # ulh.setFormatter(formatter) # logger.addHandler(ulh) utils.setup_logging(options.verbose - options.quiet, handlers=[fh, ulh], quiet_stdout=True) try: (provider, game_date) = options.game.split("/", 1) except (ValueError, AttributeError): if options.game in session.PROVIDERS: provider = options.game game_date = datetime.now().date() else: provider = list(config.settings.profile.providers.keys())[0] game_date = dateutil.parser.parse(options.game) logger.debug("mlbstreamer starting") entries = Dropdown.get_palette_entries() entries.update(ScrollingListBox.get_palette_entries()) entries.update(DataTable.get_palette_entries()) # raise Exception(entries) palette = Palette("default", **entries) screen = urwid.raw_display.Screen() screen.set_terminal_properties(256) view = ScheduleView(provider, game_date) log_console = widgets.ConsoleWindow() # log_box = urwid.BoxAdapter(urwid.LineBox(log_console), 10) pile = urwid.Pile([("weight", 5, urwid.LineBox(view)), ("weight", 1, urwid.LineBox(log_console))]) def global_input(key): if key in ('q', 'Q'): raise urwid.ExitMainLoop() else: return False state.loop = urwid.MainLoop(pile, palette, screen=screen, unhandled_input=global_input, pop_ups=True) ulh.connect(state.loop.watch_pipe(log_console.log_message)) logger.info("mlbstreamer starting") if options.verbose: logger.setLevel(logging.DEBUG) state.loop.run()
def main(): parser = argparse.ArgumentParser() parser.add_argument("--pipeline-uri", type=str, help="A URI to JSON with instructions") parser.add_argument("--pipeline", type=str, help="JSON with instructions") parser.add_argument( "--prisma-path", type=str, help="PRISMA Scene Local Path" # TODO: remove ) parser.add_argument( "--prisma-uri", type=str, help="PRISMA Scene S3 URI" # TODO: remove ) parser.add_argument( "--prisma-stac-id", type=str, help="STAC Item ID to process from the STAC collection") parser.add_argument( "--prisma-collection-id", type=str, default=PRISMA_ARCHIVE_COLLECTION_ID, ) parser.add_argument( "--stac-api-uri", type=str, default=os.environ.get("STAC_API_URI", "http://franklin:9090"), ) parser.add_argument("--s3-bucket", type=str, default=os.environ.get("S3_BUCKET", "aviris-data")) parser.add_argument( "--s3-prefix", type=str, default=os.environ.get("S3_PREFIX", "aviris-scene-cogs-l2"), ) parser.add_argument("--temp-dir", type=str, default=os.environ.get("TEMP_DIR", None)) parser.add_argument("--output-format", type=str, default=os.environ.get("GDAL_OUTPUT_FORMAT", "COG")) parser.add_argument( "--keep-temp-dir", action="store_true", help= "If provided, script does not delete temporary directory before script exits. Useful for debugging.", ) parser.add_argument( "--force", action="store_true", help= "If provided, force reingest StacItem even though this it is already present in the catalog.", ) parser.add_argument("--skip-upload", action="store_true", help="Skips upload to S3", default=False) try: warpMemoryLimit = int(os.environ.get("GDAL_WARP_MEMORY_LIMIT", None)) except TypeError: warpMemoryLimit = None # TODO: replace it with parser.parse_args() later cli_args, cli_unknown = parser.parse_known_args() # parse all cli arguments args = CliConfig(cli_args, cli_unknown) # Create tmpdir temp_dir = Path(args.temp_dir if args.temp_dir is not None else mkdtemp()) temp_dir.mkdir(parents=True, exist_ok=True) s3 = boto3.client("s3") stac_client = STACClient(args.stac_api_uri) prisma_uri = S3Uri(args.prisma_uri) product_name_derived = os.path.splitext(Path(prisma_uri.key).name)[0] local_archive = Path(temp_dir, Path(prisma_uri.key).name) try: if not local_archive.exists(): logger.info( f'Downloading PRISMA archive {prisma_uri.url} to {str(local_archive)}...' ) s3.download_file(prisma_uri.bucket, prisma_uri.key, str(local_archive)) else: logger.info( f'Skipping downloading PRISMA archive {prisma_uri.url}, it already exists {str(local_archive)}' ) h5_path = Path(Path(temp_dir), f'{product_name_derived}.he5') if not h5_path.exists(): logger.info(f'Extracting PRISMA archive {str(local_archive)}...') with zipfile.ZipFile(local_archive, 'r') as zip_ref: zip_ref.extractall(str(temp_dir)) else: logger.info( f'Skipping extraction of the PRISMA archive {str(local_archive)}, file is already extracted {str(h5_path)}' ) logger.info(f'Reading {str(h5_path)}...') h5 = h5py.File(str(h5_path)) data = h5['HDFEOS']['SWATHS']['PRS_L2D_HCO']['Data Fields'] # UInt16 swir = data['SWIR_Cube'] # UInt16 vnir = data['VNIR_Cube'] # ERR Matrix is in Bytes swir_error = data['SWIR_PIXEL_L2_ERR_MATRIX'] vnir_error = data['VNIR_PIXEL_L2_ERR_MATRIX'] swirT = np.swapaxes(swir, 1, 2) vnirT = np.swapaxes(vnir, 1, 2) swir_errorT = np.swapaxes(swir_error, 1, 2) vnir_errorT = np.swapaxes(vnir_error, 1, 2) rows, cols = swirT.shape[0], swirT.shape[1] product_name = os.path.splitext(str( h5.attrs['Product_Name'].decode()))[0] # X = Lon, Y = Lat # LatLon projection, required for the STAC catalog extent_ll, _, _ = raster_extent(ll_y=h5.attrs['Product_LLcorner_lat'], ll_x=h5.attrs['Product_LLcorner_long'], lr_y=h5.attrs['Product_LRcorner_lat'], lr_x=h5.attrs['Product_LRcorner_long'], ul_y=h5.attrs['Product_ULcorner_lat'], ul_x=h5.attrs['Product_ULcorner_long'], ur_y=h5.attrs['Product_URcorner_lat'], ur_x=h5.attrs['Product_URcorner_long'], cols=cols, rows=rows) # X = easting, Y = northing extent, res, affine_transform = raster_extent( ll_y=h5.attrs['Product_LLcorner_lat'], ll_x=h5.attrs['Product_LLcorner_long'], lr_y=h5.attrs['Product_LRcorner_lat'], lr_x=h5.attrs['Product_LRcorner_long'], ul_y=h5.attrs['Product_ULcorner_lat'], ul_x=h5.attrs['Product_ULcorner_long'], ur_y=h5.attrs['Product_URcorner_lat'], ur_x=h5.attrs['Product_URcorner_long'], cols=cols, rows=rows) cog_proj = osr.SpatialReference() cog_proj.SetUTM(int(h5.attrs['Projection_Id'].decode())) cog_proj.AutoIdentifyEPSG() def create_geotiff(path, array, geo_transform=affine_transform, projection=cog_proj, dataType=gdal.GDT_UInt16, driver=gdal.GetDriverByName("GTiff")): ds = driver.Create(str(path), array.shape[1], array.shape[0], array.shape[2], dataType, ['COMPRESS=DEFLATE', 'BIGTIFF=YES']) ds.SetGeoTransform(geo_transform) ds.SetProjection(projection.ExportToWkt()) for i in range(1, array.shape[2]): ds.GetRasterBand(i).WriteArray(array[::, ::, i]) ds.FlushCache() cog_path = str(Path(path.parent, f'cog_{path.name}')) warp_opts = gdal.WarpOptions(callback=warp_callback, warpOptions=["NUM_THREADS=ALL_CPUS"], creationOptions=[ "NUM_THREADS=ALL_CPUS", "COMPRESS=DEFLATE", "BIGTIFF=YES", "TILED=YES" ], multithread=True, warpMemoryLimit=warpMemoryLimit, format=args.output_format) logger.info(f'Converting {str(path)} to {cog_path}...') # NOTE: # We can't directly write TIFFs on S3 as the result of the gdal.Warp operation # see: https://github.com/OSGeo/gdal/issues/1189 with timing("GDAL Warp"): gdal.Warp(str(cog_path), str(path), options=warp_opts) return cog_path swir_path = create_geotiff(Path(temp_dir, "SWIR_Cube.tiff"), swirT) vnir_path = create_geotiff(Path(temp_dir, "VNIR_Cube.tiff"), vnirT) swir_error_path = create_geotiff( Path(temp_dir, "SWIR_PIXEL_L2_ERR_MATRIX.tiff"), swir_errorT) vnir_error_path = create_geotiff( Path(temp_dir, "VNIR_PIXEL_L2_ERR_MATRIX.tiff"), vnir_errorT) def key(path_string): return f'prisma-scene-cogs/{product_name}/{Path(path_string).name}' # prep all upload links s3_uri_swir = f's3://{args.s3_bucket}/{key(swir_path)}' s3_uri_vnir = f's3://{args.s3_bucket}/{key(vnir_path)}' s3_uri_swir_error = f's3://{args.s3_bucket}/{key(swir_error_path)}' s3_uri_vnir_error = f's3://{args.s3_bucket}/{key(vnir_error_path)}' upload_paths = [ ('SWIR_Cube', swir_path, s3_uri_swir), ('VNIR_Cube', vnir_path, s3_uri_vnir), ('SWIR_PIXEL_L2_ERR_MATRIX', swir_error_path, s3_uri_swir_error), ('VNIR_PIXEL_L2_ERR_MATRIX', vnir_error_path, s3_uri_vnir_error) ] if (not args.skip_upload): for _, local_path, s3_uri in upload_paths: logger.info(f'Uploading {local_path} to {s3_uri}') s3.upload_file( str(local_path), args.s3_bucket, str(key(local_path)), Callback=ProgressPercentage(str(local_path)), Config=TransferConfig(multipart_threshold=1 * GB), ) swir_λs = h5.attrs['List_Cw_Swir'] swir_flags = h5.attrs['List_Cw_Swir_Flags'] swir_freqs = zip( swir_λs[swir_flags == 1], map(lambda x: x[0], filter(lambda x: x[1] == 1, enumerate(swir_flags)))) swir_freqs_sorted = sorted([(b, f * 0.001) for f, b in swir_freqs], key=lambda x: x[0]) vnir_λs = h5.attrs['List_Cw_Vnir'] vnir_flags = h5.attrs['List_Cw_Vnir_Flags'] vnir_freqs = zip( vnir_λs[vnir_flags == 1], map(lambda x: x[0], filter(lambda x: x[1] == 1, enumerate(vnir_flags)))) vnir_freqs_sorted = sorted([(b, f * 0.001) for f, b in vnir_freqs], key=lambda x: x[0]) λs = np.hstack([swir_λs, vnir_λs]) flags = np.hstack([swir_flags, vnir_flags]) band_freqs = sorted( zip( λs[flags == 1], map(lambda x: x[0], filter(lambda x: x[1] == 1, enumerate(flags))))) sorted_freqs = sorted([(b, f * 0.001) for f, b in band_freqs], key=lambda x: x[0]) eo_swir_bands = [{ 'name': str(b), 'center_wavelength': float(f) } for (b, f) in swir_freqs_sorted] eo_vnir_bands = [{ 'name': str(b), 'center_wavelength': float(f) } for (b, f) in vnir_freqs_sorted] start_datetime = str(h5.attrs['Product_StartTime'].decode()) end_datetime = str(h5.attrs['Product_StopTime'].decode()) wavelength_min = float(min(sorted_freqs, key=lambda e: e[1])[1]) wavelength_max = float(max(sorted_freqs, key=lambda e: e[1])[1]) cog_item_id = f'{PRISMA_COG_COLLECTION.id}_{product_name}' cog_item = pystac.Item( id=cog_item_id, stac_extensions=COG_ITEM_EXTENSIONS, geometry=shapely.geometry.mapping( shapely.geometry.box(*extent_ll)), datetime=dateutil.parser.isoparse(start_datetime), bbox=list(map(lambda c: float(c), extent_ll)), collection=PRISMA_COG_COLLECTION.id, properties={ 'start_datetime': start_datetime, 'end_datetime': end_datetime, 'hsi:wavelength_min': wavelength_min, 'hsi:wavelength_max': wavelength_max, 'proj:epsg': int(cog_proj.GetAttrValue('AUTHORITY', 1)) }) # add item assets for eo_bands, (key, _, uri) in zip( [eo_swir_bands, eo_swir_bands, eo_swir_bands, eo_swir_bands], upload_paths): cog_item.add_asset(key=key, asset=pystac.Asset( href=uri, media_type=pystac.MediaType.GEOTIFF, properties={'eo:bands': eo_bands})) finally: if not args.keep_temp_dir: logger.info(f'Removing temp dir: {temp_dir}') shutil.rmtree(temp_dir, ignore_errors=True) # Create COG Collection if it doesn't exist if not stac_client.has_collection(PRISMA_COG_COLLECTION.id): stac_client.post_collection(PRISMA_COG_COLLECTION) # Add COG Item to AVIRIS L2 STAC Collection logger.info(f'POST Item {cog_item.id} to {args.stac_api_uri}') item_data = stac_client.post_collection_item(PRISMA_COG_COLLECTION.id, cog_item) if item_data.get('id', None): logger.info(f"Success: {item_data['id']}") activation_output(item_data['id']) else: logger.error(f'Failure: {item_data}') return -1
def main(): today = datetime.now(pytz.timezone('US/Eastern')).date() parser = argparse.ArgumentParser() parser.add_argument("-d", "--date", help="game date", type=valid_date, default=today) parser.add_argument("-g", "--game-number", help="number of team game on date (for doubleheaders)", default=1, type=int) parser.add_argument("-b", "--begin", help="begin playback at this offset from start", nargs="?", metavar="offset_from_game_start", type=begin_arg_to_offset, const=0) parser.add_argument("-r", "--resolution", help="stream resolution", default="720p") parser.add_argument("-s", "--save-stream", help="save stream to file", nargs="?", const=True) parser.add_argument("--no-cache", help="do not use response cache", action="store_true") parser.add_argument("-v", "--verbose", action="store_true", help="verbose logging") parser.add_argument("--init-config", help="initialize configuration", action="store_true") parser.add_argument("game", metavar="game", nargs="?", help="team abbreviation or MLB game ID") options, args = parser.parse_known_args() global logger logger = logging.getLogger("mlbstreamer") if options.verbose: logger.setLevel(logging.DEBUG) formatter = logging.Formatter( "%(asctime)s [%(levelname)8s] %(message)s", datefmt='%Y-%m-%d %H:%M:%S') handler = logging.StreamHandler(sys.stdout) handler.setFormatter(formatter) logger.addHandler(handler) else: logger.addHandler(logging.NullHandler()) if options.init_config: config.settings.init_config() sys.exit(0) config.settings.load() if not options.game: parser.error("option game") state.session = MLBSession.new(no_cache=options.no_cache) preferred_stream = None date = None if options.game.isdigit(): game_specifier = int(options.game) else: game_specifier = (options.date, options.game, options.game_number) try: proc = play_stream( game_specifier, options.resolution, offset=options.begin, preferred_stream=preferred_stream, output=options.save_stream, ) proc.wait() except MLBPlayInvalidArgumentError as e: raise argparse.ArgumentTypeError(str(e)) except MLBPlayException as e: logger.error(e)
default=os.environ.get('ES_HOSTS', 'http://10.80.121.231:9200'), help='ElasticSearch server(s)') parser.add_argument('--index', '-i', dest='elastic_index', action='store', default='o365-%s' % hex(abs(hash(json.dumps(sys.argv[1:]))))[2:10], help='ElasticSearch index name') parser.add_argument("paths", nargs=REMAINDER, help='Target audit log file(s)', metavar='paths') args, extra = parser.parse_known_args(sys.argv[1:]) es = Elasticsearch(args.elastic_server, index=args.elastic_index) tzinfos = {"AEST": gettz("Australia/Brisbane")} print(f"Using server: {args.elastic_server}") print(f"Using index: {args.elastic_index}") def convert_key(string): s1 = re.sub('([^\.])([A-Z][a-z]+)', r'\1_\2', string.replace(' ', '_')) s2 = re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() s3 = re.sub('_+', r'_', s2.replace('.', '_')) return s3.replace('extended_properties', '').replace('target_updated_properties', 'target_updated') \ .replace('parameters', '').replace('additional_details', '')
def main(): today = datetime.now(pytz.timezone('US/Eastern')).date() init_parser = argparse.ArgumentParser(add_help=False) init_parser.add_argument("--init-config", help="initialize configuration", action="store_true") init_parser.add_argument("-p", "--profile", help="use alternate config profile") options, args = init_parser.parse_known_args() if options.init_config: config.settings.init_config() sys.exit(0) config.settings.load() if options.profile: config.settings.set_profile(options.profile) parser = argparse.ArgumentParser( description=init_parser.format_help(), formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("-b", "--begin", help="begin playback at this offset from start", nargs="?", metavar="offset_from_game_start", type=begin_arg_to_offset, const=0) parser.add_argument("-r", "--resolution", help="stream resolution", default=config.settings.profile.default_resolution) parser.add_argument("-s", "--save-stream", help="save stream to file", nargs="?", const=True) parser.add_argument("--no-cache", help="do not use response cache", action="store_true") group = parser.add_mutually_exclusive_group() group.add_argument("-v", "--verbose", action="count", default=0, help="verbose logging") group.add_argument("-q", "--quiet", action="count", default=0, help="quiet logging") parser.add_argument("game", metavar="game", nargs="?", help="team abbreviation or MLB game ID") options, args = parser.parse_known_args(args) try: (provider, game) = options.game.split("/", 1) except ValueError: game = options.game #.split(".", 1)[1] provider = list(config.settings.profile.providers.keys())[0] if game.isdigit(): game_specifier = int(game) else: game_specifier = game utils.setup_logging(options.verbose - options.quiet) if not options.game: parser.error("option game") state.session = session.new(provider) preferred_stream = None date = None try: proc = play_stream(game_specifier, options.resolution, offset=options.begin, preferred_stream=preferred_stream, output=options.save_stream, verbose=options.verbose) proc.wait() except MLBPlayInvalidArgumentError as e: raise argparse.ArgumentTypeError(str(e)) except MLBPlayException as e: logger.error(e)
import warnings # Python 3.6 deprecate invalid character-pairs \A, \* ... in non # raw-strings and other things. Let's not re-introduce them warnings.filterwarnings('error', '.*invalid escape sequence.*', category=DeprecationWarning) warnings.filterwarnings( 'default', '.*inspect.getargspec\(\) is deprecated.*', category=DeprecationWarning) from matplotlib import test parser = argparse.ArgumentParser(add_help=False) parser.add_argument('--no-network', action='store_true', help='Run tests without network connection') parser.add_argument('--recursionlimit', type=int, default=0, help='Specify recursionlimit for test run') args, extra_args = parser.parse_known_args() if args.no_network: from matplotlib.testing import disable_internet disable_internet.turn_off_internet() extra_args.extend(['-m', 'not network']) print('Python byte-compilation optimization level:', sys.flags.optimize) retcode = test(argv=extra_args, switch_backend_warn=False, recursionlimit=args.recursionlimit) sys.exit(retcode)
def init(): parser = argparse.ArgumentParser( description= "Module that loads given datasets and trains and evaluates one or more neural network models on that." ) parser.add_argument('dataset', type=str, help="The filepath to the dataset/s.") parser.add_argument( '--models', type=str, help= "A list of the models that are going to be trained and evaluated. Default is all available." ) parser.add_argument( '--args', type=str, help= "A list of arguments to be passed on to the models. In the format key1=value1,key2=value2.1;value2.2" ) parser.add_argument('--quiet', dest='quiet', action="store_true", help="Do not plot graphs, but save them as images.") parser.set_defaults(quiet=False) parser.add_argument( '--saveModel', type=str, help= "Location to save the model architecture, weights, training history, evaluation scores and prediction graphs. DO NOT include file extension, just filename!" ) parser.add_argument( '--loadModel', type=str, help= "Location to to load a saved model architecture and weights. Include the file extension!" ) parser.add_argument('--shuffle', dest='shuffle', action="store_true", help="Shuffle the generated dataset and labels.") parser.set_defaults(shuffle=False) parser.add_argument( '--trim-batch', dest='trim', action="store_true", help= "Trim each dataset so that its length is divisible by the batch size.") parser.set_defaults(trim=False) parser.add_argument( '--no-replace', dest='noReplace', action="store_true", help= "If a model is already saved with the same filename, don't train and save the new model." ) parser.set_defaults(noReplace=False) parser.add_argument('--no-train', dest='train', action="store_false", help="Do not train, only evaluathe the model.") parser.set_defaults(train=True) args, _ = parser.parse_known_args() if args.noReplace and args.saveModel is not None: if os.path.exists(args.saveModel + modelExtension): print("The file already exists and the 'no-replace' flag is used.") return #do not replace it givenModels = [x.lower() for x in args.models.split(',')] if args.models else None modelArgs = {} pairs = args.args.split(',') if args.args else [] for pair in pairs: key, value = pair.split('=') try: value = float(value) if value == int(value): value = int(value) except ValueError: if ':' in value: value = [int(i) for i in value.split(':')] pass modelArgs[key] = value print("Processed model arguments", modelArgs) run(args.dataset, givenModels, modelArgs=modelArgs, quiet=args.quiet, saveModel=args.saveModel, loadModel=args.loadModel, shuffle=args.shuffle, trim=args.trim, train=args.train)