def do(self):
usagestring = "DECam_clusters_upload.py date <options>"
tstart = time.time()
# read in the options from the param file and the command line
# some convoluted syntax here, making it so param file is not required
parser = self.add_options(usage=usagestring)
options, args = parser.parse_known_args()
config = configparser.ConfigParser()
config.read("%s/settings.ini" % djangoSettings.PROJECT_DIR)
parser = self.add_options(usage=usagestring, config=config)
options, args = parser.parse_known_args()
self.options = options
try:
nsn = self.main()
except Exception as e:
print(e)
nsn = 0
smtpserver = "%s:%s" % (options.SMTP_HOST, options.SMTP_PORT)
from_addr = "*****@*****.**" % options.SMTP_LOGIN
subject = "QUB Transient Upload Failure"
print("Sending error email")
html_msg = "Alert : YSE_PZ Failed to upload transients from PSST in QUB_data.py\n"
html_msg += "Error : %s"
sendemail(from_addr, options.dbemail, subject, html_msg % (e),
options.SMTP_LOGIN, options.dbemailpassword, smtpserver)
print('QUB -> YSE_PZ took %.1f seconds for %i transients' %
(time.time() - tstart, nsn))
def main(argv=None):
"""
The main function which creates the pipeline and runs it.
"""
parser = argparse.ArgumentParser()
# Here we add some specific command line arguments we expect.
# This defaults the output table in your BigQuery you'll have
# to create the example_data dataset yourself using bq mk temp
parser.add_argument('--output', dest='output', required=False)
# Parse arguments from the command line.
known_args, pipeline_args = parser.parse_known_args(argv)
table_spec = bigquery.TableReference(
projectId=os.getenv('GOOGLE_PROJECT_ID'),
datasetId=os.getenv('GOOGLE_DATASET_ID'),
tableId=os.getenv('GOOGLE_TABLE_ID'))
pipeline_options = PipelineOptions(flags=pipeline_args, )
pipeline = beam.Pipeline(options=pipeline_options)
(pipeline
| 'Get Data From Business Central ' >> beam.Create(get_data_from_bc365())
| 'Format Dates' >> beam.ParDo(FormatElements())
| 'Write To BigQuery' >> beam.io.WriteToBigQuery(
table_spec,
write_disposition=beam.io.BigQueryDisposition.WRITE_TRUNCATE,
create_disposition=beam.io.BigQueryDisposition.CREATE_NEVER,
custom_gcs_temp_location=SYNC_BUCKET))
pipeline.run().wait_until_finish()
def main():
parser = argparse.ArgumentParser(
description="craigslist car finder", parents=())
parser.add_argument("-c", "--city", default='austin',
help='which city to search for')
parser.add_argument("-b", "--brand", default='honda',
help='car brand')
parser.add_argument("-m", "--model", default='civic',
help='car model')
parser.add_argument("-p", "--minimum_price", default='4000')
parser.add_argument("-y", "--minimum_year", default='2007')
parser.add_argument("-d", "--maximum_odometer", default='100000',
help='maximum miles travelled by car before purchase')
parser.add_argument("-t", "--blacklist_titles", nargs='+', default=['salvage', 'rebuilt'],
help='List unacceptable states for car, e.g. You may want to filter out cars that \
have been totalled or salvaged')
parser.add_argument("-w", "--week_range", default=2,
help='number of weeks to search car listings for starting from now')
parser.add_argument("-l", "--max_results", default=20,
help='limit to this number of results for cars returned')
parser.add_argument("-o", "--output",
help='write matching cars to file')
parser.add_argument("-v", "--verbose", action='store_true',
help='print debug output')
try:
args, extra_args = parser.parse_known_args()
except Exception, e:
print e
sys.exit(1)
def main():
# from .state import *
from . import utils
import argparse
global options
parser = argparse.ArgumentParser()
group = parser.add_mutually_exclusive_group()
group.add_argument("-v",
"--verbose",
action="count",
default=0,
help="verbose logging")
group.add_argument("-q",
"--quiet",
action="count",
default=0,
help="quiet logging")
options, args = parser.parse_known_args()
utils.setup_logging(options.verbose - options.quiet)
# state.session = MLBStreamSession.new()
# raise Exception(state.session.token)
raise Exception(PROVIDERS)
def main():
import argparse
parser = argparse.ArgumentParser(description='Warden IMAPReceiver configuration file parser')
parser.add_argument('--config', help="Path to the Warden IMAPReceiver configuration file.", dest='config')
parser.add_argument('--server', help='Run continually in server mode (default is to run once)', action='store_true', default=False)
args, unknown = parser.parse_known_args(sys.argv)
imapreceiver_configuration_file = os.path.abspath(os.path.expanduser(args.config))
if not os.path.exists(imapreceiver_configuration_file):
log.error('The IMAPReceiver config file specified ("%s") does not exist!' % imapreceiver_configuration_file)
sys.exit(1)
config = ConfigParser.SafeConfigParser()
config.read(imapreceiver_configuration_file)
imapreceiver = IMAPReceiver(config)
if args.server:
imapreceiver.start()
else:
imapreceiver.connect_to_mailbox()
imapreceiver.check_mails()
def main():
# Spawn threads, one for each feed.
# gsb-host and mongod-host should be passed on command line
parser = argparse.ArgumentParser()
parser.add_argument('-g',
'--gsb-url',
help='GSB Agent URL',
dest='gsb_url')
parser.add_argument('-m',
'--mongod-url',
help='Mongo DB Server URL',
dest='mongod_url')
args = parser.parse_known_args()
# Fetch API keys from apikeys.conf
with open('apikeys.conf') as f:
keys_raw = f.read().split('\n')
keys_d = dict()
for i in keys_raw:
if i is not None and i is not '':
keys_d[i.split(':')[0]] = i.split(':')[1]
feedProcessor = FeedProcessor(args, keys_d)
signal.signal(signal.SIGINT, feedProcessor.sigint_handler)
connect("threatint", host=args[0].mongod_url)
logging.basicConfig(filename="url_record.log", level=logging.INFO)
for i in sorted(dir(feedProcessor)):
if i.endswith('_worker'):
# Worker methods that fetch a feed and add to DB end with '_worker'. Start them all.
new_thread = threading.Thread(target=getattr(feedProcessor, i))
try:
new_thread.start()
logging.info("Started thread with target {}.".format(i))
except RuntimeError:
logging.exception(
"Could not start thread with target {}. Already started?".
format(i))
# Start a thread that queries the DB one by one and updates DNS query values.
while sigint_exit == False: # Hack to keep main thread active to catch SIGINT to kill the entire process.
time.sleep(2)
sys.exit(0)
def parse_arguments():
# parse input arguments
parser = argparse.ArgumentParser()
parser.add_argument(
'--search-query',
type=str,
default=DEF_QUERY,
help=
'query used for arxiv API. See http://arxiv.org/help/api/user-manual#detailed_examples'
)
parser.add_argument('--start-index',
type=int,
default=0,
help='0 = most recent API result')
parser.add_argument('--max-index',
type=int,
default=3000,
help='upper bound on the paper index we will fetch')
parser.add_argument('--results-per-iteration',
type=int,
default=100,
help='passed to arxiv API')
parser.add_argument(
'--wait-time',
type=float,
default=5.0,
help='wait time allows being gentle on the arxiv API (in seconds)')
parser.add_argument(
'--break-on-no-added',
type=int,
default=1,
help=
'break out early if all returned query papers are already in db? 1=yes, 0=no'
)
args, unknown = parser.parse_known_args()
return args
def init():
parser = argparse.ArgumentParser(description="Generates a dataset by compiling generated data properties using a certain dataset model")
parser.add_argument('--model', type=str, default='matrix', help='The name of the dataset model to use. Defaults to matrix.')
parser.add_argument('properties', type=str, default='openPrice,closePrice,gasPrice', help='A list of the names of the properties to use, separated by a comma.')
parser.add_argument('targets', type=str, default='highPrice', help='A list of target property names, separated by a comma.')
parser.add_argument('--start', type=str, default=None, help='The start date. YYYY-MM-DD-HH')
parser.add_argument('--end', type=str, default=None, help='The end date. YYYY-MM-DD-HH')
parser.add_argument('--filename', type=str, default=None, help='The target filename / dir to save the pickled dataset to. Defaults to "data/dataset.pickle"')
parser.add_argument('--overwrite', dest='overwrite', action='store_true', help="If the filename already exists, overwrite it.")
parser.add_argument('--ratio', type=str, default='1', help='On how many fragments to split the main dataset. For example, "1:2:3" will create three datasets with sizes proportional to what given.')
parser.add_argument('--shuffle', dest='shuffle', action="store_true", help="Shuffle the generated dataset and labels.")
parser.set_defaults(shuffle=False)
parser.set_defaults(overwrite=False)
args, _ = parser.parse_known_args()
if len(_) != 0:
raise ValueError("Provided flags %s cannot be understood." % str(_))
if args.filename == None:
filename = "data/dataset_" + str(args.start) + "-" + str(args.end) + ".pickle"
else: filename = args.filename
start = args.start
end = args.end
start = dateutil.parser.parse(start) if start is not None else None
end = dateutil.parser.parse(end) if end is not None else None
try:
ratio = [int(x) for x in args.ratio.split(':')]
except ValueError:
print("Error while reading the given ratio. Did you format it in the correct way?")
return
run(args.model, args.properties.split(','), args.targets.split(','), filename, start=start, end=end, ratio=ratio, shuffle=args.shuffle, overwrite=args.overwrite)
def main():
global options
global logger
today = datetime.now(pytz.timezone('US/Eastern')).date()
init_parser = argparse.ArgumentParser()
init_parser.add_argument("-p",
"--profile",
help="use alternate config profile")
options, args = init_parser.parse_known_args()
config.load(merge_default=True)
if options.profile:
config.settings.set_profile(options.profile)
player.Player.load()
parser = argparse.ArgumentParser()
group = parser.add_mutually_exclusive_group()
group.add_argument("-v",
"--verbose",
action="count",
default=0,
help="verbose logging")
group.add_argument("-q",
"--quiet",
action="count",
default=0,
help="quiet logging")
parser.add_argument("-d",
"--debug-console",
help="show logging console (disables task manager UI)",
action="store_true")
parser.add_argument("spec",
metavar="SPECIFIER",
help="media specifier",
nargs="?")
options, args = parser.parse_known_args(args)
state.options = AttrDict(vars(options))
logger = logging.getLogger()
providers.load()
model.init()
sh = logging.StreamHandler()
state.logger = setup_logging(options.verbose - options.quiet,
quiet_stdout=False)
providers.load_config()
with db_session(optimistic=False):
model.MediaFeed.purge_all(
min_items=config.settings.profile.cache.min_items,
max_items=config.settings.profile.cache.max_items,
max_age=config.settings.profile.cache.max_age)
spec = None
logger.debug(f"{PACKAGE_NAME} starting")
state.asyncio_loop = asyncio.get_event_loop()
state.task_manager = tasks.TaskManager()
state.task_manager_task = state.asyncio_loop.create_task(
state.task_manager.start())
log_file = os.path.join(config.CONFIG_DIR, f"{PACKAGE_NAME}.log")
fh = logging.FileHandler(log_file)
add_log_handler(fh)
logging.getLogger("panwid").setLevel(logging.INFO)
action, provider, selection, opts = providers.parse_spec(options.spec)
if selection:
run_cli(action, provider, selection, **opts)
else:
run_gui(action, provider, **opts)
except ImportError:
pass
# The warnings need to be before any of matplotlib imports, but after
# dateutil.parser and setuptools (if present) which has syntax error with
# the warnings enabled. Filtering by module does not work as this will be
# raised by Python itself so `module=matplotlib.*` is out of question.
import warnings
# Python 3.6 deprecate invalid character-pairs \A, \* ... in non
# raw-strings and other things. Let's not re-introduce them
warnings.filterwarnings('error', '.*invalid escape sequence.*',
category=DeprecationWarning)
warnings.filterwarnings(
'default',
r'.*inspect.getargspec\(\) is deprecated.*',
category=DeprecationWarning)
from matplotlib import test
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument('--recursionlimit', type=int, default=0,
help='Specify recursionlimit for test run')
args, extra_args = parser.parse_known_args()
print('Python byte-compilation optimization level:', sys.flags.optimize)
retcode = test(argv=extra_args, recursionlimit=args.recursionlimit)
sys.exit(retcode)
def main():
global options
global logger
today = datetime.now(pytz.timezone('US/Eastern')).date()
init_parser = argparse.ArgumentParser()
init_parser.add_argument("-c",
"--config-file",
help="use alternate config file")
init_parser.add_argument("-p",
"--profile",
help="use alternate config profile")
options, args = init_parser.parse_known_args()
# -c used to refer to a config dir
config_file = None
if options.config_file:
config_file = os.path.expanduser(options.config_file)
if os.path.isdir(config_file):
config_file = os.path.join(config_file,
config.Config.DEFAULT_CONFIG_FILE)
logger.warning("`-c` should refer to a file, not a directory. "
f"using `{config_file}`")
config.load(config_file, merge_default=True)
if options.profile:
for p in options.profile.split(","):
config.settings.include_profile(p)
programs.load()
parser = argparse.ArgumentParser()
group = parser.add_mutually_exclusive_group()
group.add_argument("-v",
"--verbose",
action="count",
default=0,
help="verbose logging")
group.add_argument("-q",
"--quiet",
action="count",
default=0,
help="quiet logging")
parser.add_argument("uri", metavar="URI", help="media URI", nargs="?")
options, args = parser.parse_known_args(args)
if options.verbose:
try:
import ipdb
pdb_on_exception("ipdb")
except ImportError:
pdb_on_exception()
state.options = AttrDict(vars(options))
state.options.config_file = config_file
logging.captureWarnings(True)
logger = logging.getLogger()
sh = logging.StreamHandler()
state.logger = setup_logging(options.verbose - options.quiet,
quiet_stdout=False)
state.task_manager = tasks.TaskManager()
providers.load()
model.init()
providers.load_config(default=state.app_data.selected_provider)
providers.apply_settings()
spec = None
logger.debug(f"{PACKAGE_NAME} starting")
if config.settings.profile.downloads.max_age > 0:
with db_session(optimistic=False):
model.MediaDownload.purge(
age=config.settings.profile.downloads.max_age)
state.start_task_manager()
# state.task_manager_task = state.event_loop.create_task(state.task_manager.start())
log_file = os.path.join(config.settings.CONFIG_DIR, f"{PACKAGE_NAME}.log")
fh = logging.FileHandler(log_file)
add_log_handler(fh)
logging.getLogger("panwid.dropdown").setLevel(logging.INFO)
logging.getLogger("panwid.keymap").setLevel(logging.INFO)
logging.getLogger("panwid.datatable").setLevel(logging.INFO)
logging.getLogger("aio_mpv_jsonipc").setLevel(logging.INFO)
action, provider, selection, opts = providers.parse_uri(options.uri)
if selection:
state.tui_enabled = False
rc = run_cli(action, provider, selection, **opts)
else:
state.tui_enabled = True
rc = run_gui(action, provider, **opts)
state.stop_task_manager()
return rc
def main():
global options
global logger
parser = argparse.ArgumentParser()
parser.add_argument("-v", "--verbose", action="store_true")
options, args = parser.parse_known_args()
log_file = os.path.join(config.CONFIG_DIR, "mlbstreamer.log")
formatter = logging.Formatter(
"%(asctime)s [%(module)16s:%(lineno)-4d] [%(levelname)8s] %(message)s",
datefmt="%Y-%m-%d %H:%M:%S"
)
fh = logging.FileHandler(log_file)
fh.setLevel(logging.DEBUG)
fh.setFormatter(formatter)
logger = logging.getLogger("mlbstreamer")
logger.setLevel(logging.INFO)
logger.addHandler(fh)
ulh = UrwidLoggingHandler()
ulh.setLevel(logging.DEBUG)
ulh.setFormatter(formatter)
logger.addHandler(ulh)
logger.debug("mlbstreamer starting")
config.settings.load()
state.session = MLBSession.new()
entries = Dropdown.get_palette_entries()
entries.update(ScrollingListBox.get_palette_entries())
entries.update(DataTable.get_palette_entries())
# raise Exception(entries)
palette = Palette("default", **entries)
screen = urwid.raw_display.Screen()
screen.set_terminal_properties(256)
view = ScheduleView()
log_console = widgets.ConsoleWindow()
# log_box = urwid.BoxAdapter(urwid.LineBox(log_console), 10)
pile = urwid.Pile([
("weight", 1, urwid.LineBox(view)),
(10, urwid.LineBox(log_console))
])
def global_input(key):
if key in ('q', 'Q'):
raise urwid.ExitMainLoop()
else:
return False
state.loop = urwid.MainLoop(
pile,
palette,
screen=screen,
unhandled_input=global_input,
pop_ups=True
)
ulh.connect(state.loop.watch_pipe(log_console.log_message))
logger.info("mlbstreamer starting")
if options.verbose:
logger.setLevel(logging.DEBUG)
state.loop.run()
def main():
today = datetime.now().date()
parser = argparse.ArgumentParser()
parser.add_argument("-d",
"--date",
help="game date",
type=valid_date,
default=today)
parser.add_argument("-b",
"--beginning",
help="play live streams from beginning",
nargs="?",
metavar="offset_from_game_start",
type=int,
const=-10)
parser.add_argument("-r",
"--resolution",
help="stream resolution",
default="720p")
parser.add_argument("-s",
"--save-stream",
help="save stream to file",
nargs="?",
const=True)
parser.add_argument("-v",
"--verbose",
action="store_true",
help="verbose logging")
parser.add_argument("--init-config",
help="initialize configuration",
action="store_true")
parser.add_argument("game",
metavar="game",
nargs="?",
help="team abbreviation or MLB game ID")
options, args = parser.parse_known_args()
logger = logging.getLogger("mlbstreamer")
if options.verbose:
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter(
"%(asctime)s [%(levelname)8s] %(message)s",
datefmt='%Y-%m-%d %H:%M:%S')
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(formatter)
logger.addHandler(handler)
else:
logger.addHandler(logging.NullHandler())
if options.init_config:
config.settings.init_config()
sys.exit(0)
config.settings.load()
if not options.game:
parser.error("option game")
state.session = MLBSession.new()
if options.game.isdigit():
game_id = int(options.game)
else:
season = today.year
teams_url = ("http://statsapi.mlb.com/api/v1/teams"
"?sportId={sport}&season={season}".format(sport=1,
season=season))
teams = {
team["fileCode"]: team["id"]
for team in state.session.get(teams_url).json()["teams"]
}
if options.game not in teams:
print("'%s' not a valid team code, must be one of:\n%s" %
(options.game, " ".join(teams)))
sys.exit(1)
schedule = state.session.schedule(start=options.date,
end=options.date,
sport_id=1,
team_id=teams[options.game])
game_id = schedule["dates"][0]["games"][0]["gamePk"]
try:
proc = play_stream(game_id,
options.resolution,
offset_from_beginning=options.beginning,
output=options.save_stream)
proc.wait()
except MLBPlayException as e:
logger.error(e)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--pipeline-uri",
type=str,
help="A URI to JSON with instructions")
parser.add_argument("--pipeline", type=str, help="JSON with instructions")
parser.add_argument(
"--sentinel-stac-id",
type=str,
help="STAC Item ID to process from the STAC collection")
parser.add_argument(
"--sentinel-collection-id",
type=str,
default=SENTINEL_ARCHIVE_COLLECTION_ID,
)
parser.add_argument(
"--stac-api-uri",
type=str,
default=os.environ.get("STAC_API_URI", "http://franklin:9090"),
)
parser.add_argument(
"--stac-api-uri-sentinel",
type=str,
default=os.environ.get("STAC_API_URI_SENTINEL",
"https://earth-search.aws.element84.com/v0"),
)
parser.add_argument("--s3-bucket",
type=str,
default=os.environ.get("S3_BUCKET",
"sentinel-s2-data"))
parser.add_argument(
"--s3-prefix",
type=str,
default=os.environ.get("S3_PREFIX", "aviris-scene-cogs-l2"),
)
parser.add_argument("--temp-dir",
type=str,
default=os.environ.get("TEMP_DIR", None))
parser.add_argument("--output-format",
type=str,
default=os.environ.get("GDAL_OUTPUT_FORMAT", "COG"))
parser.add_argument(
"--keep-temp-dir",
action="store_true",
help=
"If provided, script does not delete temporary directory before script exits. Useful for debugging.",
)
parser.add_argument(
"--force",
action="store_true",
help=
"If provided, force reingest StacItem even though this it is already present in the catalog.",
)
try:
warpMemoryLimit = int(os.environ.get("GDAL_WARP_MEMORY_LIMIT", None))
except TypeError:
warpMemoryLimit = None
# TODO: replace it with parser.parse_args() later
cli_args, cli_unknown = parser.parse_known_args()
# parse all cli arguments
args = CliConfig(cli_args, cli_unknown)
s3 = boto3.client("s3")
stac_client_sentinel = STACClient(args.stac_api_uri_sentinel)
stac_client = STACClient(args.stac_api_uri)
collection = stac_client_sentinel.get_collection(
args.sentinel_collection_id)
SENTINEL_COG_COLLECTION = pystac.Collection(
SENTINEL_COG_COLLECTION_ID,
"Sentinel-2a and Sentinel-2b imagery, processed to Level 2A (Surface Reflectance) and converted to Cloud-Optimized GeoTIFFs",
collection.extent,
stac_extensions=COG_COLLECTION_EXTENSIONS)
SENTINEL_COG_COLLECTION.links = []
SENTINEL_COG_COLLECTION.properties = {}
SENTINEL_COG_COLLECTION.properties['eo:bands'] = SENTINEL_BANDS
SENTINEL_COG_COLLECTION.properties[
'hsi:wavelength_min'] = SENTINEL_WAVELENGTH_MIN
SENTINEL_COG_COLLECTION.properties[
'hsi:wavelength_max'] = SENTINEL_WAVELENGTH_MAX
# GET STAC Item from SENTINEL Catalog
item = stac_client_sentinel.get_collection_item(
args.sentinel_collection_id, args.sentinel_stac_id)
assets = item.assets
bands_map = {
'B01': vsis3(strip_scheme(assets['B01'].href)),
'B02': vsis3(strip_scheme(assets['B02'].href)),
'B03': vsis3(strip_scheme(assets['B03'].href)),
'B04': vsis3(strip_scheme(assets['B04'].href)),
'B05': vsis3(strip_scheme(assets['B05'].href)),
'B06': vsis3(strip_scheme(assets['B06'].href)),
'B07': vsis3(strip_scheme(assets['B07'].href)),
'B08': vsis3(strip_scheme(assets['B08'].href)),
'B8A': vsis3(strip_scheme(assets['B8A'].href)),
'B09': vsis3(strip_scheme(assets['B09'].href)),
'B11': vsis3(strip_scheme(assets['B11'].href)),
'B12': vsis3(strip_scheme(assets['B12'].href)),
'AOT': vsis3(strip_scheme(assets['AOT'].href)),
# 'WVP': vsis3(strip_scheme(assets['WVP'].href)),
# 'SCL': vsis3(strip_scheme(assets['SCL'].href))
}
# we don't need assets here, since the gather scripts knows what and how to download by the sentinel path
properties = item.properties
datetime = dateutil.parser.isoparse(properties['datetime'])
# here "href": "s3://sentinel-s2-l2a/tiles/31/V/CE/2021/8/19/0/R60m/B01.jp2"
# path is tiles/31/V/CE/2021/8/19/0
sentintel_path = 'tiles/{}/{}/{}/{}/{}/{}/{}'.format(
properties['sentinel:utm_zone'], properties['sentinel:latitude_band'],
properties['sentinel:grid_square'], str(datetime.year),
str(datetime.month), str(datetime.day),
properties['sentinel:sequence'])
# Create new COG STAC Item
cog_item_id = "{}_{}".format(SENTINEL_COG_COLLECTION.id, item.id)
cog_item = pystac.Item(
cog_item_id,
item.geometry,
item.bbox,
item.datetime,
item.properties,
stac_extensions=COG_ITEM_EXTENSIONS,
collection=SENTINEL_COG_COLLECTION.id,
)
cog_item.properties['eo:bands'] = SENTINEL_COG_COLLECTION.properties[
'eo:bands']
cog_item.properties[
'hsi:wavelength_min'] = SENTINEL_COG_COLLECTION.properties[
'hsi:wavelength_min']
cog_item.properties[
'hsi:wavelength_max'] = SENTINEL_COG_COLLECTION.properties[
'hsi:wavelength_max']
cog_item.properties['proj:epsg'] = '4326'
# Create COG Collection if it doesn't exist
if not stac_client.has_collection(SENTINEL_COG_COLLECTION.id):
stac_client.post_collection(SENTINEL_COG_COLLECTION)
if not args.force:
# Exit early if COG STAC Item already exists
try:
stac_client.get_collection_item(SENTINEL_COG_COLLECTION.id,
cog_item_id)
logger.info(f'STAC Item {cog_item_id} already exists. Exiting.')
activation_output(cog_item_id)
return
except requests.exceptions.HTTPError:
pass
_, s3_uri = gather_sentinel(
f'{cog_item_id}.tiff',
f's3://{args.s3_bucket}/{args.s3_prefix}/{sentintel_path}/', bands_map)
# Add assets to COG STAC Item
idx = 0
cog_item.add_asset(
f'{args.output_asset_name}_{idx}',
pystac.Asset(s3_uri, media_type=pystac.MediaType.COG, roles=["data"]),
)
# Add COG Item to AVIRIS L2 STAC Collection
logger.info(f"POST Item {cog_item.id} to {args.stac_api_uri}")
item_data = stac_client.post_collection_item(SENTINEL_COG_COLLECTION.id,
cog_item)
if item_data.get('id', None):
logger.info(f"Success: {item_data['id']}")
activation_output(item_data['id'])
else:
logger.error(f"Failure: {item_data}")
return -1
plt.savefig(filename, dpi = 1500)
print("Saved accuracy graph at %s." % filename)
if __name__ == "__main__": #if this is the main file, parse the command args
parser = argparse.ArgumentParser(description="Module that loads given datasets and trains and evaluates one or more neural network models on that.")
parser.add_argument('dataset', type=str, help="The filepath to the dataset/s.")
parser.add_argument('--models', type=str, help="A list of the models that are going to be trained and evaluated. Default is all available.")
parser.add_argument('--args', type=str, help="A list of arguments to be passed on to the models. In the format key1=value1,key2=value2.1;value2.2")
parser.add_argument('--quiet', dest='quiet', action="store_true", help="Do not plot graphs, but save them as images.")
parser.set_defaults(quiet=False)
parser.add_argument('--shuffle', dest='shuffle', action="store_true", help="Shuffle the generated dataset and labels.")
parser.set_defaults(shuffle=False)
parser.add_argument('--trim-batch', dest='trim', action="store_true", help="Trim each dataset so that its length is divisible by the batch size.")
parser.set_defaults(trim=False)
args, _ = parser.parse_known_args()
givenModels = args.models.split(',') if args.models else None
modelArgs = {}
pairs = args.args.split(',')
for pair in pairs:
key, value = pair.split('=')
try:
value = float(value)
if value == int(value):
value = int(value)
except ValueError:
if ':' in value:
value = [int(i) for i in value.split(':')]
def main():
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(dest='test_type')
# Host-side test args.
host_cmd_parser = subparsers.add_parser(
'host-cmd',
help='Runs a host-side test. Pass the host-side command to run after '
'"--". If --use-vm is passed, hostname and port for the device '
'will be 127.0.0.1:9222.')
host_cmd_parser.set_defaults(func=host_cmd)
host_cmd_parser.add_argument(
'--deploy-chrome',
action='store_true',
help='Will deploy a locally built Chrome binary to the device before '
'running the host-cmd.')
# GTest args.
# TODO(bpastene): Rename 'vm-test' arg to 'gtest'.
gtest_parser = subparsers.add_parser('vm-test',
help='Runs a device-side gtest.')
gtest_parser.set_defaults(func=device_test)
gtest_parser.add_argument(
'--test-exe',
type=str,
required=True,
help='Path to test executable to run inside the device. If the value is '
'%s, the sanity test that ships with the device image runs instead. '
'This test smokes-check the system browser (eg: loads a simple '
'webpage, executes some javascript), so a fully-built Chrome binary '
'that can get deployed to the device is expected to be available in '
'the out-dir.' % SANITY_TEST_TARGET)
# GTest args. Some are passed down to the test binary in the device. Others
# are parsed here since they might need tweaking or special handling.
gtest_parser.add_argument(
'--test-launcher-summary-output',
type=str,
help='When set, will pass the same option down to the test and retrieve '
'its result file at the specified location.')
# Shard args are parsed here since we might also specify them via env vars.
gtest_parser.add_argument('--test-launcher-shard-index',
type=int,
default=os.environ.get('GTEST_SHARD_INDEX', 0),
help='Index of the external shard to run.')
gtest_parser.add_argument('--test-launcher-total-shards',
type=int,
default=os.environ.get('GTEST_TOTAL_SHARDS', 1),
help='Total number of external shards.')
gtest_parser.add_argument(
'--stop-ui',
action='store_true',
help='Will stop the UI service in the device before running the test.')
gtest_parser.add_argument(
'--trace-dir',
type=str,
help='When set, will pass down to the test to generate the trace and '
'retrieve the trace files to the specified location.')
gtest_parser.add_argument(
'--env-var',
nargs=2,
action='append',
default=[],
help='Env var to set on the device for the duration of the test. '
'Expected format is "--env-var SOME_VAR_NAME some_var_value". Specify '
'multiple times for more than one var.')
# Tast test args.
# pylint: disable=line-too-long
tast_test_parser = subparsers.add_parser(
'tast',
help='Runs a device-side set of Tast tests. For more details, see: '
'https://chromium.googlesource.com/chromiumos/platform/tast/+/master/docs/running_tests.md'
)
tast_test_parser.set_defaults(func=device_test)
tast_test_parser.add_argument(
'--suite-name',
type=str,
required=True,
help='Name to apply to the set of Tast tests to run. This has no effect '
'on what is executed, but is used mainly for test results reporting '
'and tracking (eg: flakiness dashboard).')
tast_test_parser.add_argument(
'--test-launcher-summary-output',
type=str,
help='Generates a simple GTest-style JSON result file for the test run.'
)
# TODO(bpastene): Change all uses of "--conditional" to use "--attr-expr".
tast_test_parser.add_argument(
'--conditional',
'--attr-expr',
type=str,
dest='conditional',
help='A boolean expression whose matching tests will run '
'(eg: ("dep:chrome")).')
tast_test_parser.add_argument(
'--strip-chrome',
action='store_true',
help='Strips symbols from the browser before deploying to the device.')
tast_test_parser.add_argument(
'--test',
'-t',
action='append',
dest='tests',
help='A Tast test to run in the device (eg: "ui.ChromeLogin").')
add_common_args(gtest_parser, tast_test_parser, host_cmd_parser)
args, unknown_args = parser.parse_known_args()
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARN)
if not args.use_vm and not args.device:
# If we're not running on a VM, but haven't specified a hostname, assume
# we're on a lab bot and are trying to run a test on a lab DUT. See if the
# magic lab DUT hostname resolves to anything. (It will in the lab and will
# not on dev machines.)
try:
socket.getaddrinfo(LAB_DUT_HOSTNAME, None)
except socket.gaierror:
logging.error('The default DUT hostname of %s is unreachable.',
LAB_DUT_HOSTNAME)
return 1
if args.use_vm:
if not os.path.exists('/dev/kvm'):
logging.error(
'/dev/kvm is missing. Is KVM installed on this machine?')
return 1
elif not os.access('/dev/kvm', os.W_OK):
logging.error(
'/dev/kvm is not writable as current user. Perhaps you should be '
'root?')
return 1
args.cros_cache = os.path.abspath(args.cros_cache)
return args.func(args, unknown_args)
def init():
np.set_printoptions(precision=3, linewidth=180)
parser = argparse.ArgumentParser(
description=
"Tool that can read historical data from the db or from a file and visualize it as a graph."
)
parser.add_argument(
'data',
type=str,
nargs='*',
help='A pickled file or file list, output from training.')
parser.add_argument(
'--keys',
type=str,
default=None,
help='Filter out only one or multiple keys, separated by a comma.')
parser.add_argument('--best',
type=int,
default=None,
help='Only display the best N results.')
parser.add_argument('--detail',
dest='detail',
action='store_true',
help='Display the scores in detail.')
parser.set_defaults(detail=False)
parser.add_argument('--no-display',
dest='display',
action='store_false',
help='Do not plot any graphs.')
parser.set_defaults(display=True)
args, _ = parser.parse_known_args()
files = args.data
for i in range(len(files)):
if '\n' in files[i]:
s = files[i].split('\n')
subfiles = [f for f in s if f != '' and not f.isspace()]
files[i] = None
files.extend(subfiles)
files = [f for f in files if f is not None]
#print(files)
histories = []
data = {}
titles = files.copy()
keys = args.keys.split(',') if args.keys is not None else None
for f in files:
with open(f, 'rb') as fi:
element = pickle.load(fi)
hist = element['history']
histories.append(hist)
for i, hist in enumerate(histories):
if keys is not None:
hist2 = {}
for key in keys:
hist2[key] = hist[key]
hist = hist2
for key in hist:
if key not in data:
data[key] = []
filename = basename(files[i]) #extract filename from path
data[key].append({'file': filename, 'data': hist[key]})
for key in data:
if key == 'val_loss' or key == 'loss' or key == 'rmse' or key == 'custom':
func = min
data[key] = sortResults(data[key], ascending=True, func=func)
elif key == 'R2' or key == 'sign':
func = max
data[key] = sortResults(data[key], ascending=False, func=func)
else:
raise ValueError("Unexpected and unknown metric '%s'." % key)
print("Best for %s: are as follows" % (key))
for i in range(min(5, len(data[key]))):
val = data[key][i]['data'][-1] if func is None else func(
data[key][i]['data'])
print("#%d %s is %f at %s" %
(i + 1, key, val, data[key][i]['file']))
if args.best is not None:
data[key] = data[key][:args.best]
if args.detail:
resLines = ""
joiner = ' & '
for i, hist in enumerate(histories):
print("Detailed results for %s." % (basename(files[i])))
best = []
headers = []
for key in (hist if keys is None else keys):
best.append(getCompareFuncBasedOnKey(key)(hist[key]))
headers.append(key)
print(headers)
print(best)
commLine = '%' + basename(files[i]) + '\n'
resLine = str.join(joiner, ("%f" % val for val in best)) + '\\\\\n'
resLines += commLine
resLines += resLine
print(str.join(joiner, headers))
print(resLine)
print("All result lines for conveninece:")
print(resLines)
if args.display:
drawAccuracyGraph(data)
def main():
# https://sfbay.craigslist.org/search/eby/cto?
# sort=pricedsc&
# hasPic=1&
# search_distance=10&
# postal=94610&
# min_price=4000&
# max_price=10000&
# auto_make_model=toyota+corolla&
# min_auto_year=2008&
# min_auto_miles=100000&
# max_auto_miles=120000&
# condition=30&
# condition=40&
# auto_cylinders=2&
# auto_title_status=1&
# auto_transmission=2
parser = argparse.ArgumentParser(description="craigslist car finder", parents=())
parser.add_argument("-c", "--city", default='sfbay', help='which city to search for')
parser.add_argument("-d", "--sort_results", default='pricedsc', help='how to sort results')
parser.add_argument("-e", "--search_distance", default='40', help='maximum distance from search zip')
parser.add_argument("-f", "--postal", default='94610', help='search zip')
parser.add_argument("-p", "--min_price", default='2000')
parser.add_argument("-o", "--max_price", default='16000')
parser.add_argument("-b", "--make", default='kia', help='car make (brand)')
parser.add_argument("-m", "--model", default='soul', help='car model')
parser.add_argument("-y", "--min_auto_year", default='2008')
parser.add_argument("-a", "--min_auto_miles", default='50000',
help='minimum miles travelled by car before purchase')
parser.add_argument("-i", "--max_auto_miles", default='150000',
help='maximum miles travelled by car before purchase')
parser.add_argument("-c1", "--condition_1", default='30') # 30 = excellent
parser.add_argument("-c2", "--condition_2", default='40') # 40 = good
parser.add_argument("-g1", "--auto_cylinders", default='2') # 2 = 4 cylinders
parser.add_argument("-t1", "--auto_title_status", default='1') # 1 = clean
parser.add_argument("-x1", "--auto_transmission", default='2') # 2 = automatic, 1 = manual
parser.add_argument("-l", "--max_results", default=1000, help='limit to this number of results for cars returned')
parser.add_argument("-w", "--week_range", default=2,
help='number of weeks to search car listings for starting from now')
parser.add_argument("-v", "--verbose", default='False', help='print debug output')
parser.add_argument("-t", "--blacklist_titles", nargs='+', default=['salvage', 'rebuilt'],
help='List unacceptable states for car, e.g. You may want to filter out cars that '
'have been totalled or salvaged')
# parser.add_argument("-o", "--output", help='write matching cars to file')
try:
args, extra_args = parser.parse_known_args()
if args.verbose == 'True':
verbose = True
else:
verbose = False
if verbose:
print('\n\nargs:', args)
print('\n\nextra_args:', extra_args)
except Exception as e:
print(e)
sys.exit(1)
all_cars = get_craigslist_cars(args.city, args.sort_results, args.search_distance, args.postal,
args.min_price, args.max_price, args.make, args.model, args.min_auto_year,
args.min_auto_miles, args.max_auto_miles, args.condition_1, args.condition_2,
args.auto_cylinders, args.auto_title_status, args.auto_transmission,
args.max_results, verbose=verbose)
if verbose:
print('\n\nall_cars:')
for i, car_attributes in enumerate(all_cars):
print('\n\ncar #: {}'.format(i))
for key, value in car_attributes.items():
print('key: {}; value: {}'.format(key, value))
filtered_cars = filter_cars(all_cars, args.max_auto_miles, args.blacklist_titles, args.week_range, verbose=verbose)
print('\n\nfiltered_cars:')
final_car_attributes_list = []
for i, car_attributes in enumerate(filtered_cars):
print('\n\ncar #: {}'.format(i))
final_car_attributes_list.extend(car_attributes.keys())
for key, value in car_attributes.items():
if key == 'posting_body':
continue
else:
print('key: {}; value: {}'.format(key, value))
final_car_attributes_list = list(set(final_car_attributes_list))
print('\n\nfinal_car_attributes_list:\n', final_car_attributes_list)
car_row_dict_list = []
for i, car_attributes in enumerate(filtered_cars):
car_row_dict = collections.OrderedDict()
for car_attribute in final_car_attributes_list:
print('\n\n******************')
print('car_attribute:', car_attribute)
print('car_attributes[car_attribute]:', car_attributes[car_attribute])
try:
if isinstance(car_attributes[car_attribute], str) or \
car_attributes[car_attribute] != car_attributes[car_attribute]:
print('treat as string')
car_row_dict[car_attribute] = car_attributes[car_attribute]
else:
try:
print('treat as list')
car_row_dict[car_attribute] = car_attributes[car_attribute][0]
except IndexError as e:
print('\ncar_attributes[car_attribute]:', car_attributes[car_attribute])
print('str(e):', str(e))
car_row_dict[car_attribute] = np.nan
except KeyError:
car_row_dict[car_attribute] = np.nan
print('car_row_dict[car_attribute]:', car_row_dict[car_attribute])
car_row_dict_list.append(car_row_dict)
df = pd.DataFrame.from_dict(car_row_dict_list)
def do_it(element):
element = element[21:25]
return element
df['model_make_year'] = df['posting_title'].apply(lambda element: do_it(element))
def do_it(element):
element = element.replace('\'', '\'\'\'').replace('\n', ' ').replace('favorite this post', '').\
replace('hide this posting unhide', '').strip()
return element
df['posting_title'] = df['posting_title'].apply(lambda element: do_it(element))
def do_it(element):
element = element.replace('\'', '\'\'\'').replace('\n', ' ').replace('QR Code Link to This Post', '').strip()
return element
df['posting_body'] = df['posting_body'].apply(lambda element: do_it(element))
print(df.columns)
df = df[['model_make_year', 'odometer', 'price', 'time_posted', 'fuel', 'condition', 'transmission', 'title status', 'url',
'posting_title', 'drive', 'cylinders', 'VIN', 'type', 'size', 'paint color', 'attribute']] # ,
# 'posting_body']]
df.sort_values(by=['model_make_year', 'odometer', 'price', 'time_posted'], inplace=True)
with pd.option_context('display.max_rows', None, 'display.max_columns', None):
print('\n\ndf.head():\n', df.head())
df.to_csv('results.csv')
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-f",
"--fields",
dest="inputfields",
default=None,
help="fields to emit (default all fields)")
parser.add_argument("-t",
"--types",
dest="typenames",
default=None,
help="type converters for emitted fields")
parser.add_argument("-i",
"--insep",
dest="insep",
default=",",
help="input field separator (default ',')")
parser.add_argument("-a",
"--array",
dest="emit_arrays",
default=False,
action="store_true",
help="emit arrays instead of objects")
parser.add_argument("-H",
"--header",
dest="array_header",
default=False,
action="store_true",
help="when emitting arrays, also emit a header array")
parser.add_argument("--fullhelp",
dest="full_help",
action="store_true",
default=False,
help="display full usage")
(options, args) = parser.parse_known_args()
if options.full_help:
usage()
return 0
options.inputfields = (options.inputfields.split(",")
if options.inputfields else [])
options.typenames = (options.typenames.split(",")
if options.typenames else [])
if len(args) > 2:
usage(sys.argv[0])
return 1
if len(args) >= 1:
inf = open(args[0], "r")
else:
inf = sys.stdin
if len(args) == 2:
outf = open(args[1], "w")
else:
outf = sys.stdout
reader = csv.DictReader(inf, delimiter=options.insep)
if not options.inputfields:
options.inputfields = reader.fieldnames
try:
generate_type_converters(options)
except ValueError as exc:
usage(exc.args)
return 1
return translate(reader, options, outf)
def inference(l_args, s_ticker):
import flair
parser = argparse.ArgumentParser(
prog='infer',
description=
"""Print quick sentiment inference from last tweets that contain
the ticker. This model splits the text into character-level
tokens and uses the DistilBERT model to make predictions. DistilBERT is a distilled
version of the powerful BERT transformer model. Not only time period of these,
but also frequency. [Source: Twitter]""")
parser.add_argument('-n',
"--num",
action="store",
dest="n_num",
type=int,
default=100,
choices=range(10, 101),
help='num of latest tweets to infer from.')
(ns_parser, l_unknown_args) = parser.parse_known_args(l_args)
if l_unknown_args:
print(
f"The following args couldn't be interpreted: {l_unknown_args}\n")
return
# Get tweets using Twitter API
params = {
'q': '$' + s_ticker,
'tweet_mode': 'extended',
'lang': 'en',
'count': str(ns_parser.n_num)
}
# Request Twitter API
response = requests.get(
'https://api.twitter.com/1.1/search/tweets.json',
params=params,
headers={'authorization': 'Bearer ' + cfg.API_TWITTER_BEARER_TOKEN})
# Create dataframe
df_tweets = pd.DataFrame()
# Check that the API response was successful
if response.status_code == 200:
for tweet in response.json()['statuses']:
row = get_data(tweet)
df_tweets = df_tweets.append(row, ignore_index=True)
# Load sentiment model
sentiment_model = flair.models.TextClassifier.load('en-sentiment')
print("")
# We will append probability/sentiment preds later
probs = []
sentiments = []
for s_tweet in df_tweets['text'].to_list():
tweet = clean_tweet(s_tweet, s_ticker)
# Make sentiment prediction
sentence = flair.data.Sentence(tweet)
sentiment_model.predict(sentence)
# Extract sentiment prediction (POSITIVE/NEGATIVE) and confidence (0-1)
probs.append(sentence.labels[0].score)
sentiments.append(sentence.labels[0].value)
# Add probability and sentiment predictions to tweets dataframe
df_tweets['probability'] = probs
df_tweets['sentiment'] = sentiments
# Add sentiment estimation (probability positive for POSITIVE sentiment, and negative for NEGATIVE sentiment)
df_tweets['sentiment_estimation'] = df_tweets.apply(
lambda row: row['probability'] *
(-1, 1)[row['sentiment'] == 'POSITIVE'],
axis=1).cumsum()
# Cumulative sentiment_estimation
df_tweets['prob_sen'] = df_tweets.apply(
lambda row: row['probability'] *
(-1, 1)[row['sentiment'] == 'POSITIVE'],
axis=1)
# Percentage of confidence
if df_tweets['sentiment_estimation'].values[-1] > 0:
n_pos = df_tweets[df_tweets['prob_sen'] > 0]['prob_sen'].sum()
n_pct = round(100 * n_pos / df_tweets['probability'].sum())
else:
n_neg = abs(df_tweets[df_tweets['prob_sen'] < 0]['prob_sen'].sum())
n_pct = round(100 * n_neg / df_tweets['probability'].sum())
# Parse tweets
dt_from = dateutil.parser.parse(df_tweets['created_at'].values[-1])
dt_to = dateutil.parser.parse(df_tweets['created_at'].values[0])
print(f"From: {dt_from.strftime('%Y-%m-%d %H:%M:%S')}")
print(f"To: {dt_to.strftime('%Y-%m-%d %H:%M:%S')}")
print(f"{len(df_tweets)} tweets were analyzed.")
dt_delta = (dt_to - dt_from)
n_freq = dt_delta.total_seconds() / len(df_tweets)
print(f"Frequency of approx 1 tweet every {round(n_freq)} seconds.")
s_sen = f"{('NEGATIVE', 'POSITIVE')[int(df_tweets['sentiment_estimation'].values[-1] > 0)]}"
s_conf = f"{round(100*df_tweets['sentiment_estimation'].values[-1]/df_tweets['probability'].sum())}%"
print(f"The sentiment of {s_ticker} is: {s_sen} ({s_conf})")
print("")
def sentiment(l_args, s_ticker):
import flair
parser = argparse.ArgumentParser(
prog='sen',
description=
"""Plot in-depth sentiment predicted from tweets from last days
that contain pre-defined ticker. This model splits the text into character-level
tokens and uses the DistilBERT model to make predictions. DistilBERT is a distilled
version of the powerful BERT transformer model. Note that a big num of tweets extracted per
hour in conjunction with a high number of days in the past, will make the
algorithm take a long period of time to estimate sentiment. [Source: Twitter]"""
)
# in reality this argument could be 100, but after testing it takes too long to compute which may not be acceptable
parser.add_argument('-n',
"--num",
action="store",
dest="n_tweets",
type=int,
default=10,
choices=range(10, 61),
help='num of tweets to extract per hour.')
parser.add_argument('-d',
"--days",
action="store",
dest="n_days_past",
type=int,
default=7,
choices=range(1, 8),
help='num of days in the past to extract tweets.')
(ns_parser, l_unknown_args) = parser.parse_known_args(l_args)
if l_unknown_args:
print(
f"The following args couldn't be interpreted: {l_unknown_args}\n")
return
# Setup API request params and headers
headers = {'authorization': f'Bearer {cfg.API_TWITTER_BEARER_TOKEN}'}
params = {
'query': f'({s_ticker}) (lang:en)',
'max_results': str(ns_parser.n_tweets),
'tweet.fields': 'created_at,lang'
}
# Date format string required by twitter
dtformat = '%Y-%m-%dT%H:%M:%SZ'
# Algorithm to extract
dt_recent = datetime.now() - timedelta(seconds=20)
dt_old = dt_recent - timedelta(days=ns_parser.n_days_past)
print(
f"From {dt_recent.date()} retrieving {ns_parser.n_tweets*24} tweets ({ns_parser.n_tweets} tweets/hour)"
)
df_tweets = pd.DataFrame()
while True:
# Iterate until we haven't passed the old number of days
if dt_recent < dt_old:
break
# Update past datetime
dt_past = dt_recent - timedelta(minutes=60)
if dt_past.day < dt_recent.day:
print(
f"From {dt_past.date()} retrieving {ns_parser.n_tweets*24} tweets ({ns_parser.n_tweets} tweets/hour)"
)
# Assign from and to datetime parameters for the API
params['start_time'] = dt_past.strftime(dtformat)
params['end_time'] = dt_recent.strftime(dtformat)
# Send API request
response = requests.get(
'https://api.twitter.com/2/tweets/search/recent',
params=params,
headers=headers)
# Update recent datetime
dt_recent = dt_past
# If response from API request is a success
if response.status_code == 200:
# Iteratively append our tweet data to our dataframe
for tweet in response.json()['data']:
row = get_data(tweet)
df_tweets = df_tweets.append(row, ignore_index=True)
# Load sentiment model
print("")
sentiment_model = flair.models.TextClassifier.load('en-sentiment')
print("")
# Append probability and sentiment preds later
probs = []
sentiments = []
for s_tweet in df_tweets['text'].to_list():
tweet = clean_tweet(s_tweet, s_ticker)
# Make sentiment prediction
sentence = flair.data.Sentence(tweet)
sentiment_model.predict(sentence)
# Extract sentiment prediction (POSITIVE/NEGATIVE) and confidence (0-1)
probs.append(sentence.labels[0].score)
sentiments.append(sentence.labels[0].value)
# Add probability and sentiment predictions to tweets dataframe
df_tweets['probability'] = probs
df_tweets['sentiment'] = sentiments
# Sort tweets per date
df_tweets.sort_index(ascending=False, inplace=True)
# Add sentiment estimation (probability positive for POSITIVE sentiment, and negative for NEGATIVE sentiment)
df_tweets['sentiment_estimation'] = df_tweets.apply(
lambda row: row['probability'] *
(-1, 1)[row['sentiment'] == 'POSITIVE'],
axis=1).cumsum()
# Cumulative sentiment_estimation
df_tweets['prob_sen'] = df_tweets.apply(
lambda row: row['probability'] *
(-1, 1)[row['sentiment'] == 'POSITIVE'],
axis=1)
# Percentage of confidence
if df_tweets['sentiment_estimation'].values[-1] > 0:
n_pos = df_tweets[df_tweets['prob_sen'] > 0]['prob_sen'].sum()
n_pct = round(100 * n_pos / df_tweets['probability'].sum())
else:
n_neg = abs(df_tweets[df_tweets['prob_sen'] < 0]['prob_sen'].sum())
n_pct = round(100 * n_neg / df_tweets['probability'].sum())
s_sen = f"{('NEGATIVE', 'POSITIVE')[int(df_tweets['sentiment_estimation'].values[-1] > 0)]}"
#df_tweets.to_csv(r'notebooks/tweets.csv', index=False)
df_tweets.reset_index(inplace=True)
# Plotting
plt.subplot(211)
plt.title(
f"Twitter's {s_ticker} sentiment over time is {s_sen} ({n_pct} %)")
plt.plot(df_tweets.index,
df_tweets['sentiment_estimation'].values,
lw=3,
c='cyan')
plt.xlim(df_tweets.index[0], df_tweets.index[-1])
plt.grid(b=True,
which='major',
color='#666666',
linestyle='-',
lw=1.5,
alpha=0.5)
plt.minorticks_on()
plt.grid(b=True, which='minor', color='#999999', linestyle='-', alpha=0.2)
plt.ylabel('Cumulative Sentiment')
l_xticks = list()
l_xlabels = list()
l_xticks.append(0)
l_xlabels.append(df_tweets['created_at'].values[0].split(' ')[0])
n_day = datetime.strptime(df_tweets['created_at'].values[0],
"%Y-%m-%d %H:%M:%S").day
n_idx = 0
n_next_idx = 0
for n_next_idx, dt_created in enumerate(df_tweets['created_at']):
if datetime.strptime(dt_created, "%Y-%m-%d %H:%M:%S").day > n_day:
l_xticks.append(n_next_idx)
l_xlabels.append(
df_tweets['created_at'].values[n_next_idx].split(' ')[0])
l_val_days = df_tweets['sentiment_estimation'].values[
n_idx:n_next_idx] - df_tweets['sentiment_estimation'].values[
n_idx]
plt.plot(range(n_idx, n_next_idx), l_val_days, lw=3, c='tab:blue')
n_day_avg = np.mean(l_val_days)
if n_day_avg > 0:
plt.hlines(n_day_avg,
n_idx,
n_next_idx,
linewidth=2.5,
linestyle='--',
color='green',
lw=3)
else:
plt.hlines(n_day_avg,
n_idx,
n_next_idx,
linewidth=2.5,
linestyle='--',
color='red',
lw=3)
n_idx = n_next_idx
n_day += 1
l_val_days = df_tweets['sentiment_estimation'].values[n_idx:] - df_tweets[
'sentiment_estimation'].values[n_idx]
plt.plot(range(n_idx, len(df_tweets)), l_val_days, lw=3, c='tab:blue')
n_day_avg = np.mean(l_val_days)
if n_day_avg > 0:
plt.hlines(n_day_avg,
n_idx,
len(df_tweets),
linewidth=2.5,
linestyle='--',
color='green',
lw=3)
else:
plt.hlines(n_day_avg,
n_idx,
len(df_tweets),
linewidth=2.5,
linestyle='--',
color='red',
lw=3)
l_xticks.append(len(df_tweets))
datetime.strptime(dt_created, "%Y-%m-%d %H:%M:%S") + timedelta(days=1)
l_xlabels.append(
datetime.strftime(
datetime.strptime(
df_tweets['created_at'].values[len(df_tweets) - 1],
"%Y-%m-%d %H:%M:%S") + timedelta(days=1), "%Y-%m-%d"))
plt.xticks(l_xticks, l_xlabels)
plt.axhspan(plt.gca().get_ylim()[0], 0, facecolor='r', alpha=0.1)
plt.axhspan(0, plt.gca().get_ylim()[1], facecolor='g', alpha=0.1)
plt.subplot(212)
plt.bar(df_tweets[df_tweets['prob_sen'] > 0].index,
df_tweets[df_tweets['prob_sen'] > 0]['prob_sen'].values,
color='green')
plt.bar(df_tweets[df_tweets['prob_sen'] < 0].index,
df_tweets[df_tweets['prob_sen'] < 0]['prob_sen'].values,
color='red')
for l_x in l_xticks[1:]:
plt.vlines(l_x, -1, 1, linewidth=2, linestyle='--', color='k', lw=3)
plt.xlim(df_tweets.index[0], df_tweets.index[-1])
plt.xticks(l_xticks, l_xlabels)
plt.grid(b=True, which='major', color='#666666', linestyle='-')
plt.ylabel('Sentiment')
plt.xlabel("Time")
plt.show()
def main():
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(dest='test_type')
# Host-side test args.
host_cmd_parser = subparsers.add_parser(
'host-cmd',
help='Runs a host-side test. Pass the host-side command to run after '
'"--". If --use-vm is passed, hostname and port for the device '
'will be 127.0.0.1:9222.')
host_cmd_parser.set_defaults(func=host_cmd)
host_cmd_parser.add_argument(
'--deploy-chrome',
action='store_true',
help=
'Will deploy a locally built ash-chrome binary to the device before '
'running the host-cmd.')
host_cmd_parser.add_argument(
'--deploy-lacros',
action='store_true',
help='Deploy a lacros-chrome instead of ash-chrome.')
# GTest args.
# TODO(bpastene): Rename 'vm-test' arg to 'gtest'.
gtest_parser = subparsers.add_parser('vm-test',
help='Runs a device-side gtest.')
gtest_parser.set_defaults(func=device_test)
gtest_parser.add_argument(
'--test-exe',
type=str,
required=True,
help='Path to test executable to run inside the device.')
# GTest args. Some are passed down to the test binary in the device. Others
# are parsed here since they might need tweaking or special handling.
gtest_parser.add_argument(
'--test-launcher-summary-output',
type=str,
help='When set, will pass the same option down to the test and retrieve '
'its result file at the specified location.')
gtest_parser.add_argument(
'--stop-ui',
action='store_true',
help='Will stop the UI service in the device before running the test.')
gtest_parser.add_argument(
'--trace-dir',
type=str,
help='When set, will pass down to the test to generate the trace and '
'retrieve the trace files to the specified location.')
gtest_parser.add_argument(
'--env-var',
nargs=2,
action='append',
default=[],
help='Env var to set on the device for the duration of the test. '
'Expected format is "--env-var SOME_VAR_NAME some_var_value". Specify '
'multiple times for more than one var.')
# Tast test args.
# pylint: disable=line-too-long
tast_test_parser = subparsers.add_parser(
'tast',
help='Runs a device-side set of Tast tests. For more details, see: '
'https://chromium.googlesource.com/chromiumos/platform/tast/+/master/docs/running_tests.md'
)
tast_test_parser.set_defaults(func=device_test)
tast_test_parser.add_argument(
'--suite-name',
type=str,
required=True,
help='Name to apply to the set of Tast tests to run. This has no effect '
'on what is executed, but is used mainly for test results reporting '
'and tracking (eg: flakiness dashboard).')
tast_test_parser.add_argument(
'--test-launcher-summary-output',
type=str,
help='Generates a simple GTest-style JSON result file for the test run.'
)
# TODO(bpastene): Change all uses of "--conditional" to use "--attr-expr".
tast_test_parser.add_argument(
'--conditional',
'--attr-expr',
type=str,
dest='conditional',
help='A boolean expression whose matching tests will run '
'(eg: ("dep:chrome")).')
tast_test_parser.add_argument(
'--strip-chrome',
action='store_true',
help='Strips symbols from ash-chrome before deploying to the device.')
tast_test_parser.add_argument(
'--deploy-lacros',
action='store_true',
help='Deploy a lacros-chrome instead of ash-chrome.')
tast_test_parser.add_argument(
'--tast-var',
action='append',
dest='tast_vars',
help='Runtime variables for Tast tests, and the format are expected to '
'be "key=value" pairs.')
tast_test_parser.add_argument(
'--test',
'-t',
action='append',
dest='tests',
help='A Tast test to run in the device (eg: "ui.ChromeLogin").')
tast_test_parser.add_argument(
'--gtest_filter',
type=str,
help="Similar to GTest's arg of the same name, this will filter out the "
"specified tests from the Tast run. However, due to the nature of Tast's "
'cmd-line API, this will overwrite the value(s) of "--test" above.')
add_common_args(gtest_parser, tast_test_parser, host_cmd_parser)
args = sys.argv[1:]
unknown_args = []
# If a '--' is present in the args, treat everything to the right of it as
# args to the test and everything to the left as args to this test runner.
# Otherwise treat all known args as args to this test runner and all unknown
# args as test args.
if '--' in args:
unknown_args = args[args.index('--') + 1:]
args = args[0:args.index('--')]
if unknown_args:
args = parser.parse_args(args=args)
else:
args, unknown_args = parser.parse_known_args()
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARN)
if not args.use_vm and not args.device:
# If we're not running on a VM, but haven't specified a hostname, assume
# we're on a lab bot and are trying to run a test on a lab DUT. See if the
# magic lab DUT hostname resolves to anything. (It will in the lab and will
# not on dev machines.)
try:
socket.getaddrinfo(LAB_DUT_HOSTNAME, None)
except socket.gaierror:
logging.error('The default DUT hostname of %s is unreachable.',
LAB_DUT_HOSTNAME)
return 1
args.cros_cache = os.path.abspath(args.cros_cache)
return args.func(args, unknown_args)
'Output directory, if not provided will output to current working directory',
default=None)
parser.add_argument('--verbosity', help='Verbosity (default=0)', default=0)
parser.add_argument('--override',
help='Overwrite NetCDF files (default=True)',
default=True)
parser.add_argument(
'--acolite_path',
help=
'Path to ACOLITE code (if not already in Python path) (default=None)',
default=None)
parser.add_argument('--satellite',
help='Specifiy satellite (default=Sentinel-2A)',
default='Sentinel-2A')
args, unknown = parser.parse_known_args()
## acolite path
if args.acolite_path is not None:
sys.path.append(os.path.expanduser(args.acolite_path))
try:
import acolite as ac
except:
print(
'Could not import acolite - make sure it is in your Python path, or provide it with --acolite_path'
)
exit(1)
## parse output directory
if args.output is not None:
args.output = os.path.expanduser(args.output)
def main():
global options
global logger
today = datetime.now(pytz.timezone('US/Eastern')).date()
init_parser = argparse.ArgumentParser()
init_parser.add_argument("-p",
"--profile",
help="use alternate config profile")
options, args = init_parser.parse_known_args()
config.settings.load()
if options.profile:
config.settings.set_profile(options.profile)
parser = argparse.ArgumentParser()
# parser.add_argument("-d", "--date", help="game date",
# type=utils.valid_date,
# default=today)
parser.add_argument("-r",
"--resolution",
help="stream resolution",
default=config.settings.profile.default_resolution)
group = parser.add_mutually_exclusive_group()
group.add_argument("-v",
"--verbose",
action="count",
default=0,
help="verbose logging")
group.add_argument("-q",
"--quiet",
action="count",
default=0,
help="quiet logging")
parser.add_argument("game",
metavar="game",
help="game specifier",
nargs="?")
options, args = parser.parse_known_args()
log_file = os.path.join(config.CONFIG_DIR, "mlbstreamer.log")
# formatter = logging.Formatter(
# "%(asctime)s [%(module)16s:%(lineno)-4d] [%(levelname)8s] %(message)s",
# datefmt="%Y-%m-%d %H:%M:%S"
# )
fh = logging.FileHandler(log_file)
fh.setLevel(logging.DEBUG)
# fh.setFormatter(formatter)
logger = logging.getLogger("mlbstreamer")
# logger.setLevel(logging.INFO)
# logger.addHandler(fh)
ulh = UrwidLoggingHandler()
# ulh.setLevel(logging.DEBUG)
# ulh.setFormatter(formatter)
# logger.addHandler(ulh)
utils.setup_logging(options.verbose - options.quiet,
handlers=[fh, ulh],
quiet_stdout=True)
try:
(provider, game_date) = options.game.split("/", 1)
except (ValueError, AttributeError):
if options.game in session.PROVIDERS:
provider = options.game
game_date = datetime.now().date()
else:
provider = list(config.settings.profile.providers.keys())[0]
game_date = dateutil.parser.parse(options.game)
logger.debug("mlbstreamer starting")
entries = Dropdown.get_palette_entries()
entries.update(ScrollingListBox.get_palette_entries())
entries.update(DataTable.get_palette_entries())
# raise Exception(entries)
palette = Palette("default", **entries)
screen = urwid.raw_display.Screen()
screen.set_terminal_properties(256)
view = ScheduleView(provider, game_date)
log_console = widgets.ConsoleWindow()
# log_box = urwid.BoxAdapter(urwid.LineBox(log_console), 10)
pile = urwid.Pile([("weight", 5, urwid.LineBox(view)),
("weight", 1, urwid.LineBox(log_console))])
def global_input(key):
if key in ('q', 'Q'):
raise urwid.ExitMainLoop()
else:
return False
state.loop = urwid.MainLoop(pile,
palette,
screen=screen,
unhandled_input=global_input,
pop_ups=True)
ulh.connect(state.loop.watch_pipe(log_console.log_message))
logger.info("mlbstreamer starting")
if options.verbose:
logger.setLevel(logging.DEBUG)
state.loop.run()
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--pipeline-uri",
type=str,
help="A URI to JSON with instructions")
parser.add_argument("--pipeline", type=str, help="JSON with instructions")
parser.add_argument(
"--prisma-path",
type=str,
help="PRISMA Scene Local Path" # TODO: remove
)
parser.add_argument(
"--prisma-uri",
type=str,
help="PRISMA Scene S3 URI" # TODO: remove
)
parser.add_argument(
"--prisma-stac-id",
type=str,
help="STAC Item ID to process from the STAC collection")
parser.add_argument(
"--prisma-collection-id",
type=str,
default=PRISMA_ARCHIVE_COLLECTION_ID,
)
parser.add_argument(
"--stac-api-uri",
type=str,
default=os.environ.get("STAC_API_URI", "http://franklin:9090"),
)
parser.add_argument("--s3-bucket",
type=str,
default=os.environ.get("S3_BUCKET", "aviris-data"))
parser.add_argument(
"--s3-prefix",
type=str,
default=os.environ.get("S3_PREFIX", "aviris-scene-cogs-l2"),
)
parser.add_argument("--temp-dir",
type=str,
default=os.environ.get("TEMP_DIR", None))
parser.add_argument("--output-format",
type=str,
default=os.environ.get("GDAL_OUTPUT_FORMAT", "COG"))
parser.add_argument(
"--keep-temp-dir",
action="store_true",
help=
"If provided, script does not delete temporary directory before script exits. Useful for debugging.",
)
parser.add_argument(
"--force",
action="store_true",
help=
"If provided, force reingest StacItem even though this it is already present in the catalog.",
)
parser.add_argument("--skip-upload",
action="store_true",
help="Skips upload to S3",
default=False)
try:
warpMemoryLimit = int(os.environ.get("GDAL_WARP_MEMORY_LIMIT", None))
except TypeError:
warpMemoryLimit = None
# TODO: replace it with parser.parse_args() later
cli_args, cli_unknown = parser.parse_known_args()
# parse all cli arguments
args = CliConfig(cli_args, cli_unknown)
# Create tmpdir
temp_dir = Path(args.temp_dir if args.temp_dir is not None else mkdtemp())
temp_dir.mkdir(parents=True, exist_ok=True)
s3 = boto3.client("s3")
stac_client = STACClient(args.stac_api_uri)
prisma_uri = S3Uri(args.prisma_uri)
product_name_derived = os.path.splitext(Path(prisma_uri.key).name)[0]
local_archive = Path(temp_dir, Path(prisma_uri.key).name)
try:
if not local_archive.exists():
logger.info(
f'Downloading PRISMA archive {prisma_uri.url} to {str(local_archive)}...'
)
s3.download_file(prisma_uri.bucket, prisma_uri.key,
str(local_archive))
else:
logger.info(
f'Skipping downloading PRISMA archive {prisma_uri.url}, it already exists {str(local_archive)}'
)
h5_path = Path(Path(temp_dir), f'{product_name_derived}.he5')
if not h5_path.exists():
logger.info(f'Extracting PRISMA archive {str(local_archive)}...')
with zipfile.ZipFile(local_archive, 'r') as zip_ref:
zip_ref.extractall(str(temp_dir))
else:
logger.info(
f'Skipping extraction of the PRISMA archive {str(local_archive)}, file is already extracted {str(h5_path)}'
)
logger.info(f'Reading {str(h5_path)}...')
h5 = h5py.File(str(h5_path))
data = h5['HDFEOS']['SWATHS']['PRS_L2D_HCO']['Data Fields']
# UInt16
swir = data['SWIR_Cube']
# UInt16
vnir = data['VNIR_Cube']
# ERR Matrix is in Bytes
swir_error = data['SWIR_PIXEL_L2_ERR_MATRIX']
vnir_error = data['VNIR_PIXEL_L2_ERR_MATRIX']
swirT = np.swapaxes(swir, 1, 2)
vnirT = np.swapaxes(vnir, 1, 2)
swir_errorT = np.swapaxes(swir_error, 1, 2)
vnir_errorT = np.swapaxes(vnir_error, 1, 2)
rows, cols = swirT.shape[0], swirT.shape[1]
product_name = os.path.splitext(str(
h5.attrs['Product_Name'].decode()))[0]
# X = Lon, Y = Lat
# LatLon projection, required for the STAC catalog
extent_ll, _, _ = raster_extent(ll_y=h5.attrs['Product_LLcorner_lat'],
ll_x=h5.attrs['Product_LLcorner_long'],
lr_y=h5.attrs['Product_LRcorner_lat'],
lr_x=h5.attrs['Product_LRcorner_long'],
ul_y=h5.attrs['Product_ULcorner_lat'],
ul_x=h5.attrs['Product_ULcorner_long'],
ur_y=h5.attrs['Product_URcorner_lat'],
ur_x=h5.attrs['Product_URcorner_long'],
cols=cols,
rows=rows)
# X = easting, Y = northing
extent, res, affine_transform = raster_extent(
ll_y=h5.attrs['Product_LLcorner_lat'],
ll_x=h5.attrs['Product_LLcorner_long'],
lr_y=h5.attrs['Product_LRcorner_lat'],
lr_x=h5.attrs['Product_LRcorner_long'],
ul_y=h5.attrs['Product_ULcorner_lat'],
ul_x=h5.attrs['Product_ULcorner_long'],
ur_y=h5.attrs['Product_URcorner_lat'],
ur_x=h5.attrs['Product_URcorner_long'],
cols=cols,
rows=rows)
cog_proj = osr.SpatialReference()
cog_proj.SetUTM(int(h5.attrs['Projection_Id'].decode()))
cog_proj.AutoIdentifyEPSG()
def create_geotiff(path,
array,
geo_transform=affine_transform,
projection=cog_proj,
dataType=gdal.GDT_UInt16,
driver=gdal.GetDriverByName("GTiff")):
ds = driver.Create(str(path), array.shape[1], array.shape[0],
array.shape[2], dataType,
['COMPRESS=DEFLATE', 'BIGTIFF=YES'])
ds.SetGeoTransform(geo_transform)
ds.SetProjection(projection.ExportToWkt())
for i in range(1, array.shape[2]):
ds.GetRasterBand(i).WriteArray(array[::, ::, i])
ds.FlushCache()
cog_path = str(Path(path.parent, f'cog_{path.name}'))
warp_opts = gdal.WarpOptions(callback=warp_callback,
warpOptions=["NUM_THREADS=ALL_CPUS"],
creationOptions=[
"NUM_THREADS=ALL_CPUS",
"COMPRESS=DEFLATE", "BIGTIFF=YES",
"TILED=YES"
],
multithread=True,
warpMemoryLimit=warpMemoryLimit,
format=args.output_format)
logger.info(f'Converting {str(path)} to {cog_path}...')
# NOTE:
# We can't directly write TIFFs on S3 as the result of the gdal.Warp operation
# see: https://github.com/OSGeo/gdal/issues/1189
with timing("GDAL Warp"):
gdal.Warp(str(cog_path), str(path), options=warp_opts)
return cog_path
swir_path = create_geotiff(Path(temp_dir, "SWIR_Cube.tiff"), swirT)
vnir_path = create_geotiff(Path(temp_dir, "VNIR_Cube.tiff"), vnirT)
swir_error_path = create_geotiff(
Path(temp_dir, "SWIR_PIXEL_L2_ERR_MATRIX.tiff"), swir_errorT)
vnir_error_path = create_geotiff(
Path(temp_dir, "VNIR_PIXEL_L2_ERR_MATRIX.tiff"), vnir_errorT)
def key(path_string):
return f'prisma-scene-cogs/{product_name}/{Path(path_string).name}'
# prep all upload links
s3_uri_swir = f's3://{args.s3_bucket}/{key(swir_path)}'
s3_uri_vnir = f's3://{args.s3_bucket}/{key(vnir_path)}'
s3_uri_swir_error = f's3://{args.s3_bucket}/{key(swir_error_path)}'
s3_uri_vnir_error = f's3://{args.s3_bucket}/{key(vnir_error_path)}'
upload_paths = [
('SWIR_Cube', swir_path, s3_uri_swir),
('VNIR_Cube', vnir_path, s3_uri_vnir),
('SWIR_PIXEL_L2_ERR_MATRIX', swir_error_path, s3_uri_swir_error),
('VNIR_PIXEL_L2_ERR_MATRIX', vnir_error_path, s3_uri_vnir_error)
]
if (not args.skip_upload):
for _, local_path, s3_uri in upload_paths:
logger.info(f'Uploading {local_path} to {s3_uri}')
s3.upload_file(
str(local_path),
args.s3_bucket,
str(key(local_path)),
Callback=ProgressPercentage(str(local_path)),
Config=TransferConfig(multipart_threshold=1 * GB),
)
swir_λs = h5.attrs['List_Cw_Swir']
swir_flags = h5.attrs['List_Cw_Swir_Flags']
swir_freqs = zip(
swir_λs[swir_flags == 1],
map(lambda x: x[0],
filter(lambda x: x[1] == 1, enumerate(swir_flags))))
swir_freqs_sorted = sorted([(b, f * 0.001) for f, b in swir_freqs],
key=lambda x: x[0])
vnir_λs = h5.attrs['List_Cw_Vnir']
vnir_flags = h5.attrs['List_Cw_Vnir_Flags']
vnir_freqs = zip(
vnir_λs[vnir_flags == 1],
map(lambda x: x[0],
filter(lambda x: x[1] == 1, enumerate(vnir_flags))))
vnir_freqs_sorted = sorted([(b, f * 0.001) for f, b in vnir_freqs],
key=lambda x: x[0])
λs = np.hstack([swir_λs, vnir_λs])
flags = np.hstack([swir_flags, vnir_flags])
band_freqs = sorted(
zip(
λs[flags == 1],
map(lambda x: x[0],
filter(lambda x: x[1] == 1, enumerate(flags)))))
sorted_freqs = sorted([(b, f * 0.001) for f, b in band_freqs],
key=lambda x: x[0])
eo_swir_bands = [{
'name': str(b),
'center_wavelength': float(f)
} for (b, f) in swir_freqs_sorted]
eo_vnir_bands = [{
'name': str(b),
'center_wavelength': float(f)
} for (b, f) in vnir_freqs_sorted]
start_datetime = str(h5.attrs['Product_StartTime'].decode())
end_datetime = str(h5.attrs['Product_StopTime'].decode())
wavelength_min = float(min(sorted_freqs, key=lambda e: e[1])[1])
wavelength_max = float(max(sorted_freqs, key=lambda e: e[1])[1])
cog_item_id = f'{PRISMA_COG_COLLECTION.id}_{product_name}'
cog_item = pystac.Item(
id=cog_item_id,
stac_extensions=COG_ITEM_EXTENSIONS,
geometry=shapely.geometry.mapping(
shapely.geometry.box(*extent_ll)),
datetime=dateutil.parser.isoparse(start_datetime),
bbox=list(map(lambda c: float(c), extent_ll)),
collection=PRISMA_COG_COLLECTION.id,
properties={
'start_datetime': start_datetime,
'end_datetime': end_datetime,
'hsi:wavelength_min': wavelength_min,
'hsi:wavelength_max': wavelength_max,
'proj:epsg': int(cog_proj.GetAttrValue('AUTHORITY', 1))
})
# add item assets
for eo_bands, (key, _, uri) in zip(
[eo_swir_bands, eo_swir_bands, eo_swir_bands, eo_swir_bands],
upload_paths):
cog_item.add_asset(key=key,
asset=pystac.Asset(
href=uri,
media_type=pystac.MediaType.GEOTIFF,
properties={'eo:bands': eo_bands}))
finally:
if not args.keep_temp_dir:
logger.info(f'Removing temp dir: {temp_dir}')
shutil.rmtree(temp_dir, ignore_errors=True)
# Create COG Collection if it doesn't exist
if not stac_client.has_collection(PRISMA_COG_COLLECTION.id):
stac_client.post_collection(PRISMA_COG_COLLECTION)
# Add COG Item to AVIRIS L2 STAC Collection
logger.info(f'POST Item {cog_item.id} to {args.stac_api_uri}')
item_data = stac_client.post_collection_item(PRISMA_COG_COLLECTION.id,
cog_item)
if item_data.get('id', None):
logger.info(f"Success: {item_data['id']}")
activation_output(item_data['id'])
else:
logger.error(f'Failure: {item_data}')
return -1
def main():
today = datetime.now(pytz.timezone('US/Eastern')).date()
parser = argparse.ArgumentParser()
parser.add_argument("-d",
"--date",
help="game date",
type=valid_date,
default=today)
parser.add_argument("-g",
"--game-number",
help="number of team game on date (for doubleheaders)",
default=1,
type=int)
parser.add_argument("-b",
"--begin",
help="begin playback at this offset from start",
nargs="?",
metavar="offset_from_game_start",
type=begin_arg_to_offset,
const=0)
parser.add_argument("-r",
"--resolution",
help="stream resolution",
default="720p")
parser.add_argument("-s",
"--save-stream",
help="save stream to file",
nargs="?",
const=True)
parser.add_argument("--no-cache",
help="do not use response cache",
action="store_true")
parser.add_argument("-v",
"--verbose",
action="store_true",
help="verbose logging")
parser.add_argument("--init-config",
help="initialize configuration",
action="store_true")
parser.add_argument("game",
metavar="game",
nargs="?",
help="team abbreviation or MLB game ID")
options, args = parser.parse_known_args()
global logger
logger = logging.getLogger("mlbstreamer")
if options.verbose:
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter(
"%(asctime)s [%(levelname)8s] %(message)s",
datefmt='%Y-%m-%d %H:%M:%S')
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(formatter)
logger.addHandler(handler)
else:
logger.addHandler(logging.NullHandler())
if options.init_config:
config.settings.init_config()
sys.exit(0)
config.settings.load()
if not options.game:
parser.error("option game")
state.session = MLBSession.new(no_cache=options.no_cache)
preferred_stream = None
date = None
if options.game.isdigit():
game_specifier = int(options.game)
else:
game_specifier = (options.date, options.game, options.game_number)
try:
proc = play_stream(
game_specifier,
options.resolution,
offset=options.begin,
preferred_stream=preferred_stream,
output=options.save_stream,
)
proc.wait()
except MLBPlayInvalidArgumentError as e:
raise argparse.ArgumentTypeError(str(e))
except MLBPlayException as e:
logger.error(e)
default=os.environ.get('ES_HOSTS',
'http://10.80.121.231:9200'),
help='ElasticSearch server(s)')
parser.add_argument('--index',
'-i',
dest='elastic_index',
action='store',
default='o365-%s' %
hex(abs(hash(json.dumps(sys.argv[1:]))))[2:10],
help='ElasticSearch index name')
parser.add_argument("paths",
nargs=REMAINDER,
help='Target audit log file(s)',
metavar='paths')
args, extra = parser.parse_known_args(sys.argv[1:])
es = Elasticsearch(args.elastic_server, index=args.elastic_index)
tzinfos = {"AEST": gettz("Australia/Brisbane")}
print(f"Using server: {args.elastic_server}")
print(f"Using index: {args.elastic_index}")
def convert_key(string):
s1 = re.sub('([^\.])([A-Z][a-z]+)', r'\1_\2', string.replace(' ', '_'))
s2 = re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
s3 = re.sub('_+', r'_', s2.replace('.', '_'))
return s3.replace('extended_properties', '').replace('target_updated_properties', 'target_updated') \
.replace('parameters', '').replace('additional_details', '')
def main():
today = datetime.now(pytz.timezone('US/Eastern')).date()
init_parser = argparse.ArgumentParser(add_help=False)
init_parser.add_argument("--init-config",
help="initialize configuration",
action="store_true")
init_parser.add_argument("-p",
"--profile",
help="use alternate config profile")
options, args = init_parser.parse_known_args()
if options.init_config:
config.settings.init_config()
sys.exit(0)
config.settings.load()
if options.profile:
config.settings.set_profile(options.profile)
parser = argparse.ArgumentParser(
description=init_parser.format_help(),
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("-b",
"--begin",
help="begin playback at this offset from start",
nargs="?",
metavar="offset_from_game_start",
type=begin_arg_to_offset,
const=0)
parser.add_argument("-r",
"--resolution",
help="stream resolution",
default=config.settings.profile.default_resolution)
parser.add_argument("-s",
"--save-stream",
help="save stream to file",
nargs="?",
const=True)
parser.add_argument("--no-cache",
help="do not use response cache",
action="store_true")
group = parser.add_mutually_exclusive_group()
group.add_argument("-v",
"--verbose",
action="count",
default=0,
help="verbose logging")
group.add_argument("-q",
"--quiet",
action="count",
default=0,
help="quiet logging")
parser.add_argument("game",
metavar="game",
nargs="?",
help="team abbreviation or MLB game ID")
options, args = parser.parse_known_args(args)
try:
(provider, game) = options.game.split("/", 1)
except ValueError:
game = options.game #.split(".", 1)[1]
provider = list(config.settings.profile.providers.keys())[0]
if game.isdigit():
game_specifier = int(game)
else:
game_specifier = game
utils.setup_logging(options.verbose - options.quiet)
if not options.game:
parser.error("option game")
state.session = session.new(provider)
preferred_stream = None
date = None
try:
proc = play_stream(game_specifier,
options.resolution,
offset=options.begin,
preferred_stream=preferred_stream,
output=options.save_stream,
verbose=options.verbose)
proc.wait()
except MLBPlayInvalidArgumentError as e:
raise argparse.ArgumentTypeError(str(e))
except MLBPlayException as e:
logger.error(e)
import warnings
# Python 3.6 deprecate invalid character-pairs \A, \* ... in non
# raw-strings and other things. Let's not re-introduce them
warnings.filterwarnings('error', '.*invalid escape sequence.*',
category=DeprecationWarning)
warnings.filterwarnings(
'default',
'.*inspect.getargspec\(\) is deprecated.*',
category=DeprecationWarning)
from matplotlib import test
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument('--no-network', action='store_true',
help='Run tests without network connection')
parser.add_argument('--recursionlimit', type=int, default=0,
help='Specify recursionlimit for test run')
args, extra_args = parser.parse_known_args()
if args.no_network:
from matplotlib.testing import disable_internet
disable_internet.turn_off_internet()
extra_args.extend(['-m', 'not network'])
print('Python byte-compilation optimization level:', sys.flags.optimize)
retcode = test(argv=extra_args, switch_backend_warn=False,
recursionlimit=args.recursionlimit)
sys.exit(retcode)
def init():
parser = argparse.ArgumentParser(
description=
"Module that loads given datasets and trains and evaluates one or more neural network models on that."
)
parser.add_argument('dataset',
type=str,
help="The filepath to the dataset/s.")
parser.add_argument(
'--models',
type=str,
help=
"A list of the models that are going to be trained and evaluated. Default is all available."
)
parser.add_argument(
'--args',
type=str,
help=
"A list of arguments to be passed on to the models. In the format key1=value1,key2=value2.1;value2.2"
)
parser.add_argument('--quiet',
dest='quiet',
action="store_true",
help="Do not plot graphs, but save them as images.")
parser.set_defaults(quiet=False)
parser.add_argument(
'--saveModel',
type=str,
help=
"Location to save the model architecture, weights, training history, evaluation scores and prediction graphs. DO NOT include file extension, just filename!"
)
parser.add_argument(
'--loadModel',
type=str,
help=
"Location to to load a saved model architecture and weights. Include the file extension!"
)
parser.add_argument('--shuffle',
dest='shuffle',
action="store_true",
help="Shuffle the generated dataset and labels.")
parser.set_defaults(shuffle=False)
parser.add_argument(
'--trim-batch',
dest='trim',
action="store_true",
help=
"Trim each dataset so that its length is divisible by the batch size.")
parser.set_defaults(trim=False)
parser.add_argument(
'--no-replace',
dest='noReplace',
action="store_true",
help=
"If a model is already saved with the same filename, don't train and save the new model."
)
parser.set_defaults(noReplace=False)
parser.add_argument('--no-train',
dest='train',
action="store_false",
help="Do not train, only evaluathe the model.")
parser.set_defaults(train=True)
args, _ = parser.parse_known_args()
if args.noReplace and args.saveModel is not None:
if os.path.exists(args.saveModel + modelExtension):
print("The file already exists and the 'no-replace' flag is used.")
return #do not replace it
givenModels = [x.lower()
for x in args.models.split(',')] if args.models else None
modelArgs = {}
pairs = args.args.split(',') if args.args else []
for pair in pairs:
key, value = pair.split('=')
try:
value = float(value)
if value == int(value):
value = int(value)
except ValueError:
if ':' in value:
value = [int(i) for i in value.split(':')]
pass
modelArgs[key] = value
print("Processed model arguments", modelArgs)
run(args.dataset,
givenModels,
modelArgs=modelArgs,
quiet=args.quiet,
saveModel=args.saveModel,
loadModel=args.loadModel,
shuffle=args.shuffle,
trim=args.trim,
train=args.train)
