def get_options(script='config_file', ignore_config=False):
"""
config parser wrapper. Used to generate options object that can be
propagated throughout all member classes of the trainer class
:param options:
"""
confFile = loadConfigFile("./config/paths.yaml")
dataPath = confFile['CREMI_data']
p = configargparse.ArgParser(
default_config_files=['./config/%s.conf' % (script)])
# where to save the net
def_net_name = 'V5_BN_times100_ft'
if ignore_config:
# do not use config specified by -c
p.add('-c', '--my-config')
else:
p.add('-c', '--my-config', is_config_file=True)
p.add('--net_name', default=def_net_name)
p.add('--net_arch', default="ID_v5_hydra_BN")
p.add('--no-save_net', dest='save_net_b', action='store_false')
p.add('--val_name',
default='') # empty strings have catastrophic consequences!!!!
p.add('--validation_b', default=False, action='store_true'
) # empty strings have catastrophic consequences!!!!
# reload existing net
p.add('--load_net', dest='load_net_b', action='store_true')
p.add('--load_net_path', default='None')
p.add('--load_init_net_path', default='None')
p.add('--gpu', default='gpu0')
# train data paths
def_train_version = 'second_repr' # def change me
p.add('--dataset', default="Polygon")
p.add('--train_version', default=def_train_version)
p.add('--seed_method',
type=str,
default="timo",
help='available metods: gt, timo, grid',
dest='seed_method')
p.add('--s_minsize', type=int, default=0)
p.add('--input_data_path', type=str, default="None")
p.add('--all_edges', type=str, default="")
# valid data paths
def_valid_version = 'first_repr'
p.add('--valid_version', default=def_valid_version)
p.add('--max_processes', default=2, type=int)
p.add('--max_num_gpus', default=4, type=int)
p.add('--defect_slices_b', default=False,
action='store_true') # if true defect slices will be excluded
# training general
p.add('--no-val', dest='val_b', action='store_false')
p.add('--export_quick_eval', action='store_true')
p.add('--save_counter', default=1000, type=int)
p.add('--observation_counter', default=5, type=int)
p.add('--dummy_data', dest='dummy_data_b', action='store_true')
p.add('--global_edge_len', default=300, type=int)
p.add('--fast_reset', action='store_true')
p.add('--clip_method', default='clip') # rescale, exp, clip
p.add('--claim_aug', default='None')
p.add('--padding_b', action='store_true')
p.add('--master_training', action='store_true')
p.add('--master', action='store_true')
p.add('--validation_slave', action='store_true')
p.add('--merge_seeds', dest='merge_seeds', action='store_true')
p.add('--train_merge', dest='train_merge', action='store_true')
p.add('--dropout_b', action='store_true', default=False)
p.add('--bnorm_b', action='store_true', default=False)
p.add('--weight_by_pathlength_b', action='store_true', default=False)
p.add('--weight_by_distance_b', action='store_true', default=False)
p.add('--weight_by_RI_b', action='store_true', default=False)
p.add('--weight_by_importance', action='store_true', default=False)
p.add('--fully_conf_valildation_b', action='store_true', default=False)
p.add('--discount_factor', default=0.4, type=float)
p.add('--future_discount_factor', default=0.999, type=float)
# pre-training
p.add('--pre_train_iter', default=600000, type=int)
p.add('--regularization', default=10.**1, type=float)
p.add('--network_channels', default=1, type=int)
p.add('--claim_channels', default=2, type=int)
p.add('--batch_size', default=16, type=int)
p.add('--quick_eval', action='store_true')
p.add('--no-augment_pretraining',
dest='augment_pretraining',
action='store_false')
p.add('--create_holes', action='store_true', default=False)
# RNN
p.add('--n_recurrent_hidden', default=128, type=int)
p.add('--feature_map_size_reduction', default=1, type=int)
p.add('--backtrace_length', default=5, type=int)
p.add('--n_batch_errors', default=1, type=int)
p.add('--weight_fct', default="hard", type=str)
# lesion study
p.add('--lesion_remove_hidden', action='store_true')
# fine-tuning
p.add('--batch_size_ft', default=4, type=int)
p.add('--no-aug-ft', dest='augment_ft', action='store_false')
p.add('--optimizer', default="nesterov", type=str)
p.add('--stochastic_update_b', default=False, action='store_true')
p.add('--learningrate', default=0.000001, type=float)
p.add('--lr_decrase', default=0.95, type=float)
p.add('--lr_decrease_counter', default=100, type=int)
# experience replay
p.add('--no_bash_backup', action='store_true')
p.add('--lowercomplete_e', default=0., type=float)
p.add('--raw_path', default="None", type=str)
p.add('--membrane_path', default="None", type=str)
p.add('--label_path', default="None", type=str)
p.add('--height_gt_path', default="None", type=str)
# dataprovider
# polygon
p.add('--dashes_on_b', action='store_true')
p.add('--dash_len', default=5, type=int)
p.add('--hole_length', default=5, type=int)
# validation
p.add('--slices_total', type=int, default=10)
p.add('--start_slice_z', type=int, default=100)
p.add('--reset_after_fine_tune', action='store_true')
p.add('--no-ft', dest='fine_tune_b', action='store_false')
p.add('--reset-ft', dest='rs_ft', action='store_true')
p.add('--reset_pretraining', dest='reset_pretraining', action='store_true')
p.add('--margin', default=0.5, type=float)
# clip_method="exp20"
p.add('--exp_bs', default=16, type=int)
p.add('--exp_ft_bs', default=8, type=int)
p.add('--exp_warmstart', default=1000, type=int)
p.add('--exp_acceptance_rate', default=0.1, type=float)
p.add('--no-exp_height', dest='exp_height', action='store_false')
p.add('--no-exp_save', dest='exp_save', action='store_false')
p.add('--exp_mem_size', default=20000, type=int)
p.add('--exp_load', default="None", type=str)
p.add('--no-exp_loss', dest='exp_loss', action='store_false')
p.add('--exp_wlast', default=1., type=float)
p.add('--max_iter', default=10000000000000, type=int)
p.add('--scale_height_factor', default=100, type=float)
p.add('--ahp', dest='add_height_penalty', action='store_true')
p.add('--max_penalty_pixel', default=3, type=float)
options = p.parse_args()
# options.slices = range(2000)
# options.fc_prec = False
# options.dp_already_exits = True
if options.input_data_path == "None":
options.input_data_path = '%s/data/volumes/%sinput_%s.h5' % (
dataPath, options.all_edges, options.train_version)
if options.raw_path == "None":
options.raw_path = '%s/data/volumes/raw_%s.h5' % (
dataPath, options.train_version)
if options.membrane_path == "None":
options.membrane_path = '%s/data/volumes/membranes_%s.h5' % (
dataPath, options.train_version)
if options.label_path == "None":
options.label_path = '%s/data/volumes/label_%s.h5' % (
dataPath, options.train_version)
if options.height_gt_path == "None":
options.height_gt_path = '%s/data/volumes/height_%s.h5' % (
dataPath, options.train_version)
options.save_net_path = dataPath + '/data/nets/' + options.net_name + '/'
print('saving files to ', options.net_name)
# parse validation config
if not ignore_config and options.val_name != '':
options.val_options = get_options(script=options.val_name,
ignore_config=True)
return options
def __init__(self) -> None:
"""
Initialize the configuration settings and provide sensible default
values if possible.
"""
self._parser = configargparse.ArgParser(
description='ANN-SoLo: Approximate nearest neighbor spectral '
'library searching\n'
'================================================'
'=================\n\n'
'Bittremieux et al. Fast open modification spectral '
'library searching through approximate nearest '
'neighbor indexing. Journal of Proteome Research 17, '
'3464-3474 (2018).\n\nBittremieux et al. Extremely '
'fast and accurate open modification spectral library '
'searching of high-resolution mass spectra using '
'feature hashing and graphics processing units. '
'Journal of Proteome Research 18, 3792-3799 (2019).'
'\n\nOfficial code website: '
'https://github.com/bittremieux/ANN-SoLo\n\n',
default_config_files=['config.ini'],
args_for_setting_config_path=['-c', '--config'],
formatter_class=NewlineTextHelpFormatter)
# IO
self._parser.add_argument('spectral_library_filename',
help='spectral library file '
'(supported formats: splib)')
self._parser.add_argument('query_filename',
help='query file (supported formats: mgf)')
self._parser.add_argument(
'out_filename',
help='name of the mzTab output file containing the search results')
# PREPROCESSING
# Spectral library resolution to round mass values.
self._parser.add_argument(
'--resolution',
default=None,
type=int,
help='spectral library resolution; masses will be rounded to the '
'given number of decimals (default: no rounding)')
# Minimum and maximum fragment peak mass values.
self._parser.add_argument(
'--min_mz',
default=11,
type=int,
help='minimum m/z value (inclusive, default: %(default)s m/z)')
self._parser.add_argument(
'--max_mz',
default=2010,
type=int,
help='maximum m/z value (inclusive, default: %(default)s m/z)')
# Remove peaks around the precursor mass from fragment spectra.
self._parser.add_argument(
'--remove_precursor',
action='store_true',
help='remove peaks around the precursor mass '
'(default: no peaks are removed)')
self._parser.add_argument(
'--remove_precursor_tolerance',
default=0,
type=float,
help='the window (in m/z) around the precursor mass to remove '
'peaks (default: %(default)s m/z)')
# Minimum fragment peak intensity to filter out noise peaks.
self._parser.add_argument(
'--min_intensity',
default=0.01,
type=float,
help='remove peaks with a lower intensity relative to the maximum '
'intensity (default: %(default)s)')
# Minimum number of fragment peaks or mass range (m/z).
self._parser.add_argument(
'--min_peaks',
default=10,
type=int,
help='discard spectra with less peaks (default: %(default)s)')
self._parser.add_argument(
'--min_mz_range',
default=250,
type=float,
help='discard spectra with a smaller mass range '
'(default: %(default)s m/z)')
# Maximum number of fragment peaks to use for each query spectrum.
self._parser.add_argument(
'--max_peaks_used',
default=50,
type=int,
help='only use the specified most intense peaks for the query '
'spectra (default: %(default)s)')
# Maximum number of fragment peaks to use for each library spectrum.
self._parser.add_argument(
'--max_peaks_used_library',
default=50,
type=int,
help='only use the specified most intense peaks for the library '
'spectra (default: %(default)s)')
# Manner in which to scale the peak intensities.
self._parser.add_argument(
'--scaling',
default='rank',
type=str,
choices=['sqrt', 'rank'],
help='to reduce the influence of very intense peaks, scale the'
' peaks by their square root or by their rank '
'(default: %(default)s)')
# MATCHING
# Maximum SSM precursor mass tolerance.
self._parser.add_argument(
'--precursor_tolerance_mass',
type=float,
required=True,
help='precursor mass tolerance (small window for the first level '
'of the cascade search)')
self._parser.add_argument(
'--precursor_tolerance_mode',
type=str,
choices=['Da', 'ppm'],
required=True,
help='precursor mass tolerance unit (options: %(choices)s)')
self._parser.add_argument(
'--precursor_tolerance_mass_open',
type=float,
help='precursor mass tolerance (wide window for the second level '
'of the cascade search)')
self._parser.add_argument(
'--precursor_tolerance_mode_open',
type=str,
choices=['Da', 'ppm'],
help='precursor mass tolerance unit (options: %(choices)s)')
# Fragment peak matching.
self._parser.add_argument('--fragment_mz_tolerance',
type=float,
required=True,
help='fragment mass tolerance (m/z)')
# Shifted dot product.
self._parser.add_argument(
'--allow_peak_shifts',
action='store_true',
help='use the shifted dot product instead of the standard dot '
'product')
# Maximum FDR.
self._parser.add_argument(
'--fdr',
default=0.01,
type=float,
help='FDR threshold to accept identifications during the cascade '
'search (default: %(default)s)')
self._parser.add_argument(
'--fdr_tolerance_mass',
default=0.1,
type=float,
help='mass difference bin width for the group FDR calculation '
'during the second cascade level (default: %(default)s Da)')
self._parser.add_argument(
'--fdr_tolerance_mode',
default='Da',
type=str,
choices=['Da', 'ppm'],
help='mass difference bin unit for the group FDR calculation '
'during the second cascade level (default: %(default)s)')
self._parser.add_argument(
'--fdr_min_group_size',
default=20,
type=int,
help='minimum group size for the group FDR calculation during the '
'second cascade level (default: %(default)s)')
# MODE
# Use an ANN index or the conventional brute-force mode.
self._parser.add_argument(
'--mode',
default='ann',
type=str,
choices=['ann', 'bf'],
help="search using an approximate nearest neighbors or the "
"traditional (brute-force) mode; 'bf': brute-force, 'ann': "
"approximate nearest neighbors (default: %(default)s)")
# Bin size for the ANN index (Dalton).
self._parser.add_argument(
'--bin_size',
default=0.04,
type=float,
help='ANN vector bin width (default: %(default)s Da)')
# ANN vector length after hashing.
self._parser.add_argument(
'--hash_len',
default=800,
type=int,
help='ANN vector length (default: %(default)s)')
# Number of candidates to retrieve from the ANN index for each query.
self._parser.add_argument(
'--num_candidates',
default=1024,
type=int,
help='number of candidates to retrieve from the ANN index for each'
' query (default: %(default)s), maximum 1024 when using GPU '
'indexing')
# Number of query spectra to process simultaneously.
self._parser.add_argument(
'--batch_size',
default=16384,
type=int,
help='number of query spectra to process simultaneously '
'(default: %(default)s)')
# Custom FAISS parameters.
# Number of lists in the IVF.
self._parser.add_argument('--num_list',
default=256,
type=int,
help='number of partitions in the ANN index '
'(default: %(default)s)')
# Number of lists to inspect during querying.
self._parser.add_argument(
'--num_probe',
default=128,
type=int,
help='number of partitions in the ANN index to inspect during '
'querying (default: %(default)s), maximum 1024 when using '
'GPU indexing')
# Don't try to use the GPU.
self._parser.add_argument(
'--no_gpu',
action='store_true',
help="don't use the GPU for ANN searching (default: GPU is used "
"if available)")
# Filled in 'parse', contains the specified settings.
self._namespace = None
#!/usr/bin/env python
import sys
import configargparse
import os
import numpy as N
from ebosscat import *
p = configargparse.ArgParser()
p.add('-c',
'--my-config',
required=True,
is_config_file=True,
help='config file path')
p.add('--collate', required=True, help='Collate file with targets')
p.add('--geometry', required=True, help='Mangle geometry file of the survey')
p.add('--vetos_dir', help='Directory containing veto masks')
p.add('--zcatalog', required=True, help='fits file containing redshifts')
p.add('--version', required=True)
p.add('--outdir', required=True, help='Output directory')
p.add('--target', required=True, help='Type of targets')
p.add('--cap', default='Both', help='Cap: North, South or Both')
p.add('--completeness', help='Type of completeness')
p.add('--min_comp', type=float, help='Minimal value for completeness')
p.add('--apply_vetos',
action='store_true',
help='Apply veto masks? It is slower')
p.add('--start_over', action='store_true', help='Re-start from beginning')
p.add('--zmin', type=float, help='Minimum redshift range')
p.add('--zmax', type=float, help='Maximum redshift range')
def __init__(self): super(opts, self).__init__() self.parser = configargparse.ArgParser(default_config_files=[])
import configargparse
global config
parser = configargparse.ArgParser(
default_config_files=["/etc/upsilon-custodian/custodian.cfg"])
parser.add("--dbUser")
parser.add("--dbPass", default="")
parser.add("--dbName")
parser.add("--dbHost")
parser.add("--amqpHost")
parser.add("--amqpExchange")
parser.add("--amqpQueue", default="upsilon-custodian")
parser.add("--promPort", default=1300)
parser.add("--promOnly", action='store_true')
config = parser.parse_args()
def getRuntimeConfig():
return config
valid_ref_dir = os.path.join(os.path.sep, sys.prefix,
'valid_reference_fasta_files')
bwa_index_dir = os.path.join(os.path.sep, sys.prefix, 'bwa_indexes')
last_index_dir = os.path.join(os.path.sep, sys.prefix, 'last_indexes')
if not os.path.exists(logfolder):
os.makedirs(logfolder)
if not os.path.exists(valid_ref_dir):
os.makedirs(valid_ref_dir)
if not os.path.exists(bwa_index_dir):
os.makedirs(bwa_index_dir)
if not os.path.exists(last_index_dir):
os.makedirs(last_index_dir)
parser = \
configargparse.ArgParser(description='minup: A program to analyse minION fast5 files in real-time or post-run.'
, default_config_files=[config_file])
parser.add(
'-dbh',
'--mysql-host',
type=str,
dest='dbhost',
required=False,
default='localhost',
help="The location of the MySQL database. default is 'localhost'.",
)
parser.add(
'-dbu',
'--mysql-username',
type=str,
dest='dbusername',
def main():
#p = configargparse.YAMLConfigFileParser() #TODO: Yaml would be better
p = configargparse.ArgParser(default_config_files=['settings.conf'])
p.add('-c',
'--config-file',
required=False,
is_config_file=True,
help='config file path')
p.add('-d', '--data-provider', required=False)
p.add("-s",
"--sim",
dest="sim",
help="use simulation mode for testing without hardware",
action="store_true",
default=False)
p.add("--port", dest="port", help="listen to daq on port ", default=None)
p.add("-t",
"--timewindow",
dest="time_window",
help="time window for the measurement in s (default 5s)",
type=float,
default=5.0)
p.add("-m",
"--measurement-duration",
dest="meas_duration",
required=False,
help="Duration of measurement in seconds",
type=float)
#p.add("-d", "--debug", dest="log_level", help="switch to loglevel debug", action="store_const", const=logging.DEBUG, default=logging.INFO)
p.add("-n",
"--nostatus",
dest="write_daq_status",
help="do not write DAQ status messages to RAW data files",
action="store_false",
default=True)
p.add("-v",
"--version",
dest="version",
help="show current version",
action="store_true",
default=False)
# Consumers:
p.add("--raw",
dest="raw_consumer",
help="View raw DAQ data",
action="store_true",
default=False)
p.add("-P",
"--data-path",
dest="data_path",
help="Store measurement data in the specified directory",
type=str,
default=None)
p.add(
"-M",
"--MySQL",
nargs=3,
metavar=('HOST', 'USER', 'DATABASE'),
help=
"Stream measurement data to database. Arguments are: host user database - in this order"
)
p.add(
"--MySQL-db-user-id",
help=
"ID of user in 'users' table to be associated with the current measurements",
dest="mysql_db_user_id",
default=0,
required=False)
p.add("-D",
"--Django",
nargs=1,
metavar=("USER"),
help="Initialize Django consumer with USER",
default=None)
p.add("-G",
"--GUI",
dest="GUI",
help="Invoke GUI",
action="store_true",
default=False)
# Analyzers
p.add("--buffer-size",
dest="buf_size",
help="Buffer size for analyzed data",
type=int,
default=255)
p.add("--rate",
dest="rate_analyzer",
help="Analyze rates",
action="store_true",
default=False)
p.add("--pulse",
dest="pulse_analyzer",
help="Analyze pulses",
action="store_true",
default=False)
p.add("--decay",
dest="decay_analyzer",
help="Analyze decays",
action="store_true",
default=False)
p.add("--velocity",
dest="velocity_analyzer",
help="Analyze velocity",
action="store_true",
default=False)
options = vars(p.parse_args())
consumers = []
# consumers.append(DummyConsumer())
if options.get("raw_consumer"):
consumers.append(DummyConsumer())
if options.get('data_path') is not None:
consumers.append(
FileConsumer(data_dir=options.get("data_path"), logger=logger))
if options.get("Django") is not None:
try:
import django
from django.conf import settings
from muonic_django.consumer import Consumer as DjangoConsumer
except:
raise ImportError("Django consumer optional dependency missing")
os.environ.setdefault("DJANGO_SETTINGS_MODULE",
"muonic_webapp.settings")
django.setup()
consumers.append(
DjangoConsumer(simulation=options.get("sim"),
username=options.get("Django")[0]))
if options.get("MySQL") is not None:
try:
from muonic_mysql.consumer import MySqlConsumer
except:
raise ImportError("MySQL consumer optional dependency missing")
consumers.append(MySqlConsumer(options))
analyzers = []
# analyzers.append(DummyAnalyzer(consumers=[bf], **options))
if options.get("GUI"):
# TODO: move analyzers, consumers, bf to application.py, maybe __init__ -> property or so?
try:
from muonic_gui.gui import Application
except:
raise ImportError("muonic_gui optional dependency missing")
try:
from PyQt4.QtGui import QApplication
except:
raise ImportError("Qt4 optional dependency missing")
root = QApplication([])
root.setQuitOnLastWindowClosed(True)
app = Application(logger=logger, opts=options, consumers=consumers)
app.showMaximized()
root.exec()
# gui_consumer = GuiConsumer(logger=logger)
# consumers.append(gui_consumer)
#
# bf = BufferedConsumer(1000, *consumers)
#
else:
# if options.get("decay_analyzer") and options.get("velocity_analyzer"):
# raise RuntimeError("Cannot analyze decay and velocity")
bf = [BufferedConsumer(options.get("buf_size"), *consumers)]
if options.get("rate_analyzer"):
analyzers.append(RateAnalyzer(consumers=bf, **options))
if options.get("pulse_analyzer"):
analyzers.append(PulseAnalyzer(consumers=bf, **options))
if options.get("decay_analyzer"):
analyzers.append(DecayAnalyzer(consumers=bf, **options))
if options.get("velocity_analyzer"):
analyzers.append(VelocityAnalyzer(consumers=bf, **options))
app = App(options=options, analyzers=analyzers, logger=logger)
app.run()
def get_args():
# f**k PEP8
configpath = os.path.join(os.path.dirname(__file__), '../config/config.ini')
parser = configargparse.ArgParser(default_config_files=[configpath], auto_env_var_prefix='POGOMAP_')
parser.add_argument('-a', '--auth-service', type=str.lower, action='append',
help='Auth Services, either one for all accounts or one per account: ptc or google. Defaults all to ptc.')
parser.add_argument('-u', '--username', action='append',
help='Usernames, one per account.')
parser.add_argument('-p', '--password', action='append',
help='Passwords, either single one for all accounts or one per account.')
parser.add_argument('-l', '--location', type=parse_unicode,
help='Location, can be an address or coordinates')
parser.add_argument('-j', '--jitter', help='Apply random -9m to +9m jitter to location',
action='store_true', default=False)
parser.add_argument('-st', '--step-limit', help='Steps', type=int,
default=12)
parser.add_argument('-sd', '--scan-delay',
help='Time delay between requests in scan threads',
type=float, default=10)
parser.add_argument('-ld', '--login-delay',
help='Time delay between each login attempt',
type=float, default=5)
parser.add_argument('-lr', '--login-retries',
help='Number of logins attempts before refreshing a thread',
type=int, default=3)
parser.add_argument('-mf', '--max-failures',
help='Maximum number of failures to parse locations before an account will go into a two hour sleep',
type=int, default=5)
parser.add_argument('-msl', '--min-seconds-left',
help='Time that must be left on a spawn before considering it too late and skipping it. eg. 600 would skip anything with < 10 minutes remaining. Default 0.',
type=int, default=0)
parser.add_argument('-dc', '--display-in-console',
help='Display Found Pokemon in Console',
action='store_true', default=False)
parser.add_argument('-H', '--host', help='Set web server listening host',
default='127.0.0.1')
parser.add_argument('-P', '--port', type=int,
help='Set web server listening port', default=5000)
parser.add_argument('-L', '--locale',
help='Locale for Pokemon names (default: {},\
check {} for more)'.
format(config['LOCALE'], config['LOCALES_DIR']), default='en')
parser.add_argument('-c', '--china',
help='Coordinates transformer for China',
action='store_true')
parser.add_argument('-m', '--mock', type=str,
help='Mock mode - point to a fpgo endpoint instead of using the real PogoApi, ec: http://127.0.0.1:9090',
default='')
parser.add_argument('-ns', '--no-server',
help='No-Server Mode. Starts the searcher but not the Webserver.',
action='store_true', default=False)
parser.add_argument('-os', '--only-server',
help='Server-Only Mode. Starts only the Webserver without the searcher.',
action='store_true', default=False)
parser.add_argument('-nsc', '--no-search-control',
help='Disables search control',
action='store_false', dest='search_control', default=True)
parser.add_argument('-fl', '--fixed-location',
help='Hides the search bar for use in shared maps.',
action='store_true', default=False)
parser.add_argument('-k', '--gmaps-key',
help='Google Maps Javascript API Key',
required=True)
parser.add_argument('--spawnpoints-only', help='Only scan locations with spawnpoints in them.',
action='store_true', default=False)
parser.add_argument('-C', '--cors', help='Enable CORS on web server',
action='store_true', default=False)
parser.add_argument('-D', '--db', help='Database filename',
default='pogom.db')
parser.add_argument('-cd', '--clear-db',
help='Deletes the existing database before starting the Webserver.',
action='store_true', default=False)
parser.add_argument('-np', '--no-pokemon',
help='Disables Pokemon from the map (including parsing them into local db)',
action='store_true', default=False)
parser.add_argument('-ng', '--no-gyms',
help='Disables Gyms from the map (including parsing them into local db)',
action='store_true', default=False)
parser.add_argument('-nk', '--no-pokestops',
help='Disables PokeStops from the map (including parsing them into local db)',
action='store_true', default=False)
parser.add_argument('-ss', '--spawnpoint-scanning',
help='Use spawnpoint scanning (instead of hex grid). Scans in a circle based on step_limit when on DB', nargs='?', const='nofile', default=False)
parser.add_argument('--dump-spawnpoints', help='dump the spawnpoints from the db to json (only for use with -ss)',
action='store_true', default=False)
parser.add_argument('-pd', '--purge-data',
help='Clear pokemon from database this many hours after they disappear \
(0 to disable)', type=int, default=0)
parser.add_argument('-px', '--proxy', help='Proxy url (e.g. socks5://127.0.0.1:9050)')
parser.add_argument('--db-type', help='Type of database to be used (default: sqlite)',
default='sqlite')
parser.add_argument('--db-name', help='Name of the database to be used')
parser.add_argument('--db-user', help='Username for the database')
parser.add_argument('--db-pass', help='Password for the database')
parser.add_argument('-pu','--page-user',help='username to protect your server, default admin',default='admin')
parser.add_argument('-pp','--page-pass',help='password for protecting your server')
parser.add_argument('--db-host', help='IP or hostname for the database')
parser.add_argument('--db-port', help='Port for the database', type=int, default=3306)
parser.add_argument('--db-max_connections', help='Max connections (per thread) for the database',
type=int, default=5)
parser.add_argument('--db-threads', help='Number of db threads; increase if the db queue falls behind',
type=int, default=1)
parser.add_argument('-wh', '--webhook', help='Define URL(s) to POST webhook information to',
nargs='*', default=False, dest='webhooks')
parser.add_argument('-gi', '--gym-info', help='Get all details about gyms (causes an additional API hit for every gym)',
action='store_true', default=False)
parser.add_argument('--webhook-updates-only', help='Only send updates (pokémon & lured pokéstops)',
action='store_true', default=False)
parser.add_argument('--wh-threads', help='Number of webhook threads; increase if the webhook queue falls behind',
type=int, default=1)
parser.add_argument('--ssl-certificate', help='Path to SSL certificate file')
parser.add_argument('--ssl-privatekey', help='Path to SSL private key file')
parser.add_argument('-ps', '--print-status', action='store_true',
help='Show a status screen instead of log messages. Can switch between status and logs by pressing enter.', default=False)
parser.add_argument('-el', '--encrypt-lib', help='Path to encrypt lib to be used instead of the shipped ones')
verbosity = parser.add_mutually_exclusive_group()
verbosity.add_argument('-v', '--verbose', help='Show debug messages from PomemonGo-Map and pgoapi. Optionally specify file to log to.', nargs='?', const='nofile', default=False, metavar='filename.log')
verbosity.add_argument('-vv', '--very-verbose', help='Like verbose, but show debug messages from all modules as well. Optionally specify file to log to.', nargs='?', const='nofile', default=False, metavar='filename.log')
verbosity.add_argument('-d', '--debug', help='Depreciated, use -v or -vv instead.', action='store_true')
parser.set_defaults(DEBUG=False)
args = parser.parse_args()
if args.only_server:
if args.location is None:
parser.print_usage()
print(sys.argv[0] + ": error: arguments -l/--location is required")
sys.exit(1)
else:
errors = []
num_auths = 1
num_usernames = 0
num_passwords = 0
if (args.username is None):
errors.append('Missing `username` either as -u/--username or in config')
else:
num_usernames = len(args.username)
if (args.location is None):
errors.append('Missing `location` either as -l/--location or in config')
if (args.password is None):
errors.append('Missing `password` either as -p/--password or in config')
else:
num_passwords = len(args.password)
if (args.step_limit is None):
errors.append('Missing `step_limit` either as -st/--step-limit or in config')
if args.auth_service is None:
args.auth_service = ['ptc']
else:
num_auths = len(args.auth_service)
if num_usernames > 1:
if num_passwords > 1 and num_usernames != num_passwords:
errors.append('The number of provided passwords ({}) must match the username count ({})'.format(num_passwords, num_usernames))
if num_auths > 1 and num_usernames != num_auths:
errors.append('The number of provided auth ({}) must match the username count ({})'.format(num_auths, num_usernames))
if len(errors) > 0:
parser.print_usage()
print(sys.argv[0] + ": errors: \n - " + "\n - ".join(errors))
sys.exit(1)
# Fill the pass/auth if set to a single value
if num_passwords == 1:
args.password = [args.password[0]] * num_usernames
if num_auths == 1:
args.auth_service = [args.auth_service[0]] * num_usernames
# Make our accounts list
args.accounts = []
# Make the accounts list
for i, username in enumerate(args.username):
args.accounts.append({'username': username, 'password': args.password[i], 'auth_service': args.auth_service[i]})
return args
def parse(cls, description):
parser = configargparse.ArgParser(description=description,
default_config_files=[
os.path.join(
os.path.expanduser(d),
"covmatic.conf")
for d in ("~", ".")
])
parser.add_argument('-I',
'--ip',
metavar='address',
type=str,
required=False,
help="the robot's IP address or hostname")
parser.add_argument('-A',
'--app',
metavar='path',
type=str,
default="C:/Program Files/Opentrons/Opentrons.exe",
help="the Opentrons App filepath")
parser.add_argument('-S',
'--station',
metavar='name',
type=str,
default="A",
help="the default station type")
parser.add_argument('-L',
'--lang',
metavar='lang',
type=str,
default="ENG",
help="the default message language")
parser.add_argument('-K',
'--ssh-key',
metavar='file',
type=str,
default=os.path.expanduser("~/ot2_ssh_key"),
help="SSH key file path")
parser.add_argument('-P',
'--pwd',
metavar='pwd',
type=str,
default="",
help="the SSH key passphrase")
parser.add_argument('-U',
'--user',
metavar='name',
type=str,
default="root",
help="the SSH username")
parser.add_argument('--barcode-port',
metavar='port',
type=int,
default=5002,
help="the barcode server port")
parser.add_argument(
'--no-wait-log',
dest="wait_log",
action='store_false',
help=
'make the protocol start as soon as possible, without waiting for the first log request'
)
parser.add_argument('--disable_watchdog',
dest="watchdog_enable",
action='store_false',
help='disable watchdog for execution')
parser.add_argument(
'--pcr-app',
metavar="path",
type=str,
default=
"C:/PCR_BioRad/APIs/BioRad_CFX_API_v1.4/BioRad.Example_Application.exe",
help="the PCR app filepath")
parser.add_argument(
'--pcr-results',
metavar="path",
type=str,
default=
"C:/PCR_BioRad/json_results/????????_Data_??-??-????_??-??-??_Result.json",
help="the PCR results filepath (scheme)")
parser.add_argument('--pcr-pcrd',
metavar="path",
type=str,
default="C:/PCR_BioRad/pcrd_results",
help="the PCR pcrd folder")
parser.add_argument(
'--web-app-fixed',
metavar='url',
type=str,
default="",
help=
"the Web App URL to be opened independently without building path with the station name"
)
parser.add_argument('--web-app',
metavar='url',
type=str,
default="",
help="the Web App URL")
parser.add_argument(
'--log-local',
metavar='path',
type=str,
default="",
help=
"local file path for run log file. Use '{}' to add date time info."
)
parser.add_argument(
'--tip-log-remote',
metavar='path',
type=str,
default="/var/lib/jupyter/notebooks/outputs/tip_log.json",
help="remote file path for tip log file")
parser.add_argument('--tip-log-local',
metavar='path',
type=str,
default=os.path.join(os.path.dirname(__file__),
"tip_log.json"),
help="local file path for tip log file")
parser.add_argument('--temp-data-dir',
metavar='path',
type=str,
default=os.path.join(os.path.dirname(__file__),
"temp_data"),
help="directory for temporary data")
parser.add_argument(
'--copan48-json-remote',
metavar='path',
type=str,
default="",
help=
"remote file path for copan48 configuration (this is used for rack configuration)"
)
parser.add_argument(
'--copan48-json-local',
metavar='path',
type=str,
default="",
help=
"local file path for copan48 configuration (if both remote and local paths are given, this is used for rack configuration)"
)
parser.add_argument(
'--magnet-json-remote',
metavar='path',
type=str,
default="",
help=
"remote file path for magnet configuration (this is used for magnet configuration)"
)
parser.add_argument(
'--magnet-json-local',
metavar='path',
type=str,
default="",
help=
"local file path for magnet configuration (if both remote and local paths are given, this is used for magnet configuration)"
)
parser.add_argument(
'--start-at',
action='store_true',
help=
'enable the start_at protocol argument in the upload protocol window'
)
parser.add_argument('--protocol-remote',
metavar='path',
type=str,
default="/var/lib/jupyter/notebooks/protocol.py",
help="remote file path for protocol file")
parser.add_argument('--protocol-local',
metavar='path',
type=str,
default=os.path.join(os.path.dirname(__file__),
".tmp_protocol.py"),
help="local file path for protocol file")
parser.add_argument('--api-prefix',
metavar='prefix',
type=str,
default="/api",
help="the prefix for localwebserver API paths")
parser.add_argument('--pcr-backup',
metavar='path',
type=str,
default=os.path.expanduser("~/.pcr_backup"),
help="the backup folder for PCR result files")
parser.add_argument('--desktop-file',
metavar='path',
type=str,
default=desktop_file(),
help="(setup) the desktop file path for the GUI")
parser.add_argument(
'--tempdeck-desktop-file',
metavar='path',
type=str,
default=tempdeck_desktop_file(),
help="(setup) the desktop file path for the Tempdeck GUI")
parser.add_argument(
'--tempdeck-desktop',
action='store_true',
help='(setup) create a desktop file for the Tempdeck GUI')
parser.add_argument('--python',
metavar='path',
type=str,
default=os.sys.executable,
help="(setup) the Python instance file path")
parser.add_argument(
'--protocols-to-show',
type=str,
default="",
help="Limit protocols name to show to these, comma separated")
return cls.reset(**parser.parse_known_args()[0].__dict__)
def parse_args():
defaultconfigfiles = []
default_tokenfile = None
if '-cf' not in sys.argv and '--config' not in sys.argv:
defaultconfigfiles = [
os.getenv('MAD_CONFIG',
os.path.join(mapadroid.MAD_ROOT, 'configs/config.ini'))
]
if '-td' not in sys.argv and '--token_dispenser' not in sys.argv:
default_tokenfile = os.getenv(
'MAD_CONFIG',
os.path.join(mapadroid.MAD_ROOT, 'configs/token-dispensers.ini'))
parser = configargparse.ArgParser(default_config_files=defaultconfigfiles,
auto_env_var_prefix='THERAIDMAPPER_')
parser.add_argument('-cf',
'--config',
is_config_file=True,
help='Set configuration file')
parser.add_argument('-asi',
'--apk_storage_interface',
default='fs',
help='APK Storage Interface')
# MySQL
# TODO - Depercate this
parser.add_argument('-dbm',
'--db_method',
required=False,
default="rm",
help='LEGACY: DB scheme to be used')
parser.add_argument('-dbip',
'--dbip',
required=False,
help='IP or hostname of MySql Server')
parser.add_argument('-dbport',
'--dbport',
type=int,
default=3306,
help='Port for MySQL Server')
parser.add_argument('-dbuser',
'--dbusername',
required=False,
help='Username for MySQL login')
parser.add_argument('-dbpassw',
'--dbpassword',
required=False,
help='Password for MySQL login')
parser.add_argument('-dbname',
'--dbname',
required=False,
help='Name of MySQL Database')
parser.add_argument(
'-dbps',
'--db_poolsize',
type=int,
default=2,
help='Size of MySQL pool (open connections to DB). Default: 2')
# Websocket Settings (RGC receiver)
parser.add_argument('-wsip',
'--ws_ip',
required=False,
default="0.0.0.0",
type=str,
help='IP for websocket to listen on. Default: 0.0.0.0')
parser.add_argument(
'-wsport',
'--ws_port',
required=False,
type=int,
default=8080,
help='Port of the websocket to listen on. Default: 8080')
parser.add_argument(
'-wsct',
'--websocket_command_timeout',
required=False,
type=int,
default=30,
help=
'The max time to wait for a command to return (in seconds). Default: 30 seconds'
)
# MITM Receiver (PD receiver)
parser.add_argument(
'-mrip',
'--mitmreceiver_ip',
required=False,
default="0.0.0.0",
type=str,
help='IP to listen on for proto data (MITM data). Default: 0.0.0.0')
parser.add_argument(
'-mrport',
'--mitmreceiver_port',
required=False,
default=8000,
help='Port to listen on for proto data (MITM data). Default: 8000')
parser.add_argument(
'-mrdw',
'--mitmreceiver_data_workers',
type=int,
default=2,
help='Amount of workers to work off the data that queues up. Default: 2'
)
parser.add_argument(
'-miptt',
'--mitm_ignore_proc_time_thresh',
type=int,
default=0,
help='Ignore MITM data having a timestamp too far in the past.'
'Specify in seconds. Default: 0 (off)')
parser.add_argument(
'-mipb',
'--mitm_ignore_pre_boot',
default=False,
type=bool,
help='Ignore MITM data having a timestamp pre MAD\'s startup time')
parser.add_argument(
'-mspass',
'--mitm_status_password',
default='',
help='Header Authorization password for MITM /status/ page')
# Walk Settings
parser.add_argument(
'--enable_worker_specific_extra_start_stop_handling',
default=False,
help=
'Option to enable/disable extra handling for the start/stop routine of workers. Default: '
'False')
parser.add_argument(
'-mvd',
'--maximum_valid_distance',
required=False,
type=int,
default=50,
help=
'The maximum distance for a scan of a location to be considered a valid/correct scan of'
' that location in meters. Default: 50m')
# Job Processor
parser.add_argument('-jobdtwh',
'--job_dt_wh',
action='store_true',
default=False,
help='Send job status to discord. Default: False')
parser.add_argument('-jobdtwhurl',
'--job_dt_wh_url',
required=False,
default="",
type=str,
help='Discord Webhook URL for job messages')
parser.add_argument(
'-jobdtsdtyp',
'--job_dt_send_type',
required=False,
default="SUCCESS|FAILURE|NOCONNECT|TERMINATED",
type=str,
help='Kind of Job Messages to send - separated by pipe | '
'(Default: SUCCESS|FAILURE|NOCONNECT|TERMINATED)')
parser.add_argument(
'-jobrtnc',
'--job_restart_notconnect',
required=False,
type=int,
default=0,
help=
'Restart job if device is not connected (in minutes). Default: 0 (Off)'
)
parser.add_argument(
'-jtc',
'--job_thread_count',
type=int,
default=1,
help='Amount of threads to work off the device jobs. Default: 1')
# Runtypes
parser.add_argument('-os',
'--only_scan',
action='store_true',
default=True,
help='Use this instance only for scanning')
parser.add_argument(
'-otc',
'--ocr_thread_count',
type=int,
default=2,
help=
'Amount of threads/processes to be used for screenshot-analysis. Default: 2'
)
parser.add_argument(
'-or',
'--only_routes',
action='store_true',
default=False,
help='Only calculate routes, then exit the program. No scanning.')
parser.add_argument('-cm',
'--config_mode',
action='store_true',
default=False,
help='Run in ConfigMode')
parser.add_argument("-sn",
"--status-name",
default="mad",
help=("Enable status page database update using"
" STATUS_NAME as main worker name."))
parser.add_argument('-nec',
'--no_event_checker',
action="store_true",
help='Disable event checker task')
parser.add_argument('-ut',
'--unit_tests',
action='store_true',
default=False,
help='Run unit tests then quit',
dest='unit_tests')
# Path Settings
parser.add_argument('-tmp',
'--temp_path',
default='temp',
help='Temp Folder for OCR Scanning. Default: temp')
parser.add_argument(
'-upload',
'--upload_path',
default=os.path.join(mapadroid.MAD_ROOT, 'upload'),
help=
'Path for uploaded Files via madmin and for device installation. Default: '
'/absolute/path/to/upload')
parser.add_argument(
'--file_path',
help=
'Defines directory to save worker stats- and position files and calculated routes',
default='files')
# Should be deprecated but may be required for *REALLY* old systems
parser.add_argument('-mf',
'--mappings',
default=os.getenv(
'MAD_CONFIG',
os.path.join(mapadroid.MAD_ROOT,
'configs/mappings.json')),
help='Set mappings file')
# other settings
parser.add_argument(
'-w',
'--weather',
action='store_true',
default=False,
help='Read weather and post to db - if supported! (Default: False)')
parser.add_argument(
'-hlat',
'--home_lat',
default='0.0',
type=float,
help=
('Set Lat from the center of your scan location.'
'Especially for using MADBOT (User submitted Raidscreens). Default: 0.0'
))
parser.add_argument(
'-hlng',
'--home_lng',
default='0.0',
type=float,
help=
('Set Lng from the center of your scan location.'
'Especially for using MADBOT (User submitted Raidscreens). Default: 0.0'
))
parser.add_argument('-L',
'--language',
default='en',
help=('Set Language for MadMin / Quests. Default: en'))
# MADmin
parser.add_argument('-dm',
'--disable_madmin',
action='store_true',
default=False,
help='Disable Madmin on the instance')
parser.add_argument('-mmbp',
'--madmin_base_path',
default='/',
help='Base path for madmin')
parser.add_argument('-mmip',
'--madmin_ip',
default='0.0.0.0',
help='MADmin listening interface (Default: 0.0.0.0)')
parser.add_argument('-mmprt',
'--madmin_port',
default='5000',
help='MADmin web port (Default: 5000)')
parser.add_argument('-mmuser',
'--madmin_user',
default='',
help='Username for MADmin Frontend.')
parser.add_argument('-mmpassword',
'--madmin_password',
default='',
help='Password for MADmin Frontend.')
parser.add_argument(
'-mmsc',
'--madmin_sort',
default='6',
help='MADmin sort column Raid/Gym (5= Modify / 6 = Create) (Default: 6)'
)
parser.add_argument('-mmt',
'--madmin_time',
default='24',
help='MADmin clock format (12/24) (Default: 24)')
parser.add_argument('-mmnrsp',
'--madmin_noresponsive',
action='store_false',
default=True,
help='MADmin deactivate responsive tables')
parser.add_argument(
'-qpub',
'--quests_public',
action='store_true',
default=False,
help=
'Enables MADmin /quests_pub, /get_quests, and pushassets endpoints for public quests'
'overview')
parser.add_argument(
'-ods',
'--outdated_spawnpoints',
type=int,
default=3,
help='Define when a spawnpoint is out of date (in days). Default: 3.')
parser.add_argument(
'--quest_stats_fences',
default="",
help=
"Comma separated list of geofences for stop/quest statistics (Empty: all)"
)
# Statistics
parser.add_argument('-stat',
'--statistic',
action='store_true',
default=False,
help='Activate system statistics (Default: False)')
parser.add_argument(
'-stco',
'--stat_gc',
action='store_true',
default=False,
help='Store collected objects (garbage collector) (Default: False)')
parser.add_argument(
'-stiv',
'--statistic_interval',
default=60,
type=int,
help='Store new local stats every N seconds (Default: 60)')
# Game Stats
parser.add_argument('-ggs',
'--game_stats',
action='store_true',
default=False,
help='Generate worker stats')
parser.add_argument(
'-ggrs',
'--game_stats_raw',
action='store_true',
default=False,
help='Generate worker raw stats (only with --game_stats)')
parser.add_argument(
'-gsst',
'--game_stats_save_time',
default=300,
type=int,
help='Number of seconds until worker information is saved to database')
parser.add_argument(
'-rds',
'--raw_delete_shiny',
default=0,
help=
'Delete shiny mon in raw stats older then x days (0 = Disable (Default))'
)
# ADB
parser.add_argument('-adb',
'--use_adb',
action='store_true',
default=False,
help='Use ADB for "device control" (Default: False)')
parser.add_argument('-adbservip',
'--adb_server_ip',
default='127.0.0.1',
help='IP address of ADB server (Default: 127.0.0.1)')
parser.add_argument('-adpservprt',
'--adb_server_port',
type=int,
default=5037,
help='Port of ADB server (Default: 5037)')
# Webhook
parser.add_argument('-wh',
'--webhook',
action='store_true',
default=False,
help='Activate webhook support')
parser.add_argument(
'-whurl',
'--webhook_url',
default='',
help='URL endpoint/s for webhooks (seperated by commas) with [<type>] '
'for restriction like [mon|weather|raid]http://example.org/foo/bar '
'- urls have to start with http*')
parser.add_argument('-whser',
'--webhook_submit_exraids',
action='store_true',
default=False,
help='Send Ex-raids to the webhook if detected')
parser.add_argument(
'-whea',
'--webhook_excluded_areas',
default="",
help=
'Comma-separated list of area names to exclude elements from within to be sent to a '
'webhook')
parser.add_argument(
'-pwhn',
'--pokemon_webhook_nonivs',
action='store_true',
default=False,
help='Send non-IVd pokemon even if they are on Global Mon List')
parser.add_argument(
'-qwhf',
'--quest_webhook_flavor',
choices=['default', 'poracle'],
default='default',
help='Webhook format for Quests: default or poracle compatible')
parser.add_argument(
'-whst',
'--webhook_start_time',
default=0,
help=
'Debug: Set initial timestamp to fetch changed elements from the DB to send via WH.'
)
parser.add_argument(
'-whmps',
'--webhook_max_payload_size',
default=0,
type=int,
help=
'Split up the payload into chunks and send multiple requests. Default: 0 (unlimited)'
)
# Dynamic Rarity
parser.add_argument(
'-rh',
'--rarity_hours',
type=int,
default=72,
help=
'Set the number of hours for the calculation of pokemon rarity (Default: 72)'
)
parser.add_argument(
'-ruf',
'--rarity_update_frequency',
type=int,
default=60,
help='Update frequency for dynamic rarity in minutes (Default: 60)')
# Logging
parser.add_argument(
'--no_file_logs',
action='store_true',
default=False,
help=
"Disable file logging (Default: file logging is enabled by default)")
parser.add_argument('--log_path',
default="logs/",
help="Defines directory to save log files to.")
parser.add_argument('--log_filename',
default='%Y%m%d_%H%M_<SN>.log',
help=("Defines the log filename to be saved."
" Allows date formatting, and replaces <SN>"
" with the instance's status name. Read the"
" python time module docs for details."
" Default: %%Y%%m%%d_%%H%%M_<SN>.log."))
parser.add_argument(
"--log_file_rotation",
default="50 MB",
help=("This parameter expects a human-readable value like"
" '18:00', 'sunday', 'weekly', 'monday at 12:00' or"
" a maximum file size like '100 MB' or '0.5 GB'."
" Set to '0' to disable completely. (Default: 50 MB)"))
parser.add_argument('--log_level',
help=("Forces a certain log level. By default"
" it's set to INFO while being modified"
" by the -v command to show DEBUG logs."
" Custom log levels like DEBUG[1-5] can"
" be used too."))
verbose = parser.add_mutually_exclusive_group()
verbose.add_argument('-v',
action='count',
default=0,
dest='verbose',
help=("Show debug messages. Has no effect, if"
"--log_level has been set."))
parser.add_argument(
"--log_file_level",
help="File logging level. See description for --log_level.")
parser.add_argument("--log_file_retention",
default="10",
help=("Amount of days to keep file logs. Set to 0 to"
" keep them forever (Default: 10)"))
parser.add_argument('--no_log_colors',
action="store_true",
default=False,
help=("Disable colored logs."))
parser.set_defaults(DEBUG=False)
# MADAPKs
parser.add_argument('-td',
'--token_dispenser',
default=default_tokenfile,
help='Token dispenser config (MAD)')
parser.add_argument('-tdu',
'--token_dispenser_user',
default='',
help='Token dispenser config (User)')
parser.add_argument(
'-gu',
'--gmail_user',
default='',
help='Google Mail User for interacting with the Google Play Store')
parser.add_argument(
'-gp',
'--gmail_passwd',
default='',
help=
'Google Mail Password for interacting with the Google Play Store. Must be an app'
' password or 2fa will be triggered (this should be enabled on your account anyways'
)
# Auto-Configuration
parser.add_argument(
'-acna',
'--autoconfig_no_auth',
action='store_true',
default=False,
help='MAD PoGo auth is not required during autoconfiguration',
dest='autoconfig_no_auth')
# Redis cache
parser.add_argument(
'-ec',
'--enable_cache',
action='store_true',
default=False,
help=('Enable caching for incoming data to prevent re-inserting '
'data to the DB over and over again'))
parser.add_argument('-ch',
'--cache_host',
default='localhost',
help=('Redis host used by caching'))
parser.add_argument('-cp',
'--cache_port',
default=6379,
help=('Redis port used by caching'))
parser.add_argument(
'-cdb',
'--cache_database',
default=0,
help=
('Redis database. Use different numbers (0-15) if you are running multiple instances'
))
if "MODE" in os.environ and os.environ["MODE"] == "DEV":
args = parser.parse_known_args()[0]
else:
args = parser.parse_args()
# Allow status name and date formatting in log filename.
args.log_filename = strftime(args.log_filename)
args.log_filename = args.log_filename.replace('<sn>', '<SN>')
args.log_filename = args.log_filename.replace('<SN>', args.status_name)
return args
def main():
def display_system_info():
if args.enable_internet:
place = get_location()
addr = get_ip()
else:
place = '?'
addr = None
DisplayMsg.show(
Message.SYSTEM_INFO(
info={
'version': version,
'location': place,
'ip': addr,
'engine_name': engine_name,
'user_name': user_name
}))
def compute_legal_fens(g):
"""
Computes a list of legal FENs for the given game.
:param g: The game
:return: A list of legal FENs
"""
fens = []
for move in g.legal_moves:
g.push(move)
fens.append(g.board_fen())
g.pop()
return fens
def probe_tablebase(game):
if not gaviota:
return None
score = gaviota.probe_dtm(game)
if score is not None:
Observable.fire(Event.NEW_SCORE(score='tb', mate=score))
return score
def think(game, tc):
"""
Starts a new search on the current game.
If a move is found in the opening book, fire an event in a few seconds.
:return:
"""
start_clock()
book_move = searchmoves.book(bookreader, game)
if book_move:
Observable.fire(Event.NEW_SCORE(score='book', mate=None))
Observable.fire(Event.BEST_MOVE(result=book_move, inbook=True))
else:
probe_tablebase(game)
while not engine.is_waiting():
time.sleep(0.1)
logging.warning('engine is still not waiting')
engine.position(copy.deepcopy(game))
uci_dict = tc.uci()
uci_dict['searchmoves'] = searchmoves.all(game)
engine.go(uci_dict)
def analyse(game):
"""
Starts a new ponder search on the current game.
:return:
"""
probe_tablebase(game)
engine.position(copy.deepcopy(game))
engine.ponder()
def observe(game):
"""
Starts a new ponder search on the current game.
:return:
"""
start_clock()
analyse(game)
def stop_search():
"""
Stop current search.
:return:
"""
engine.stop()
def stop_clock():
if interaction_mode in (Mode.NORMAL, Mode.OBSERVE, Mode.REMOTE):
time_control.stop()
DisplayMsg.show(Message.CLOCK_STOP())
else:
logging.warning('wrong mode: {}'.format(interaction_mode))
def stop_search_and_clock():
stop_clock()
stop_search()
def start_clock():
if interaction_mode in (Mode.NORMAL, Mode.OBSERVE, Mode.REMOTE):
time_control.start(game.turn)
DisplayMsg.show(
Message.CLOCK_START(turn=game.turn, time_control=time_control))
else:
logging.warning('wrong mode: {}'.format(interaction_mode))
def check_game_state(game, play_mode):
"""
Check if the game has ended or not ; it also sends Message to Displays if the game has ended.
:param game:
:param play_mode:
:return: True is the game continues, False if it has ended
"""
result = None
if game.is_stalemate():
result = GameResult.STALEMATE
if game.is_insufficient_material():
result = GameResult.INSUFFICIENT_MATERIAL
if game.is_seventyfive_moves():
result = GameResult.SEVENTYFIVE_MOVES
if game.is_fivefold_repetition():
result = GameResult.FIVEFOLD_REPETITION
if game.is_checkmate():
result = GameResult.MATE
if result is None:
return True
else:
DisplayMsg.show(
Message.GAME_ENDS(result=result,
play_mode=play_mode,
game=game.copy()))
return False
def user_move(move):
logging.debug('user move [%s]', move)
if move not in game.legal_moves:
logging.warning('Illegal move [%s]', move)
else:
handle_move(move=move)
def process_fen(fen):
nonlocal last_computer_fen
nonlocal last_legal_fens
nonlocal searchmoves
nonlocal legal_fens
# Check for same position
if (fen == game.board_fen()
and not last_computer_fen) or fen == last_computer_fen:
logging.debug('Already in this fen: ' + fen)
# Check if we have to undo a previous move (sliding)
elif fen in last_legal_fens:
if interaction_mode == Mode.NORMAL:
if (play_mode == PlayMode.USER_WHITE and game.turn == chess.BLACK) or \
(play_mode == PlayMode.USER_BLACK and game.turn == chess.WHITE):
stop_search()
game.pop()
logging.debug(
'User move in computer turn, reverting to: ' +
game.board_fen())
elif last_computer_fen:
last_computer_fen = None
game.pop()
game.pop()
logging.debug(
'User move while computer move is displayed, reverting to: '
+ game.board_fen())
else:
logging.error("last_legal_fens not cleared: " +
game.board_fen())
elif interaction_mode == Mode.REMOTE:
if (play_mode == PlayMode.USER_WHITE and game.turn == chess.BLACK) or \
(play_mode == PlayMode.USER_BLACK and game.turn == chess.WHITE):
game.pop()
logging.debug('User move in remote turn, reverting to: ' +
game.board_fen())
elif last_computer_fen:
last_computer_fen = None
game.pop()
game.pop()
logging.debug(
'User move while remote move is displayed, reverting to: '
+ game.board_fen())
else:
logging.error('last_legal_fens not cleared: ' +
game.board_fen())
else:
game.pop()
logging.debug('Wrong color move -> sliding, reverting to: ' +
game.board_fen())
legal_moves = list(game.legal_moves)
user_move(legal_moves[last_legal_fens.index(fen)])
if interaction_mode == Mode.NORMAL or interaction_mode == Mode.REMOTE:
legal_fens = []
else:
legal_fens = compute_legal_fens(game)
# legal move
elif fen in legal_fens:
time_control.add_inc(game.turn)
legal_moves = list(game.legal_moves)
user_move(legal_moves[legal_fens.index(fen)])
last_legal_fens = legal_fens
if interaction_mode == Mode.NORMAL or interaction_mode == Mode.REMOTE:
legal_fens = []
else:
legal_fens = compute_legal_fens(game)
# Player had done the computer or remote move on the board
elif last_computer_fen and fen == game.board_fen():
last_computer_fen = None
if check_game_state(
game, play_mode) and interaction_mode in (Mode.NORMAL,
Mode.REMOTE):
# finally reset all alternative moves see: handle_move()
nonlocal searchmoves
searchmoves.reset()
time_control.add_inc(not game.turn)
if time_control.mode != TimeMode.FIXED:
start_clock()
DisplayMsg.show(Message.COMPUTER_MOVE_DONE_ON_BOARD())
legal_fens = compute_legal_fens(game)
else:
legal_fens = []
last_legal_fens = []
# Check if this is a previous legal position and allow user to restart from this position
else:
game_history = copy.deepcopy(game)
if last_computer_fen:
game_history.pop()
while game_history.move_stack:
game_history.pop()
if game_history.board_fen() == fen:
logging.debug("Current game FEN : {}".format(
game.fen()))
logging.debug("Undoing game until FEN: {}".format(fen))
stop_search_and_clock()
while len(game_history.move_stack) < len(game.move_stack):
game.pop()
last_computer_fen = None
last_legal_fens = []
if (interaction_mode == Mode.REMOTE or interaction_mode == Mode.NORMAL) and \
((play_mode == PlayMode.USER_WHITE and game_history.turn == chess.BLACK)
or (play_mode == PlayMode.USER_BLACK and game_history.turn == chess.WHITE)):
legal_fens = []
if interaction_mode == Mode.NORMAL:
searchmoves.reset()
if check_game_state(game, play_mode):
think(game, time_control)
else:
legal_fens = compute_legal_fens(game)
if interaction_mode == Mode.ANALYSIS or interaction_mode == Mode.KIBITZ:
analyse(game)
elif interaction_mode == Mode.OBSERVE or interaction_mode == Mode.REMOTE:
observe(game)
start_clock()
DisplayMsg.show(Message.USER_TAKE_BACK())
break
def set_wait_state():
if interaction_mode == Mode.NORMAL:
nonlocal play_mode
play_mode = PlayMode.USER_WHITE if game.turn == chess.WHITE else PlayMode.USER_BLACK
def handle_move(move, ponder=None, inbook=False):
nonlocal game
nonlocal last_computer_fen
nonlocal searchmoves
fen = game.fen()
turn = game.turn
# clock must be stoped BEFORE the "book_move" event cause SetNRun resets the clock display
if interaction_mode == Mode.NORMAL:
stop_clock()
elif interaction_mode == Mode.REMOTE or interaction_mode == Mode.OBSERVE:
stop_search_and_clock()
elif interaction_mode == Mode.ANALYSIS or interaction_mode == Mode.KIBITZ:
stop_search()
# engine or remote move
if (interaction_mode == Mode.NORMAL or interaction_mode == Mode.REMOTE) and \
((play_mode == PlayMode.USER_WHITE and game.turn == chess.BLACK) or
(play_mode == PlayMode.USER_BLACK and game.turn == chess.WHITE)):
last_computer_fen = game.board_fen()
game.push(move)
if inbook:
DisplayMsg.show(Message.BOOK_MOVE())
searchmoves.add(move)
text = Message.COMPUTER_MOVE(move=move,
ponder=ponder,
fen=fen,
turn=turn,
game=game.copy(),
time_control=time_control,
wait=inbook)
DisplayMsg.show(text)
else:
last_computer_fen = None
game.push(move)
if inbook:
DisplayMsg.show(Message.BOOK_MOVE())
searchmoves.reset()
if interaction_mode == Mode.NORMAL:
if check_game_state(game, play_mode):
think(game, time_control)
text = Message.USER_MOVE(move=move,
fen=fen,
turn=turn,
game=game.copy())
elif interaction_mode == Mode.REMOTE:
if check_game_state(game, play_mode):
observe(game)
text = Message.USER_MOVE(move=move,
fen=fen,
turn=turn,
game=game.copy())
elif interaction_mode == Mode.OBSERVE:
if check_game_state(game, play_mode):
observe(game)
text = Message.REVIEW_MOVE(move=move,
fen=fen,
turn=turn,
game=game.copy(),
mode=interaction_mode)
else: # interaction_mode in (Mode.ANALYSIS, Mode.KIBITZ):
if check_game_state(game, play_mode):
analyse(game)
text = Message.REVIEW_MOVE(move=move,
fen=fen,
turn=turn,
game=game.copy(),
mode=interaction_mode)
DisplayMsg.show(text)
def transfer_time(time_list):
def num(ts):
try:
return int(ts)
except ValueError:
return 1
if len(time_list) == 1:
secs = num(time_list[0])
time_control = TimeControl(TimeMode.FIXED, seconds_per_move=secs)
text = dgttranslate.text('B00_tc_fixed', '{:2d}'.format(secs))
elif len(time_list) == 2:
mins = num(time_list[0])
finc = num(time_list[1])
if finc == 0:
time_control = TimeControl(TimeMode.BLITZ,
minutes_per_game=mins)
text = dgttranslate.text('B00_tc_blitz', '{:2d}'.format(mins))
else:
time_control = TimeControl(TimeMode.FISCHER,
minutes_per_game=mins,
fischer_increment=finc)
text = dgttranslate.text('B00_tc_fisch',
'{:2d} {:2d}'.format(mins, finc))
else:
time_control = TimeControl(TimeMode.BLITZ, minutes_per_game=5)
text = dgttranslate.text('B00_tc_blitz', ' 5')
return time_control, text
def get_engine_level_dict(engine_level):
from engine import get_installed_engines
installed_engines = get_installed_engines(engine.get_shell(),
engine.get_file())
for index in range(0, len(installed_engines)):
eng = installed_engines[index]
if eng['file'] == engine.get_file():
level_list = sorted(eng['level_dict'])
try:
level_index = level_list.index(engine_level)
return eng['level_dict'][level_list[level_index]]
except ValueError:
break
return {}
# Enable garbage collection - needed for engine swapping as objects orphaned
gc.enable()
# Command line argument parsing
parser = configargparse.ArgParser(default_config_files=[
os.path.join(os.path.dirname(__file__), "picochess.ini")
])
parser.add_argument("-e",
"--engine",
type=str,
help="UCI engine executable path",
default=None)
parser.add_argument("-el",
"--engine-level",
type=str,
help="UCI engine level",
default=None)
parser.add_argument(
"-d",
"--dgt-port",
type=str,
help="enable dgt board on the given serial port such as /dev/ttyUSB0")
parser.add_argument(
"-b",
"--book",
type=str,
help="Opening book - full name of book in 'books' folder",
default='h-varied.bin')
parser.add_argument(
"-t",
"--time",
type=str,
default='5 0',
help=
"Time settings <FixSec> or <StMin IncSec> like '10'(fixed) or '5 0'(blitz) '3 2'(fischer)"
)
parser.add_argument("-g",
"--enable-gaviota",
action='store_true',
help="enable gavoita tablebase probing")
parser.add_argument("-leds",
"--enable-revelation-leds",
action='store_true',
help="enable Revelation leds")
parser.add_argument(
"-l",
"--log-level",
choices=['notset', 'debug', 'info', 'warning', 'error', 'critical'],
default='warning',
help="logging level")
parser.add_argument("-lf",
"--log-file",
type=str,
help="log to the given file")
parser.add_argument("-rs",
"--remote-server",
type=str,
help="remote server running the engine")
parser.add_argument("-ru",
"--remote-user",
type=str,
help="remote user on server running the engine")
parser.add_argument("-rp",
"--remote-pass",
type=str,
help="password for the remote user")
parser.add_argument("-rk",
"--remote-key",
type=str,
help="key file used to connect to the remote server")
parser.add_argument("-pf",
"--pgn-file",
type=str,
help="pgn file used to store the games",
default='games.pgn')
parser.add_argument("-pu",
"--pgn-user",
type=str,
help="user name for the pgn file",
default=None)
parser.add_argument("-ar",
"--auto-reboot",
action='store_true',
help="reboot system after update")
parser.add_argument("-web",
"--web-server",
dest="web_server_port",
nargs="?",
const=80,
type=int,
metavar="PORT",
help="launch web server")
parser.add_argument("-m",
"--email",
type=str,
help="email used to send pgn files",
default=None)
parser.add_argument("-ms",
"--smtp-server",
type=str,
help="Adress of email server",
default=None)
parser.add_argument("-mu",
"--smtp-user",
type=str,
help="Username for email server",
default=None)
parser.add_argument("-mp",
"--smtp-pass",
type=str,
help="Password for email server",
default=None)
parser.add_argument("-me",
"--smtp-encryption",
action='store_true',
help="use ssl encryption connection to smtp-Server")
parser.add_argument("-mf",
"--smtp-from",
type=str,
help="From email",
default='*****@*****.**')
parser.add_argument("-mk",
"--mailgun-key",
type=str,
help="key used to send emails via Mailgun Webservice",
default=None)
parser.add_argument("-bc",
"--beep-config",
choices=['none', 'some', 'all'],
help="sets standard beep config",
default='some')
parser.add_argument(
"-beep",
"--beep-level",
type=int,
default=0x03,
help="sets (some-)beep level from 0(=no beeps) to 15(=all beeps)")
parser.add_argument("-uvoice",
"--user-voice",
type=str,
help="voice for user",
default=None)
parser.add_argument("-cvoice",
"--computer-voice",
type=str,
help="voice for computer",
default=None)
parser.add_argument("-inet",
"--enable-internet",
action='store_true',
help="enable internet lookups")
parser.add_argument("-nook",
"--disable-ok-message",
action='store_true',
help="disable ok confirmation messages")
parser.add_argument("-v",
"--version",
action='version',
version='%(prog)s version {}'.format(version),
help="show current version",
default=None)
parser.add_argument("-pi",
"--dgtpi",
action='store_true',
help="use the dgtpi hardware")
parser.add_argument("-lang",
"--language",
choices=['en', 'de', 'nl', 'fr', 'es'],
default='en',
help="picochess language")
parser.add_argument("-c",
"--console",
action='store_true',
help="use console interface")
args = parser.parse_args()
if args.engine is None:
el = read_engine_ini()
args.engine = el[0][
'file'] # read the first engine filename and use it as standard
# Enable logging
if args.log_file:
handler = RotatingFileHandler('logs' + os.sep + args.log_file,
maxBytes=1024 * 1024,
backupCount=9)
logging.basicConfig(
level=getattr(logging, args.log_level.upper()),
format=
'%(asctime)s.%(msecs)03d %(levelname)5s %(module)10s - %(funcName)s: %(message)s',
datefmt="%Y-%m-%d %H:%M:%S",
handlers=[handler])
logging.getLogger('chess.uci').setLevel(
logging.INFO) # don't want to get so many python-chess uci messages
logging.debug('#' * 20 + ' PicoChess v' + version + ' ' + '#' * 20)
# log the startup parameters but hide the password fields
p = copy.copy(vars(args))
p['mailgun_key'] = p['remote_key'] = p['remote_pass'] = p[
'smtp_pass'] = '******'
logging.debug('startup parameters: {}'.format(p))
# Update
if args.enable_internet:
update_picochess(args.auto_reboot)
gaviota = None
if args.enable_gaviota:
try:
gaviota = chess.gaviota.open_tablebases('tablebases/gaviota')
logging.debug('Tablebases gaviota loaded')
except OSError:
logging.error('Tablebases gaviota doesnt exist')
gaviota = None
# The class dgtDisplay talks to DgtHw/DgtPi or DgtVr
dgttranslate = DgtTranslate(args.beep_config, args.beep_level,
args.language)
DgtDisplay(args.disable_ok_message, dgttranslate).start()
# Launch web server
if args.web_server_port:
WebServer(args.web_server_port).start()
dgtserial = DgtSerial(args.dgt_port, args.enable_revelation_leds,
args.dgtpi)
if args.console:
# Enable keyboard input and terminal display
logging.debug('starting picochess in virtual mode')
KeyboardInput(dgttranslate, args.dgtpi).start()
TerminalDisplay().start()
DgtVr(dgtserial, dgttranslate).start()
else:
# Connect to DGT board
logging.debug('starting picochess in board mode')
if args.dgtpi:
DgtPi(dgtserial, dgttranslate).start()
DgtHw(dgtserial, dgttranslate).start()
# Save to PGN
PgnDisplay(args.pgn_file,
net=args.enable_internet,
email=args.email,
mailgun_key=args.mailgun_key,
smtp_server=args.smtp_server,
smtp_user=args.smtp_user,
smtp_pass=args.smtp_pass,
smtp_encryption=args.smtp_encryption,
smtp_from=args.smtp_from).start()
if args.pgn_user:
user_name = args.pgn_user
else:
if args.email:
user_name = args.email.split('@')[0]
else:
user_name = 'Player'
# Create ChessTalker for speech output
talker = None
if args.user_voice or args.computer_voice:
logging.debug("initializing ChessTalker [%s, %s]",
str(args.user_voice), str(args.computer_voice))
talker = chesstalker.chesstalker.ChessTalker(args.user_voice,
args.computer_voice)
talker.start()
else:
logging.debug('ChessTalker disabled')
# Gentlemen, start your engines...
engine = UciEngine(args.engine,
hostname=args.remote_server,
username=args.remote_user,
key_file=args.remote_key,
password=args.remote_pass)
try:
engine_name = engine.get().name
except AttributeError:
logging.error('no engines started')
sys.exit(-1)
# Startup - internal
game = chess.Board() # Create the current game
legal_fens = compute_legal_fens(game) # Compute the legal FENs
all_books = get_opening_books()
try:
book_index = [book['file'] for book in all_books].index(args.book)
except ValueError:
logging.warning("selected book not present, defaulting to %s",
all_books[7]['file'])
book_index = 7
bookreader = chess.polyglot.open_reader(all_books[book_index]['file'])
searchmoves = AlternativeMover()
interaction_mode = Mode.NORMAL
play_mode = PlayMode.USER_WHITE
last_computer_fen = None
last_legal_fens = []
game_declared = False # User declared resignation or draw
engine.startup(get_engine_level_dict(args.engine_level))
# Startup - external
time_control, time_text = transfer_time(args.time.split())
time_text.beep = False
if args.engine_level:
level_text = dgttranslate.text('B00_level', args.engine_level)
level_text.beep = False
else:
level_text = None
DisplayMsg.show(
Message.STARTUP_INFO(
info={
'interaction_mode': interaction_mode,
'play_mode': play_mode,
'books': all_books,
'book_index': book_index,
'level_text': level_text,
'time_control': time_control,
'time_text': time_text
}))
DisplayMsg.show(
Message.ENGINE_STARTUP(shell=engine.get_shell(),
file=engine.get_file(),
has_levels=engine.has_levels(),
has_960=engine.has_chess960()))
system_info_thread = threading.Timer(0, display_system_info)
system_info_thread.start()
# Event loop
logging.info('evt_queue ready')
while True:
try:
event = evt_queue.get()
except queue.Empty:
pass
else:
logging.debug('received event from evt_queue: %s', event)
for case in switch(event):
if case(EventApi.FEN):
process_fen(event.fen)
break
if case(EventApi.KEYBOARD_MOVE):
move = event.move
logging.debug('keyboard move [%s]', move)
if move not in game.legal_moves:
logging.warning('illegal move [%s]', move)
else:
g = copy.deepcopy(game)
g.push(move)
fen = g.fen().split(' ')[0]
if event.flip_board:
fen = fen[::-1]
DisplayMsg.show(Message.KEYBOARD_MOVE(fen=fen))
break
if case(EventApi.LEVEL):
if event.options:
engine.startup(event.options, False)
DisplayMsg.show(Message.LEVEL(level_text=event.level_text))
break
if case(EventApi.NEW_ENGINE):
config = ConfigObj('picochess.ini')
config['engine'] = event.eng['file']
config.write()
old_file = engine.get_file()
engine_shutdown = True
# Stop the old engine cleanly
engine.stop()
# Closeout the engine process and threads
# The all return non-zero error codes, 0=success
if engine.quit(): # Ask nicely
if engine.terminate(): # If you won't go nicely....
if engine.kill(): # Right that does it!
logging.error('engine shutdown failure')
DisplayMsg.show(Message.ENGINE_FAIL())
engine_shutdown = False
if engine_shutdown:
# Load the new one and send args.
# Local engines only
engine_fallback = False
engine = UciEngine(event.eng['file'])
try:
engine_name = engine.get().name
except AttributeError:
# New engine failed to start, restart old engine
logging.error(
"new engine failed to start, reverting to %s",
old_file)
engine_fallback = True
event.options = {
} # Reset options. This will load the last(=strongest?) level
engine = UciEngine(old_file)
try:
engine_name = engine.get().name
except AttributeError:
# Help - old engine failed to restart. There is no engine
logging.error('no engines started')
sys.exit(-1)
# Schedule cleanup of old objects
gc.collect()
engine.startup(event.options)
# All done - rock'n'roll
if not engine_fallback:
DisplayMsg.show(
Message.ENGINE_READY(
eng=event.eng,
engine_name=engine_name,
eng_text=event.eng_text,
has_levels=engine.has_levels(),
has_960=engine.has_chess960(),
ok_text=event.ok_text))
else:
DisplayMsg.show(Message.ENGINE_FAIL())
set_wait_state()
# Go back to analysing or observing
if interaction_mode == Mode.ANALYSIS or interaction_mode == Mode.KIBITZ:
analyse(game)
if interaction_mode == Mode.OBSERVE or interaction_mode == Mode.REMOTE:
observe(game)
break
if case(EventApi.SETUP_POSITION):
logging.debug("setting up custom fen: {0}".format(
event.fen))
if engine.has_chess960():
engine.option('UCI_Chess960', event.uci960)
engine.send()
else: # start normal new game if engine can't handle the user wish
event.uci960 = False
logging.warning('engine doesnt support 960 mode')
if game.move_stack:
if game.is_game_over() or game_declared:
DisplayMsg.show(
Message.GAME_ENDS(result=GameResult.ABORT,
play_mode=play_mode,
game=game.copy()))
game = chess.Board(event.fen, event.uci960)
legal_fens = compute_legal_fens(game)
stop_search_and_clock()
time_control.reset()
interaction_mode = Mode.NORMAL
last_computer_fen = None
last_legal_fens = []
searchmoves.reset()
DisplayMsg.show(
Message.START_NEW_GAME(time_control=time_control,
game=game.copy()))
game_declared = False
set_wait_state()
break
if case(EventApi.PAUSE_RESUME):
if engine.is_thinking():
stop_clock()
engine.stop(show_best=True)
else:
if time_control.is_ticking():
stop_clock()
else:
start_clock()
break
if case(EventApi.ALTERNATIVE_MOVE):
if last_computer_fen:
last_computer_fen = None
game.pop()
DisplayMsg.show(Message.ALTERNATIVE_MOVE())
think(game, time_control)
break
if case(EventApi.SWITCH_SIDES):
if interaction_mode == Mode.NORMAL:
user_to_move = False
last_legal_fens = []
if engine.is_thinking():
stop_clock()
engine.stop(show_best=False)
user_to_move = True
if event.engine_finished:
last_computer_fen = None
move = game.pop()
user_to_move = True
else:
move = chess.Move.null()
if user_to_move:
last_legal_fens = []
play_mode = PlayMode.USER_WHITE if game.turn == chess.WHITE else PlayMode.USER_BLACK
else:
play_mode = PlayMode.USER_WHITE if game.turn == chess.BLACK else PlayMode.USER_BLACK
if not user_to_move and check_game_state(
game, play_mode):
time_control.reset_start_time()
think(game, time_control)
legal_fens = []
else:
start_clock()
legal_fens = compute_legal_fens(game)
text = dgttranslate.text(play_mode.value)
DisplayMsg.show(
Message.PLAY_MODE(play_mode=play_mode,
play_mode_text=text))
if event.engine_finished:
DisplayMsg.show(Message.SWITCH_SIDES(move=move))
break
if case(EventApi.NEW_GAME):
stop_search_and_clock()
if game.move_stack:
logging.debug(
'starting a new game with code: {}'.format(
event.pos960))
if not (game.is_game_over() or game_declared):
DisplayMsg.show(
Message.GAME_ENDS(result=GameResult.ABORT,
play_mode=play_mode,
game=game.copy()))
game = chess.Board()
if event.pos960 != 518: # 518 is normal game setup
game.set_chess960_pos(event.pos960)
legal_fens = compute_legal_fens(game)
last_legal_fens = []
# interaction_mode = Mode.NORMAL @todo
last_computer_fen = None
time_control.reset()
searchmoves.reset()
DisplayMsg.show(
Message.START_NEW_GAME(time_control=time_control,
game=game.copy()))
game_declared = False
set_wait_state()
break
if case(EventApi.DRAWRESIGN):
if not game_declared: # in case user leaves kings in place while moving other pieces
stop_search_and_clock()
DisplayMsg.show(
Message.GAME_ENDS(result=event.result,
play_mode=play_mode,
game=game.copy()))
game_declared = True
break
if case(EventApi.REMOTE_MOVE):
if interaction_mode == Mode.REMOTE:
handle_move(move=chess.Move.from_uci(event.move))
legal_fens = compute_legal_fens(game)
break
if case(EventApi.BEST_MOVE):
handle_move(move=event.result.bestmove,
ponder=event.result.ponder,
inbook=event.inbook)
break
if case(EventApi.NEW_PV):
# illegal moves can occur if a pv from the engine arrives at the same time as a user move.
if game.is_legal(event.pv[0]):
DisplayMsg.show(
Message.NEW_PV(pv=event.pv,
mode=interaction_mode,
fen=game.fen(),
turn=game.turn))
else:
logging.info(
'illegal move can not be displayed. move:%s fen=%s',
event.pv[0], game.fen())
break
if case(EventApi.NEW_SCORE):
if event.score == 'book':
score = 'book'
elif event.score == 'tb':
score = 'tb {0}'.format(event.mate)
else:
try:
score = int(event.score)
if game.turn == chess.BLACK:
score *= -1
except ValueError:
score = event.score
logging.debug('could not convert score ' + score)
except TypeError:
score = 'm {0}'.format(event.mate)
DisplayMsg.show(
Message.NEW_SCORE(score=score,
mate=event.mate,
mode=interaction_mode))
break
if case(EventApi.SET_INTERACTION_MODE):
if interaction_mode in (Mode.NORMAL, Mode.OBSERVE,
Mode.REMOTE):
stop_clock(
) # only stop, if the clock is really running
interaction_mode = event.mode
if engine.is_thinking():
stop_search() # dont need to stop, if pondering
if engine.is_pondering(
) and interaction_mode == Mode.NORMAL:
stop_search(
) # if change from ponder modes to normal, also stops the pondering
set_wait_state()
DisplayMsg.show(
Message.INTERACTION_MODE(mode=event.mode,
mode_text=event.mode_text,
ok_text=event.ok_text))
break
if case(EventApi.SET_OPENING_BOOK):
config = ConfigObj('picochess.ini')
config['book'] = event.book['file']
config.write()
logging.debug("changing opening book [%s]",
event.book['file'])
bookreader = chess.polyglot.open_reader(event.book['file'])
DisplayMsg.show(
Message.OPENING_BOOK(book_text=event.book_text,
ok_text=event.ok_text))
break
if case(EventApi.SET_TIME_CONTROL):
time_control = event.time_control
config = ConfigObj('picochess.ini')
if time_control.mode == TimeMode.BLITZ:
config['time'] = '{:d} 0'.format(
time_control.minutes_per_game)
elif time_control.mode == TimeMode.FISCHER:
config['time'] = '{:d} {:d}'.format(
time_control.minutes_per_game,
time_control.fischer_increment)
elif time_control.mode == TimeMode.FIXED:
config['time'] = '{:d}'.format(
time_control.seconds_per_move)
config.write()
DisplayMsg.show(
Message.TIME_CONTROL(time_text=event.time_text,
ok_text=event.ok_text))
break
if case(EventApi.OUT_OF_TIME):
stop_search_and_clock()
DisplayMsg.show(
Message.GAME_ENDS(result=GameResult.OUT_OF_TIME,
play_mode=play_mode,
game=game.copy()))
break
if case(EventApi.SHUTDOWN):
if talker:
talker.say_event(event)
DisplayMsg.show(
Message.GAME_ENDS(result=GameResult.ABORT,
play_mode=play_mode,
game=game.copy()))
shutdown(args.dgtpi)
break
if case(EventApi.REBOOT):
if talker:
talker.say_event(event)
DisplayMsg.show(
Message.GAME_ENDS(result=GameResult.ABORT,
play_mode=play_mode,
game=game.copy()))
reboot()
break
if case(EventApi.DGT_BUTTON):
DisplayMsg.show(Message.DGT_BUTTON(button=event.button))
break
if case(EventApi.DGT_FEN):
DisplayMsg.show(Message.DGT_FEN(fen=event.fen))
break
if case(): # Default
logging.warning("event not handled : [%s]", event)
evt_queue.task_done()
def parse_settings(root_path):
config['ROOT_PATH'] = root_path
parser = configargparse.ArgParser(default_config_files=[get_path('config/config.ini')])
parser.add_argument('-d', '--debug', help='Debug Mode', action='store_true', default=False)
parser.add_argument('-H', '--host', help='Set web server listening host', default='127.0.0.1')
parser.add_argument('-P', '--port', type=int, help='Set web server listening port', default=4000)
parser.add_argument('-m', '--manager_count', type=int, default=1,
help='Number of Manager processes to start.')
parser.add_argument('-M', '--manager_name', type=parse_unicode, action='append', default=[],
help='Names of Manager processes to start.')
parser.add_argument('-k', '--key', type=parse_unicode, action='append', default=[None],
help='Specify a Google API Key to use.')
parser.add_argument('-f', '--filters', type=parse_unicode, action='append', default=['filters.json'],
help='Filters configuration file. default: filters.json', )
parser.add_argument('-a', '--alarms', type=parse_unicode, action='append', default=['alarms.json'],
help='Alarms configuration file. default: alarms.json', )
parser.add_argument('-gf', '--geofences', type=parse_unicode, action='append', default=[None],
help='Alarms configuration file. default: None')
parser.add_argument('-l', '--location', type=parse_unicode, action='append', default=[None],
help='Location, can be an address or coordinates')
parser.add_argument('-L', '--locale', type=parse_unicode, action='append', default=['en'],
choices=['de', 'en', 'es', 'fr', 'it', 'zh_hk'],
help='Locale for Pokemon and Move names: default en, check locale folder for more options')
parser.add_argument('-u', '--units', type=parse_unicode, default=['imperial'], action='append',
choices=['metric', 'imperial'],
help='Specify either metric or imperial units to use for distance measurements. ')
parser.add_argument('-tl', '--timelimit', type=int, default=[0], action='append',
help='Minimum number of seconds remaining on a pokemon to send a notify')
parser.add_argument('-tz', '--timezone', type=str, action='append', default=[None],
help='Timezone used for notifications. Ex: "America/Los_Angeles"')
args = parser.parse_args()
if args.debug:
log.setLevel(logging.DEBUG)
logging.getLogger().setLevel(logging.DEBUG)
logging.getLogger('PokeAlarm').setLevel(logging.DEBUG)
logging.getLogger('Manager').setLevel(logging.DEBUG)
log.debug("Debug mode enabled!")
config['HOST'] = args.host
config['PORT'] = args.port
config['QUIET'] = False
config['DEBUG'] = args.debug
# Check to make sure that the same number of arguements are included
for list_ in [args.key, args.filters, args.alarms, args.geofences, args.location,
args.locale, args.units, args.timelimit, args.timezone]:
if len(list_) > 1: # Remove defaults from the list
list_.pop(0)
size = len(list_)
if size != 1 and size != args.manager_count:
log.critical("\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n" +
"Incorrect number of arguments applied: must be either 1 for all processes or else the " +
"number of arguments must match the number of processes. Process will exit.")
log.critical(list_)
sys.exit(1)
# Attempt to parse the timezones
for i in range(len(args.timezone)):
if str(args.timezone[i]).lower() == "none":
args.timezone[i] = None
continue
try:
log.info(args.timezone[i])
args.timezone[i] = pytz.timezone(args.timezone[i])
except pytz.exceptions.UnknownTimeZoneError:
log.error("Invalid timezone. For a list of valid timezones, " +
"see https://en.wikipedia.org/wiki/List_of_tz_database_time_zones")
sys.exit(1)
# Construct the managers
for m_ct in range(args.manager_count):
m = Manager(
name=args.manager_name[m_ct] if m_ct < len(args.manager_name) else "Manager_{}".format(m_ct),
google_key=args.key[m_ct] if len(args.key) > 1 else args.key[0],
filters=args.filters[m_ct] if len(args.filters) > 1 else args.filters[0],
geofences=args.geofences[m_ct] if len(args.geofences) > 1 else args.geofences[0],
alarms=args.alarms[m_ct] if len(args.alarms) > 1 else args.alarms[0],
location=args.location[m_ct] if len(args.location) > 1 else args.location[0],
locale=args.locale[m_ct] if len(args.locale) > 1 else args.locale[0],
units=args.units[m_ct] if len(args.units) > 1 else args.units[0],
time_limit=args.timelimit[m_ct] if len(args.timelimit) > 1 else args.timelimit[0],
timezone=args.timezone[m_ct] if len(args.timezone) > 1 else args.timezone[0]
)
if m.get_name() not in managers:
# Add the manager to the map
managers[m.get_name()] = m
else:
log.critical("\n\n\n !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n" +
"Names of Manager processes must be unique (regardless of capitalization)! Process will exit.")
sys.exit(1)
import urllib2
import json
# import memcache
import hashlib
# Unbuffered IO
# sys.stdin = os.fdopen(sys.stdin.fileno(), 'w', 0) # MS
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 1) # MS
sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', 0) # MS
config_file = script_dir = os.path.dirname(
os.path.realpath('__file__')) + '/' + 'minup_posix.config'
parser = \
configargparse.ArgParser(description='interaction: A program to provide real time interaction for minION runs.'
, default_config_files=[config_file])
parser.add(
'-dbh',
'--mysql-host',
type=str,
dest='dbhost',
required=False,
default='localhost',
help="The location of the MySQL database. default is 'localhost'.",
)
parser.add(
'-dbu',
'--mysql-username',
type=str,
dest='dbusername',
required=True,
def get_args():
# Pre-check to see if the -cf or --config flag is used on the command line.
# If not, we'll use the env var or default value. This prevents layering of
# config files as well as a missing config.ini.
defaultconfigfiles = []
if '-cf' not in sys.argv and '--config' not in sys.argv:
defaultconfigfiles = [os.getenv('POGOMAP_CONFIG', os.path.join(
os.path.dirname(__file__), '../config/config.ini'))]
parser = configargparse.ArgParser(
default_config_files=defaultconfigfiles,
auto_env_var_prefix='POGOMAP_')
parser.add_argument('-cf', '--config',
is_config_file=True, help='Set configuration file')
parser.add_argument('-a', '--auth-service', type=str.lower,
action='append', default=[],
help=('Auth Services, either one for all accounts ' +
'or one per account: ptc or google. Defaults ' +
'all to ptc.'))
parser.add_argument('-u', '--username', action='append', default=[],
help='Usernames, one per account.')
parser.add_argument('-p', '--password', action='append', default=[],
help=('Passwords, either single one for all ' +
'accounts or one per account.'))
parser.add_argument('-w', '--workers', type=int,
help=('Number of search worker threads to start. ' +
'Defaults to the number of accounts specified.'))
parser.add_argument('-asi', '--account-search-interval', type=int,
default=0,
help=('Seconds for accounts to search before ' +
'switching to a new account. 0 to disable.'))
parser.add_argument('-ari', '--account-rest-interval', type=int,
default=7200,
help=('Seconds for accounts to rest when they fail ' +
'or are switched out.'))
parser.add_argument('-ac', '--accountcsv',
help=('Load accounts from CSV file containing ' +
'"auth_service,username,passwd" lines.'))
parser.add_argument('-bh', '--beehive',
help=('Use beehive configuration for multiple ' +
'accounts, one account per hex. Make sure ' +
'to keep -st under 5, and -w under the total ' +
'amount of accounts available.'),
action='store_true', default=False)
parser.add_argument('-wph', '--workers-per-hive',
help=('Only referenced when using --beehive. Sets ' +
'number of workers per hive. Default value ' +
'is 1.'),
type=int, default=1)
parser.add_argument('-l', '--location', type=parse_unicode,
help='Location, can be an address or coordinates.')
# Default based on the average elevation of cities around the world.
# Source: https://www.wikiwand.com/en/List_of_cities_by_elevation
parser.add_argument('-alt', '--altitude',
help='Default altitude in meters.',
type=int, default=507)
parser.add_argument('-altv', '--altitude-variance',
help='Variance for --altitude in meters',
type=int, default=1)
parser.add_argument('-uac', '--use-altitude-cache',
help=('Query the Elevation API for each step,' +
' rather than only once, and store results in' +
' the database.'),
action='store_true', default=False)
parser.add_argument('-nj', '--no-jitter',
help=("Don't apply random -9m to +9m jitter to " +
"location."),
action='store_true', default=False)
parser.add_argument('-st', '--step-limit', help='Steps.', type=int,
default=12)
parser.add_argument('-sd', '--scan-delay',
help='Time delay between requests in scan threads.',
type=float, default=10)
parser.add_argument('--spawn-delay',
help=('Number of seconds after spawn time to wait ' +
'before scanning to be sure the Pokemon ' +
'is there.'),
type=float, default=10)
parser.add_argument('-enc', '--encounter',
help='Start an encounter to gather IVs and moves.',
action='store_true', default=False)
parser.add_argument('-cs', '--captcha-solving',
help='Enables captcha solving.',
action='store_true', default=False)
parser.add_argument('-ck', '--captcha-key',
help='2Captcha API key.')
parser.add_argument('-cds', '--captcha-dsk',
help='Pokemon Go captcha data-sitekey.',
default="6LeeTScTAAAAADqvhqVMhPpr_vB9D364Ia-1dSgK")
parser.add_argument('-mcd', '--manual-captcha-domain',
help='Domain to where captcha tokens will be sent.',
default="http://127.0.0.1:5000")
parser.add_argument('-mcr', '--manual-captcha-refresh',
help='Time available before captcha page refreshes.',
type=int, default=30)
parser.add_argument('-mct', '--manual-captcha-timeout',
help='Maximum time captchas will wait for manual ' +
'captcha solving. On timeout, if enabled, 2Captcha ' +
'will be used to solve captcha. Default is 0.',
type=int, default=0)
parser.add_argument('-ed', '--encounter-delay',
help=('Time delay between encounter pokemon ' +
'in scan threads.'),
type=float, default=1)
encounter_list = parser.add_mutually_exclusive_group()
encounter_list.add_argument('-ewht', '--encounter-whitelist',
action='append', default=[],
help=('List of Pokemon to encounter for ' +
'more stats.'))
encounter_list.add_argument('-eblk', '--encounter-blacklist',
action='append', default=[],
help=('List of Pokemon to NOT encounter for ' +
'more stats.'))
encounter_list.add_argument('-ewhtf', '--encounter-whitelist-file',
default='', help='File containing a list of '
'Pokemon to encounter for'
' more stats.')
encounter_list.add_argument('-eblkf', '--encounter-blacklist-file',
default='', help='File containing a list of '
'Pokemon to NOT encounter for'
' more stats.')
parser.add_argument('-ld', '--login-delay',
help='Time delay between each login attempt.',
type=float, default=6)
parser.add_argument('-lr', '--login-retries',
help=('Number of times to retry the login before ' +
'refreshing a thread.'),
type=int, default=3)
parser.add_argument('-mf', '--max-failures',
help=('Maximum number of failures to parse ' +
'locations before an account will go into a ' +
'sleep for -ari/--account-rest-interval ' +
'seconds.'),
type=int, default=5)
parser.add_argument('-me', '--max-empty',
help=('Maximum number of empty scans before an ' +
'account will go into a sleep for ' +
'-ari/--account-rest-interval seconds.' +
'Reasonable to use with proxies.'),
type=int, default=0)
parser.add_argument('-bsr', '--bad-scan-retry',
help=('Number of bad scans before giving up on a ' +
'step. Default 2, 0 to disable.'),
type=int, default=2)
parser.add_argument('-msl', '--min-seconds-left',
help=('Time that must be left on a spawn before ' +
'considering it too late and skipping it. ' +
'For example 600 would skip anything with ' +
'< 10 minutes remaining. Default 0.'),
type=int, default=0)
parser.add_argument('-dc', '--display-in-console',
help='Display Found Pokemon in Console.',
action='store_true', default=False)
parser.add_argument('-H', '--host', help='Set web server listening host.',
default='127.0.0.1')
parser.add_argument('-P', '--port', type=int,
help='Set web server listening port.', default=5000)
parser.add_argument('-L', '--locale',
help=('Locale for Pokemon names (default: {}, check ' +
'{} for more).').format(config['LOCALE'],
config['LOCALES_DIR']),
default='en')
parser.add_argument('-c', '--china',
help='Coordinates transformer for China.',
action='store_true')
parser.add_argument('-m', '--mock', type=str,
help=('Mock mode - point to a fpgo endpoint instead ' +
'of using the real PogoApi, ec: ' +
'http://127.0.0.1:9090'),
default='')
parser.add_argument('-ns', '--no-server',
help=('No-Server Mode. Starts the searcher but not ' +
'the Webserver.'),
action='store_true', default=False)
parser.add_argument('-os', '--only-server',
help=('Server-Only Mode. Starts only the Webserver ' +
'without the searcher.'),
action='store_true', default=False)
parser.add_argument('-nsc', '--no-search-control',
help='Disables search control.',
action='store_false', dest='search_control',
default=True)
parser.add_argument('-fl', '--fixed-location',
help='Hides the search bar for use in shared maps.',
action='store_true', default=False)
parser.add_argument('-k', '--gmaps-key',
help='Google Maps Javascript API Key.',
required=True)
parser.add_argument('--skip-empty',
help=('Enables skipping of empty cells in normal ' +
'scans - requires previously populated ' +
'database (not to be used with -ss)'),
action='store_true', default=False)
parser.add_argument('-C', '--cors', help='Enable CORS on web server.',
action='store_true', default=False)
parser.add_argument('-D', '--db', help='Database filename for SQLite.',
default='pogom.db')
parser.add_argument('-cd', '--clear-db',
help=('Deletes the existing database before ' +
'starting the Webserver.'),
action='store_true', default=False)
parser.add_argument('-np', '--no-pokemon',
help=('Disables Pokemon from the map (including ' +
'parsing them into local db.)'),
action='store_true', default=False)
parser.add_argument('-ng', '--no-gyms',
help=('Disables Gyms from the map (including ' +
'parsing them into local db).'),
action='store_true', default=False)
parser.add_argument('-nk', '--no-pokestops',
help=('Disables PokeStops from the map (including ' +
'parsing them into local db).'),
action='store_true', default=False)
parser.add_argument('-ss', '--spawnpoint-scanning',
help=('Use spawnpoint scanning (instead of hex ' +
'grid). Scans in a circle based on step_limit ' +
'when on DB.'),
nargs='?', const='nofile', default=False)
parser.add_argument('-speed', '--speed-scan',
help=('Use speed scanning to identify spawn points ' +
'and then scan closest spawns.'),
action='store_true', default=False)
parser.add_argument('-kph', '--kph',
help=('Set a maximum speed in km/hour for scanner ' +
'movement.'),
type=int, default=35)
parser.add_argument('-ldur', '--lure-duration',
help=('Change duration for lures set on pokestops. ' +
'This is useful for events that extend lure ' +
'duration.'), type=int, default=30)
parser.add_argument('--dump-spawnpoints',
help=('Dump the spawnpoints from the db to json ' +
'(only for use with -ss).'),
action='store_true', default=False)
parser.add_argument('-pd', '--purge-data',
help=('Clear Pokemon from database this many hours ' +
'after they disappear (0 to disable).'),
type=int, default=0)
parser.add_argument('-px', '--proxy',
help='Proxy url (e.g. socks5://127.0.0.1:9050)',
action='append')
parser.add_argument('-pxsc', '--proxy-skip-check',
help='Disable checking of proxies before start.',
action='store_true', default=False)
parser.add_argument('-pxt', '--proxy-timeout',
help='Timeout settings for proxy checker in seconds.',
type=int, default=5)
parser.add_argument('-pxd', '--proxy-display',
help=('Display info on which proxy being used ' +
'(index or full). To be used with -ps.'),
type=str, default='index')
parser.add_argument('-pxf', '--proxy-file',
help=('Load proxy list from text file (one proxy ' +
'per line), overrides -px/--proxy.'))
parser.add_argument('-pxr', '--proxy-refresh',
help=('Period of proxy file reloading, in seconds. ' +
'Works only with -pxf/--proxy-file. ' +
'(0 to disable).'),
type=int, default=0)
parser.add_argument('-pxo', '--proxy-rotation',
help=('Enable proxy rotation with account changing ' +
'for search threads (none/round/random).'),
type=str, default='none')
parser.add_argument('--db-type',
help='Type of database to be used (default: sqlite).',
default='sqlite')
parser.add_argument('--db-name', help='Name of the database to be used.')
parser.add_argument('--db-user', help='Username for the database.')
parser.add_argument('--db-pass', help='Password for the database.')
parser.add_argument('--db-host', help='IP or hostname for the database.')
parser.add_argument(
'--db-port', help='Port for the database.', type=int, default=3306)
parser.add_argument('--db-max_connections',
help='Max connections (per thread) for the database.',
type=int, default=5)
parser.add_argument('--db-threads',
help=('Number of db threads; increase if the db ' +
'queue falls behind.'),
type=int, default=1)
parser.add_argument('-wh', '--webhook',
help='Define URL(s) to POST webhook information to.',
default=None, dest='webhooks', action='append')
parser.add_argument('-gi', '--gym-info',
help=('Get all details about gyms (causes an ' +
'additional API hit for every gym).'),
action='store_true', default=False)
parser.add_argument('--disable-clean', help='Disable clean db loop.',
action='store_true', default=False)
parser.add_argument('--webhook-updates-only',
help='Only send updates (Pokemon & lured pokestops).',
action='store_true', default=False)
parser.add_argument('--wh-threads',
help=('Number of webhook threads; increase if the ' +
'webhook queue falls behind.'),
type=int, default=1)
parser.add_argument('-whc', '--wh-concurrency',
help=('Async requests pool size.'), type=int,
default=25)
parser.add_argument('-whr', '--wh-retries',
help=('Number of times to retry sending webhook ' +
'data on failure.'),
type=int, default=3)
parser.add_argument('-wht', '--wh-timeout',
help='Timeout (in seconds) for webhook requests.',
type=float, default=1.0)
parser.add_argument('-whbf', '--wh-backoff-factor',
help=('Factor (in seconds) by which the delay ' +
'until next retry will increase.'),
type=float, default=0.25)
parser.add_argument('-whlfu', '--wh-lfu-size',
help='Webhook LFU cache max size.', type=int,
default=1000)
parser.add_argument('-whsu', '--webhook-scheduler-updates',
help=('Send webhook updates with scheduler status ' +
'(use with -wh).'),
action='store_true', default=True)
parser.add_argument('--ssl-certificate',
help='Path to SSL certificate file.')
parser.add_argument('--ssl-privatekey',
help='Path to SSL private key file.')
parser.add_argument('-ps', '--print-status',
help=('Show a status screen instead of log ' +
'messages. Can switch between status and ' +
'logs by pressing enter. Optionally specify ' +
'"logs" to startup in logging mode.'),
nargs='?', const='status', default=False,
metavar='logs')
parser.add_argument('-slt', '--stats-log-timer',
help='In log view, list per hr stats every X seconds',
type=int, default=0)
parser.add_argument('-sn', '--status-name', default=None,
help=('Enable status page database update using ' +
'STATUS_NAME as main worker name.'))
parser.add_argument('-spp', '--status-page-password', default=None,
help='Set the status page password.')
parser.add_argument('-hk', '--hash-key', default=None, action='append',
help='Key for hash server')
parser.add_argument('-tut', '--complete-tutorial', action='store_true',
help=("Complete ToS and tutorial steps on accounts " +
"if they haven't already."),
default=False)
parser.add_argument('-novc', '--no-version-check', action='store_true',
help='Disable API version check.',
default=False)
parser.add_argument('-vci', '--version-check-interval', type=int,
help='Interval to check API version in seconds ' +
'(Default: in [60, 300]).',
default=random.randint(60, 300))
parser.add_argument('-el', '--encrypt-lib',
help=('Path to encrypt lib to be used instead of ' +
'the shipped ones.'))
parser.add_argument('-odt', '--on-demand_timeout',
help=('Pause searching while web UI is inactive ' +
'for this timeout (in seconds).'),
type=int, default=0)
parser.add_argument('--disable-blacklist',
help=('Disable the global anti-scraper IP blacklist.'),
action='store_true', default=False)
parser.add_argument('-tp', '--trusted-proxies', default=[],
action='append',
help=('Enables the use of X-FORWARDED-FOR headers ' +
'to identify the IP of clients connecting ' +
'through these trusted proxies.'))
verbosity = parser.add_mutually_exclusive_group()
verbosity.add_argument('-v', '--verbose',
help=('Show debug messages from RocketMap ' +
'and pgoapi. Optionally specify file ' +
'to log to.'),
nargs='?', const='nofile', default=False,
metavar='filename.log')
verbosity.add_argument('-vv', '--very-verbose',
help=('Like verbose, but show debug messages ' +
'from all modules as well. Optionally ' +
'specify file to log to.'),
nargs='?', const='nofile', default=False,
metavar='filename.log')
parser.set_defaults(DEBUG=False)
args = parser.parse_args()
if args.only_server:
if args.location is None:
parser.print_usage()
print(sys.argv[0] +
": error: arguments -l/--location is required.")
sys.exit(1)
else:
# If using a CSV file, add the data where needed into the username,
# password and auth_service arguments.
# CSV file should have lines like "ptc,username,password",
# "username,password" or "username".
if args.accountcsv is not None:
# Giving num_fields something it would usually not get.
num_fields = -1
with open(args.accountcsv, 'r') as f:
for num, line in enumerate(f, 1):
fields = []
# First time around populate num_fields with current field
# count.
if num_fields < 0:
num_fields = line.count(',') + 1
csv_input = []
csv_input.append('')
csv_input.append('<username>')
csv_input.append('<username>,<password>')
csv_input.append('<ptc/google>,<username>,<password>')
# If the number of fields is differend this is not a CSV.
if num_fields != line.count(',') + 1:
print(sys.argv[0] +
": Error parsing CSV file on line " + str(num) +
". Your file started with the following " +
"input, '" + csv_input[num_fields] +
"' but now you gave us '" +
csv_input[line.count(',') + 1] + "'.")
sys.exit(1)
field_error = ''
line = line.strip()
# Ignore blank lines and comment lines.
if len(line) == 0 or line.startswith('#'):
continue
# If number of fields is more than 1 split the line into
# fields and strip them.
if num_fields > 1:
fields = line.split(",")
fields = map(str.strip, fields)
# If the number of fields is one then assume this is
# "username". As requested.
if num_fields == 1:
# Empty lines are already ignored.
args.username.append(line)
# If the number of fields is two then assume this is
# "username,password". As requested.
if num_fields == 2:
# If field length is not longer than 0 something is
# wrong!
if len(fields[0]) > 0:
args.username.append(fields[0])
else:
field_error = 'username'
# If field length is not longer than 0 something is
# wrong!
if len(fields[1]) > 0:
args.password.append(fields[1])
else:
field_error = 'password'
# If the number of fields is three then assume this is
# "ptc,username,password". As requested.
if num_fields == 3:
# If field 0 is not ptc or google something is wrong!
if (fields[0].lower() == 'ptc' or
fields[0].lower() == 'google'):
args.auth_service.append(fields[0])
else:
field_error = 'method'
# If field length is not longer then 0 something is
# wrong!
if len(fields[1]) > 0:
args.username.append(fields[1])
else:
field_error = 'username'
# If field length is not longer then 0 something is
# wrong!
if len(fields[2]) > 0:
args.password.append(fields[2])
else:
field_error = 'password'
if num_fields > 3:
print(('Too many fields in accounts file: max ' +
'supported are 3 fields. ' +
'Found {} fields').format(num_fields))
sys.exit(1)
# If something is wrong display error.
if field_error != '':
type_error = 'empty!'
if field_error == 'method':
type_error = (
'not ptc or google instead we got \'' +
fields[0] + '\'!')
print(sys.argv[0] +
": Error parsing CSV file on line " + str(num) +
". We found " + str(num_fields) + " fields, " +
"so your input should have looked like '" +
csv_input[num_fields] + "'\nBut you gave us '" +
line + "', your " + field_error +
" was " + type_error)
sys.exit(1)
errors = []
num_auths = len(args.auth_service)
num_usernames = 0
num_passwords = 0
if len(args.username) == 0:
errors.append(
'Missing `username` either as -u/--username, csv file ' +
'using -ac, or in config.')
else:
num_usernames = len(args.username)
if args.location is None:
errors.append(
'Missing `location` either as -l/--location or in config.')
if len(args.password) == 0:
errors.append(
'Missing `password` either as -p/--password, csv file, ' +
'or in config.')
else:
num_passwords = len(args.password)
if args.step_limit is None:
errors.append(
'Missing `step_limit` either as -st/--step-limit or ' +
'in config.')
if num_auths == 0:
args.auth_service = ['ptc']
num_auths = len(args.auth_service)
if num_usernames > 1:
if num_passwords > 1 and num_usernames != num_passwords:
errors.append((
'The number of provided passwords ({}) must match the ' +
'username count ({})').format(num_passwords,
num_usernames))
if num_auths > 1 and num_usernames != num_auths:
errors.append((
'The number of provided auth ({}) must match the ' +
'username count ({}).').format(num_auths, num_usernames))
if len(errors) > 0:
parser.print_usage()
print(sys.argv[0] + ": errors: \n - " + "\n - ".join(errors))
sys.exit(1)
# Fill the pass/auth if set to a single value.
if num_passwords == 1:
args.password = [args.password[0]] * num_usernames
if num_auths == 1:
args.auth_service = [args.auth_service[0]] * num_usernames
# Make the accounts list.
args.accounts = []
for i, username in enumerate(args.username):
args.accounts.append({'username': username,
'password': args.password[i],
'auth_service': args.auth_service[i]})
# Make max workers equal number of accounts if unspecified, and disable
# account switching.
if args.workers is None:
args.workers = len(args.accounts)
args.account_search_interval = None
# Disable search interval if 0 specified.
if args.account_search_interval == 0:
args.account_search_interval = None
# Make sure we don't have an empty account list after adding command
# line and CSV accounts.
if len(args.accounts) == 0:
print(sys.argv[0] +
": Error: no accounts specified. Use -a, -u, and -p or " +
"--accountcsv to add accounts.")
sys.exit(1)
if args.encounter_whitelist_file:
with open(args.encounter_whitelist_file) as f:
args.encounter_whitelist = [get_pokemon_id(name) for name in
f.read().splitlines()]
elif args.encounter_blacklist_file:
with open(args.encounter_blacklist_file) as f:
args.encounter_blacklist = [get_pokemon_id(name) for name in
f.read().splitlines()]
else:
args.encounter_blacklist = [int(i) for i in
args.encounter_blacklist]
args.encounter_whitelist = [int(i) for i in
args.encounter_whitelist]
# Decide which scanning mode to use.
if args.spawnpoint_scanning:
args.scheduler = 'SpawnScan'
elif args.skip_empty:
args.scheduler = 'HexSearchSpawnpoint'
elif args.speed_scan:
args.scheduler = 'SpeedScan'
else:
args.scheduler = 'HexSearch'
# Disable webhook scheduler updates if webhooks are disabled
if args.webhooks is None:
args.webhook_scheduler_updates = False
return args
def main(sysargs=sys.argv[1:]):
p = configargparse.ArgParser()
# These are safe for command line usage (no accent in Dale)
LEAGUES, DIVISIONS, ALLTEAMS = get_league_division_team_data()
p.add('-v',
'--version',
required=False,
default=False,
action='store_true',
help='Print program name and version number and exit')
p.add('-c',
'--config',
required=False,
is_config_file=True,
help='config file path')
# Winning streaks or losing streaks
g = p.add_mutually_exclusive_group()
g.add('--winning',
action='store_true',
default=False,
help='Find winning streaks')
g.add('--losing',
action='store_true',
default=False,
help='Find losing streaks')
# Pick our team
h = p.add_mutually_exclusive_group()
h.add('--team',
choices=ALLTEAMS,
action='append',
help='Specify our team (use flag multiple times for multiple teams)')
h.add(
'--division',
choices=DIVISIONS,
action='append',
help=
'Specify our division (use flag multiple times for multiple divisions)'
)
h.add('--league',
choices=LEAGUES,
action='append',
help='Specify our league')
# Pick versus team
k = p.add_mutually_exclusive_group()
k.add(
'--versus-team',
choices=ALLTEAMS,
action='append',
help='Specify versus team (use flag multiple times for multiple teams)'
)
k.add(
'--versus-division',
choices=DIVISIONS,
action='append',
help=
'Specify versus division (use flag multiple times for multiple divisions)'
)
k.add('--versus-league',
choices=LEAGUES,
action='append',
help='Specify versus league')
# Specify what season data to view
p.add(
'--season',
required=False,
action='append',
help=
'Specify season (use flag multiple times for multiple seasons, no --seasons flag means all data)'
)
# Minimum number of wins to be considered a streak
p.add(
'--min',
required=False,
type=int,
default=3,
help=
'Minimum number of wins to be considered a streak (defaults to 3, make this higher if looking at multiple teams)'
)
p.add('--text',
action='store_true',
default=True,
help='Print streak data in plain text format')
p.add('--markdown',
action='store_true',
default=False,
help='Print streak data in Markdown table format')
p.add(
'--output',
required=False,
type=str,
default='',
help=
'Specify the name of the Markdown output file, for use with --markdown flag'
)
# Pick format for streak data
m = p.add_mutually_exclusive_group()
m.add('--long',
action='store_true',
default=False,
help='Print streak data in long format (one table per streak)')
m.add('--short',
action='store_true',
default=False,
help='Print streak data in short format (one line per streak)')
# Pick name format (nickname vs full name)
m = p.add_mutually_exclusive_group()
m.add('--nickname',
action='store_true',
default=False,
help='Print team nicknames (e.g., Sunbeams)')
m.add('--fullname',
action='store_true',
default=False,
help='Print full team names (e.g., Hellmouth Sunbeams)')
# -----
# Print help, if no arguments provided
if len(sys.argv[1:]) == 0:
p.print_help()
exit(0)
# Parse arguments
options = p.parse_args(sys.argv[1:])
# If the user asked for the version,
# print the version number and exit.
if options.version:
from . import _program, __version__
print(_program, __version__)
sys.exit(0)
# If user did not specify winning/losing, use default (winning)
if (not options.winning) and (not options.losing):
options.winning = True
# If the user did not specify long/short table format, use default (short)
if (not options.long) and (not options.short):
options.short = True
# If user did not specify a name format, use short
if (not options.nickname) and (not options.fullname):
options.nickname = True
# Make sure output file is an absolute path
if options.output != '':
options.output = os.path.abspath(options.output)
# If the user specified a division or a league,
# turn that into a list of teams for them
if options.division:
divteams = []
for div in options.division:
divteams += division_to_teams(div)
options.team = divteams
options.division = None
if options.league:
leateams = []
for lea in options.league:
leateams += league_to_teams(lea)
options.team = leateams
options.league = None
# Same for versus
if options.versus_division:
vdivteams = []
for div in options.versus_division:
vdivteams += division_to_teams(div)
options.versus_team = vdivteams
options.versus_division = None
if options.versus_league:
vleateams = []
for lea in options.versus_league:
vleateams += league_to_teams(lea)
options.versus_team = vleateams
options.versus_league = None
# If nothing was supplied for our team/division/league, use all teams
if not options.team and not options.division and not options.league:
options.team = ALLTEAMS
# If nothing was supplied for versus team/division/league, use all teams
if not options.versus_team and not options.versus_division and not options.versus_league:
options.versus_team = ALLTEAMS
# If nothing was provided for seasons, set it to 'all'
if not options.season:
options.season = ['all']
else:
try:
_ = [int(j) for j in options.season]
except ValueError:
raise Exception(
"Error: you must provide integers to the --season flag: --season 1 --season 2"
)
if options.markdown:
v = MarkdownView(options)
v.table()
else:
v = TextView(options)
v.table()
from __future__ import print_function
import configargparse
from file_markov import FileMarkov
if __name__ == "__main__":
p = configargparse.ArgParser(ignore_unknown_config_file_keys=True)
p.add('-c', '--my-config', is_config_file=True, help='config file path')
p.add('-f', '--file', help='File to use as a base ()')
p.add('-e', '--encoding', default='utf8', help='Encoding for file')
p.add('--state-size', default=1, type=int, help='State size for markov chain')
p.add('--max-sentences', default=10, type=int, help='Number of sentences to return')
options = p.parse_args()
m = FileMarkov( options.file, encoding=options.encoding, state_size=options.state_size )
for i in m.get_sentences( max_sentences=options.max_sentences ):
print( i )
import torch.nn as nn
from torch.autograd import Variable
import torch.backends.cudnn as cudnn
import torchnet as tnt
from torchnet.engine import Engine
from torchnet.logger import VisdomPlotLogger, VisdomLogger
from tqdm import tqdm
from models import DenseNet
from datasets import ImageNet, CIFAR10, CIFAR100
import os
import copy
import math
import random
import numpy as np
parser = configargparse.ArgParser(default_config_files=[])
parser.add('--config',
required=True,
is_config_file=True,
help='config file path')
parser.add('--batch-size',
type=int,
default=256,
metavar='N',
help='input batch size for training (default: 256)')
parser.add('--num-batch-splits',
type=int,
default=1,
metavar='split',
help='split batch size for training (default: 1)')
parser.add('--dataset',
from funlib.run.run_singularity import run_singularity
import subprocess as sp
import configargparse
import logging
import os
import random
import re
bsub_stdout_regex = re.compile("Job <(\d+)> is submitted*")
logger = logging.getLogger(__name__)
p = configargparse.ArgParser(default_config_files=['~/.pysub'])
p.add('-p',
required=True,
help="The command to run" + " e.g. ``python train.py``.")
p.add('-c',
'--num_cpus',
required=False,
type=int,
help="Number of CPUs to request, default 5.",
default=5)
p.add('-g',
'--num_gpus',
required=False,
type=int,
help="Number of GPUs to request, default 1.",
default=1)
def parse_args():
p = configargparse.ArgParser(
description=("Compute coefficients for the DeMaSk predictor."),
epilog="""
For a set of DMS datasets, load the DMS scores, aligned homologs,
and pre-computed substitution matrix, and fit a linear model,
saving the coefficients to a file for later prediction on new
proteins.
Dataset files must be tab-delimited with column names.
Columns containing residue position, WT residue, variant
residue, and score must be present in any order, and other
columns will be ignored. Entries for which the WT and variant
residues are not among the 20 canonical amino acids or are the
same are ignored.
DMS scores should already be normalized as desired. For the
default DeMaSk matrix, variant scores were rank-normalized per
protein, and then the wild-type fitness levels subtracted, so that
scores represent delta fitness.
""",
)
p.add(
"-d",
"--datadir",
required=True,
help=
"Name of directory containing (only) DMS data files, one per protein.",
)
p.add(
"-a",
"--alndir",
required=True,
help=("Name of the directory containing alignment files, each"
"of which must be named as the basename of a DMS data file"
"followed by the '.a2m' extension."),
)
dirname = os.path.dirname(__file__)
matrix = os.path.join(dirname, "..", "data", "matrix.txt")
p.add(
"-m",
"--matrix",
metavar="FILE",
default=matrix,
help=("File containing the directional substitution matrix. "
"Defaults to the file at 'demask/matrix/matrix.txt'."),
)
p.add(
"-o",
"--outfile",
required=True,
help="Name of the file in which to save the coefficients.",
)
p.add(
"--columns",
required=False,
default="pos,WT,var,score",
help=
("An optional comma-separated list of DMS data column names to read in place of "
"'pos', 'WT', 'var', and 'score', respectively. For example, "
"'Position,wt,mutant,Fitness'. Must apply to all files provided."),
)
config = os.path.join(dirname, "..", "config.ini")
p.add(
"-c",
"--config",
is_config_file=True,
metavar="FILE",
default=config,
help=
"Configuration file. Defaults to 'config.ini' in the demask directory.",
)
p.add(
"-n",
"--nseqs",
type=int,
default=500,
help=("Maximum number of supporting sequences per query sequence. "
"Defaults to 500."),
)
p.add(
"-w",
"--weight_threshold",
type=float,
default=None,
help=
("Sequence identity threshold used for "
"sequence weighting, e.g. 0.8. Sequences are weighted by the inverse "
"of the number of sequences within this percent identity. If None "
"(default), sequence weighting is not used."),
)
args, unknown = p.parse_known_args()
return args
# ------------------------
print(hparams)
print()
trainer.fit(model)
trainer.test()
if __name__ == "__main__":
# ------------------------
# TRAINING ARGUMENTS
# ------------------------
# these are project-wide arguments
config_path = Path(__file__).absolute().parent.parent / "config/config.yml"
parser = configargparse.ArgParser(default_config_files=[str(config_path)],
description="Hyper-parameters.")
parser.add_argument("--config",
is_config_file=True,
default=False,
help="config file path")
# args
parser.add_argument("--max_epochs", default=20, type=int)
parser.add_argument("--max_iters", default=None, type=int)
parser.add_argument("--resume_from_checkpoint",
type=str,
help="resume from checkpoint")
parser.add_argument("--seed", type=int, default=1, help="seed")
parser.add_argument("--gpus", type=int, default=0, help="how many gpus")
parser.add_argument("--use_16bit",
type=bool,
def parse_args(root_dir):
"""Parse command line arguments.
Args:
root_dir : Path to the root directory,
where the configs folder can be found.
Return:
args : parsed argument object.
"""
parser = configargparse.ArgParser()
# =========================================================================
# training args
subparsers = parser.add_subparsers(dest="mode")
train_config_path = os.path.join(root_dir, 'configs', 'train_config.yaml')
parser_train = subparsers.add_parser(
'train',
config_file_parser_class=configargparse.YAMLConfigFileParser,
default_config_files=[train_config_path])
add_train_options(parser_train)
# =========================================================================
# Inference args
infer_config_path = os.path.join(root_dir, 'configs', 'infer_config.yaml')
parser_infer = subparsers.add_parser(
'denoise',
config_file_parser_class=configargparse.YAMLConfigFileParser,
default_config_files=[infer_config_path])
add_inference_options(parser_infer)
# =========================================================================
# Evaluation args
parser_eval = subparsers.add_parser(
'eval',
config_file_parser_class=configargparse.YAMLConfigFileParser,
default_config_files=[infer_config_path])
add_eval_options(parser_eval)
# =========================================================================
args, extra = parser.parse_known_args()
if args.mode == "denoise":
check_dependence(args.deletebg, args.gen_bigwig, parser,
"--deletebg requires --gen_bigwig")
if args.mode == "train":
if not (args.val_chrom or args.holdout_chrom):
if args.train_h5_files or args.val_h5_files:
check_dependence(args.train_h5_files,
args.val_h5_files,
parser,
"Specify both --train_h5_file and "
"--val_h5_file.")
else:
parser.error("val_chrom and holdout_chrom are required for \
training.")
check_dependence(args.cleanbw, args.cleanpeakfile, parser,
"cleanbw and cleanpeakfile are required for \
training")
if args.mode == "eval":
check_dependence(args.cleanbw, args.cleanpeakfile, parser,
"cleanbw and cleanpeakfile are required for \
eval")
if not(args.distributed) and (args.gpu_idx is None):
parser.error("Either specify which GPU to run atacworks on \
through --gpu_idx, or pass the flag --distributed \
to run on ALL available GPUs.")
return args
def main():
parser = configargparse.ArgParser(
description='PyTorch Training',
formatter_class=configargparse.ArgumentDefaultsRawHelpFormatter)
parser = parseArgs(parser)
args = parser.parse_args()
print('{}'.format(args))
# TODO: we would need to fix a lot of other seeds too!
# torch.manual_seed(args.seed)
# --> Currently not reproducible runs, but
# since we use multiple runs and average, the results should be close
# to the ones reported in the paper
if os.path.isdir(args.expdir) and not args.test_only:
print('WARNING: path already exists!')
for i in range(2, 99):
path = args.expdir + str(i)
if not os.path.isdir(path):
args.expdir = path
print('set new expdir:', path)
break
if args.ensemble:
args.test_only = True
if not os.path.isdir(args.expdir):
mkdir_p(args.expdir)
writer = SummaryWriter(os.path.join(args.expdir, 'runs'))
if not args.test_only:
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M")
cmd_log = open(os.path.join(args.expdir, 'cmd.log'), 'a')
cmd_log.write('{}\n'.format(timestamp))
cmd_log.write('{}\n'.format(args))
cmd_log.write(parser.format_values())
start_time = time.time()
if args.resume is not None and args.resume == 'default':
args.resume = os.path.join(args.expdir, 'checkpoints',
'best_model.pth.tar')
print('try to resume: {}'.format(args.resume))
# Initialize the model for this run
model = initialize_model(args.model_name,
args.n_classes,
use_pretrained=args.finetune,
pool_type=args.pool_type,
gmp_lambda=args.gmp_lambda,
lse_r=args.lse_r,
last_conv_stride=args.last_conv_stride,
normalize=args.normalize)
# Print the model we just instantiated
print(model)
### Data augmentation and normalization for training ###
# pxl mean: 185.2293097, std: 61.8039951063
m = 185.2293097 / 255
s = 61.8039951063 / 255
data_transforms = {
'train':
transforms.Compose([
# TODO: maybe more random scale?
transforms.RandomResizedCrop(args.input_size),
transforms.RandomRotation(degrees=5),
# what's with hue and saturation as with the patch-wise?
transforms.ColorJitter(brightness=0.1, contrast=0.1),
transforms.ToTensor(),
transforms.Normalize([m, m, m], [s, s, s])
]),
#'val': #test_transform
'val':
transforms.Compose([
transforms.Resize(args.input_size),
transforms.CenterCrop(args.input_size),
transforms.ToTensor(),
transforms.Normalize([m, m, m], [s, s, s])
]),
}
print("Initializing Datasets and Dataloaders...")
data_dir = {}
data_dir['train'] = args.traindir
data_dir['val'] = args.traindir if not args.valdir else args.valdir
labels = {}
labels['train'] = args.train_labels
labels['val'] = args.val_labels
# Create training and validation datasets
image_datasets = {x: dataset_label.DatasetLabel(data_dir[x], labels[x],
args.suffix,
transform=data_transforms[x]) \
for x in ['train', 'val']}
# Create training and validation dataloaders
dataloaders_dict = {x: torch.utils.data.DataLoader(image_datasets[x],
batch_size=args.batch_size\
if x == 'train' else\
args.batch_size_val,
shuffle=True if \
x == 'train' else False,
num_workers=args.workers,
pin_memory=False,
drop_last=True) \
for x in ['train', 'val']}
# Detect if we have a GPU available
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# Send the model to GPU (has to be done before constructing optim
model = model.to(device)
# Observe that all parameters are being optimized
if not args.finetune:
# we have to train the full model from scratch
# with this we set all parameters for update
finetune_start = 0
else:
finetune_start = args.finetune_start
assert (finetune_start == 0)
print('set all remaining parameters to requires_grad=True')
if args.model_name == 'poolnet':
fe_params = {'params': model.feature_extractor.parameters()}
# shall we treat the pool layer differently?
# --> doesnt seem to be needed, or?
if args.pool_lr_multiplier != 0:
pool_params = {
'params': model.pool.parameters(),
'lr': args.pool_lr_multiplier * args.lr,
'weight_decay': 0
}
else:
pool_params = {'params': model.pool.parameters()}
if args.class_lr_multiplier != 0:
fc_params = {
'params': model.fc.parameters(),
'lr': args.class_lr_multiplier * args.lr
}
else:
fc_params = {'params': model.fc.parameters()}
params_to_update = [fe_params, pool_params, fc_params]
else:
params_to_update = []
for name, param in model.named_parameters():
if param.requires_grad == True:
params_to_update.append(param)
if args.optimizer == 'adam':
optimizer = optim.Adam(params_to_update,
lr=args.lr,
weight_decay=args.weight_decay,
amsgrad=True)
elif args.optimizer == 'sgd':
optimizer = optim.SGD(params_to_update,
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay,
nesterov=args.nesterov)
else:
raise ValueError('unknown optimizer: {}'.format(args.optimizer))
if args.normalize == 'learn':
optimizer.add_param_group({'params': model.normalize.parameters()})
best_acc = 0
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
best_acc = checkpoint['best_acc']
model.load_state_dict(checkpoint['state_dict'])
model.to(device)
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
else:
raise ValueError("=> no checkpoint found at '{}'".format(
args.resume))
######################################################################
# Run Training and Validation Step
# --------------------------------
#
# Finally, the last step is to setup the loss for the model, then run the
# training and validation function for the set number of epochs. Notice,
# depending on the number of epochs this step may take a while on a CPU.
# Also, the default learning rate is not optimal for all of the models, so
# to achieve maximum accuracy it would be necessary to tune for each model
# separately.
#
# Setup the loss fxn
criterion = nn.CrossEntropyLoss()
# Train and evaluate
if not args.test_only:
model = train_model(model,
dataloaders_dict,
criterion,
optimizer,
start_epoch=args.start_epoch,
n_epochs=args.n_epochs,
finetune_start=finetune_start,
lr=args.lr,
lr_scheduler=args.lr_scheduler,
start_exp_decay=args.start_exp_decay,
print_freq=args.print_freq,
expdir=args.expdir,
check_epoch=args.check_epoch,
device=device,
best_acc=best_acc,
writer=writer)
model.eval()
# test model
test_dataset = dataset_label.DatasetLabel(args.testdir,
args.test_labels,
args.suffix,
transform=data_transforms['val'])
test_dataloader = torch.utils.data.DataLoader(test_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
if len(args.ensemble) > 0:
if '*' in args.ensemble[0] or '?' in args.ensemble[0]:
ensemble = glob.glob(args.ensemble[0])
else:
ensemble = args.ensemble
assert (len(ensemble) > 0)
print('ensemble of:', ensemble)
all_pred = []
all_outputs = []
all_best_val = []
all_test_acc = []
for e, ens_file in enumerate(ensemble):
print('ensemble {} / {}'.format(e + 1, len(ensemble)))
if os.path.isfile(ens_file):
print("=> loading ensemble checkpoint '{}'".format(ens_file))
checkpoint = torch.load(ens_file)
all_best_val.append(checkpoint['best_acc'].cpu().numpy())
model.load_state_dict(checkpoint['state_dict'])
model.to(device)
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded ensemble checkpoint '{}' (epoch {})".format(
ens_file, checkpoint['epoch']))
else:
raise ValueError('=> no checkpoint found at'
' {}'.format(ens_file))
_, _, _, test_acc, _, pred, outputs = run_epoch(model,
test_dataloader,
'test',
optimizer,
criterion,
0,
device,
compute_extra=True)
all_test_acc.append(test_acc)
all_pred.append(torch.cat(pred).cpu().reshape(-1, 1))
all_outputs.append(outputs)
print('single test accuracy: {}'.format(test_acc))
with open(os.path.join(args.expdir, 'test.log'), 'a') as f:
f.write('acc: {}\n'.format(test_acc))
np_labels = np.array(list(zip(*test_dataset.samples))[1])
labels = torch.from_numpy(np_labels)
# all predictions of all classifiers
all_pred = torch.cat(all_pred, dim=1)
# now let's take majority
maj, _ = all_pred.mode(dim=1)
maj_acc = torch.sum(maj == labels)
maj_acc = maj_acc.double() / float(len(test_dataset))
print('majority test accuracy: {}'.format(maj_acc))
with open(os.path.join(args.expdir, 'maj.log'), 'a') as f:
f.write('acc: {}\n'.format(maj_acc))
# via softmax
all_outputs = torch.stack(all_outputs) # n_ensembles x N x C
soft = nn.functional.softmax(all_outputs, 2)
new_out = torch.sum(soft, 0) # -> N x C
_, preds = torch.max(new_out, 1)
soft_acc = torch.sum(preds == labels).double() / float(
len(test_dataset))
print('softmax test accuracy: {}'.format(soft_acc))
with open(os.path.join(args.expdir, 'soft.log'), 'a') as f:
f.write('soft: {}\n'.format(soft_acc))
# weighted softmax
all_best_val = np.array(all_best_val)
all_best_val /= all_best_val.sum() # l1 norm
for i in range(len(all_best_val)):
soft[i, :, :] *= all_best_val[i]
new_out = torch.sum(soft, 0) # -> N x C
_, preds = torch.max(new_out, 1)
wsoft_acc = torch.sum(preds == labels).double() / len(test_dataset)
print('weighted softmax test accuracy: {}'.format(wsoft_acc))
with open(os.path.join(args.expdir, 'wsoft.log'), 'a') as f:
f.write('wsoft: {}\n'.format(wsoft_acc))
else:
_, _, _, test_acc = run_epoch(
model,
test_dataloader,
'test',
optimizer,
criterion,
0,
#eval_stats,
device)
print('test accuracy: {}'.format(test_acc))
with open(os.path.join(args.expdir, 'test.log'), 'a') as f:
f.write('acc: {}\n'.format(test_acc))
if not args.test_only:
curr_time = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M")
cmd_log.write('Finished: {}\n'.format(curr_time))
time_elapsed = time.time() - start_time
cmd_log.write('Duration {:.0f}m {:.0f}s\n'.format(
time_elapsed // 60, time_elapsed % 60))
cmd_log.close()
def parse_config():
default_config = pathlib.Path(__file__).stem + ".ini"
config = configargparse.ArgParser(default_config_files=[default_config])
config.add_argument(
'-c',
'--config',
is_config_file=True,
help="Specify path to Configuration file. Default is {0}.ini".format(
pathlib.Path(__file__).stem),
metavar='CONFIG')
config.add_argument(
'-v',
'--verbosity',
action='store',
help="Specify logging level for script. Default is %(default)s.",
choices=['warning', 'info', 'debug'],
default='warning')
config.add_argument('-f',
'--file',
action='store',
help='Write log messages to a file',
metavar='LOGFILE')
config.add_argument(
'--source_files',
action='store',
help=
"Specify path to directory containing source markdown files. Default is to use a "
"\"source\" folder in the current directory. ",
default=pathlib.Path.joinpath(pathlib.Path(__file__).parent, "source"),
metavar='DIRECTORY')
config.add_argument(
'--target_files',
action='store',
help=
"Specify path to directory where processed markdown files should be saved. Default is to "
"use a \"dest\" folder in the current directory. ",
default=pathlib.Path.joinpath(pathlib.Path(__file__).parent, "dest"),
metavar='DIRECTORY')
config.add_argument(
'-p',
'--process',
action='store',
help=
"Determine whether to process all source files or only recently modified files. Default "
"is %(default)s.",
choices=['all', 'modified'],
default='all')
config.add_argument(
'-m',
'--modified',
action='store',
type=int,
help=
"Specify in minutes what is the time limit for recently modified files. Default is "
"%(default)s.",
default=60,
metavar='MINUTES')
if len(sys.argv) == 1:
print(
'Script is being executed without any parameters and will use built-in defaults. Re-run script with -h '
'parameter to understand options available.')
options = config.parse_known_args()
# Convert tuple of parsed arguments into a dictionary. There are two values within this tuple.
# [0] represents recognized arguments. [1] represents unrecognized arguments on command-line or config file.
option_values = vars(options[0])
# Assign dictionary values to variables.
config_file = option_values.get("config")
source_files = option_values.get("source_files")
target_files = option_values.get("target_files")
logging_level = option_values.get("verbosity")
log_file = option_values.get("file")
process_type = option_values.get("process")
modified_time = option_values.get("modified")
# Reset logging levels as per config
logger = logging.getLogger()
logger.setLevel(logging_level.upper())
# Configure file-based logging
if log_file is None:
logging.debug(
"No log file set. All log messages will print to Console only")
else:
filelogger = logging.FileHandler("{0}".format(log_file))
filelogformatter = logging.Formatter(
'%(asctime)s %(levelname)-8s %(funcName)s():%(lineno)i: %(message)s',
datefmt="%Y-%m-%d %H:%M:%S")
filelogger.setFormatter(filelogformatter)
logger.addHandler(filelogger)
logging.warning("Outputting to log file")
# Check if specified config file exists else bail
if config_file is None:
config_file = default_config
logging.debug(
"No configuration file specified. Using the default configuration file %s",
default_config)
elif pathlib.Path(config_file).exists():
logging.debug("Found configuration file %s", config_file)
else:
logging.exception('Did not find the specified configuration file %s',
config_file)
raise FileNotFoundError
# Check if somehow modified_time is set to NIL when processing modified files.
if process_type == 'modified' and not modified_time:
raise ValueError(
"Script is set to process only recently modified files. But the modified time parameter is "
"incorrectly defined.")
# Print values of other parameters in debug mode
if process_type == 'all' and modified_time:
logging.debug(
"Script is set to process all files. Modified time parameter (if any) will have no effect."
)
elif process_type == 'modified' and modified_time:
logging.debug(
"Script is set to only process files modified in last %s minutes",
modified_time)
return config_file, source_files, target_files, log_file, process_type, modified_time
def get_args():
default_config = []
if '-cf' not in sys.argv and '--config' not in sys.argv:
default_config = [
os.path.join(os.path.dirname(__file__), '../config/config.ini')
]
parser = configargparse.ArgParser(default_config_files=default_config)
parser.add_argument('-cf',
'--config',
is_config_file=True,
help='Set configuration file.')
parser.add_argument('-v',
'--verbose',
help='Run in the verbose mode.',
action='store_true')
parser.add_argument('--log-path',
help='Directory where log files are saved.',
default='logs')
parser.add_argument('--download-path',
help='Directory where download files are saved.',
default='downloads')
parser.add_argument('-pj',
'--proxy-judge',
help='URL for AZenv script used to test proxies.',
default='http://pascal.hoez.free.fr/azenv.php')
group = parser.add_argument_group('Database')
group.add_argument('--db-name',
help='Name of the database to be used.',
required=True)
group.add_argument('--db-user',
help='Username for the database.',
required=True)
group.add_argument('--db-pass',
help='Password for the database.',
required=True)
group.add_argument('--db-host',
help='IP or hostname for the database.',
default='127.0.0.1')
group.add_argument('--db-port',
help='Port for the database.',
type=int,
default=3306)
group = parser.add_argument_group('Proxy Sources')
group.add_argument('-Pf',
'--proxy-file',
help='Filename of proxy list to verify.',
default=None)
group.add_argument('-Ps',
'--proxy-scrap',
help='Scrap webpages for proxy lists.',
default=False,
action='store_true')
group.add_argument('-Pp',
'--proxy-protocol',
help=('Specify proxy protocol we are testing. ' +
'Default: socks.'),
default='socks',
choices=('http', 'socks', 'all'))
group.add_argument('-Pri',
'--proxy-refresh-interval',
help=('Refresh proxylist from configured sources '
'every X minutes. Default: 180.'),
default=180,
type=int)
group.add_argument('-Psi',
'--proxy-scan-interval',
help=('Scan proxies from database every X minutes. '
'Default: 60.'),
default=60,
type=int)
group.add_argument('-Pic',
'--proxy-ignore-country',
help=('Ignore proxies from countries in this list. '
'Default: ["china"]'),
default=['china'],
action='append')
group = parser.add_argument_group('Output')
group.add_argument('-Oi',
'--output-interval',
help=('Output working proxylist every X minutes. '
'Default: 60.'),
default=60,
type=int)
group.add_argument('-Ol',
'--output-limit',
help=('Maximum number of proxies to output. '
'Default: 100.'),
default=100,
type=int)
group.add_argument('-Onp',
'--output-no-protocol',
help='Proxy URL format will not include protocol.',
default=False,
action='store_true')
group.add_argument('-Oh',
'--output-http',
help=('Output filename for working HTTP proxies. '
'To disable: None/False.'),
default='working_http.txt')
group.add_argument('-Os',
'--output-socks',
help=('Output filename for working SOCKS proxies. '
'To disable: None/False.'),
default='working_socks.txt')
group.add_argument('-Okc',
'--output-kinancity',
help=('Output filename for KinanCity proxylist. '
'Default: None (disabled).'),
default=None)
group.add_argument('-Opc',
'--output-proxychains',
help=('Output filename for ProxyChains proxylist. '
'Default: None (disabled).'),
default=None)
group.add_argument('-Orm',
'--output-rocketmap',
help=('Output filename for RocketMap proxylist. '
'Default: None (disabled).'),
default=None)
group = parser.add_argument_group('Proxy Tester')
group.add_argument('-Tr',
'--tester-retries',
help=('Maximum number of web request attempts. '
'Default: 5.'),
default=5,
type=int)
group.add_argument('-Tbf',
'--tester-backoff-factor',
help=('Time factor (in seconds) by which the delay '
'until next retry will increase. Default: 0.5.'),
default=0.5,
type=float)
group.add_argument('-Tt',
'--tester-timeout',
help='Connection timeout in seconds. Default: 5.',
default=5,
type=float)
group.add_argument('-Tmc',
'--tester-max-concurrency',
help=('Maximum concurrent proxy testing threads. '
'Default: 100.'),
default=100,
type=int)
group.add_argument('-Tda',
'--tester-disable-anonymity',
help='Disable anonymity proxy test.',
default=False,
action='store_true')
group.add_argument('-Tni',
'--tester-notice-interval',
help=('Print proxy tester statistics every X seconds. '
'Default: 60.'),
default=60,
type=int)
group.add_argument('-Tpv',
'--tester-pogo-version',
help='PoGo API version currently required by Niantic.',
default='0.101.1')
group = parser.add_argument_group('Proxy Scrapper')
group.add_argument('-Sr',
'--scrapper-retries',
help=('Maximum number of web request attempts. '
'Default: 3.'),
default=3,
type=int)
group.add_argument('-Sbf',
'--scrapper-backoff-factor',
help=('Time factor (in seconds) by which the delay '
'until next retry will increase. Default: 0.5.'),
default=0.5,
type=float)
group.add_argument('-St',
'--scrapper-timeout',
help='Connection timeout in seconds. Default: 5.',
default=5,
type=float)
group.add_argument('-Sp',
'--scrapper-proxy',
help=('Use this proxy for webpage scrapping. '
'Format: <proto>://[<user>:<pass>@]<ip>:<port> '
'Default: None.'),
default=None)
args = parser.parse_args()
return args
def create_parser() -> configargparse.ArgParser:
"""
Initialize the command line argument parser.
"""
parser = configargparse.ArgParser(
default_config_files=[
"/etc/lookout/analyzer.conf", "~/.config/lookout/analyzer.conf"
],
formatter_class=ArgumentDefaultsHelpFormatterNoNone,
auto_env_var_prefix="lookout_")
slogging.add_logging_args(parser)
subparsers = parser.add_subparsers(help="Commands", dest="command")
def add_parser(name, help):
return subparsers.add_parser(
name,
help=help,
formatter_class=ArgumentDefaultsHelpFormatterNoNone)
list_parser = add_parser("list", "Print globally available analyzers.")
list_parser.set_defaults(handler=list_analyzers)
run_parser = add_parser(
"run",
"Launch a new service with the specified (one or more) analyzers.")
run_parser.set_defaults(handler=run_analyzers)
add_analyzer_arg(run_parser)
run_parser.add("-c",
"--config",
is_config_file=True,
help="Path to the configuration file with option defaults.")
run_parser.add("-s",
"--server",
required=True,
help="Lookout server address, e.g. localhost:1234.")
run_parser.add("-w",
"--workers",
type=int,
default=1,
help="Number of threads which process Lookout events.")
add_model_repository_args(run_parser)
run_parser.add_argument(
"--request-server",
default="auto",
help="Address of the data retrieval service. \"same\" means --server.")
init_parser = add_parser("init", "Initialize the model repository.")
init_parser.set_defaults(handler=init_repo)
add_model_repository_args(init_parser)
tool_parser = add_parser("tool", "Invoke the tooling of a given analyzer.")
tool_parser.set_defaults(handler=run_analyzer_tool)
tool_parser.add("analyzer",
help="Fully qualified package name with an analyzer.")
tool_parser.add("args", nargs=argparse.REMAINDER)
package_parser = add_parser(
"package",
"Package several analyzers to a Docker container and write a sample Docker Compose config "
"for Lookout.")
package_parser.set_defaults(handler=package_cmdline_entry)
add_analyzer_arg(package_parser)
package_parser.add("-w",
"--workdir",
help="Generate files in this directory.",
default=tempfile.mkdtemp(prefix="lookout_package_"))
package_parser.add("--requirements",
help="Path to a custom requirements.txt")
package_parser.add("-r",
"--repo",
help="GitHub repository name to watch. "
"Example: \"src-d/lookout\".",
required=True)
package_parser.add(
"-u",
"--user",
help="GitHub user name which will send review comments.",
required=True)
paturl = "https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/" # noqa
package_parser.add("-t",
"--token",
help="GitHub token for -u/--user. See " + paturl,
required=True)
package_parser.add("-y",
"--yes",
help="Run the commands in the end.",
action="store_true")
package_parser.add("-n",
"--no",
help="Do not run the commands in the end.",
action="store_true")
return parser
""" Configuration
Author: Thomas Rampelberg
Date: 2/24/2017
\\//
\/apor IO
"""
import configargparse
parser = configargparse.ArgParser(
config_file_parser_class=configargparse.YAMLConfigFileParser,
default_config_files=['/code/config.yaml'])
backend_help = 'Name/path combination for the backend to use. ' + \
'example: "backend;;http://demo.vapor.io:5000"'
parser.add('-c', '--my-config', is_config_file=True, help='config file path')
parser.add('--port', env_var='PORT', default=5050, help='Port to listen on.')
parser.add('--backend',
env_var='BACKEND',
nargs='+',
default=['backend;;http://synse-server:5000'],
help=backend_help)
parser.add('--cert-bundle',
env_var='CERT_BUNDLE',
help='Client bundle to use for mutual auth. ' +
'(should contain both the private key and certificate')
parser.add('--ssl-verify',
env_var='SSL_VERIFY',
def parse_args(args):
"""Parse command line parameters using qary.ini for the default values
Args:
args ([str]): command line parameters as list of strings
Returns:
:obj:`argparse.Namespace`: command line parameters namespace
"""
print(f"args: {args}")
parser = configargparse.ArgParser(
default_config_files=[
'~/qary.ini',
# '~/qary.ini',
# '~/nlpiabot.ini',
# '~/nlpia.ini',
# os.path.join(BASE_DIR, '*.ini'),
os.path.join(DATA_DIR, '*.ini'),
],
description="Command line bot application. Try `$ bot how do you work?`")
parser.add('-c', '--config', required=False, is_config_file=True,
help="Config file path (default: ~/qary.ini)")
parser.add_argument(
'-d', '--debug',
help="Set DEBUG logging level and raise more exceptions immediately.",
dest="debug",
default=str(DEFAULT_CONFIG['debug'])[0].lower() in 'fty1p',
action='store_true')
parser.add_argument(
'--version',
action='version',
version='qary {ver}'.format(ver=__version__))
parser.add_argument(
'--name',
default=None, # DEFAULT_CONFIG['name'],
dest="nickname",
help="IRC nick or CLI command name for the bot",
type=str,
metavar="STR")
parser.add_argument(
'-n',
'--num_top_replies',
default=None, # DEFAULT_CONFIG['num_top_replies'],
dest="num_top_replies",
help="Limit on the number of top (high score) replies that are randomly selected from.",
type=int,
metavar="INT")
parser.add_argument(
'-u',
'--use_cuda',
help="Use CUDA and GPU to speed up transformer inference.",
dest='use_cuda',
default=USE_CUDA,
action='store_true')
parser.add_argument(
'-p',
'--persist',
help="DEPRECATED: Don't exit. Retain language model in memory and maintain dialog until user says 'exit' or 'quit'",
dest='persist',
default=str(DEFAULT_CONFIG['persist'])[0].lower() in 'fty1p',
action='store_true')
parser.add_argument(
'-b',
'--bots',
default=None, # DEFAULT_CONFIG['bots'], # None so config.ini can populate defaults
dest="bots",
help="Comma-separated list of bot personalities to load. Defaults: pattern,parul,search_fuzzy,time,eliza",
type=str,
metavar="STR")
parser.add_argument(
'-q',
'--quiet',
dest="verbosity",
help="Quiet: set loglevel to ERROR",
action='store_const',
const=logging.ERROR)
parser.add_argument(
'-qq',
'--very_quiet',
dest="verbosity",
help="Very quiet: set loglevel to FATAL",
action='store_const',
const=logging.FATAL)
parser.add_argument(
'-v',
'--verbose',
dest="verbosity",
help="Verbose: set loglevel to INFO",
action='store_const',
const=logging.INFO)
parser.add_argument(
'-vv',
'--very_verbose',
dest="verbosity",
help="Verty verbose: set loglevel to DEBUG",
action='store_const',
const=logging.DEBUG)
parser.add_argument(
'-l',
'--loglevel',
dest="loglevel",
help="Raw integer loglevel (10=debug, 20=info, 30=warn, 40=error, 50=fatal)",
type=int,
default=DEFAULT_CONFIG['loglevel'])
parser.add_argument(
'--spacy_lang',
default=None, # None allows ini to set default
dest="spacy_lang",
help="SpaCy language model: en_core_web_sm, en_core_web_md, or en_core_web_lg",
type=str,
metavar="STR")
parser.add_argument(
'--wiki_title_max_words',
default=DEFAULT_CONFIG['wiki_title_max_words'],
dest="wiki_title_max_words",
help='Maximum n-gram length (in tokens) for wikipedia article title guesses.',
type=int,
metavar="INT")
parser.add_argument(
'-s',
'--score_weights',
default=DEFAULT_CONFIG['score_weights'],
dest="score_weights",
help='Dictionary of weights: {"spell": .5, "sentiment": .5, "semantics": .5}',
type=str,
metavar="DICT_STR")
parser.add_argument(
'--semantics',
type=float,
default=1.0,
dest='semantics',
metavar='FLOAT',
help='set weight of the semantic quality score')
parser.add_argument(
'--sentiment',
type=float,
default=0.5,
dest='sentiment',
metavar='FLOAT',
help='set weight of the sentiment quality score')
parser.add_argument(
'--spell',
type=float,
default=0.2,
dest='spell',
metavar='FLOAT',
help='set weight of the spell quality score')
parser.add_argument(
'words',
type=str,
nargs='*',
help="Words to pass to bot as an utterance or conversational statement requiring a bot reply or action.")
parser.add_argument(
'--qa_model',
help="Select which model qa_bots will use",
dest='qa_model',
default=DEFAULT_CONFIG['qa_model'],
type=str,
metavar='STR')
parsed_args = parser.parse_args(args)
print(parsed_args)
return parsed_args
from .conversion import sanatize
from dacapo.training_stats import TrainingStats
from dacapo.validation_scores import ValidationScores
from pymongo import MongoClient, ASCENDING
from pymongo.errors import DuplicateKeyError
import configargparse
import json
parser = configargparse.ArgParser(
default_config_files=['~/.config/dacapo', './dacapo.conf'])
parser.add('--mongo_db_host',
help="Name of the MongoDB host for stats and scores")
parser.add('--mongo_db_name',
help="Name of the MongoDB database for stats and scores")
class MongoDbStore:
def __init__(self):
"""Create a MongoDB sync. Used to sync runs, tasks, models,
optimizers, trainig stats, and validation scores."""
options = parser.parse_known_args()[0]
self.db_host = options.mongo_db_host
self.db_name = options.mongo_db_name
self.client = MongoClient(self.db_host)
self.database = self.client[self.db_name]
self.__open_collections()
self.__init_db()
def sync_run(self,
def get_args():
# f**k PEP8
defaultconfigpath = os.getenv(
'POGOMAP_CONFIG',
os.path.join(os.path.dirname(__file__), '../config/config.ini'))
parser = configargparse.ArgParser(default_config_files=[defaultconfigpath],
auto_env_var_prefix='POGOMAP_')
parser.add_argument('-cf',
'--config',
is_config_file=True,
help='Configuration file')
parser.add_argument(
'-a',
'--auth-service',
type=str.lower,
action='append',
default=[],
help=
'Auth Services, either one for all accounts or one per account: ptc or google. Defaults all to ptc.'
)
parser.add_argument('-u',
'--username',
action='append',
default=[],
help='Usernames, one per account.')
parser.add_argument(
'-p',
'--password',
action='append',
default=[],
help='Passwords, either single one for all accounts or one per account.'
)
parser.add_argument(
'-w',
'--workers',
type=int,
help=
'Number of search worker threads to start. Defaults to the number of accounts specified.'
)
parser.add_argument(
'-asi',
'--account-search-interval',
type=int,
default=0,
help=
'Seconds for accounts to search before switching to a new account. 0 to disable.'
)
parser.add_argument(
'-ari',
'--account-rest-interval',
type=int,
default=7200,
help='Seconds for accounts to rest when they fail or are switched out')
parser.add_argument(
'-ac',
'--accountcsv',
help=
'Load accounts from CSV file containing "auth_service,username,passwd" lines'
)
parser.add_argument('-l',
'--location',
type=parse_unicode,
help='Location, can be an address or coordinates')
parser.add_argument('-j',
'--jitter',
help='Apply random -9m to +9m jitter to location',
action='store_true',
default=False)
parser.add_argument('-st',
'--step-limit',
help='Steps',
type=int,
default=12)
parser.add_argument('-sd',
'--scan-delay',
help='Time delay between requests in scan threads',
type=float,
default=10)
parser.add_argument('-enc',
'--encounter',
help='Start an encounter to gather IVs and moves',
action='store_true',
default=False)
parser.add_argument(
'-ed',
'--encounter-delay',
help='Time delay between encounter pokemon in scan threads',
type=float,
default=1)
encounter_list = parser.add_mutually_exclusive_group()
encounter_list.add_argument(
'-ewht',
'--encounter-whitelist',
action='append',
default=[],
help='List of pokemon to encounter for more stats')
encounter_list.add_argument(
'-eblk',
'--encounter-blacklist',
action='append',
default=[],
help='List of pokemon to NOT encounter for more stats')
parser.add_argument('-ld',
'--login-delay',
help='Time delay between each login attempt',
type=float,
default=5)
parser.add_argument(
'-lr',
'--login-retries',
help='Number of logins attempts before refreshing a thread',
type=int,
default=3)
parser.add_argument(
'-mf',
'--max-failures',
help=
'Maximum number of failures to parse locations before an account will go into a two hour sleep',
type=int,
default=5)
parser.add_argument(
'-msl',
'--min-seconds-left',
help=
'Time that must be left on a spawn before considering it too late and skipping it. eg. 600 would skip anything with < 10 minutes remaining. Default 0.',
type=int,
default=0)
parser.add_argument('-dc',
'--display-in-console',
help='Display Found Pokemon in Console',
action='store_true',
default=False)
parser.add_argument('-H',
'--host',
help='Set web server listening host',
default='127.0.0.1')
parser.add_argument('-P',
'--port',
type=int,
help='Set web server listening port',
default=5000)
parser.add_argument('-L',
'--locale',
help='Locale for Pokemon names (default: {},\
check {} for more)'.format(config['LOCALE'],
config['LOCALES_DIR']),
default='en')
parser.add_argument('-c',
'--china',
help='Coordinates transformer for China',
action='store_true')
parser.add_argument(
'-m',
'--mock',
type=str,
help=
'Mock mode - point to a fpgo endpoint instead of using the real PogoApi, ec: http://127.0.0.1:9090',
default='')
parser.add_argument(
'-ns',
'--no-server',
help='No-Server Mode. Starts the searcher but not the Webserver.',
action='store_true',
default=False)
parser.add_argument(
'-os',
'--only-server',
help=
'Server-Only Mode. Starts only the Webserver without the searcher.',
action='store_true',
default=False)
parser.add_argument('-nsc',
'--no-search-control',
help='Disables search control',
action='store_false',
dest='search_control',
default=True)
parser.add_argument('-fl',
'--fixed-location',
help='Hides the search bar for use in shared maps.',
action='store_true',
default=False)
parser.add_argument('-k',
'--gmaps-key',
help='Google Maps Javascript API Key',
required=True)
parser.add_argument(
'--skip-empty',
help=
'Enables skipping of empty cells in normal scans - requires previously populated database (not to be used with -ss)',
action='store_true',
default=False)
parser.add_argument('-C',
'--cors',
help='Enable CORS on web server',
action='store_true',
default=False)
parser.add_argument('-D',
'--db',
help='Database filename',
default='pogom.db')
parser.add_argument(
'-cd',
'--clear-db',
help='Deletes the existing database before starting the Webserver.',
action='store_true',
default=False)
parser.add_argument(
'-np',
'--no-pokemon',
help=
'Disables Pokemon from the map (including parsing them into local db)',
action='store_true',
default=False)
parser.add_argument(
'-ng',
'--no-gyms',
help=
'Disables Gyms from the map (including parsing them into local db)',
action='store_true',
default=False)
parser.add_argument(
'-nk',
'--no-pokestops',
help=
'Disables PokeStops from the map (including parsing them into local db)',
action='store_true',
default=False)
parser.add_argument(
'-ss',
'--spawnpoint-scanning',
help=
'Use spawnpoint scanning (instead of hex grid). Scans in a circle based on step_limit when on DB',
nargs='?',
const='nofile',
default=False)
parser.add_argument(
'--dump-spawnpoints',
help='dump the spawnpoints from the db to json (only for use with -ss)',
action='store_true',
default=False)
parser.add_argument(
'-pd',
'--purge-data',
help='Clear pokemon from database this many hours after they disappear \
(0 to disable)',
type=int,
default=0)
parser.add_argument('-px',
'--proxy',
help='Proxy url (e.g. socks5://127.0.0.1:9050)',
action='append')
parser.add_argument('-pxsc',
'--proxy-skip-check',
help='Disable checking of proxies before start',
action='store_true',
default=False)
parser.add_argument('-pxt',
'--proxy-timeout',
help='Timeout settings for proxy checker in seconds ',
type=int,
default=5)
parser.add_argument(
'-pxd',
'--proxy-display',
help=
'Display info on which proxy beeing used (index or full) To be used with -ps',
type=str,
default='index')
parser.add_argument('--db-type',
help='Type of database to be used (default: sqlite)',
default='sqlite')
parser.add_argument('--db-name', help='Name of the database to be used')
parser.add_argument('--db-user', help='Username for the database')
parser.add_argument('--db-pass', help='Password for the database')
parser.add_argument('--db-host', help='IP or hostname for the database')
parser.add_argument('--db-port',
help='Port for the database',
type=int,
default=3306)
parser.add_argument('--db-max_connections',
help='Max connections (per thread) for the database',
type=int,
default=5)
parser.add_argument(
'--db-threads',
help='Number of db threads; increase if the db queue falls behind',
type=int,
default=1)
parser.add_argument('-wh',
'--webhook',
help='Define URL(s) to POST webhook information to',
nargs='*',
default=False,
dest='webhooks')
parser.add_argument(
'-gi',
'--gym-info',
help=
'Get all details about gyms (causes an additional API hit for every gym)',
action='store_true',
default=False)
parser.add_argument('--disable-clean',
help='Disable clean db loop',
action='store_true',
default=False)
parser.add_argument('--webhook-updates-only',
help='Only send updates (pokémon & lured pokéstops)',
action='store_true',
default=False)
parser.add_argument(
'--wh-threads',
help=
'Number of webhook threads; increase if the webhook queue falls behind',
type=int,
default=1)
parser.add_argument('--ssl-certificate',
help='Path to SSL certificate file')
parser.add_argument('--ssl-privatekey',
help='Path to SSL private key file')
parser.add_argument(
'-ps',
'--print-status',
action='store_true',
help=
'Show a status screen instead of log messages. Can switch between status and logs by pressing enter.',
default=False)
parser.add_argument(
'-sn',
'--status-name',
default=None,
help=
'Enable status page database update using STATUS_NAME as main worker name'
)
parser.add_argument('-spp',
'--status-page-password',
default=None,
help='Set the status page password')
parser.add_argument(
'-el',
'--encrypt-lib',
help='Path to encrypt lib to be used instead of the shipped ones')
parser.add_argument(
'-odt',
'--on-demand_timeout',
help=
'Pause searching while web UI is inactive for this timeout(in seconds)',
type=int,
default=0)
verbosity = parser.add_mutually_exclusive_group()
verbosity.add_argument(
'-v',
'--verbose',
help=
'Show debug messages from PomemonGo-Map and pgoapi. Optionally specify file to log to.',
nargs='?',
const='nofile',
default=False,
metavar='filename.log')
verbosity.add_argument(
'-vv',
'--very-verbose',
help=
'Like verbose, but show debug messages from all modules as well. Optionally specify file to log to.',
nargs='?',
const='nofile',
default=False,
metavar='filename.log')
verbosity.add_argument('-d',
'--debug',
help='Deprecated, use -v or -vv instead.',
action='store_true')
parser.set_defaults(DEBUG=False)
args = parser.parse_args()
if args.only_server:
if args.location is None:
parser.print_usage()
print(sys.argv[0] + ": error: arguments -l/--location is required")
sys.exit(1)
else:
# If using a CSV file, add the data where needed into the username,password and auth_service arguments.
# CSV file should have lines like "ptc,username,password", "username,password" or "username".
if args.accountcsv is not None:
# Giving num_fields something it would usually not get
num_fields = -1
with open(args.accountcsv, 'r') as f:
for num, line in enumerate(f, 1):
fields = []
# First time around populate num_fields with current field count.
if num_fields < 0:
num_fields = line.count(',') + 1
csv_input = []
csv_input.append('')
csv_input.append('<username>')
csv_input.append('<username>,<password>')
csv_input.append('<ptc/google>,<username>,<password>')
# If the number of fields is differend this is not a CSV
if num_fields != line.count(',') + 1:
print(
sys.argv[0] + ": Error parsing CSV file on line " +
str(num) +
". Your file started with the following input, '" +
csv_input[num_fields] + "' but now you gave us '" +
csv_input[line.count(',') + 1] + "'.")
sys.exit(1)
field_error = ''
line = line.strip()
# Ignore blank lines and comment lines
if len(line) == 0 or line.startswith('#'):
continue
# If number of fields is more than 1 split the line into fields and strip them
if num_fields > 1:
fields = line.split(",")
fields = map(str.strip, fields)
# If the number of fields is one then assume this is "username". As requested..
if num_fields == 1:
# Empty lines are already ignored.
args.username.append(line)
# If the number of fields is two then assume this is "username,password". As requested..
if num_fields == 2:
# If field length is not longer then 0 something is wrong!
if len(fields[0]) > 0:
args.username.append(fields[0])
else:
field_error = 'username'
# If field length is not longer then 0 something is wrong!
if len(fields[1]) > 0:
args.password.append(fields[1])
else:
field_error = 'password'
# If the number of fields is three then assume this is "ptc,username,password". As requested..
if num_fields == 3:
# If field 0 is not ptc or google something is wrong!
if fields[0].lower() == 'ptc' or fields[0].lower(
) == 'google':
args.auth_service.append(fields[0])
else:
field_error = 'method'
# If field length is not longer then 0 something is wrong!
if len(fields[1]) > 0:
args.username.append(fields[1])
else:
field_error = 'username'
# If field length is not longer then 0 something is wrong!
if len(fields[2]) > 0:
args.password.append(fields[2])
else:
field_error = 'password'
if num_fields > 3:
print 'Too many fields in accounts file: max supported are 3 fields. Found {} fields'.format(
num_fields)
sys.exit(1)
# If something is wrong display error.
if field_error != '':
type_error = 'empty!'
if field_error == 'method':
type_error = 'not ptc or google instead we got \'' + fields[
0] + '\'!'
print(
sys.argv[0] + ": Error parsing CSV file on line " +
str(num) + ". We found " + str(num_fields) +
" fields, so your input should have looked like '"
+ csv_input[num_fields] + "'\nBut you gave us '" +
line + "', your " + field_error + " was " +
type_error)
sys.exit(1)
errors = []
num_auths = len(args.auth_service)
num_usernames = 0
num_passwords = 0
if len(args.username) == 0:
errors.append(
'Missing `username` either as -u/--username, csv file using -ac, or in config'
)
else:
num_usernames = len(args.username)
if args.location is None:
errors.append(
'Missing `location` either as -l/--location or in config')
if len(args.password) == 0:
errors.append(
'Missing `password` either as -p/--password, csv file, or in config'
)
else:
num_passwords = len(args.password)
if args.step_limit is None:
errors.append(
'Missing `step_limit` either as -st/--step-limit or in config')
if num_auths == 0:
args.auth_service = ['ptc']
num_auths = len(args.auth_service)
if num_usernames > 1:
if num_passwords > 1 and num_usernames != num_passwords:
errors.append(
'The number of provided passwords ({}) must match the username count ({})'
.format(num_passwords, num_usernames))
if num_auths > 1 and num_usernames != num_auths:
errors.append(
'The number of provided auth ({}) must match the username count ({})'
.format(num_auths, num_usernames))
if len(errors) > 0:
parser.print_usage()
print(sys.argv[0] + ": errors: \n - " + "\n - ".join(errors))
sys.exit(1)
# Fill the pass/auth if set to a single value
if num_passwords == 1:
args.password = [args.password[0]] * num_usernames
if num_auths == 1:
args.auth_service = [args.auth_service[0]] * num_usernames
# Make our accounts list
args.accounts = []
# Make the accounts list
for i, username in enumerate(args.username):
args.accounts.append({
'username': username,
'password': args.password[i],
'auth_service': args.auth_service[i]
})
# Make max workers equal number of accounts if unspecified, and disable account switching
if args.workers is None:
args.workers = len(args.accounts)
args.account_search_interval = None
# Disable search interval if 0 specified
if args.account_search_interval == 0:
args.account_search_interval = None
# Make sure we don't have an empty account list after adding command line and CSV accounts
if len(args.accounts) == 0:
print(
sys.argv[0] +
": Error: no accounts specified. Use -a, -u, and -p or --accountcsv to add accounts"
)
sys.exit(1)
args.encounter_blacklist = [int(i) for i in args.encounter_blacklist]
args.encounter_whitelist = [int(i) for i in args.encounter_whitelist]
# Decide which scanning mode to use
if args.spawnpoint_scanning:
args.scheduler = 'SpawnScan'
elif args.skip_empty:
args.scheduler = 'HexSearchSpawnpoint'
else:
args.scheduler = 'HexSearch'
return args
def main():
parser = configargparse.ArgParser(
default_config_files=CONFIG_FILES,
description="Act on voice commands using Google's speech recognition")
parser.add_argument('-I',
'--input-device',
default='default',
help='Name of the audio input device')
parser.add_argument('-O',
'--output-device',
default='default',
help='Name of the audio output device')
parser.add_argument('-T',
'--trigger',
default='gpio',
choices=['clap', 'gpio', 'ok-google'],
help='Trigger to use')
parser.add_argument(
'--cloud-speech',
action='store_true',
help='Use the Cloud Speech API instead of the Assistant API')
parser.add_argument(
'-L',
'--language',
default='en-US',
help='Language code to use for speech (default: en-US)')
parser.add_argument('-l',
'--led-fifo',
default='/tmp/status-led',
help='Status led control fifo')
parser.add_argument('-p',
'--pid-file',
help='File containing our process id for monitoring')
parser.add_argument(
'--audio-logging',
action='store_true',
help='Log all requests and responses to WAV files in /tmp')
parser.add_argument('--assistant-always-responds',
action='store_true',
help='Play Assistant responses for local actions.'
' You should make sure that you have IFTTT applets for'
' your actions to get the correct response, and also'
' that your actions do not call say().')
parser.add_argument('--assistant-secrets',
default=os.path.expanduser('~/assistant.json'),
help='Path to client secrets for the Assistant API')
parser.add_argument('--cloud-speech-secrets',
default=os.path.expanduser('~/cloud_speech.json'),
help='Path to service account credentials for the '
'Cloud Speech API')
parser.add_argument('--trigger-sound',
default=None,
help='Sound when trigger is activated (WAV format)')
args = parser.parse_args()
create_pid_file(args.pid_file)
aiy.i18n.set_locale_dir(LOCALE_DIR)
aiy.i18n.set_language_code(args.language, gettext_install=True)
player = aiy.audio.get_player()
if args.cloud_speech:
credentials_file = os.path.expanduser(args.cloud_speech_secrets)
if not os.path.exists(credentials_file) and os.path.exists(
OLD_SERVICE_CREDENTIALS):
credentials_file = OLD_SERVICE_CREDENTIALS
recognizer = speech.CloudSpeechRequest(credentials_file)
else:
credentials = try_to_get_credentials(
os.path.expanduser(args.assistant_secrets))
recognizer = speech.AssistantSpeechRequest(credentials)
status_ui = StatusUi(player, args.led_fifo, args.trigger_sound)
# switch = GpioSwitch([action.reboot, action.shutdown])
# switch.start()
# The ok-google trigger is handled with the Assistant Library, so we need
# to catch this case early.
if args.trigger == 'ok-google':
if args.cloud_speech:
print('trigger=ok-google only works with the Assistant, not with '
'the Cloud Speech API.')
sys.exit(1)
do_assistant_library(args, credentials, player, status_ui)
else:
recorder = aiy.audio.get_recorder()
with recorder:
do_recognition(args, recorder, recognizer, player, status_ui)