def insert_data(self, booklist = []):
''' Directly use the database to get book list.
Parameters:
booklist -- The gtk.liststore into which the books will be added
'''
# Read the config file
try:
config = load_config.load_config()
self.calibre_base = HOME_DIR + "/" + config.calibre_db
logging.info(self.calibre_base)
except:
return
self.booklist = booklist
conn = sqlite3.connect(self.calibre_base)
c = conn.cursor()
# Get author title pair.
c.execute('select name, title from authors, books, books_authors_link \
where books.id=books_authors_link.book \
AND authors.id=books_authors_link.author \
;')
# Insert data into booklist
for row in c:
name = row[0]
author = []
author.append(name[-1]) # Last part
author.append(", ") # Decoration
author.append(' '.join(name[0:-1])) # All except last part adding a space between them
author = ''.join(author) # Join all elements into a string
self.booklist.append(['', row[0], row[1],
'', '', '', '0', 0, 0, 'e-book'])
return self.booklist
def insert_data(self, booklist = []):
''' Directly use the database to get book list.
Parameters:
booklist -- The gtk.liststore into which the books will be added
'''
# Read the config file
try:
config = load_config.load_config()
self.calibre_base = HOME_DIR + "/" + config.calibre_db
logging.info(self.calibre_base)
except:
return
self.booklist = booklist
conn = sqlite3.connect(self.calibre_base)
c = conn.cursor()
# Get author title pair.
c.execute('select name, title from authors, books, books_authors_link \
where books.id=books_authors_link.book \
AND authors.id=books_authors_link.author \
;')
# Insert data into booklist
for row in c:
name = row[0]
author = []
author.append(name[-1]) # Last part
author.append(", ") # Decoration
author.append(' '.join(name[0:-1])) # All except last part adding a space between them
author = ''.join(author) # Join all elements into a string
self.booklist.append(['', row[0], row[1],
'', '', '', '0', 0, 0, 'e-book'])
return self.booklist
def main():
"""Client main function"""
config = {'id': os.getpid()}
config = load_config.load_config(config)
for i in range(3):
client = asyncClientClass.ClientTask(os.getpid(), config)
client.start()
def main():
parser = argparse.ArgumentParser(
description="""char-parrot: a character-level language model
using a GRU- or LSTM-based RNN, implemented with PyTorch
[Text generation script]""")
parser.add_argument("project_dir",
help="""Path to the project directory containing the
relevant model.ini configuration file. See
sample_project/model.ini for a commented example"""
)
parser.add_argument("-l",
"--load-file",
help="""Load previously saved model state from
project_dir/LOAD_FILE. The current configuration
must be consistent with that of the model
which generated this file""",
required=True)
parser.add_argument("-s",
'--seed',
help="""Seed used to predict the first character.
Must be at least as long as the number of time steps
specified in the config file""",
required=True)
parser.add_argument("-n",
"--length",
help="Length of sequence to predict and print.",
required=False,
default=250)
parser.add_argument("-t",
"--temperature",
help="""Temperature to use when predicting the
next character. Lower is more greedy, higher is
more random""",
required=False,
default=1)
args = parser.parse_args()
os.chdir(args.project_dir)
config = load_config()
char_parrot = model.CharParrot(
model_type=config['model_type'],
dataset_file=config['dataset_file'],
case_sensitive=bool(int(config['case_sensitive'])),
time_steps=int(config['time_steps']),
batch_size=int(config['batch_size']),
hidden_size=int(config['hidden_size']),
nb_layers=int(config['nb_layers']),
dropout=float(config['dropout']),
learning_rate=float(config['learning_rate']),
zero_hidden=bool(int(config['zero_hidden'])),
save_file=None)
char_parrot.load(args.load_file, True)
char_parrot.generate(args.seed, int(args.length),
int(config['time_steps']), float(args.temperature))
def count_experiments(series: Union[dict, List[dict]]) -> int:
if type(series) != list:
series = [series]
n = 0
for item in series:
if type(item) is list:
# Implicit series
n += count_experiments(item)
elif 'series' in item:
# Composite experiment with explicit series
n += 1 + sum(
count_experiments(load_config(path))
for path in item['series'])
else:
# Single experiment
n += 1
if 'base_experiment' in item:
n += count_experiments(load_config(item['base_experiment']))
return n
def __init__(
self,
box_roi_pool,
box_head,
box_predictor,
# Faster R-CNN training
fg_iou_thresh,
bg_iou_thresh,
batch_size_per_image,
positive_fraction,
bbox_reg_weights,
# Faster R-CNN inference
score_thresh,
nms_thresh,
detections_per_img,
# Mask
mask_roi_pool=None,
mask_head=None,
mask_predictor=None,
keypoint_roi_pool=None,
keypoint_head=None,
keypoint_predictor=None,
):
super(RoIHeads, self).__init__()
self.box_similarity = box_ops.box_iou
# assign ground-truth boxes for each proposal
self.proposal_matcher = det_utils.Matcher(
fg_iou_thresh, bg_iou_thresh, allow_low_quality_matches=False)
self.fg_bg_sampler = det_utils.BalancedPositiveNegativeSampler(
batch_size_per_image, positive_fraction)
if bbox_reg_weights is None:
bbox_reg_weights = (10., 10., 5., 5.)
self.box_coder = det_utils.BoxCoder(bbox_reg_weights)
self.box_roi_pool = box_roi_pool
self.box_head = box_head
self.box_predictor = box_predictor
self.score_thresh = score_thresh
self.nms_thresh = nms_thresh
self.detections_per_img = detections_per_img
self.mask_roi_pool = mask_roi_pool
self.mask_head = mask_head
self.mask_predictor = mask_predictor
self.keypoint_roi_pool = keypoint_roi_pool
self.keypoint_head = keypoint_head
self.keypoint_predictor = keypoint_predictor
self.boundaries, self.types_limit = load_config()
def simulation_originale():
CONFIG = load_config()
simulation = SimulationOriginale(
float(CONFIG['CONSTANTE']['X']), float(CONFIG['CONSTANTE']['Y']),
float(CONFIG['CONSTANTE']['Z']), float(CONFIG['CONSTANTE']['W']),
float(CONFIG['VARIABLE']['lambda']), float(CONFIG['VARIABLE']['mu']),
float(CONFIG['VARIABLE']['alpha']), float(CONFIG['VARIABLE']['beta']),
int(CONFIG['SIMULATION']['nombre_serveurs']),
float(CONFIG['SIMULATION']['temps_simulation']))
simulation.simulation_magasin()
def init_db() -> MySQLDatabase:
db_config = asyncio.run(load_config())['DataBase']
# 定义数据库对象
db = MySQLDatabase(
database=db_config['DB'],
user=db_config['User'],
password=db_config['Password'],
host=db_config['Host'],
port=db_config['Port']
)
return db
def master(comm):
size = comm.Get_size()
cfg = load_config('sconfig')
file_size = int(cfg['file_size'])
mb_per_img = cfg['image_size']
batch_size = int(cfg['batch_size'])
files = glob.glob(cfg['path'] + '/swmr_file*')
for fil in files:
os.remove(fil)
av_files = []
def classification_embed2d(config: dict, run_args: dict):
base_experiment, _ = experiment_main(
load_config(config['base_experiment']), run_args)
encoder = base_experiment.model.get_encoder()
encoder.requires_grad = False
exp_params = config['exp_params']
c, h, w = get_example_shape(exp_params['data'])
model = create_model(**config['model_params'],
width=w,
height=h,
channels=c,
encoder=encoder)
return ClassificationExperiment(model, params=exp_params)
def create_viz_production(results_table_name='final_model_eval'):
"""
Creates visualizations for models that are trained on the full training data set,
and are used in production.
Parameters
----------
results_table_name : str
The name of a SQL table which contains results (from the test set) about the final
models.
"""
# set up required variables
local_paths_env = load_local_paths('../pipeline/local_paths.yaml')
env = load_psql_env(local_paths_env['pgpass_path'])
ignition = load_config(local_paths_env['ignition_path'] +
'_1_baseline_ignition.yaml')
# open sql connection
connection = SQLConn(env)
connection.open()
# pull data from table
query = f"select * from results.{results_table_name};"
results_df = pd.read_sql_query(query, connection.conn)
results_df['label'] = results_df['review_group']
# create directory for visualizations
vis_dir = f'{local_paths_env["store_visualizations"]}/production_citations'
# create folders to store visualizations
if not os.path.exists(vis_dir):
os.makedirs(vis_dir)
# precision recall plots
for rg in tqdm(results_df['review_group'].unique()):
plot = plot_precision_recall_curve_best(results_df,
rg,
plot_baseline=False)
plt.savefig(f'{vis_dir}/pr_curve-{rg}.png')
plt.close()
# stacked bar workload
plot = workload_relative_stackedbar(results_df)
plt.savefig(f'{vis_dir}/workload_relative.png')
plt.close()
plot_average_workload_reduction(results_df)
plt.savefig(f'{vis_dir}/workload_average.png')
plt.close()
def main():
parser = argparse.ArgumentParser(
description="""char-parrot: a character-level language model
using a GRU- or LSTM-based RNN, implemented with PyTorch
[Training script]""")
parser.add_argument("project_dir",
help="""Path to the project directory containing the
relevant model.ini configuration file. See
sample_project/model.ini for a commented example"""
)
parser.add_argument("-e",
"--epochs",
help="Number of training epochs",
required=False,
default=10)
parser.add_argument(
"-s",
"--save-file",
help="""Save model state to project_dir/SAVE_FILE after
every epoch, overwriting any existing file""",
required=False)
parser.add_argument(
"-l",
"--load-file",
help="""Load model state from project_dir/LOAD_FILE. The
current configuration must be consistent with
that of the model which generated this file""",
required=False)
args = parser.parse_args()
os.chdir(args.project_dir)
config = load_config()
char_parrot = model.CharParrot(
model_type=config['model_type'],
dataset_file=config['dataset_file'],
case_sensitive=bool(int(config['case_sensitive'])),
time_steps=int(config['time_steps']),
batch_size=int(config['batch_size']),
hidden_size=int(config['hidden_size']),
nb_layers=int(config['nb_layers']),
dropout=float(config['dropout']),
learning_rate=float(config['learning_rate']),
zero_hidden=bool(int(config['zero_hidden'])),
save_file=args.save_file)
if args.load_file:
char_parrot.load(args.load_file)
char_parrot.train(int(args.epochs))
def run():
__CONFIG__ = load_config()
print('Preparing base folder ... ')
if not os.path.exists(os.path.join(__CONFIG__['base-folder'], 'classificator')):
os.makedirs(os.path.join(__CONFIG__['base-folder'], 'classificator'))
print('Loading data ...')
max_dataset_size = __CONFIG__['max-dataset-size'] if __CONFIG__['max-dataset-size'] > 0 else None
positive_documents_train, positive_documents_test = load_documents(os.path.join(__CONFIG__['input-folder'],
'positive'), max_dataset_size, __CONFIG__)
unlabeled_documents_train, unlabeled_documents_test = load_documents(os.path.join(__CONFIG__['input-folder'],
'negative'), max_dataset_size, __CONFIG__)
positive_documents_train, unlabeled_documents_train = unskew(positive_documents_train, unlabeled_documents_train)
positive_documents_test, unlabeled_documents_test = unskew(positive_documents_test, unlabeled_documents_test)
print("Training ...")
tokenizer = SimpleGermanTokenizer()
gt = []
for document in positive_documents_train:
gt.append('medical')
for document in unlabeled_documents_train:
gt.append('nonmedical')
pipeline = Pipeline([
('vectorizer', TfidfVectorizer(tokenizer=tokenizer.tokenize)),
('classifier', MultinomialNB()) ])
pipeline.fit(positive_documents_train + unlabeled_documents_train, gt)
model_path = os.path.join(__CONFIG__['base-folder'], 'classificator', __CONFIG__['classifier-name'] + '.pickle')
print("Saving model to %s ..." % model_path)
with open(model_path, 'wb') as file:
pickle.dump(pipeline, file)
print('Testing ...')
gt_test = []
for document in tqdm.tqdm(positive_documents_test):
gt_test.append('medical')
for document in tqdm.tqdm(unlabeled_documents_test):
gt_test.append('nonmedical')
predictions = pipeline.predict(positive_documents_test+unlabeled_documents_test)
print(confusion_matrix(gt_test, predictions))
print('F1: %s' % f1_score(gt_test, predictions, pos_label='medical'))
print('Accuracy: %s' % accuracy_score(gt_test, predictions))
print('Precision: %s' % precision_score(gt_test, predictions, pos_label='medical'))
print('Recall: %s' % recall_score(gt_test, predictions, pos_label='medical'))
def init_celery() -> Celery:
celery_config = asyncio.run(load_config())['CeleryRedis']
# 配置uri
broker_uri = backend_uri = "redis://{User}:{Password}@{Host}:{Port}/{DB}".format(
User=celery_config['User'],
Password=quote_plus(celery_config['Password']),
Host=celery_config['Host'],
Port=celery_config['Port'],
DB=celery_config['DB'])
# 配置Celery
celery_obj = Celery('celery_app', broker=broker_uri, backend=backend_uri)
return celery_obj
def hparam_search(config: dict, run_args: dict):
import ray
from ray import tune
run_config = load_config(config['experiment'])
run_config = generate_run_config(run_config)
run_config['trainer_params'] = deep_merge(
run_config['trainer_params'].copy(),
{
'max_steps': config['num_train_steps'],
#'val_check_interval': config['num_train_steps'],
'limit_val_batches': config['num_val_steps'],
#'log_every_n_steps': 1,
#'max_epochs': config.get('num_epochs', 1),
})
if config.get('randomize_seed', False):
print('Warning: randomizing seed for each trial')
run_config['manual_seed'] = tune.sample_from(
lambda spec: np.random.randint(0, 64_000))
ray.init(num_gpus=1)
print("ray.get_gpu_ids(): {}".format(ray.get_gpu_ids()))
analysis = tune.run(
tune.with_parameters(experiment_main,
run_args=dict(**run_args, enable_tune=True)),
name=run_config['entrypoint'],
config=run_config,
local_dir=run_args['save_dir'],
num_samples=config['num_samples'],
resources_per_trial=deep_merge({
'cpu': 6,
'gpu': 1,
}, config.get('resources_per_trial', {})),
)
metric = config.get('metric', 'val/loss')
scope = config.get('scope', 'last')
best_config = get_best_config(analysis=analysis,
metric=metric,
scope=scope)
# Restore original trainer_params, which were overridden
# so the hparam search is shorter than a full experiment.
best_config['trainer_params'] = config['trainer_params']
print('Best config:')
print(best_config)
experiment_main(best_config, run_args)
def __init__(self, comm, filt):
self.cfg = load_config('sconfig')
self.comm = comm
self.rank = comm.Get_rank()
self.size = comm.Get_size()
self.filt = filt
self.master_msg = msg(self.rank)
self.total_read = np.zeros(self.size - 1)
self.total_events = np.zeros(self.size - 1)
self.total_file_size = 0
self.cum_event_num = 0
self.file0_length = 0
self.final_length = -1
self.sd_eof = False
# self.master_dump = open('dump/master_dump.txt', 'w')
self.eof_lock = False
self.last_timestamps = []
if self.filt:
self.hit_probability = float(self.cfg['hit_probability'])
else:
self.hit_probability = 1
self.diode_lp = np.zeros(self.size - 1)
def search_calibre(self, needle, booklist = []):
''' Search the calibre database on the parameters given.
The DB is searched twice for title and name. Adding an OR clause to
the search made te query v e r y s l o w to execute. Much
quicker to do two searches.
'''
import string
mybooklist = booklist
try:
config = load_config.load_config()
self.calibre_base = HOME_DIR + "/" + config.calibre_db
#logging.info(self.calibre_base)
except:
return
self.booklist = booklist
conn = sqlite3.connect(self.calibre_base)
c = conn.cursor()
# Get author title pair.
query = "select name, title from authors, books, books_authors_link \
where books.id=books_authors_link.book \
AND authors.id=books_authors_link.author \
AND title LIKE '%%%s%%';" % (needle)
c.execute(query)
# Insert data into booklist
for row in c:
mybooklist.append(['', row[0], row[1], \
'', '', '', '0', 0, 0, 'e-book'])
query = "select name, title from authors, books, books_authors_link \
where books.id=books_authors_link.book \
AND authors.id=books_authors_link.author \
AND name LIKE '%%%s%%';" % (needle)
c.execute(query)
# Insert data into booklist
for row in c:
mybooklist.append(['', row[0], row[1], \
'', '', '', '0', 0, 0, 'e-book'])
return mybooklist
def search_calibre(self, needle, booklist = []):
''' Search the calibre database on the parameters given.
The DB is searched twice for title and name. Adding an OR clause to
the search made te query v e r y s l o w to execute. Much
quicker to do two searches.
'''
import string
mybooklist = booklist
try:
config = load_config.load_config()
self.calibre_base = HOME_DIR + "/" + config.calibre_db
#logging.info(self.calibre_base)
except:
return
self.booklist = booklist
conn = sqlite3.connect(self.calibre_base)
c = conn.cursor()
# Get author title pair.
query = "select name, title from authors, books, books_authors_link \
where books.id=books_authors_link.book \
AND authors.id=books_authors_link.author \
AND title LIKE '%%%s%%';" % (needle)
c.execute(query)
# Insert data into booklist
for row in c:
mybooklist.append(['', row[0], row[1], \
'', '', '', '0', 0, 0, 'e-book'])
query = "select name, title from authors, books, books_authors_link \
where books.id=books_authors_link.book \
AND authors.id=books_authors_link.author \
AND name LIKE '%%%s%%';" % (needle)
c.execute(query)
# Insert data into booklist
for row in c:
mybooklist.append(['', row[0], row[1], \
'', '', '', '0', 0, 0, 'e-book'])
return mybooklist
def simulation_originale_masse():
"""
Fait des simulations un grand nombre de fois et crée un fichier csv avec les résultats.
"""
CONFIG = load_config()
with open('simulation_originale1.csv', 'w') as myfile:
wr = csv.writer(myfile, quoting=csv.QUOTE_ALL)
for a in range(10, 36):
recettes = []
for i in range(100):
simulation = SimulationOriginale(
float(CONFIG['CONSTANTE']['X']),
float(CONFIG['CONSTANTE']['Y']),
float(CONFIG['CONSTANTE']['Z']),
float(CONFIG['CONSTANTE']['W']),
float(CONFIG['VARIABLE']['lambda']),
float(CONFIG['VARIABLE']['mu']),
float(CONFIG['VARIABLE']['alpha']),
float(CONFIG['VARIABLE']['beta']), a,
float(CONFIG['SIMULATION']['temps_simulation']))
simulation.simulation_magasin()
recettes.append(simulation.recette)
wr.writerow(recettes)
def main():
import argparse
from load_config import load_config
parser = argparse.ArgumentParser()
parser.add_argument('pickle_dump')
parser.add_argument('output_file')
parser.add_argument('-c',
'--config_filename',
dest='config_filename',
help="Config file with private info",
default=None)
parser.add_argument(
'--max_happy_commute',
default=45,
type=float,
help=
"For plot that overlays all the commutes, what's the longest not colored red?"
)
parser.add_argument('--center_lat',
default=34.053695,
help="latitude to center on")
parser.add_argument('--center_lng',
default=-118.430208,
help="longitutde to center on")
parser.add_argument(
'--zoom',
default=11,
type=int,
help="initial zoom of maps. goes 1 (least zoomed) to 20 (most zoomed)"
)
parser.add_argument(
'--map_type',
default='roadmap',
help="initial zoom of maps. goes 1 (least zoomed) to 20 (most zoomed)"
)
parser.add_argument(
'--palette',
default='Viridis',
help="Palette to use. Must be in bokeh.palettes.all_palettes")
parser.add_argument(
'--ncolors',
type=int,
default=256,
help=
"Number of colors to use. Must be able to access bokeh.palettes.all_palettes[<palette>][<ncolors>]"
)
parser.add_argument('--cbar_min', default=15, type=float)
parser.add_argument('--cbar_max', default=75, type=float)
args = parser.parse_args()
config, timezome = load_config(args.config_filename)
api_key = config['api_key']
with open(args.pickle_dump, 'rb') as f:
print(f"Loading from {args.pickle_dump}")
data = pickle.load(f)
center_lats = np.array(data.pop('lat'))
center_longs = np.array(data.pop('long'))
names = set(k.split('_')[0] for k in data)
dx = np.max(center_longs[1:] - center_longs[:-1]) / 2
dy = np.max(center_lats[1:] - center_lats[:-1]) / 2
xcoords = [[xc - dx, xc - dx, xc + dx, xc + dx] for xc in center_longs]
ycoords = [[yc - dy, yc + dy, yc + dy, yc - dy] for yc in center_lats]
print(f"Plotting {len(xcoords)} squares")
try:
args.center_lat = float(args.center_lat)
except ValueError:
args.center_lat = (min([yc[0] for yc in ycoords]) +
max([yc[1] for yc in ycoords])) / 2
try:
args.center_lng = float(args.center_lng)
except ValueError:
args.center_lng = (min([xc[0] for xc in ycoords]) +
max([xc[2] for xc in ycoords])) / 2
plots = []
bk.output_file(args.output_file, title="Commute times"), #mode="inlne")
moptions = GMapOptions(lat=args.center_lat,
lng=args.center_lng,
zoom=args.zoom,
map_type=args.map_type)
allkeys = [
f'{name}_{destkey}'
for name, destkey in product(names, ['towork', 'tohome'])
]
dsets = {restructure_key(key): data[key] for key in allkeys}
color_mapper = LinearColorMapper(
palette=all_palettes[args.palette][args.ncolors],
low=args.cbar_min,
high=args.cbar_max)
nhappy = np.zeros(len(xcoords), dtype=int)
for name in names:
for destkey in ['towork', 'tohome']:
key = f'{name}_{destkey}'
colors = data[key]
plots.append(
plot_patches_on_gmap(xcoords,
ycoords,
api_key,
values=colors,
map_options=moptions,
title=restructure_key(key),
color_mapper=color_mapper))
msk = np.array(colors) <= args.max_happy_commute
nhappy[msk] += 1
## now overlap all the commutes:
title = f'Areas where all commutes are < {args.max_happy_commute} minutes'
plot = plot_patches_on_gmap(
list(np.array(xcoords)[nhappy < len(allkeys) - 1]),
list(np.array(ycoords)[nhappy < len(allkeys) - 1]),
api_key,
map_options=moptions,
title=title,
solid_fill='red')
data = dict(xs=list(np.array(xcoords)[nhappy == len(allkeys) - 1]),
ys=list(np.array(ycoords)[nhappy == len(allkeys) - 1]))
source_patches = bk.ColumnDataSource(data=data)
patches_glyph = plot.patches('xs',
'ys',
fill_alpha=0.25,
fill_color='orange',
source=source_patches,
line_width=0)
plots.append(plot)
## now show
grid = gridplot(plots, ncols=2)
bk.show(grid)
worker.start()
self.proxy_check_workers.append(worker)
# 启动下载线程
def _start_new_worker(self):
worker = DownloadWorker(self)
# worker.setDaemon(True)
worker.start()
self.download_workers.append(worker)
# 启动解析线程
def _start_new_extractor(self):
worker = ExtractorWorker(self)
# worker.setDaemon(True)
worker.start()
self.extractor_workers.append(worker)
if __name__ == "__main__":
fileConfig("logger_config.ini")
xq_config = load_config("xq_user_article")
services = []
xq_article_service = XQUserArticleService(xq_config)
xq_cube_service = XQUserCubeService(xq_config)
xq_stock_service = XQUserStockService(xq_config)
services.append(xq_article_service)
services.append(xq_cube_service)
services.append(xq_stock_service)
crawler = Crawler(xq_config, services)
crawler.start()
import logging, load_config, pprint
from cryptorclient import CryptorClient
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger('Cryptor Daemon')
settings = load_config.load_config('E:\Cryptor\.config\cryptor.cfg')
logger.info('Loaded Config')
client = CryptorClient(settings)
logger.info('Client created successfully')
if client.verify_environment():
logger.info('Environment is verified')
else:
logger.info('EXITING: Reason - Environment is not verified')
exit()
old_delta = client.db.delta(path_prefix='/Cryptor')
count = 0
flag = True
while(flag):
count+=1
delta = client.db.delta(old_delta['cursor'], client.settings['db_home'])
diff = delta['entries']
from trackbot import TrackBot
from load_config import load_config
config = load_config('config.json')
imei = "442283480893012"
tracker = TrackBot(
config['host'],
config['port'],
config['buffer_size'],
imei
)
# while True:
# tracker.ping_tcp_random()
while True:
print tracker.ping_api_random('123', 'Shaphil')
def main():
"""Server main function"""
config = load_config.load_config({})
server = asyncServerClass.ServerTask(config)
server.start()
server.join()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('outname')
parser.add_argument('-c',
'--config_filename',
dest='config_filename',
help="Config file with private info",
default=None)
parser.add_argument(
'--state_name',
default="California",
help=
"State to use for the boundary to distinguish land from water. Points outside the state are discarded"
)
parser.add_argument(
'--npts',
default=25,
type=int,
help=
"Number of points on each side of the grid (i.e. will use npts**2 points total)"
)
parser.add_argument('--northern_limit', default=34.219498, type=float)
parser.add_argument('--southern_limit', default=33.816168, type=float)
parser.add_argument('--eastern_limit', default=-118.254013, type=float)
parser.add_argument('--western_limit', default=-118.606110, type=float)
args = parser.parse_args()
config, timezome = load_config(args.config_filename)
CommuteTimes = CommuteTimesClass(key=config['api_key'])
commutes = config['commutes']
xv = np.linspace(args.western_limit, args.eastern_limit, args.npts)
yv = np.linspace(args.southern_limit, args.northern_limit, args.npts)
pairs = np.array(np.meshgrid(xv, yv)).T.reshape(-1, 2)
xvals, yvals = pairs.T
if args.state_name.lower() != 'none':
import shapely.vectorized
import shapely.geometry as sgeom
import cartopy.io.shapereader as shpreader
shpfilename = shpreader.natural_earth(resolution='10m',
category='cultural',
name='admin_1_states_provinces')
reader = shpreader.Reader(shpfilename)
states = list(reader.records())
state, = [
state for state in states
if state.attributes['name'] == args.state_name
]
geom = state.geometry
mask = shapely.vectorized.contains(geom, xvals, yvals)
else:
mask = np.ones(xvals.size, dtype=bool)
result = defaultdict(list)
names = commutes.keys()
for ii, (ll, la) in enumerate(tqdm(pairs)):
if mask[ii]:
try:
address = f'{la},{ll}'
res = CommuteTimes.get_commute_times(address,
commutes,
2019,
8,
7,
2,
timezone,
models=['best_guess'],
do_print=False,
do_pbar=False)
result['lat'].append(la)
result['long'].append(ll)
for key in res:
result[key].append(res[key])
except ValueError:
pass
print(f"Writing output to {args.outname}...")
with open(args.outname, 'wb') as out:
pickle.dump(result, out)
print("Done!")
logging.basicConfig(
format='%(module)s: LINE %(lineno)d: %(levelname)s: %(message)s',
level=logging.DEBUG)
#logging.disable(logging.INFO)
DEBUG = logging.debug
INFO = logging.info
# Get system platform
plat = sys.platform
# Get the real location of this script
iamhere = os.path.dirname(os.path.realpath(__file__))
#print "I am here",iamhere
# Read the config file
config = config.load_config() # For file based config
try:
db_user = config.db_user
db_pass = config.db_pass
db_base = config.db_base
db_host = config.db_host
db_lite = config.lite_db
use = config.use # What DB type to use
except:
print "\nThere There is some error in the config file.\nCannot continue!\n\n "
quit()
########################################################################
class sqlite:
def pull_results(ignition_ids,
table_name='results.evaluate_rg',
metric_col='metrics',
metrics=['precision_at_recall'],
other_cols=[
'ignition_id', 'hash_id, algorithm', 'hyperparameters',
'fold', 'recall'
]):
"""
Pull results from PSQL table into long dataframe.
Parameters
==========
ignition_ids : list
List of ignition_ids to pull into table.
table_name : str
Name of PSQL table with results.
metric_col : str
Name of column where metrics exist.
metrics : list
Metrics to be included in table. Will be parsed from jsonb.
other_cols : list
List of other columns to included in table as is.
labels : list
Labels to be included in table.
Returns
=======
results_df : pd.DataFrame
Long dataframe with results from specified ignition files, metrics, and labels.
"""
local_paths_env = load_local_paths('../pipeline/local_paths.yaml')
env = load_psql_env(local_paths_env['pgpass_path'])
ignition = load_config(local_paths_env['ignition_path'] +
'_1_baseline_ignition.yaml')
# establish SQL Connection
connection = SQLConn(env)
connection.open()
### Set up ###
results = {}
ignition_ids_sql = "('" + "','".join(ignition_ids) + "')"
other_cols_sql = ",".join(other_cols)
## Make one query for each label and store resulting df in a dict ###
i = 0
for label in ignition['classes']:
metrics_sql = f"'{label}' as label"
for metric in metrics:
metrics_sql += f",{metric_col} -> '{metric}' -> '{label.lower()}' as {metric}"
qy = f"""
select {other_cols_sql},
{metrics_sql}
from
{table_name}
where ignition_id in {ignition_ids_sql};
"""
results[label] = pd.read_sql_query(qy, connection.conn)
## Concatenate all dfs into one long df ###
results_df = pd.concat(results.values(), ignore_index=True)
connection.close()
return results_df
from pymongo import MongoClient
from urllib.parse import quote_plus
def connect_db(mongo_data: dict):
"""
Takes a loaded yaml file and returns a url to connect to.
THIS DOESN'T ESTABLISH A CONNECTION
"""
database = mongo_data['DATABASE']
database_url = mongo_data['DATABASE_URL']
port = mongo_data.get('PORT', 27017)
if all(key in mongo_data for key in ("USERNAME", "PASSWORD")): # I think this is ugly
username = quote_plus(mongo_data['USERNAME'])
password = quote_plus(mongo_data['PASSWORD'])
return MongoClient(host=[f'{database_url}:{port}'], username=username, password=password)
else:
print("NO USERNAME OR PASSWORD DETECTED. I WOULDN'T RECOMMEND RUNNING THIS IN PROD")
return MongoClient(host=[f'{database_url}:{port}'])
content_db_config = load_config('contentdb.yml')
user_db_config = load_config('userdb.yml')
content_connection = connect_db(content_db_config)
CONTENT_DB = content_connection[content_db_config['DATABASE']]
user_connection = connect_db(user_db_config)
USER_DB = user_connection[user_db_config['DATABASE']]
from pymongo import MongoClient
from load_config import load_config
from urllib.parse import quote_plus
cfg = load_config('blueprints/users/config.yml')
db = cfg['db']
(
PASSWORD = quote_plus(db['PASSWORD'])
DATABASE = db['DATABASE']
USERNAME = db['USERNAME']
DATABASE_URL = db['DATABASE_URL']
PORT = db['PORT']
conn = MongoClient(DATABASE_URL, PORT)
db = conn[DATABASE]
#auth = db.authenticate(USERNAME, PASSWORD)
server.start()
except KeyboardInterrupt:
server.stop()
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument("type", nargs="?", default="redfish")
args = parser.parse_args()
if (args.type == "redfish"):
import load_config
from redfish import app as redfish_app
# Load configuration settings
load_config.load_config()
run_rack_manager_default(app=redfish_app,
ssl_enabled=True,
host=load_config.service_host,
port=load_config.service_port,
number_threads=load_config.thread_pool)
elif (args.type == "legacy"):
from legacy import app as legacy_app
run_rack_manager_default(app=legacy_app, ssl_enabled=True, port=8000)
elif (args.type == "compile"):
from redfish import app as redfish_app
from legacy import app as legacy_app
# We just want to force the python files to compile to avoid multiple processes compiling
# the code at the same time.
from app import app
from flask import (
render_template,
redirect,
url_for,
session,
request,
Blueprint,
)
from load_config import load_config
cfg = load_config('blueprints/vault/config.yml')
vault = Blueprint(
'vault',
__name__,
template_folder='templates',
)
@vault.route('/')
def hello():
return 'Hello World'
import functools
import threading
from queue import Queue
from load_config import load_config, create_config
from output import run_txt_updater
from twitch_chat import run_twitch_chat
from vote_exec import run_vote
if __name__ == "__main__":
config = load_config("config.json")
c = create_config(config)
queue = Queue()
response_queue = Queue()
txt_update_queue = Queue()
twitch_chat_thread = threading.Thread(target=functools.partial(
run_twitch_chat, queue, response_queue, c, config))
voting_thread = threading.Thread(target=functools.partial(
run_vote, queue, response_queue, txt_update_queue, config["customize"],
config["credentials"]["nickname"]))
text_thread = threading.Thread(
target=functools.partial(run_txt_updater, txt_update_queue))
threads = [twitch_chat_thread, voting_thread, text_thread]
print("Starting!")
for t in threads:
t.start()
for t in threads:
t.join()
import random
import requests
import string
import stripe
from load_config import load_config
from mailchimp_config import mailchimp_client, mailing_list_id
from mongo import USER_DB
users = Blueprint(
'users',
__name__,
template_folder='templates',
)
users_collection = USER_DB['users']
cfg = load_config('config.yml')
STRIPE = cfg['stripe']
stripe.api_key = STRIPE['API_KEY']
SLACK = cfg['SLACK_TOKEN']
@users.route('/payment/<plan>/<coupon>', methods=['POST'])
@users.route('/payment/<plan>', methods=['POST'])
def payment_successful(plan, coupon=None):
email = request.form['stripeEmail']
session['email'] = email
if users_collection.find_one({'email': email, 'password':{'$exists': True}}):
return 'This email address is already registered.'
# Create Customer Account in Stripe
customer = stripe.Customer.create(
gettext.textdomain(APP)
_ = gettext.gettext
#logger = logging.getLogger("librarian")
logging.basicConfig(format='%(module)s: LINE %(lineno)d: %(levelname)s: %(message)s', level=logging.DEBUG)
#logging.disable(logging.INFO)
# Get system platform
plat = sys.platform
# Get the real location of this script
iamhere = os.path.dirname( os.path.realpath( __file__ ) )
#print "I am here",iamhere
# Read the config file
config = config.load_config() # For file based config
try:
db_user = config.db_user
db_pass = config.db_pass
db_base = config.db_base
db_host = config.db_host
db_lite = config.lite_db
use = config.use # What DB type to use
except:
print "\nThere There is some error in the config file.\nCannot continue!\n\n "
quit()
########################################################################
class sqlite:
' |- Home.styles.tsx\n'\
'\n'\
'- You can also provide an "-rn" flag at the and of command input.\n'\
' This option will generate the .styles file with "styled.Text``" instead of "styled.div``".\n'\
'\n'\
'- You can also run "rcs_config help" for more informations about changing defaults.'\
'\n\n'\
'https://github.com/erickvieira/react-styled-components-cli'
)
else:
if len(argv) > 2:
c_name = argv[2].lower()
if len(argv) > 3:
is_native = argv[3].lower() == '-rn'
else:
is_native = load_config()['is_native']
if len(argv) > 4:
is_jsx = argv[4].lower() == '-jsx'
if not c_name:
c_name = type
print('Generating: %s component' % c_name)
else:
print('Generating: %s %s' % (c_name, {
'p': 'page',
'c': 'component',
'ctx': 'context'
}[type]))
def loading():
dots = ['.' for i in range(0, 5)]
import os
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
from load_config import load_config
def plot_scores(params, scores):
plt.plot(params, scores)
plt.title('validation on the size of the word2vec dimensionality')
plt.ylabel('accuracy')
plt.xlabel('vector size')
plt.grid(True)
plt.savefig(base_folder + 'word2vec_dimensions_validation.pdf')
if __name__ == '__main__':
config = load_config()
base_folder = config['base_folder']
if not os.path.exists(base_folder):
os.makedirs(base_folder)
json.dump(config, open(base_folder + 'configuration.json','w'), indent=4, sort_keys=True)
scores = []
params = []
for i in range(1, 250, 10):
print("Validate parameter " + str(i))
config['lstm-layers'][0]['output-size'] = i
score = run(config)
scores.append(score)
params.append(i)
from pathlib import Path
from templates import component, index, styles, context
from load_config import load_config
ext = 'jsx' if load_config()['is_jsx'] else 'tsx'
def create_dir(dir_name='', super_dir=None):
root_path = '.' if ext == 'jsx' else './src'
if not dir_name:
raise Exception('cant find or create folder without dir_name')
else:
output_dir = None
if super_dir:
main_dir = Path(root_path) / f'{super_dir}'
main_dir.mkdir(exist_ok=True)
output_dir = main_dir / f'{dir_name.capitalize()}'
else:
output_dir = Path(root_path) / f'{dir_name.capitalize()}'
output_dir.mkdir(exist_ok=True)
return output_dir
def create_components(c_name='',
dir_path=Path('.') / f'file',
is_native=False):
if not dir_path:
raise Exception('failed to find a path to create components')
else:
component_filename = '%s.%s' % (c_name.capitalize(), ext)
index_filename = 'index.%s' % (ext)
from mpi4py import MPI
import h5py, subprocess, glob, os
import numpy as np
import time
import os, random
from load_config import load_config
cfg = load_config('sconfig')
hit_prob = float(cfg['hit_probability'])
write_limit = int(cfg['file_size'])
mb_per_img = int(cfg['image_size'])
batch_size = int(cfg['batch_size'])
bytes_per_batch = mb_per_img * batch_size * 10**6
disk_num = int(cfg['disk_num'])
path = cfg['path'] % disk_num
def read_client(comm, filt=0):
rank = comm.Get_rank()
size = comm.Get_size()
# mb_per_img = 1
# batches = 16
arr_size = mb_per_img * 500000
# path = '/drpffb/eliseo/data/xtc_lite/'
# file_name = path % rank + '/xtc_lite_%i.xtc' % rank
file_name = path + '/xtc_lite_%i.xtc' % rank
import data_utils as dutils from load_config import load_config, device cf = load_config() from logger import logger import torch, json, sys from data_utils import batch_to_wordpieces, wordpieces_to_bert_embs, build_token_to_wp_mapping, load_embeddings from sklearn.metrics import f1_score, classification_report, accuracy_score from colorama import Fore, Back, Style import random import nfgec_evaluate torch.manual_seed(123) torch.backends.cudnn.deterministic=True class ModelEvaluator(): def __init__(self, model, data_loader, word_vocab, wordpiece_vocab, hierarchy, bc, mode="train"): self.model = model self.data_loader = data_loader self.word_vocab = word_vocab self.wordpiece_vocab = wordpiece_vocab self.hierarchy = hierarchy self.bc = bc self.mode = mode self.best_f1_and_epoch = [0.0, -1]
import json
from os import listdir
from os.path import isfile, join
from load_config import load_config
if __name__ == "__main__":
DATA_PATH = './json_data/'
VECT_PATH = './vectors/'
files = [ f for f in listdir(DATA_PATH) if isfile(join(DATA_PATH, f)) ]
"""
Load base files with entities, ignored files and categories for this example.
"""
lc = load_config()
lc.read_config()
tokens = lc.get_entities()
ignored_files = lc.get_ignored()
CATEGORIES = lc.get_categories()
files = [f for f in files if f not in ignored_files and '-key' not in f]
for _file in files:
keys = {}
sfile = _file.split('.')
json_key_file = open(DATA_PATH+sfile[0]+'-key.'+sfile[1], 'r')
keys = json.loads(json_key_file.read())
json_key_file.close()
json_feature_file = open(DATA_PATH+sfile[0]+'.'+sfile[1], 'r')
features = json.loads(json_feature_file.read())
json_feature_file.close()
end_file = open(VECT_PATH+sfile[0]+'-vec.txt', 'a')
for feat_id in features:
'''
Add new borrowers to the list. Remove too I guess.
'''
import MySQLdb
import MySQLdb.cursors
import sys
import load_config
import logging
import book
import locale
import gettext
import popen2
locale.setlocale(locale.LC_ALL, '')
APP = 'librarian'
gettext.textdomain(APP)
_ = gettext.gettext
logger = logging.getLogger("librarian")
logging.basicConfig(
format='%(module)s: LINE %(lineno)d: %(levelname)s:%(message)s',
level=logging.DEBUG)
# Read the config file
config = load_config.load_config()
db_user = config.db_user
db_pass = config.db_pass
db_base = config.db_base
db_host = config.db_host
parser.add_argument('--validate',
dest="validate",
metavar='VALIDATE',
help='validation mode',
default=False)
args = parser.parse_args()
if args.smoke_test:
print(
'Executing smoke test - training will stop after a couple steps.')
cudnn.deterministic = True
cudnn.benchmark = True
decord.bridge.set_bridge('torch')
config = load_config(args.config)
total_experiments = count_experiments(config)
num_samples = args.num_samples or 1
deltas = []
for i in range(num_samples):
print(f'Running sample {i+1}/{num_samples} on cuda:{args.gpu}')
start = time.time()
run_series(config,
save_dir=args.save_dir,
num_threads=args.num_threads,
gpu=args.gpu,
exp_no=0,
total_experiments=total_experiments,
smoke_test=args.smoke_test,
def deploy():
load_config()
install_machine()
deploy_geoserver()
supervisor.update_config()
restart_services()
restart_geotaxi()
deploy_api()
