def merge_ratings():
def balance_classes(df, ndocs_per_class):
df_pos = df[df['label']==1][:int(ndocs_per_class)]
df_neg = df[df['label']==0][:int(ndocs_per_class)]
return df_pos.append(df_neg)
sub_space = lambda s: re.sub('\s+', ' ', s)
write_row = lambda l, f: f.write('\t'.join(l) + '\n')
filenames = glob('%s/*' % DATADIR)
with open(TMPFILE, 'w') as f:
write_row('id document label'.split(), f)
for filename in filenames:
for review in utils.read_json(filename):
rating = int(review['rating'])
if rating > 8: # positive 9 10
write_row([review['review_id'], sub_space(review['review']), '1'], f)
elif rating < 5: # negative 1 2 3 4
write_row([review['review_id'], sub_space(review['review']), '0'], f)
else: # neutral
pass
print('Ratings merged to %s' % TMPFILE)
df = pd.read_csv(TMPFILE, sep='\t', quoting=3)
df = df.fillna('')
np.random.seed(SEED)
df = df.iloc[np.random.permutation(len(df))]
df = balance_classes(df, NDOCS/2)
df.to_csv(RATINGSFILE, sep='\t', index=False)
print('Ratings written to %s' % RATINGSFILE)
def main():
# Check and get arguments
args = utils.check_args()
configFile = args.config
directory = args.directory
gz = args.gz
# If not config file within arguments
if not configFile:
utils.printWarning("Didn't find '--config' param. Try to read 'config.json'...")
configFile = utils.is_valid_json_file('config.json')
# Try read json config file
try:
config = utils.read_json(configFile)
config = config['config']
## TODO Exception to handle
finally:
utils.printSuccess("DONE")
# If config was loaded
if config:
try:
backupped = utils.make_backup(config, directory, gz)
finally:
utils.printSuccess("DONE")
utils.printSuccess("Close connection...")
utils.printSuccess("DONE")
def parse_raw_emdr_data(data_folder):
filenames = utils.get_filenames(data_folder)
total_orders = []
total_history = []
for f in filenames:
try:
j = json.loads(utils.read_json(f, data_folder))
rowsets = j['rowsets']
resultType = j['resultType']
if resultType=='orders':
for rowset in rowsets:
for row in rowset['rows']:
total_orders.append(row)
elif resultType=='history':
for rowset in rowsets:
typeID = rowset['typeID']
regionID = rowset['regionID']
for row in rowset['rows']:
row.append(typeID)
row.append(regionID)
total_history.append(row)
else:
print '[x] Result type is not orders or history.'
except Exception as e:
print 'Filename: ' + f
print e
return total_orders, total_history
def sentiment_count(filename):
lyrics = read_json(filename)
classifications = {'pos': 0, 'neutral': 0, 'neg': 0}
for lyric in lyrics:
classifications[lyric['sentiment']] += 1
print classifications
def load(self, filename):
data = u.read_json(filename)
if data is not None:
self._name = data[0]['name']
self._avatar = u.string_to_avatar(data[0]['avatar_name'])
self._avatar_name = data[0]['avatar_name']
self._colour = data[0]['colour']
else:
print "Error loading %s" % filename
def download(assembly_id, json, indir, outdir):
try:
url = utils.read_json('%s/%s' % (indir, json))\
['status_dict']['접수']['의안접수정보'][0]['문서'][0][1][1]
except KeyError as e:
try:
url = utils.read_json('%s/%s' % (indir, json))\
['status_dict']['접수']['의안접수정보'][0]['의안원문'][0][1][1]
except KeyError as e:
print json
print utils.read_json('%s/%s' % (indir, json))\
['status_dict']['접수']['의안접수정보'][0]
url = None
path = '%s/%s' % (outdir, json.replace('.json', '.pdf'))
if not os.path.isfile(path) and url:
urllib.urlretrieve(url, path)
print 'Downloaded %s' % path
def monitor(self): threading.Timer(self._interval, self.monitor).start() after_files = utils.directory_to_set(self._directory_path, self._extension) new_files = after_files.difference(self._before_files) if len(new_files) > 0 : for f in new_files : data_rows = utils.read_json(f) self._uploader.upload(data_rows) self._before_files = after_files
def get_order_book(self):
""" Gets the up-to-date and valid order book from the exchange, or None if
there was a problem (eg, the API service was unavailable at the
moment or the returned order book was invalid).
"""
# The URL for the API service is exchange-specific and should have been
# defined by subclasses.
json_data = read_json(self.url)
if json_data is None:
return None
order_book = self._parse_order_book_from_json(json_data)
if order_book is None or not validate_order_book(order_book):
return None
return order_book
def process(sources, output, force):
"""Download sources and process the file to the output directory.
\b
SOURCES: Source JSON file or directory of files. Required.
OUTPUT: Destination directory for generated data. Required.
"""
for path in utils.get_files(sources):
pathparts = utils.get_path_parts(path)
pathparts[0] = output.strip(os.sep)
pathparts[-1] = pathparts[-1].replace('.json', '.geojson')
outdir = os.sep.join(pathparts[:-1])
outfile = os.sep.join(pathparts)
source = utils.read_json(path)
urlfile = urlparse(source['url']).path.split('/')[-1]
if not hasattr(adapters, source['filetype']):
utils.error('Unknown filetype', source['filetype'], '\n')
continue
if os.path.isfile(outfile) and not force:
utils.error('Skipping', path, 'since generated file exists.',
'Use --force to regenerate.', '\n')
continue
utils.info('Downloading', source['url'])
try:
fp = utils.download(source['url'])
except IOError:
utils.error('Failed to download', source['url'], '\n')
continue
utils.info('Reading', urlfile)
try:
geojson = getattr(adapters, source['filetype']).read(fp, source['properties'])
except IOError:
utils.error('Failed to read', urlfile)
continue
finally:
os.remove(fp.name)
utils.make_sure_path_exists(outdir)
utils.write_json(outfile, geojson)
utils.success('Done. Processed to', outfile, '\n')
def setup_logging(save_dir,
log_config='logger/logger_config.json',
default_level=logging.INFO):
"""Setup logging configuration."""
log_config = Path(log_config)
if log_config.is_file():
config = read_json(log_config)
# Modify logging paths
for _, handler in config['handlers'].items():
if 'filename' in handler:
handler['filename'] = str(save_dir / handler['filename'])
logging.config.dictConfig(config)
else:
print('Warning: logging configuration file is not found in {}.'.format(
log_config))
logging.basicConfig(level=default_level)
def read_data_model():
loaded_encoder = tf.keras.models.load_model(
RESULT_PATH + "pretrain_gen_encoder") #"pretrain_gen_encoder"
loaded_decoder = tf.keras.models.load_model(
RESULT_PATH + "pretrain_gen_decoder") #pretrain_gen_decoder
file_path = RESULT_PATH + "train/20A_20B.csv"
te_clade_files = glob.glob(file_path)
r_dict = utils.read_json(RESULT_PATH + "r_word_dictionaries.json")
vocab_size = len(r_dict) + 1
total_te_loss = list()
for te_name in te_clade_files:
te_clade_df = pd.read_csv(te_name, sep="\t")
print(te_clade_df)
te_X = te_clade_df["X"].drop_duplicates()
te_y = te_clade_df["Y"].drop_duplicates()
print(te_X)
return loaded_encoder, loaded_decoder, te_X
def main():
if not os.path.exists(label_path):
label_file = "data/dev_label.txt"
label = read_label_from_file(label_file, frame_size=frame_size, frame_shift=frame_shift)
save_json(label, label_path)
else:
label = read_json(label_path)
features, target = sklearn_dataset(
label, task=args.task, mode='val',
frame_size=frame_size, frame_shift=frame_shift,
features_path=features_path, target_path=target_path
)
m = joblib.load(model_path)
print(m)
auc, eer = validate(m, features, target)
print('AUC :', auc)
print('ERR :', eer)
def __init__(self, profile_name):
""" Loads the JSON profile models into the tree attribute
Arguments:
----------
profile_path:
type: string
info: name of the JSON predicting profile file
"""
profile = read_json(file_name=profile_name, file_type='profile_p')
try:
self.tree = profile['tree']
self.colors = profile['colors']
self.__load_clf(self.tree)
except KeyError:
exit('Invalid JSON keys')
async def join(ctx, *args):
server_id = str(ctx.message.guild.id)
data = read_json("src/teams.json")
for id in args:
for t in data[server_id]:
found = False
if str(t.get("id")) == id:
found = True
if ctx.author.name.lower() in lowercase_players(t.get("players")):
await ctx.send("You are already a part of Team {}!".format(id))
break
if len(t.get("players")) >= max_players.get(t.get("game")):
await ctx.send("Team {} is full!".format(id))
break
t.get("players").append(ctx.author.name)
write_json(data)
await ctx.send(embed=embed_team(t))
break
if not found:
await ctx.send(em.get("team_not_found"))
def setup_logging(
save_dir: Path,
log_config: Path = Path(f"{PROJECT_DIR}/logger/logger_config.json"),
default_level: int = logging.INFO,
) -> None:
"""
Setup logging configuration
"""
if log_config.is_file():
config = read_json(log_config)
# modify logging paths based on run config
for _, handler in config["handlers"].items():
if "filename" in handler:
handler["filename"] = str(save_dir / handler["filename"])
logging.config.dictConfig(config)
else:
print("Warning: logging configuration file is not found in {}.".format(
log_config))
logging.basicConfig(level=default_level)
def make_docid2doc(int_mapped_path):
docid2doc = {}
splits = [
'dev_ent_added.json', 'test_ent_added.json', 'train_ent_added.json'
]
for split in splits:
split_file = os.path.join(int_mapped_path, split)
with codecs.open(split_file, 'r', 'utf-8') as fp:
print('Processing: {:s}'.format(split_file))
for data_json in utils.read_json(fp):
doc_id = data_json['doc_id']
docid2doc[doc_id] = {
'ents': data_json['ents'],
'text': data_json['text']
}
docid2doc_file = os.path.join(int_mapped_path, 'docid2doc.json')
print('docids2doc: {:d}'.format(len(docid2doc)))
with codecs.open(docid2doc_file, 'w', 'utf-8') as fp:
json.dump(docid2doc, fp)
print('Wrote: {:s}'.format(fp.name))
async def analytics(client):
if client.is_ready() and cfg.SAPPHIRE_ID is None:
fp = os.path.join(cfg.SCRIPT_DIR, "analytics.json")
guilds = func.get_guilds(client)
if not os.path.exists(fp):
analytics = {}
else:
analytics = utils.read_json(fp)
analytics[datetime.now(pytz.timezone(
cfg.CONFIG['log_timezone'])).strftime("%Y-%m-%d %H:%M")] = {
'nc': utils.num_active_channels(guilds),
'tt': round(cfg.TICK_TIME, 2),
'tr': main_loop.seconds,
'ng': len(guilds),
'm': round(psutil.virtual_memory().used / 1024 / 1024 / 1024,
2),
}
with concurrent.futures.ThreadPoolExecutor() as pool:
await client.loop.run_in_executor(pool, utils.write_json, fp,
analytics)
def __init__(self, args, options='', timestamp=True):
# parse default and custom cli options
for opt in options:
args.add_argument(*opt.flags, default=None, type=opt.type)
args = args.parse_args()
if args.device:
os.environ["CUDA_VISIBLE_DEVICES"] = args.device
if args.resume:
self.resume = Path(args.resume)
self.cfg_fname = self.resume.parent / 'config.json'
else:
msg_no_cfg = "Configuration file need to be specified. Add '-c config.json', for example."
assert args.config is not None, msg_no_cfg
self.resume = None
self.cfg_fname = Path(args.config)
self.is_training = args.training
# load config file and apply custom cli options
config = read_json(self.cfg_fname)
self._config = _update_config(config, options, args)
# set save_dir where trained model and log will be saved.
save_dir = Path(self.config['trainer']['save_dir'])
if not save_dir.exists():
save_dir.mkdir(exist_ok=True, parents=True)
save_dir = save_dir.resolve()
timestamp = datetime.now().strftime(
r'%Y%m%d-%H%M%S') if timestamp else ''
exper_name = self.config['name']
self._save_dir = save_dir / 'models' / exper_name / timestamp
self._log_dir = save_dir / 'log' / exper_name / timestamp
self.save_dir.mkdir(parents=True, exist_ok=True)
self.log_dir.mkdir(parents=True, exist_ok=True)
# save updated config file to the checkpoint dir
write_json(self.config, self.save_dir / 'config.json')
# configure logging module
setup_logging(self.log_dir, exper_name, timestamp, self.is_training)
async def execute(ctx, params):
author = ctx['message'].author
r = "Checking..."
s = False
m = await ctx['channel'].send(r)
if patreon_info is None:
return False, "No need to do that."
patrons = patreon_info.fetch_patrons(force_update=False)
patrons[cfg.CONFIG['admin_id']] = "sapphire"
auth_path = os.path.join(cfg.SCRIPT_DIR, "patron_auths.json")
if author.id in patrons:
auths = utils.read_json(auth_path)
auths[str(author.id)] = {"servers": [ctx['guild'].id]}
utils.write_json(auth_path, auths, indent=4)
patreon_info.update_patron_servers(patrons)
s = True
reward = patrons[author.id].title()
await func.admin_log(
"🔑 Authenticated **{}**'s {} server {} `{}`".format(
author.name, reward, ctx['guild'].name, ctx['guild'].id),
ctx['client'],
important=True)
r = "✅ Nice! This server is now a **{}** server.".format(reward)
if reward in ["Diamond", "Sapphire"]:
r += ("\nPlease give me ~{} hours to set up your private bot - "
"I'll DM you when it's ready to make the swap!".format(
12 if reward == "Sapphire" else 24))
else:
await func.admin_log(
"🔒 Failed to authenticate for **{}**".format(author.name),
ctx['client'])
r = (
"❌ Sorry it doesn't look like you're a Patron.\n"
"If you just recently became one, please make sure you've connected your discord account "
"(<https://bit.ly/2UdfYbQ>) and try again in a few minutes. "
"If it still doesn't work, let me know in the support server: <https://discord.io/DotsBotsSupport>."
)
await m.edit(content=r)
return s, "NO RESPONSE"
def setup_logging(save_dir,
log_config="logger/logger_config.json",
default_level=logging.INFO):
"""
Setup logging configuration
"""
log_config = Path(log_config)
if log_config.is_file():
config = read_json(log_config)
# modify logging paths based on run config
for _, handler in config["handlers"].items():
if "filename" in handler:
handler["filename"] = str(save_dir / handler["filename"])
logging.config.dictConfig(config)
else:
logging.basicConfig(level=default_level,
format="%(levelname)s-%(message)s")
logging.warning(
"Logging configuration file is not found in {}.".format(
log_config))
def generate_record_example_with_kp(in_path, info_path, writer):
train = glob.glob(in_path + '/*.png')
label = in_path + '/label.txt'
with open(label) as file:
file_content = file.read()
label = int(file_content)
for image in train:
img = open(image, 'rb').read()
json_file = open(info_path + image[-17:-3] + 'json', 'r')
info = json.load(json_file)
kpts = read_json(info)
feature = {
"label":
tf.train.Feature(int64_list=tf.train.Int64List(value=[label])),
'img_raw':
tf.train.Feature(bytes_list=tf.train.BytesList(value=[img])),
'kpts': tf.train.Feature(float_list=tf.train.FloatList(value=kpts))
}
example_proto = tf.train.Example(features=tf.train.Features(
feature=feature))
writer.write(example_proto.SerializeToString())
def setup_logging(save_dir,
log_config='logger/logger_config.json',
default_level=logging.INFO):
"""
Setup logging configuration
"""
log_config = Path(log_config)
if log_config.is_file():
config = read_json(log_config)
# modify logging paths based on run config
for _, handler in config['handlers'].items():
if 'filename' in handler:
handler['filename'] = str(save_dir / handler['filename'])
logging.config.dictConfig(config)
else:
print(
f"Warning: logging configuration file is not found in {log_config}."
)
logging.basicConfig(level=default_level)
return config["handlers"]["info_file_handler"]["filename"]
def time(ctx):
user = ctx.obj['USER']
city = ctx.obj['CITY']
name = utils.read_json(ctx.obj['SETTINGS'])['name'].get(user, 'error')
if city:
raise NotImplementedError('You cannot use this option yet.')
elif (user and name != 'error'):
print_date_now(name['timezone'])
elif name == 'error':
click.secho(f"Error Code {errno.EINVAL}: {os.strerror(errno.EINVAL)}",
err=True,
fg='red')
click.secho(f"This user hasn't been defined in your settings yet.",
fg='yellow')
click.secho(
f"You can add new users in '{utils.path_settings('timetravel')}'.",
fg='yellow')
else:
click.secho(f"An unknown error has occurred", err=True, fg='red')
def update_table(self, notify):
new_table_json = self.generate_table_json_from_url(self.iat_table_url)
old_table_json = read_json(self.table_file_name)
if not os.path.isfile(self.table_file_name):
write_json(self.table_file_name, new_table_json)
return
#new_table_json = '{"table": [{"cardinal_points": "18 : 0","club": "AV Speyer 03","max_score": "898.6","place": "1","score": "5056.2 : 4170.6"}],"relay": "1Gruppe+A"}'
new_table_dict = json.loads(new_table_json)
if sorted(new_table_json.decode("utf-8")) == sorted(
old_table_json.decode("utf-8")):
print "Local check: Table of " + new_table_dict[
"relay"] + " did not change"
return
else:
print "Local check: Table of " + new_table_dict[
"relay"] + " changed"
self.send_post(new_table_json, '/set_table')
if notify:
self.notify_users_about_new_placements(new_table_json,
old_table_json)
write_json(self.table_file_name, new_table_json)
def main(args):
# build model architecture
model_path= Path(args.model)
config_path = model_path.parent.joinpath('config.json')
config = read_json(config_path)
print('Loading checkpoint: {}'.format(model_path))
checkpoint = torch.load(model_path,map_location='cpu')
if len(checkpoint.keys()) == 1 and str2bool(input("Is this a fused model y/n? ")):
config['arch']['args']['batchnorm'] = False
model = getattr(module_arch,config['arch']['type'])(**dict(config['arch']['args']))
state_dict = checkpoint['state_dict']
consume_prefix_in_state_dict_if_present(state_dict,"module.")
model.load_state_dict(state_dict)
data=torch.randn([1]+config['input_size'])
device = torch.device('cpu') #'cuda' if torch.cuda.is_available() else
model.eval()
model = model.to(device)
data = data.to(device)
model(data)
res_path = parameters_extractor(model,config['extractor'],result_path=model_path.parent,fuse=args.fuse)
print("Result:\n", res_path)
def split_triple_number():
const = Const()
const.novel_tagging()
data = utils.read_json(Const.raw_test_filename)
# sentences contains 1, 2, 3, 4, and >5 triples
triples_size_1_data, triples_size_2_data, triples_size_3_data, triples_size_4_data, triples_size_5_data = [], [], [], [], []
for i, a_data in enumerate(data):
triples = set()
for triple in a_data['relationMentions']:
m1 = nltk.word_tokenize(triple['em1Text'])[-1]
m2 = nltk.word_tokenize(triple['em2Text'])[-1]
label = triple['label']
if label != 'None':
triples.add((m1, m2, label))
if len(triples) == 1:
triples_size_1_data.append(a_data)
elif len(triples) == 2:
triples_size_2_data.append(a_data)
elif len(triples) == 3:
triples_size_3_data.append(a_data)
elif len(triples) == 4:
triples_size_4_data.append(a_data)
else:
triples_size_5_data.append(a_data)
utils.write_data(open(const.raw_test_1_triple_filename, 'w'),
triples_size_1_data)
utils.write_data(open(const.raw_test_2_triple_filename, 'w'),
triples_size_2_data)
utils.write_data(open(const.raw_test_3_triple_filename, 'w'),
triples_size_3_data)
utils.write_data(open(const.raw_test_4_triple_filename, 'w'),
triples_size_4_data)
utils.write_data(open(const.raw_test_5_triple_filename, 'w'),
triples_size_5_data)
print('Sentence-1-Triple: %d' % len(triples_size_1_data))
print('Sentence-2-Triple: %d' % len(triples_size_2_data))
print('Sentence-3-Triple: %d' % len(triples_size_3_data))
print('Sentence-4-Triple: %d' % len(triples_size_4_data))
print('Sentence-5-Triple: %d' % len(triples_size_5_data))
def train_model(algorithm, training_config, output):
""" Trains a OpenCV classifier and stores it in the models folder
Arguments:
----------
algorithm:
type: string
info: name of the classifier {Eigen, Fisher, LBPH}
training_config:
type: string
info: name of the JSON with the training configuration
output:
type: string
info: name of the output model
"""
datasets = read_json(
file_name=training_config,
file_type='training_c'
)
classifier = FaceClassifier(algorithm)
classifier.train(datasets)
# Saving FaceClassifier int <-> label dict as JSON
write_json(
dictionary=classifier.properties,
file_name=output + '.json',
file_type='model'
)
# Saving FaceClassifier trained model as XML (pickle not working)
write_clf(
clf=classifier.model,
file_name=output + '.xml',
file_type='model'
)
def extract_summary(self, ):
data = read_json(self.full_path)
articles = []
abstracts = []
for item in data:
articles.appenditem['article'])
abstracts.append(item['abstract'])
data_iterator = zip(articles, abstracts)
summaries = []
references = []
for item in tqdm(data_iterator, desc="Lead:"):
article, abstract = item
summary = article[:self.extract_num]
summaries.append(summary)
references.append([abstract])
result = test_rouge(summaries, references, self.processors)
return result
def update_competitions(self, notify):
new_competitions_json = self.generate_competitions_json_from_url(
self.iat_competitions_url)
old_competitions_json = read_json(self.competition_file_name)
if not os.path.isfile(self.competition_file_name):
write_json(self.competition_file_name, new_competitions_json)
return
#new_competitions_json = '{"competitions": [{"date": "07.05.2016", "home": "TB 03 Roding", "guest": "AV Speyer 03", "location": "Roding", "score": "562.1 : 545.0 :562.0", "url": "https://www.iat.uni-leipzig.de/datenbanken/blgew1516/start.php?pid=' + "'123'" + '&protokoll=1&wkid=E3714956BFC24D6798DCD9C94B0620CC"}],"relay": "1Gruppe+A"}'
new_competitions_dict = json.loads(new_competitions_json)
if sorted(new_competitions_json.decode("utf-8")) == sorted(
old_competitions_json.decode("utf-8")):
print "Local check: Competitions of " + new_competitions_dict[
"relay"] + " did not change"
return
else:
print "Local check: Competitions of " + new_competitions_dict[
"relay"] + " changed"
self.send_post(new_competitions_json, '/set_competitions')
if notify:
self.notify_users_about_new_competitions(
new_competitions_json, old_competitions_json)
write_json(self.competition_file_name, new_competitions_json)
def analyse_video(video_path, model_name, clf_th, output): """ Identifies actors in a video and generates one with their names Arguments: ---------- video_path: type: string info: path where the video is located model_name: type: string info: name of the trained model clf_th: type: float info: confidence threshold to identify an actor output: type: string info: name of the generated video """ clf_props = read_json( file_name=model_name + '.json', file_type='model' ) # Creating and loading the trained classifier clf = FaceClassifier(clf_props['algorithm']) clf.properties = clf_props clf.model = read_clf( clf=clf, file_name=model_name + '.xml', file_type='model' ) # Generating a similar video with the names on it identify_actors(video_path, clf, clf_th, output)
async def leave(ctx, *args):
server_id = str(ctx.message.guild.id)
data = read_json("src/teams.json")
for id in args:
for t in data[server_id]:
found = False
if str(t.get("id")) == id:
found = True
if ctx.author.name.lower() not in lowercase_players(t.get("players")):
await ctx.send("You are not a part of Team {}!".format(id))
else:
remove_player(t.get("players"), ctx.author.name)
if len(t.get("players")) == 0:
data[server_id].remove(t)
await ctx.send("All players removed from Team {}.".format(id))
else:
await ctx.send("You have been removed from Team {}.".format(id))
await ctx.send(embed=embed_team(t))
write_json(data)
break
if not found:
await ctx.send(em.get("team_not_found"))
async def remove(ctx, id, *args):
server_id = str(ctx.message.guild.id)
data = read_json("src/teams.json")
for t in data[server_id]:
if str(t.get("id")) == id:
players = t.get("players")
if ctx.author.name.lower() not in lowercase_players(players):
await ctx.send(em.get("non_member"))
elif len(args) != len(set(args)):
await ctx.send(em.get("duplicate_player"))
else:
removed_players = []
args = [int(n) - 1 for n in args]
for n in args:
if 0 <= n < len(players):
removed_players.append(players[n])
else:
await ctx.send(
"Player {} is not part of the team!".format(n + 1)
)
players = [p for n, p in enumerate(players) if n not in args]
t["players"] = players
if len(players) == 0:
data[server_id].remove(t)
await ctx.send("All players removed from Team {}.".format(id))
elif len(removed_players) > 0:
await ctx.send(
"{} have been removed from Team {}.".format(
", ".join(removed_players), id
)
) if len(removed_players) > 1 else await ctx.send(
"{} has been removed from Team {}.".format(
removed_players[0], id
)
)
await ctx.send(embed=embed_team(t))
write_json(data)
return
await ctx.send(em.get("team_not_found"))
def train_model(algorithm, feats_pct, lang, output, profile_name):
""" Prepares arguments to train and saves a NodeClassif object
Arguments:
----------
algorithm:
type: string
info: name of the algorithm to train
feats_pct:
type: int
info: percentage of features to keep
lang:
type: string
info: language to perform the tokenizer process
output:
type: string
info: output file name including extension
profile_name:
type: string
info: name of the JSON training profile file
"""
if (feats_pct < 0) or (feats_pct > 100):
exit('The specified features percentage is invalid')
profile_data = read_json(file_name=profile_name, file_type='profile_t')
node_classif = NodeClassif(
algorithm=algorithm.lower(),
feats_pct=feats_pct,
lang=lang,
)
node_classif.train(profile_data)
save_object(node_classif, output, 'model')
def batch_eval_metrics(model_folder, w_config, save_path):
assert len(w_config.split('-')) == 6
print("Weight config: %s" % w_config)
metrics = ['f1', 'acc', 'nmi']
# metrics = ['f1', 'acc']
mets = {k: [] for k in metrics}
avg_mets = {k: 0 for k in metrics}
model_count = 0
eval_models = [
f for f in Path(model_folder).glob('*') if w_config in str(f)
]
txt_file_name = 'batch_eval_met_%s.txt' % w_config
txt_dir = os.path.join(save_path, txt_file_name)
txt_file = open(txt_dir, 'w')
for f in eval_models:
# for d in f.glob('*'):
# m = d / 'model_best.pth'
# if m.is_file():
# f = d; txt_file.write('%s\n' % f)
# break
txt_file.write('%s\n' % f)
print("Processing model: %s" % f)
config_file = read_json(str(f / 'config.json'))
model_best = str(f / 'model_best.pth')
config = ConfigParser(config_file, resume=model_best, testing=True)
f1, acc, nmi = eval_metrics(config)
for m, s in zip(metrics, [f1, acc, nmi]):
mets[m].append(s)
avg_mets[m] += s
model_count += 1
for k in avg_mets.keys():
avg_mets[k] /= model_count
txt_file.write(json.dumps(mets, indent=2))
txt_file.write('\n')
txt_file.write(json.dumps(avg_mets, indent=2))
txt_file.close()
def apply_threshold(file_predict, output, threshold):
"""
Apply threshold on the scores of a predicted file, reducing
the number of predicted bounding boxes.
"""
dpred = utils.read_json(file_predict)
dic = {}
discarded = 0
pb = progressbar.ProgressBar(len(dpred))
for image in sorted(dpred):
for content in dpred[image]:
if content[1] >= threshold:
if dic.has_key(image):
dic[image].append(content)
else:
dic[image] = [content]
else:
discarded += 1
pb.update()
utils.save_json(output, dic)
logger.info('Total of discarded bounding boxes: %d' % discarded)
def __init__(self, config_file):
self.config_file = Path(config_file)
self.config = read_json(self.config_file)
experiment_name = self.config['experiment_name']
out_dir = Path(self.config['trainer']['save_dir'])
timestamp = datetime.now().strftime(r'%Y%m%d_%H%M%S')
self.save_dir = out_dir / 'models' / experiment_name / timestamp
self.log_dir = out_dir / 'log' / experiment_name / timestamp
self.samples_dir = out_dir / 'samples' / experiment_name / timestamp
self.save_dir.mkdir(parents=True, exist_ok=True)
self.log_dir.mkdir(parents=True, exist_ok=True)
self.samples_dir.mkdir(parents=True, exist_ok=True)
configure_logging(self.log_dir)
self.log_levels = {
0: logging.WARNING,
1: logging.INFO,
2: logging.DEBUG,
}
def add_item(self, urls=None):
if(urls is None):
urls = [input("Url to parse : ")]
data = utils.read_json(self.database_file)
for url in urls:
try:
for builder in self.builders:
if builder.match(url):
break
else:
logger.info('The url {} is not valid'.format(url))
logger.info("Scrapping and adding {} to the database".format(url))
item_data = builder.scrapper(url, self.language).scrap()
data[builder.field_name].append(item_data)
except DatafusException as e:
logger.warning("Skipping {} for the following reason : {}".format(url, str(e)))
except Exception as e:
logger.warning("Skipping {} for the following reason : {}".format(url, str(e)))
logger.error(e)
utils.save_json(self.database_file, data)
def combine_parsed_medline_articles(saved_path='parsed_articles',
year_start=2010,
year_end=2019):
"""
Give a path to folder locating JSON files,
return all parsed paper
Input
=====
saved_path: str, path to saved JSON folder
Output
======
parsed_papers: list, list of all parsed papers
"""
paths = glob(os.path.join(saved_path, '*.json'))
parsed_papers = []
for path in paths:
papers = read_json(path)
papers = [paper for paper in papers if int(paper['pubdate']) in range(2010, 2019)]
parsed_papers.extend(papers)
return parsed_papers
def setup_logging(save_dir,
run_id='1',
log_config='logger/logger.json',
default_level=logging.INFO):
"""
Setup logging configuration
Parameters
----------
save_dir: str
logging info save root directory. By default, the name of logging file is info.log.
run_id: str
session id.
log_config: str, default='logger/logger.json'
logging config json file.
default_level: str, default=logging.INFO
logging level.
Returns
-------
None
"""
_log_config = Path(log_config)
if _log_config.is_file():
# read json file
config_dict = read_json(_log_config)
for _, handler in config_dict['handlers'].items():
if 'filename' in handler:
handler['filename'] = str(
save_dir / '{}-{}'.format(run_id, handler['filename']))
logging.config.dictConfig(config_dict)
else:
print("Warning: logging configuration file is not found in {}.".format(
_log_config))
logging.basicConfig(level=default_level)
c.close()
print "Done Classifying."
print
return classified_array
if __name__ == "__main__":
# Nome do arquivo json que queremos fazer as operações
json_filename = '../database_new.json'
# Nome do arquivo com as urls novas que queremos adicionar no json
input_filename = 'music_urls.txt'
# Leitura do arquivo json. Caso não exista, retorna uma lista vazia
database_array = read_json(json_filename)
# Leitura do arquivo de urls
urls_array = read_urls_file(input_filename) # Aqui colocamos a entrada desejada
# Leitura das urls via input do usuário
# urls_array = read_urls_input()
# Chama a função para crawl das letras retornando uma lista de dicionários
lyrics_array = crawl_lyrics(urls_array)
# Adicionamos a nova lista ao final da lista de músicas que estavam
# no banco de dados
database_array.extend(classify_lyrics(lyrics_array))
# Escrevemos no arquivo json
def process(sources, output, force):
"""Download sources and process the file to the output directory.
\b
SOURCES: Source JSON file or directory of files. Required.
OUTPUT: Destination directory for generated data. Required.
"""
catalog_features = []
failures = []
path_parts_to_skip = len(utils.get_path_parts(output))
success = True
for path in utils.get_files(sources):
try:
utils.info("Processing " + path)
pathparts = utils.get_path_parts(path)
pathparts[0] = output.strip(os.sep)
pathparts[-1] = pathparts[-1].replace('.json', '.geojson')
outdir = os.sep.join(pathparts[:-1])
outfile = os.sep.join(pathparts)
source = utils.read_json(path)
urlfile = urlparse(source['url']).path.split('/')[-1]
if not hasattr(adapters, source['filetype']):
utils.error('Unknown filetype', source['filetype'], '\n')
failures.append(path)
continue
if os.path.isfile(outfile) and \
os.path.getmtime(outfile) > os.path.getmtime(path) and not force:
utils.error('Skipping', path, 'since generated file exists.',
'Use --force to regenerate.', '\n')
with open(outfile, "rb") as f:
geojson = json.load(f)
properties = geojson['properties']
else:
utils.info('Downloading', source['url'])
try:
fp = utils.download(source['url'])
except IOError:
utils.error('Failed to download', source['url'], '\n')
failures.append(path)
continue
utils.info('Reading', urlfile)
if 'filter' in source:
filterer = BasicFilterer(source['filter'], source.get('filterOperator', 'and'))
else:
filterer = None
try:
geojson = getattr(adapters, source['filetype'])\
.read(fp, source['properties'],
filterer=filterer,
layer_name=source.get("layerName", None),
source_filename=source.get("filenameInZip", None))
except IOError, e:
utils.error('Failed to read', urlfile, str(e))
failures.append(path)
continue
except zipfile.BadZipfile, e:
utils.error('Unable to open zip file', source['url'])
failures.append(path)
continue
finally:
import traceback
import json
try:
return eval_dag(dag, filename, dag_id), dag_id
except Exception as e:
with open('error.'+str(dag_id), 'w') as err:
err.write(str(e)+'\n')
for line in traceback.format_tb(e.__traceback__):
err.write(line)
err.write(json.dumps(dag))
return (), dag_id
if __name__ == '__main__':
datafile = "ml-prove.csv"
dags = utils.read_json('test_err.json')
results = dict()
remaining_dags = [d for d in enumerate(dags) if str(d[0]) not in results]
print("Starting...", len(remaining_dags))
pprint.pprint(remaining_dags)
for e in map(lambda x: safe_dag_eval(x[1], datafile, x[0]), remaining_dags):
results[str(e[1])] = e
print(e)
print("Model %4d: Cross-validation error: %.5f (+-%.5f)" % (e[1], e[0][0], e[0][1]))
sys.stdout.flush()
print("-"*80)
best_error = sorted(results.values(), key=lambda x: x[0][0]-2*x[0][1], reverse=True)[0]
def basic_view():
view_data = json.loads(utils.read_json(utils.HEALTH_JSON_FILE))
view_data.update(json.loads(utils.read_json(utils.STATS_JSON_FILE)))
return view_data
def start(self):
self.status.set("Estado: -")
self.leng.set("Iteración: -/- y Número: -/-")
self.totaltime.set("Tiempo total: -")
self.ones.set("Total de unos: -")
self.types.set("Progreso: -")
self.startButton.config(state = 'disabled')
self.browseButton.config(state = 'disabled')
self.cancelButton.config(state = 'normal')
self.maxnumberSpinbox.config(state = 'disabled')
self.complexSpinbox.config(state = 'disabled')
if int(self.complexSpinbox.get()) in (1,2,3,4,5) and int(self.maxnumberSpinbox.get()) in (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21) and self.completeName != "":
start_time = time.time()
if self.name.get().split('.')[1] == 'csv':
self.g = generator.Generator(self.maxnumberSpinbox.get(), self.complexSpinbox.get(), utils.read_csv(self.completeName), self.cancelButton, self.types)
else:
self.g = generator.Generator(self.maxnumberSpinbox.get(), self.complexSpinbox.get(), utils.read_json(self.completeName), self.cancelButton, self.types)
self.g.count_one()
self.ones.set("Total de unos: {0}".format(len(self.g.table_uno)))
i = 0
while self.g.maxim > 1:
i += 1
self.leng.set("Iteración: {0}/{1} y Número: {2}".format(i, self.complexSpinbox.get(), self.g.maxim))
self.status.set("Estado: Generando puzzle...")
self.g.step_one()
tim = utils.sec_to(int(time.time() - start_time))
self.totaltime.set("Tiempo total: {0}h:{1}m:{2}s".format(tim[0], tim[1], tim[2]))
self.status.set("Estado: Aplicando condición uno...")
self.g.cond_dos(1)
tim = utils.sec_to(int(time.time() - start_time))
self.totaltime.set("Tiempo total: {0}h:{1}m:{2}s".format(tim[0], tim[1], tim[2]))
self.status.set("Estado: Aplicando condición dos...")
self.g.cond_dos(2)
tim = utils.sec_to(int(time.time() - start_time))
self.totaltime.set("Tiempo total: {0}h:{1}m:{2}s".format(tim[0], tim[1], tim[2]))
if self.g.maxim >= 4:
self.status.set("Estado: Aplicando condición tres...")
self.g.cond_dos(3)
tim = utils.sec_to(int(time.time() - start_time))
self.totaltime.set("Tiempo total: {0}h:{1}m:{2}s".format(tim[0], tim[1], tim[2]))
self.g.count_one()
self.ones.set("Total de unos: {0}".format(len(self.g.table_uno)))
if i == self.g.iters:
self.g.maxim -= 1
i = 0
if self.name.get().split('.')[1] == 'csv':
utils.write_csv(self.g.table_all)
else:
utils.write_json(self.g.table_all)
if self.g.cancel:
self.status.set("Estado: Cancelado")
else:
self.status.set("Estado: Completado")
self.g = None
self.startButton.config(state = 'normal')
self.browseButton.config(state = 'normal')
self.cancelButton.config(state = 'disabled')
self.maxnumberSpinbox.config(state = 'normal')
self.complexSpinbox.config(state = 'normal')
LOG.info('Connecting to the database')
connection = pymysql.connect(host=hostname, port=port, user='******', passwd='skynet', db='skynet')
cursor = connection.cursor()
try:
cursor.execute("SELECT max(parameterNumber) FROM " + parameters)
data = cursor.fetchone()
return data
finally:
cursor.close()
connection.close()
try:
utils.copy(PARAMETERS_FROM, PARAMETERS_TO)
json_object = utils.read_json(PARAMETERS_FROM)
dictionary = json_object[0]
# Get the hostname
HOSTNAME = dictionary['db-hostname']
# Build the constants we need
TABLE_STEM = dictionary['table-stem']
PARAMETERS = TABLE_STEM + '_parameters'
PORT_INT = int(dictionary['db-port'])
PORT_STR = dictionary['db-port']
max_parameter_number = get_parameter_numbers(PARAMETERS, HOSTNAME, PORT_INT)[0]
LOG.info('Max parameter number = %(max_parameter_number)s' % {'max_parameter_number': str(max_parameter_number)})
cmd = 'java -Xms15g -Xmx30g -jar /home/kevin/yabi/MergeSkyNetResults.jar -database ' + TABLE_STEM + ' -db_hostname ' + HOSTNAME + ' -db_port ' + PORT_STR
def respfunc_viewer(path):
app = QtGui.QApplication([])
pyqtgraph.setConfigOption("background", "w")
pyqtgraph.setConfigOption("foreground", "k")
win = QtGui.QMainWindow()
win.setWindowTitle("MT response function data viewer")
darea = dockarea.DockArea()
w = QtGui.QWidget()
win.setCentralWidget(darea)
taglist = QtGui.QListWidget(win)
taglist.setSelectionMode(QtGui.QAbstractItemView.ExtendedSelection)
taglist_dock = dockarea.Dock("Tags")
taglist_dock.addWidget(taglist)
darea.addDock(taglist_dock)
sitelist = QtGui.QListWidget()
sitelist.setSelectionMode(QtGui.QAbstractItemView.ExtendedSelection)
sitelist_dock = dockarea.Dock("Tree...")
sitelist_dock.addWidget(sitelist)
darea.addDock(sitelist_dock, "left", taglist_dock)
resplot = pyqtgraph.PlotWidget()
resplot_dock = dockarea.Dock("APPARENT RESISTIVITY")
resplot_dock.addWidget(resplot)
darea.addDock(resplot_dock, "left", sitelist_dock)
phaseplot = pyqtgraph.PlotWidget()
phaseplot_dock = dockarea.Dock("PHASE")
phaseplot_dock.addWidget(phaseplot)
darea.addDock(phaseplot_dock, "bottom", resplot_dock)
default_pen = [[(255,255,255,90)], dict(width=1)]
select_pen = [["r"], dict(width=1.5)]
skipflag_pen = [[(255,255,255,30)], dict(width=0.5)]
resplotitem = resplot.getPlotItem()
phaseplotitem = phaseplot.getPlotItem()
resplotitem.invertX(True)
phaseplotitem.invertX(True)
resplotitem.setLogMode(x=True, y=True)
phaseplotitem.setLogMode(x=True, y=False)
phaseplotitem.vb.setXLink(resplotitem.vb)
resplotitem.setYRange(np.log10(0.1), np.log10(1000))
phaseplotitem.setYRange(0, 90)
resvb = resplotitem.vb
phasevb = phaseplotitem.vb
data = utils.AttrDict()
tagfns = glob.glob(op.join(path, "*-cal.json"))
tag2fn = {}
fn2tag = {}
sites = set()
tagfns.sort()
data = utils.AttrDict()
with open(op.join(path, "maskedfreqs.json"), mode="r") as f:
maskedfreqs = utils.read_json(f)
maskedlines = utils.AttrDict()
datasymbols = utils.AttrDict()
psymbols = utils.AttrDict({
"xy": dict(pen=None, symbol="o", symbolBrush="b"),
"yx": dict(pen=None, symbol="s", symbolBrush="r")
})
plines = utils.AttrDict({
"xy": dict(pen="b"),
"yx": dict(pen="r")
})
plotpens = utils.AttrDict({"xy": "b", "yx": "r",})
plotsymbols = utils.AttrDict({"xy": "o", "yx": "s"})
def plot(tag):
if not hasattr(datasymbols[tag], "res_xy"):
datasymbols[tag].res_xy = pyqtgraph.PlotDataItem(data[tag].freqs, data[tag].res_xy, **psymbols.xy)
datasymbols[tag].res_yx = pyqtgraph.PlotDataItem(data[tag].freqs, data[tag].res_yx, **psymbols.yx)
datasymbols[tag].phase_xy = pyqtgraph.PlotDataItem(data[tag].freqs, data[tag].phase_xy, **psymbols.xy)
datasymbols[tag].phase_yx = pyqtgraph.PlotDataItem(data[tag].freqs, data[tag].phase_yx, **psymbols.yx)
maskedlines[tag].res_xy = pyqtgraph.PlotDataItem(data[tag].freqs, data[tag].res_xy, **plines.xy)
maskedlines[tag].res_yx = pyqtgraph.PlotDataItem(data[tag].freqs, data[tag].res_yx, **plines.yx)
maskedlines[tag].phase_xy = pyqtgraph.PlotDataItem(data[tag].freqs, data[tag].phase_xy, **plines.xy)
maskedlines[tag].phase_yx = pyqtgraph.PlotDataItem(data[tag].freqs, data[tag].phase_yx, **plines.yx)
resplotitem.addItem(datasymbols[tag].res_xy)
resplotitem.addItem(datasymbols[tag].res_yx)
resplotitem.addItem(maskedlines[tag].res_xy)
resplotitem.addItem(maskedlines[tag].res_yx)
phaseplotitem.addItem(datasymbols[tag].phase_xy)
phaseplotitem.addItem(datasymbols[tag].phase_yx)
phaseplotitem.addItem(maskedlines[tag].phase_xy)
phaseplotitem.addItem(maskedlines[tag].phase_yx)
for i, freq in enumerate(data[tag].freqs):
if maskedfreqs[tag]["masks"][i] == 0:
data[tag].freqs[i] = float(maskedfreqs[tag]["freqs"][i])
else:
data[tag].freqs[i] = np.nan
maskedlines[tag].res_xy.setData(data[tag].freqs, data[tag].res_xy)
maskedlines[tag].res_yx.setData(data[tag].freqs, data[tag].res_yx)
maskedlines[tag].phase_xy.setData(data[tag].freqs, data[tag].phase_xy)
maskedlines[tag].phase_yx.setData(data[tag].freqs, data[tag].phase_yx)
progress = QtGui.QProgressDialog("Loading data...", "Abort", 0, len(tagfns), win)
progress.setWindowModality(QtCore.Qt.WindowModal)
for i, tagfn in enumerate(tagfns):
progress.setValue(i)
tag = op.basename(tagfn).replace("-cal.json", "")
tag2fn[tag] = tagfn
fn2tag[tagfn] = tag
site = tag.split("-")[0]
sites.add(site)
data[tag] = utils.read_json(tagfn)
if not tag in maskedfreqs:
maskedfreqs[tag] = utils.AttrDict({"freqs": data[tag].freqs.copy(), "masks": np.empty_like(data[tag].freqs) * 0})
if not tag in maskedlines:
maskedlines[tag] = utils.AttrDict()
datasymbols[tag] = utils.AttrDict()
plot(tag)
if progress.wasCanceled():
break
progress.setValue(len(tagfns))
resfreqselect = pyqtgraph.LinearRegionItem([0,-1])
phasefreqselect = pyqtgraph.LinearRegionItem([0,-1])
resplotitem.addItem(resfreqselect)
phaseplotitem.addItem(phasefreqselect)
def res_region_moved():
phasefreqselect.setRegion(resfreqselect.getRegion())
def phase_region_moved():
resfreqselect.setRegion(phasefreqselect.getRegion())
resfreqselect.sigRegionChanged.connect(res_region_moved)
phasefreqselect.sigRegionChanged.connect(phase_region_moved)
def populate_tag_list(filter_sites=None):
if filter_sites:
tags = [t for t in tag2fn.keys() if t.split("-")[0] in filter_sites]
else:
tags = sorted(tag2fn.keys())
tags.sort()
taglist.clear()
for tag in tags:
# print tag
tagitem = QtGui.QListWidgetItem(taglist)
tagitem.setText(tag)
plot_per_tag_list()
print
def plot_per_tag_list():
tags = [t.text() for t in taglist.selectedItems()]
if not tags:
tags = [t.text() for t in [taglist.item(i) for i in xrange(taglist.count())]]
for plotitemtag, tagitems in datasymbols.items():
if plotitemtag in tags:
for item_name, item in tagitems.items():
item.setSymbol(plotsymbols[item_name[-2:]])
# item.setPen(None)#plotpens[item_name[-2:]])
else:
for item in tagitems.values():
item.setSymbol(None)
# item.setPen(None)
for plotitemtag, tagitems in maskedlines.items():
if plotitemtag in tags:
for item_name, item in tagitems.items():
item.setPen(plotpens[item_name[-2:]])
else:
for item in tagitems.values():
item.setPen(None)
def selected_site_names():
return [s.text() for s in sitelist.selectedItems()]
def pick_site():
newsites = selected_site_names()
populate_tag_list(newsites)
# plot_per_tag_list()
def toggle_selected_mask(value):
tags = [str(t.text()) for t in taglist.selectedItems()]
log_mask_range = resfreqselect.getRegion()
fmin = 10 ** log_mask_range[0]
fmax = 10 ** log_mask_range[1]
for tag in tags:
for i, freq in enumerate(maskedfreqs[tag]["freqs"]):
if freq >= fmin and freq <= fmax:
maskedfreqs[tag]["masks"][i] = value
plot(tag)
print log_mask_range, tags, "\n"
disable = QtGui.QPushButton("&Delete selected frequencies")
enable = QtGui.QPushButton("&Enable selected frequencies")
sitelist_dock.addWidget(disable)
sitelist_dock.addWidget(enable)
disable.clicked.connect(lambda: toggle_selected_mask(1))
enable.clicked.connect(lambda: toggle_selected_mask(0))
# def generate_key_press_event_handler(self, vb, event):
# vb.keyPressEvent(self, event)
# if event.key() is Qt.Key_X:
# toggle_selected_mask(mode="xy")
# elif event.key() is Qt.Key_Y:
# toggle_selected_mask(mode="yx")
# resplotitem.vb.keyPressEvent = lambda
populate_tag_list()
sites = sorted(list(sites))
for site in sites:
siteitem = QtGui.QListWidgetItem(sitelist)
siteitem.setText(site)
sitelist.itemSelectionChanged.connect(pick_site)
taglist.itemSelectionChanged.connect(plot_per_tag_list)
def cleanup():
with open(op.join(path, "maskedfreqs.json"), mode="w") as f:
utils.write_json(maskedfreqs, f)
win.showMaximized()
app.aboutToQuit.connect(cleanup)
app.exec_()
for e in futures.map_as_completed(inner_eval, enumerate(dags)):
results[str(e[1])] = e
res = [results[str(d[0])][0] for d in enumerate(dags)]
return res
if __name__ == '__main__':
import shelve
import pickle
results = shelve.open("results_wilt_tuned", protocol=pickle.HIGHEST_PROTOCOL)
datafile = "wilt.csv"
dags = utils.read_json('test_pop.json')
# dags = dags[36076:36077]
remaining_dags = [d for d in enumerate(dags) if str(d[0]) not in results]
print("Starting...", len(remaining_dags))
for e in map(lambda x: safe_dag_eval(x[1], datafile, x[0]), remaining_dags):
results[str(e[1])] = e
print("Model %4d: Cross-validation error: %.5f (+-%.5f)" % (e[1], e[0][0], e[0][1]))
sys.stdout.flush()
print("-"*80)
best_error = sorted(results.values(), key=lambda x: x[0][0]-2*x[0][1], reverse=True)[0]
print("Best model CV error: %.5f (+-%.5f)" % (best_error[0][0], best_error[0][1]))
import pprint
def start_prime_coordinator():
"""This function initializes an PrimeCoordinator object
:param:
:returns:
"""
if len(sys.argv)<3:
print "\n Invalid input, please enter correct parameters \n use command like: \n\t python PrimeCommand.py parameters_json.json primecommandResult.csv\n"
sys.exit(0)
#store command line arguments to local variables
json_file = sys.argv[1]
result_file = sys.argv[2]
exposure_sequence = utils.read_json(json_file)#list of exposures{mean,sd,non_rate}, read json content into memory
primeCoordinator = PrimeCoordinatorRaw()
primeCoordinator.get_counterfactual_compound_exposures(exposure_sequence)
# get the data to print/store in a file
b_output_mortality = primeCoordinator.output_baseline_mortality
b_output_mortality_num = primeCoordinator.output_baseline_mortality_num
b_total_mortality = primeCoordinator.output_baseline_mortality_total
c_output_mortality = primeCoordinator.output_counterfactual_mortality
c_output_mortality_num = primeCoordinator.output_counterfactual_mortality_num
c_total_mortality = primeCoordinator.output_counterfactual_mortality_total
total_population = primeCoordinator.output_total_population
#total_death_averted = str(int(round(primeCoordinator.output_total_death_averted))) # int
total_death_averted = str(primeCoordinator.output_total_death_averted) # decimale
total_death_baseline = str(primeCoordinator.output_total_death_baseline)
'''
These are the outputs
'''
all_mortality_outcome = primeCoordinator.output_all_mortality_exposure_outcome # [{'outcome_id':outcome_id,'name':outcome name,'baseline_death':100, 'counterfactual_death':20},{}]
all_mortality_age = primeCoordinator.output_all_mortality_age # [{'age_group_id':age_group_id,'age_group':age_group,'baseline_death':100, 'counterfactual_death':20},{}]
all_mortality_gender = primeCoordinator.output_all_mortality_gender# [{'gender':'male','baseline_death':100, 'counterfactual_death':20},{'gender':'female','baseline_death':100, 'counterfactual_death':20}]
'''
Write results into a csv file
'''
f_test_result_csv= csv.writer(open(result_file, 'wb')) # write result into csv
# write over all attributions
col1_name = 'total population'
f_test_result_csv.writerow([col1_name]) # write title
f_test_result_csv.writerow([total_population]) # write baseline and counterfactual mortalities
f_test_result_csv.writerow([]) # write seperate line
col1_name = 'total death averted'
f_test_result_csv.writerow([col1_name]) # write title
f_test_result_csv.writerow([total_death_averted]) # write baseline and counterfactual mortalities
f_test_result_csv.writerow([]) # write seperate line
# write all mortality by outcome
col1_name = 'outcome_id'
col2_name = 'name'
col3_name = 'b_mortality_sum_db'
col4_name = 'c_mortality_sum'
f_test_result_csv.writerow([col1_name,col2_name,col3_name,col4_name]) # write title
for line in all_mortality_outcome:
f_test_result_csv.writerow([line.get(col1_name),line.get(col2_name),line.get(col3_name),line.get(col4_name)]) # write baseline and counterfactual mortalities
f_test_result_csv.writerow([]) # write seperate line
# write all mortality by age
col1_name = 'age_group_id'
col2_name = 'age_group'
col3_name = 'b_mortality_sum_db'
col4_name = 'c_mortality_sum'
f_test_result_csv.writerow([col1_name,col2_name,col3_name,col4_name]) # write title
for line in all_mortality_age:
f_test_result_csv.writerow([line.get(col1_name),line.get(col2_name),line.get(col3_name),line.get(col4_name)]) # write baseline and counterfactual mortalities
f_test_result_csv.writerow([]) # write seperate line
# write all mortality by gender
col1_name = 'gender'
col2_name = 'b_mortality_sum_db'
col3_name = 'c_mortality_sum'
f_test_result_csv.writerow([col1_name,col2_name,col3_name]) # write title
for line in all_mortality_gender:
f_test_result_csv.writerow([line.get(col1_name),line.get(col2_name),line.get(col3_name)]) # write baseline and counterfactual mortalities
f_test_result_csv.writerow([]) # write seperate line
parser.add_argument('--header', action = 'store_true',
help = "Specify if the file contains a header")
args = parser.parse_args()
# read in file
if args.i[-3:] == "xls" or args.i[-4:] == "xlsx":
# make sure date and time fields are correctly processed
indexline = utils.read_columnfile(args.c)
date, time = False, False
if indexline[3] != "-":
date = indexline[3]
if indexline[4] != "-":
time = indexline[4]
lines = utils.read_excel(args.i, args.header, date, time)
elif args.i[-4:] == "json":
csvrows = utils.read_json(args.i)
else: # txt file
with open(args.i, encoding="utf-8") as fn:
lines = [x.strip().split("\t") for x in fn.readlines()]
if args.header:
lines = lines[1:]
# with open(args.o, 'w') as csvfile:
# writer = csv.writer(csvfile)
# for line in lines:
# writer.writerow(line)
# quit()
# set columns of lines in right order
if args.c:
indexline = utils.read_columnfile(args.c)
def read_routes():
input_routes = read_json(os.path.join(exec_root, '..', 'preprocessor', 'input.json'))
if len(set(r['name'] for r in input_routes['routes'])) != len(input_routes['routes']):
print('Duplicate route names in input.json. Can\'t process.')
return {r['name']: r for r in input_routes['routes']}
CUR_DIR = os.path.realpath('.')
BATCH_SIZE = int(os.getenv('CK_BATCH_SIZE', 8))
DEVICE_ID = int(os.getenv('CK_DEVICE_ID', 0))
SNAPSHOTS_DIR = 'snapshots'
SNAPSHOT_INTERVAL = int(os.getenv('CK_SNAPSHOT_INTERVAL', 1000))
TEST_INTERVAL = int(os.getenv('CK_TEST_INTERVAL', 250))
TEST_RESULTS_DIR = 'validations'
SOLVER_PROTOTXT = 'solver.prototxt'
TRAIN_PROTOTXT = 'train.prototxt'
TEST_PROTOTXT = 'test.prototxt'
TRAIN_LMDB = 'train_lmdb'
TEST_LMDB = 'test_lmdb'
LABEL_MAP_FILE = 'labelmap_kitti.prototxt'
NAME_SIZE_FILE = 'test_name_size.txt'
PREPARED_INFO = utils.read_json('info.json')
PREPARED_IMG_W = int(PREPARED_INFO['img_width'])
PREPARED_IMG_H = int(PREPARED_INFO['img_height'])
NUM_CLASSES = int(PREPARED_INFO['num_classes'])
def read_prototxt_net(file_name):
net = caffe_pb2.NetParameter()
utils.read_prototxt(file_name, net)
layers = {}
for layer in net.layer:
layers[layer.name] = layer
return net, layers
def prepare_solver_prototxt():
def test_read_json_fails_nicely(self):
self.assertIsNone(read_json('https://www.google.com/'))
self.assertIsNone(read_json('http://hopefullyneverarealurl.com/'))
