def test_retries(self, mock_get):
with self.assertLogs() as cm:
self.assertIsNone(
get_data("http://www.mocky.io/v2/5e81ea2b2f00000d002fb6d2",
max_retries=1))
self.assertIsNotNone(
get_data("http://www.mocky.io/v2/5e81ea2b2f00000d002fb6d2"))
self.assertEqual(len(cm.output), 3)
def question_two(entity, k=3):
main.get_data()
result = main.q2(entity, k=k)
if result is None:
return statement("I couldn't find anything.")
top_results = [i[0] for i in result]
results = ", ".join(top_results)
msg = "Most associated with " + entity + ": " + results
return statement(msg)
def populate_api(self):
try:
school_data = main.get_data(self.url)
main.populate_db(self.cursor, school_data)
main.commit_changes(self.conn)
except FileNotFoundError:
print('File Not Found')
def main():
rev_by_star = get_data()
X_train, Y_test, X_target, Y_target = split_data(rev_by_star)
X_train = X_train.toarray()
Y_test = Y_test.toarray()
#Y_target = trans_target(Y_target)
#X_target = trans_target(X_target)
input_num = 1000
output_num = 5
X_target = to_categorical(X_target, 5)
#Y_target = to_categorical(Y_target, 5)
#data = np.random.random((2148051, input_num))
#labels = np.random.randint(output_num, size=(2148051, 1))
#print(X_target[0])
#print(X_train.dtype, X_target.dtype)
#labels = to_categorical(labels, 5)
#print("data shape", data.dtype)
#print("label shape", labels.dtype)
print(type(X_train))
model = build_model(input_num, output_num)
model.fit(X_train,
X_target,
batch_size=128,
nb_epoch=5,
validation_split=0.25)
#model.fit(data, labels, batch_size=32, nb_epoch=10)
Y_pred = model.predict(Y_test)
Y_pred = categorical_probas_to_classes(Y_pred)
print("Accuracy is : %.2f" % ((Y_target == Y_pred).sum() * 1.0 /
(1.0 * Y_test.shape[0])))
plot_confusion_matrix(Y_pred, Y_target, "neural_network")
def test_get_data(self):
""" Test that compares the retrieved data with the expected data that's stored locally """
remote_data = get_data(
'http://www.mocky.io/v2/5e539b332e00007c002dacbe')
with open('response.json') as fin:
local_data = json.load(fin)
self.assertDictEqual(remote_data, local_data)
def result(train_fraction, test_fraction):
all_data = main.get_data(train_fraction,
test_fraction) # 0 = train 1 = test
testing_data = all_data[1]
all_models = train_bayes.get_models(all_data[0])
tp = 0
fp = 0
tn = 0
fn = 0
#p = high
#n = low
for i in testing_data:
if train_bayes.predict(all_models, i[0:len(i) - 1]) == i[-1]:
if i[-1] == 3:
tp += 1
else:
tn += 1
else:
if i[-1] == 3:
fn += 1
else:
fp += 1
# tpr = high_correct/(high_correct+high_wrong)
# far =
confusion_matrix = [[tn, fp], [fn, tp]]
return confusion_matrix
def test_false(self):
self.assertEqual(
get_data(
"https://collectionapi.metmuseum.org/public/collection/v2/departments"
),
{'message': 'Not Found'},
)
def main_function():
today_date = datetime.today().strftime('%Y-%m-%d')
data = get_data()
# print(data.head(3))
data.to_csv('results/NSE_'+today_date+'.csv')
# return this data file
return Response(print('Downloade Complete for', today_date, 'NSE Option Chain data'))
def setUpClass(cls):
logging.info('setUpClass')
# define in-memory mock data for the weather files
cls.mock_weather_df = pd.DataFrame (
{ 'ForecastSiteCode': ['','','','',''],
'ObservationTime': ['','','','',''],
'ObservationDate': ['2016-02-01',
'2016-02-01',
'2016-02-01',
'2016-02-01',
'2016-02-01'],
'WindDirection': ['','','','',''],
'WindSpeed': ['','','','',''],
'WindGust': ['','','','',''],
'Visibility': ['','','','',''],
'ScreenTemperature':
['2.1','0.1','2.8','1.6','9.8'],
'Pressure': ['','','','',''],
'SignificantWeatherCode': ['','','','',''],
'SiteName': ['','','','',''],
'Latitude': ['60.7490','40.7490','30.7490','20.7490',''],
'Longitude': ['-1.8540','-2.8540','-0.8540','-0.1854',''],
'Region':
['Orkney & Shetland',
'Orkney & Shetland',
'Orkney & Shetland',
'Orkney & Shetland',
'Highland & Eilean Siar'],
'Country': ['SCOTLAND','SCOTLAND','SCOTLAND','SCOTLAND','']} )
cls.mock_weather_df['ObservationDate'] = pd.to_datetime(cls.mock_weather_df['ObservationDate'])
cls.mock_weather_df['ScreenTemperature'] = cls.mock_weather_df['ScreenTemperature'].astype('float64')
# create utils helper object
helper = Helper()
cls.configs ={'file1': 'weather.20160201.csv', 'file2': 'weather.20160301.csv', 'path': 'raw_data/', 'separator': ','}
cls.weather_data_df = main.get_data(helper, cls.configs)
def test_get_data(self):
""" tests whether get_data method in main.py reads data from file
and return the contents in string array fromat.
use unit_test_file.txt data for test purpose.
"""
self.assertEqual(main.get_data("./test/unit_test_file.txt"),
['check', 'data', 'is', 'right'])
def adddata():
# Retrieve the name from url parameter
date_time_str = request.args.get("date", None)
date_time_obj = datetime.strptime(date_time_str, '%Y-%m-%d')
table_rows = get_data(date_time_obj.strftime('%m/%d/%Y'))
update_db(connect_to_db(), table_rows)
return "<h1>Thanks!</h1>"
def test_row_data(self):
link = 'pagina_test/lei.html'
soup = BeautifulSoup(open(link), 'html.parser')
row = soup.findAll('table')[0].findAll('tr')[3::]
alerj = get_data(row[0])
self.assertEqual(alerj['data'], '09/10/2019')
def test_getschooldata():
url = "https://api.data.gov/ed/collegescorecard/v1/schools.json?school.degrees_awarded.predominant=2," \
"3&fields=id,school.state,school.name,school.city,2018.student.size," \
"2017.student.size,2017.earnings.3_yrs_after_completion.overall_count_over_poverty_line," \
"2016.repayment.3_yr_repayment.overall"
resulting_array = main.get_data(url)
num_schools = len(resulting_array)
assert num_schools > 1000
def __init__(self, driver_obj, messages_obj, client_id):
from main import get_data
self.driver = driver_obj
self.messages = messages_obj
self.client = client_id
self.data = get_data(self.client)
self.deamon = True
threading.Thread.__init__(self)
def test_get_data_present(query_fn, testbed):
try:
main.client.set(KEY_PREFIX + 'key', 'data', 9000)
except Exception:
pytest.skip('Redis is unavailable')
data = main.get_data()
query_fn.assert_not_called()
assert data == 'data'
main.client.delete(KEY_PREFIX + 'key')
def test_get_data_not_present(query_fn, testbed):
try:
main.client.set(KEY_PREFIX + 'counter', '0', 9000)
except redis.RedisError:
pytest.skip('Redis is unavailable')
data = main.get_data(KEY_PREFIX + 'key')
query_fn.assert_called_once_with()
assert data == 'data'
assert 'data' == main.client.get(KEY_PREFIX + 'key')
main.client.delete(KEY_PREFIX + 'key')
def process_dirs(path_to_experiments):
if not os.path.isdir('results'):
os.mkdir('results')
entries = os.listdir(path_to_experiments)
dirs = [dir for dir in entries if os.path.isdir(os.path.join(path_to_experiments,dir))]
pipe = lambda dir: search_clusters(dir,*(get_data(os.path.join(path_to_experiments,dir))))
result = map(pipe,dirs)
for i,dir in enumerate(dirs):
print('%s:' % dir)
for entry in result[i]:
print ' {0}:{1}'.format(*entry)
def main():
args = Parser().parse()
use_cuda = not args.cpu and torch.cuda.is_available()
device = torch.device(f"cuda:{args.gpu}" if use_cuda else "cpu")
torch.manual_seed(args.seed)
train_loader, val_loader, test_loader = get_data(args)
# labels should be a whole number from [0, num_classes - 1]
num_labels = 10 #int(max(max(train_data.targets), max(test_data.targets))) + 1
output_size = num_labels
setup_logging(args)
if 'resnet' in args.model:
constructor = getattr(resnet, args.model)
model_stoch = constructor(True, num_labels, device).to(device)
model_det = constructor(False, num_labels, device).to(device)
elif 'vgg' in args.model:
constructor = getattr(vgg, args.model)
model_stoch = constructor(True, num_labels, device,
args.orthogonal).to(device)
model_det = constructor(False, num_labels, device,
args.orthogonal).to(device)
else:
stoch_args = [True, True, device]
det_args = [False, False, device]
model_stoch = lenet5.LeNet5(*stoch_args).to(device)
model_det = lenet5.LeNet5(*det_args).to(device)
# load saved parameters
saved_models = glob(f'/scratch/bsm92/{args.model}_{args.dataset}*.pt')
saved_det = saved_models[0] if 'det' in saved_models[0] else saved_models[1]
saved_stoch = saved_models[1 - saved_models.index(saved_det)]
it = zip([model_stoch, model_det], [saved_stoch, saved_det])
for model, param_path in it:
saved_state = torch.load(param_path, map_location=device)
if param_path[-4:] == '.tar':
saved_state = saved_state['model_state_dict']
model.load_state_dict(saved_state)
loss = torch.nn.CrossEntropyLoss()
test(args, model_det, device, test_loader, loss, 10)
test(args, model_stoch, device, test_loader, loss, 10)
def index():
if request.method == 'POST':
print("POST")
address = request.form['jobnumber']
elections = get_data(address)
html = "<div>"
html += "Address: %s </br>" % address
html += "Elections: </br>"
html += "<ul>"
for name, day, division in elections:
html += "<li>" + name + "</li>"
html += "<ul>"
html += "<li>Election day: " + day + "</li>"
html += "<li>Division: " + division + "</li>"
html += "</ul>"
html += "</ul>"
html += "</div>"
return html
return render_template('index.html')
def process_data():
page = int(request.args.get("page", 1))
per_page = PER_PAGE
offset = (page - 1) * per_page
data = get_data(
org_name=session["org_name"],
repo_count=session["repo_count"],
committees=session["committees"],
per_page=per_page,
offset=offset,
)
total = data["count"]
pagination = Pagination(
page=page,
per_page=per_page,
total=total,
css_framework="bootstrap4",
error_out=False,
)
return (data, page, per_page, pagination)
return statement("I couldn't find anything.")
return statement("I found this: " + results)
@ask.intent("GetEntity")
def question_two(entity, k=3):
main.get_data()
result = main.q2(entity, k=k)
if result is None:
return statement("I couldn't find anything.")
top_results = [i[0] for i in result]
results = ", ".join(top_results)
msg = "Most associated with " + entity + ": " + results
return statement(msg)
@ask.intent("ThirdIntent")
def question_three(query, k=3):
result = main.q3(query, k=k)
if result is None:
return statement("I couldn't find anything.")
top_results = [i[0] for i in result]
results = ", ".join(top_results)
msg = "Most associated with " + query + ": " + results
return statement(msg)
if __name__ == '__main__':
# main.initialize()
main.get_data()
app.run()
def main():
args = Parser().parse()
use_cuda = not args.cpu and torch.cuda.is_available()
device = torch.device(f"cuda:{args.gpu}" if use_cuda else "cpu")
torch.manual_seed(args.seed)
train_loader, val_loader, test_loader = get_data(args)
# labels should be a whole number from [0, num_classes - 1]
num_labels = 10 #int(max(max(train_data.targets), max(test_data.targets))) + 1
output_size = num_labels
setup_logging(args)
if 'resnet' in args.model:
constructor = getattr(resnet, args.model)
model_stoch = constructor(True, device).to(device)
model_det = constructor(False, device).to(device)
elif 'vgg' in args.model:
constructor = getattr(vgg, args.model)
model_stoch = constructor(True, device, args.orthogonal).to(device)
model_det = constructor(False, device, args.orthogonal).to(device)
else:
stoch_args = [True, True, device]
det_args = [False, False, device]
model_stoch = lenet5.LeNet5(*stoch_args).to(device)
model_det = lenet5.LeNet5(*det_args).to(device)
# load saved parameters
saved_models = glob(f'experimental_models/{args.model}*')
saved_det = saved_models[0] if 'det' in saved_models[0] else saved_models[1]
saved_stoch = saved_models[1 - saved_models.index(saved_det)]
it = zip([model_stoch, model_det], [saved_stoch, saved_det])
for model, param_path in it:
saved_state = torch.load(param_path, map_location=device)
if param_path[-4:] == '.tar':
saved_state = saved_state['model_state_dict']
model.load_state_dict(saved_state)
rows = []
det_row_prefix = [args.model, args.dataset, saved_det, False, 1]
for _ in range(args.inference_passes):
cal_results = [
*calc_calibration(args, model_det, device, test_loader,
args.batch_size, num_labels, 1)
]
rows.append(det_row_prefix + cal_results)
_log_calibration(*cal_results)
get_temp_scheduler(model_temps(model_stoch, val_only=False), args).step()
stoch_row_prefix = [args.model, args.dataset, saved_stoch, True, 1]
for num_passes in range(1, 51):
for _ in range(args.inference_passes):
cal_results = [
*calc_calibration(args, model_stoch, device, test_loader,
args.batch_size, num_labels, num_passes)
]
stoch_row_prefix[-1] = num_passes
rows.append(stoch_row_prefix + cal_results)
_log_calibration(*cal_results, prefix=f'NUM PASSES: {num_passes}')
_write_results(args, rows)
import main
import numpy as np
from scipy import stats
from matplotlib import pyplot as plt
from astroML.plotting import setup_text_plots
from astroML.stats import mean_sigma, median_sigmaG
setup_text_plots(fontsize=8, usetex=False)
if __name__ == '__main__':
quasars = main.get_data('data/I_Paris_plate_mjd_fiber.dat')
qsos, wavelengths = main.build_rest_frame(quasars)
qsos_m = main.list_to_matrix(qsos)
for i in xrange(0, 4000, 500):
fig = plt.figure(figsize=(5, 7))
fig.subplots_adjust(left=0.13, right=0.95,
bottom=0.06, top=0.95,
hspace=0.1)
ax = fig.add_subplot(1, 1, 1)
avg = np.mean(qsos_m[:, i])
std = np.std(qsos_m[:, i])
data = (qsos_m[:, i] - avg) / std
x = np.linspace(-5, 5, 1000)
pdf = stats.norm(0, 1).pdf(x)
A2, sig, crit = stats.anderson(data)
D, pD = stats.kstest(data, "norm")
W, pW = stats.shapiro(data)
import numpy as np
import seaborn as sns
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
from sklearn.semi_supervised import (
LabelPropagation,
LabelSpreading,
SelfTrainingClassifier
)
from sklearn.svm import SVC
from main import plot_decision_boundary, get_data
if __name__ == '__main__':
X_train, X_test, y_train, y_test = get_data()
X = np.concatenate([X_train, X_test], axis=0)
y = np.concatenate([y_train, -1 * np.ones_like(y_test)], axis=0)
models = (
LabelPropagation(max_iter=10000),
LabelSpreading(),
SelfTrainingClassifier(base_estimator=SVC(probability=True, gamma="auto"))
)
color_maps = ('Blues', 'Greens', 'Reds')
for model, cmap in zip(models, color_maps):
model.fit(X, y)
y_pred = model.predict(X[y == -1])
def main(cfg: DictConfig) -> None:
x, y = get_data(cfg)
assert_binary_data(x, y)
x_tr, x_te, y_tr, y_te = train_test_split(
x,
y,
test_size=cfg.evaluation.test_ratio,
random_state=_seed(cfg.seed))
mlflow.set_tracking_uri(hydra.utils.get_original_cwd() + '/mlruns')
experiment_id = get_mlflow_experiment_id(cfg.name)
with mlflow.start_run(experiment_id=experiment_id):
loss = get_loss_class(cfg)
stopping_criterion = EarlyStopping(
monitor='train_loss',
lower_is_better=True,
patience=min(10, cfg.training.max_epochs),
threshold=cfg.training.tol,
threshold_mode='rel',
sink=logger.info,
)
clf = Classifier(
module=LinearClassifier,
module__n_features=x.shape[1],
max_epochs=cfg.training.max_epochs,
criterion=loss,
predict_nonlinearity=get_loss_link(cfg),
optimizer=torch.optim.Adam,
iterator_train__batch_size=cfg.training.batch_size,
iterator_train__shuffle=True,
train_split=False,
callbacks=[('stopping_criterion', stopping_criterion)],
verbose=cfg.verbose,
)
if check_if_validation(cfg):
params = get_validation_params(cfg)
clf = GridSearchCV(
clf,
params,
refit=True,
cv=cfg.evaluation.n_cv,
scoring=negative_brier_score,
n_jobs=-1,
)
else:
clf.set_params(
lr=cfg.training.lr,
optimizer__weight_decay=cfg.training.regularization,
)
if check_if_gev_loss(loss):
clf.set_params(criterion__xi=cfg.loss.xi)
mlflow.log_param('dataset', cfg.dataset)
mlflow.log_param('loss', cfg.loss.name)
mlflow.log_param('lr', cfg.training.lr)
mlflow.log_param('max_epochs', cfg.training.max_epochs)
mlflow.log_param('tol', cfg.training.tol)
mlflow.log_param('regularization', cfg.training.regularization)
mlflow.log_param('seed', _seed(cfg.seed))
if check_if_gev_loss(loss):
mlflow.log_param('xi', cfg.loss.xi)
# Step 1: fit the base CPE model
x_tr, x_vl, y_tr, y_vl = train_test_split(x_tr, y_tr, test_size=0.3)
clf.fit(x_tr, y_tr)
# Step 2: search for the best threshold
threshold_candidates = np.arange(0.05, 1., 0.05)
validation_scores = [
f1_score(y_vl,
(clf.predict_proba(x_vl)[:, 1] > th).astype(np.uint8))
for th in threshold_candidates
]
threshold = threshold_candidates[np.argmax(validation_scores)]
y_prob = clf.predict_proba(x_te)[:, 1]
y_pred = (y_prob > threshold).astype(np.uint8)
log_metric('f1_score', f1_score(y_te, y_pred))
log_metric('roc_auc_score', roc_auc_score(y_te, y_pred))
log_metric('best_threshold', threshold)
import main as lib #parse all the info from csv lib.get_data() #here we get the rating of results rating = lib.generate_rating(lib.students) #now form a scholarship list list = lib.get_scholarship(lib.all_students, rating) #print the list to console lib.form_output(list) #and to a csv lib.form_csv(list)
sys.exit(1)
config = configuration()
config.readconfig()
if len(args) < 1:
print _("Specify at least one video.")
sys.exit(1)
for i in args:
print "----"
print _("Trying to download the video...")
print _("URL: %(url)s") % {"url" : i}
try:
data = get_data(i)
data.start()
while data.status == -1:
sleep(0.02)
if data.status == 0:
if options.saveAs != None:
saveAs = options.saveAs
else:
saveAs = os.path.join(
options.destination,
data.data[2]
)
print 'Saving file as "%(file)s"...' % {"file" : saveAs}
down = fdownload(
data.data[0],
saveAs
def set_dragon():
table = 'Dragons'
fields = 'BaseId,Name,NameJP,Rarity,ElementalType,' + \
'MinHp,MaxHp,MinAtk,MaxAtk,' + \
'Abilities11,Abilities12,Abilities21,Abilities22,' + \
'IsPlayable'
group = 'BaseId'
parse_int = ['MinHp', 'MaxHp', 'MinAtk', 'MaxAtk']
raw_data = main.get_data(table, fields, group)
names = main.load_name(FILE_NAME)
data_new = []
# data_list = []
data_dict = {}
data_updates = [False]
for i in raw_data:
item = i['title']
if item['BaseId'] and item[
'IsPlayable'] == '1' and item['BaseId'] != '210091':
uid = '{}_01'.format(item['BaseId'])
name = main.set_name(names, item, data_new, data_updates)
new_item = {
'id': uid,
'name': name,
'element': item['ElementalType'],
'rarity': item['Rarity'],
}
for k in parse_int:
new_item[k] = int(item[k])
HP_V = {'incHP1': 0, 'incHP2': 0}
STR_V = {'incSTR1': 0, 'incSTR2': 0}
res_V = {}
for a in [
'Abilities11', 'Abilities12', 'Abilities21', 'Abilities22'
]:
ability = abilities.get(item[a], '')
if ability:
new_item[a.lower()] = ability['Might']
level = a[-1]
if 'reqEle' in ability:
new_item['reqEle'] = ability['reqEle']
if 'HP' in ability:
HP_V['incHP' + level] = ability['HP']
elif 'STR' in ability:
STR_V['incSTR' + level] = ability['STR']
elif 'Hybrid' in ability:
HP_V['incHP' + level] = ability['Hybrid']
STR_V['incSTR' + level] = ability['Hybrid']
if 'res' in ability:
res_V['resEle'] = ability['resEle']
res_V['incRes'] = ability['res']
else:
new_item[a.lower()] = 0
new_item.update(HP_V)
new_item.update(STR_V)
new_item.update(res_V)
# data_list.append(new_item)
data_dict[uid] = new_item
# main.save_file('list', FILE_NAME, data_list)
main.save_file('dict', FILE_NAME, data_dict)
if data_updates[0]:
main.save_file('locales', FILE_NAME, names)
if data_new:
print(data_new)
main.download_images(FILE_NAME, data_new)
rm_lemm = [[wordnet.lemmatize(word) for word in words]
for words in rm_stop]
else:
rm_lemm = rm_stop
if stem == porter:
rm_stem = [[porter.stem(word) for word in words] for words in rm_lemm]
elif stem == snowball:
rm_stem = [[snowball.stem(word) for word in words]
for words in rm_lemm]
else:
rm_stem = rm_lemm
clean_lyrics = [' '.join([word for word in words]) for words in rm_stem]
return pd.Series(clean_lyrics)
full_dataset = get_data()
# CV
def confusion_matrix_plot_cv(tn,
fp,
fn,
tp,
save_fig=False,
cmap=plt.cm.Blues):
'''
This function prints and plots the confusion matrix.
'''
cm = np.array([[tp, fn], [fp, tn]])
classes = ['True', 'False']
plt.figure(figsize=(4, 4))
def set_adventurer():
table = 'Adventurers'
fields = 'Id,VariationId,Name,NameJP,WeaponType,Rarity,ElementalType,' + \
'MinHp3,MinHp4,MinHp5,MaxHp,PlusHp0,PlusHp1,PlusHp2,PlusHp3,PlusHp4,McFullBonusHp5,' + \
'MinAtk3,MinAtk4,MinAtk5,MaxAtk,PlusAtk0,PlusAtk1,PlusAtk2,PlusAtk3,PlusAtk4,McFullBonusAtk5,' + \
'Abilities11,Abilities12,Abilities21,Abilities22,Abilities31,Abilities32,' + \
'DefCoef,IsPlayable'
group = 'Id,VariationId'
parse_int = [
'MinHp3', 'MinHp4', 'MinHp5', 'MaxHp', 'PlusHp0', 'PlusHp1', 'PlusHp2',
'PlusHp3', 'PlusHp4', 'McFullBonusHp5', 'MinAtk3', 'MinAtk4',
'MinAtk5', 'MaxAtk', 'PlusAtk0', 'PlusAtk1', 'PlusAtk2', 'PlusAtk3',
'PlusAtk4', 'McFullBonusAtk5', 'DefCoef'
]
raw_data = main.get_data(table, fields, group)
names = main.load_name(FILE_NAME)
o_len = len(names)
data_list = []
data_dict = {}
data_new = []
for i in raw_data:
item = i['title']
if item['IsPlayable'] == '1':
uid = '{}_0{}'.format(item['Id'], item['VariationId'])
name = main.set_name(names, item, data_new)
weapon = item['WeaponType']
rarity = item['Rarity']
new_item = {
'id': uid,
'name': name,
'weapon': weapon,
'element': item['ElementalType'],
'rarity': rarity,
}
for k in parse_int:
new_item[k] = int(item[k])
inc_LV = {}
inc_Value = {}
for a in [
'Abilities11', 'Abilities12', 'Abilities21', 'Abilities22',
'Abilities31', 'Abilities32'
]:
ability = abilities.get(item[a], '')
if ability:
new_item[a.lower()] = ability['Might']
level = a[-1]
if 'STR' in ability:
inc_LV['STRLV' + level] = ABILITY_LEVEL.get(
rarity, ABILITY_LEVEL['res'])[a]
inc_Value['incSTR' + level] = ability['STR']
if 'def' in ability:
inc_LV['defLV' + level] = ABILITY_LEVEL.get(
rarity, ABILITY_LEVEL['res'])[a]
inc_Value['incDef' + level] = ability['def']
new_item.update(inc_LV)
new_item.update(inc_Value)
data_list.append(new_item)
data_dict[uid] = new_item
main.save_file('list', FILE_NAME, data_list)
main.save_file('dict', FILE_NAME, data_dict)
if len(names) != o_len:
print(data_new)
main.save_file('locales', FILE_NAME, names)
main.download_images(FILE_NAME, data_new)
def download_list(self): # Clicked on button 'open' in filechooserdialog
self.listdownloading = True
fc = self.wTree.get_widget("filechooserdialog")
pb = self.wTree.get_widget("dprogressbar")
mainDownload_button = self.wTree.get_widget("mainDownload_button")
mainClose_button = self.wTree.get_widget("mainClose_button")
os.chdir(
self.config.get(
"general",
"save_videos_in"
)
)
fc.hide()
pb.set_fraction(0)
chosen_list = fc.get_filename()
print "%(list)s selected" % {"list" : chosen_list}
print "-----"
mainDownload_button.set_sensitive(False)
mainClose_button.set_sensitive(False)
pb.set_text(_("Downloading a list..."))
file = open(
chosen_list,
"r"
)
i = 0
successful = 0 # videos downloaded successfully
while True:
pb.set_fraction(0)
pb.set_text(_("Fetching video information..."))
line = file.readline()
if not line:
break
if line[-1] == "\n":
line = line[:-1]
print "----"
print _("Trying to download %(link)s") % {"link" : line}
v_no = i+1 # example: You're downloading the v_no. video && You're downloading the 5. video
data = get_data(line)
data.start()
while data.status == -1:
gtk.main_iteration_do(False)
sleep(0.01)
gtk.main_iteration_do(True)
if data.status == 0:
saveAs = os.path.join(
self.config.get(
"general",
"save_videos_in"
),
data.data[2]
)
print _("Saving file as \"%(filename)s\"...") % {"filename" : saveAs}
down = fdownload(
data.data[0], # http://*.*/*
data.data[2] # .flv
)
down.start()
pb.set_text(_("Downloading video %(number)s") % {"number" : v_no})
progress = down.downloaded()/100
while down.get_filesize() == 0:
gtk.main_iteration_do(False)
sleep(0.01)
gtk.main_iteration_do(True)
filesize = down.get_filesize()
print _("Filesize: %(filesize)s KB") % {"filesize" : filesize}
while progress < 1:
gtk.main_iteration_do(False)
sleep(0.005)
pb.set_fraction(progress)
progress = down.downloaded()/100
gtk.main_iteration_do(True)
pb.set_fraction(1)
pb.set_text(_("Finished download #%(number)s") % {"number" : v_no})
successful += 1
if self.config.getboolean(
"general",
"convert"
) and data.data[3]:
pb.set_text(_("Converting file"))
output = convert(
saveAs,
self.config.get(
"general",
"convert_filename_extension"
),
self.config.get(
"general",
"convertcmd"
)
)
output.start()
while output.status == -1:
gtk.main_iteration_do(False)
sleep(0.01)
gtk.main_iteration_do(True)
pb.set_text(_("Converted file."))
if self.deletesourcefile:
os.remove(saveAs)
print _("Deleted input (.flv) file")
else:
print _("Wrong URL / unsupported video portal")
pb.set_text(_("Download #%(number)s failed") % {"number" : v_no})
i += 1
print "----"
pb.set_text(_("%(successful)s of %(all)s successful") % {"successful" : successful, "all" : i})
mainDownload_button.set_sensitive(True)
mainClose_button.set_sensitive(True)
self.listdownloading = False
print "-----"
def test_OK(self):
self.assertIsNotNone(
get_data("http://www.mocky.io/v2/5e539b332e00007c002dacbe"))
from main import get_data
import csv
import numpy as np
pos_path = "./review_polarity/txt_sentoken/pos/*.txt"
neg_path = "./review_polarity/txt_sentoken/neg/*.txt"
pos_data_train, pos_data_test = get_data(pos_path)
neg_data_train, neg_data_test = get_data(neg_path)
print("shape of pos data train")
print(pos_data_train.shape)
print("shape of pos data test")
print(pos_data_test.shape)
print("shape of neg data train")
print(neg_data_train.shape)
print("shape of neg data test")
print(neg_data_test.shape)
with open(
'training_data_pos.csv',
'w',
) as file:
writer = csv.writer(file)
for review in pos_data_train:
for word in review:
writer.writerow(word)
with open(
'testing_data_pos.csv',
'w',
) as file:
writer = csv.writer(file)
for review in pos_data_test:
for word in review:
writer.writerow(word)
from main import data_name_1 from main import data_name_2 from main import data_name_3 from main import get_data lines = get_data(data_name_1)
def download_single(
self,
widget,
event = None
):
entry_url = self.wTree.get_widget("entry_url")
pb = self.wTree.get_widget("dprogressbar")
mainDownload_button = self.wTree.get_widget("mainDownload_button")
mainClose_button = self.wTree.get_widget("mainClose_button")
url = entry_url.get_text()
os.chdir(
self.config.get(
"general",
"save_videos_in"
)
)
pb.set_fraction(0)
pb.set_text(_("Fetching video information..."))
if url == "":
pb.set_text(_("No URL specified"))
else:
print "----"
print _("Trying to download %(url)s") % {"url" : url}
pb.set_fraction(0)
mainDownload_button.set_sensitive(False)
mainClose_button.set_sensitive(False)
data = get_data(url)
data.start()
while data.status == -1:
gtk.main_iteration_do(False)
sleep(0.01)
gtk.main_iteration_do(True)
if data.status == 0:
saveAs = os.path.join(
self.config.get(
"general",
"save_videos_in"
),
data.data[2]
)
print "Saving file as \"%(filename)s\"..." % {"filename" : saveAs}
down = fdownload(
data.data[0],
data.data[2]
)
down.start()
pb.set_fraction(0)
pb.set_text(_("Downloading video..."))
progress = down.downloaded()/100
while down.get_filesize() == 0:
gtk.main_iteration_do(False)
sleep(0.01)
gtk.main_iteration_do(True)
filesize = down.get_filesize()
print "Filesize: %(filesize)s KB" % {"filesize" : filesize}
while progress < 1:
gtk.main_iteration_do(False)
sleep(0.01)
pb.set_fraction(progress)
progress = down.downloaded()/100
gtk.main_iteration_do(True)
pb.set_fraction(1)
pb.set_text(_("Download finished."))
if self.config.getboolean(
"general",
"convert"
) and data.data[3]:
pb.set_text(_("Converting file"))
output = convert(
saveAs,
self.config.get(
"general",
"convert_filename_extension"
),
self.config.get(
"general",
"convertcmd"
)
)
output.start()
while output.status == -1:
gtk.main_iteration_do(False)
sleep(0.01)
gtk.main_iteration_do(True)
pb.set_text(_("Converted file."))
if self.config.getboolean(
"general",
"delete_source_file_after_converting"
):
os.remove(saveAs)
print _("Deleted input (.flv) file")
else:
pb.set_text(_("Wrong URL / unsupported video portal"))
mainDownload_button.set_sensitive(True)
mainClose_button.set_sensitive(True)
print "----"
def set_adventurer():
table = "Adventurers"
fields = (
"Id,VariationId,Name,NameJP,WeaponType,Rarity,ElementalType," +
"MinHp3,MinHp4,MinHp5,MaxHp,PlusHp0,PlusHp1,PlusHp2,PlusHp3,PlusHp4,McFullBonusHp5,"
+
"MinAtk3,MinAtk4,MinAtk5,MaxAtk,PlusAtk0,PlusAtk1,PlusAtk2,PlusAtk3,PlusAtk4,McFullBonusAtk5,"
+
"Skill1Name,Skill2Name,Abilities11,Abilities12,Abilities21,Abilities22,Abilities31,Abilities32,"
+ "DefCoef,IsPlayable")
group = "Id,VariationId"
parse_int = [
"MinHp3",
"MinHp4",
"MinHp5",
"MaxHp",
"PlusHp0",
"PlusHp1",
"PlusHp2",
"PlusHp3",
"PlusHp4",
"McFullBonusHp5",
"MinAtk3",
"MinAtk4",
"MinAtk5",
"MaxAtk",
"PlusAtk0",
"PlusAtk1",
"PlusAtk2",
"PlusAtk3",
"PlusAtk4",
"McFullBonusAtk5",
"DefCoef",
]
raw_data = main.get_data(table, fields, group)
abilities = main.set_abilities()
skills = main.set_skills()
names = main.load_name(FILE_NAME)
data_list = []
data_dict = {}
data_new = []
data_updates = [False]
for i in raw_data:
item = i['title']
if item['IsPlayable'] == '1':
uid = '{}_0{}'.format(item['Id'], item['VariationId'])
name = main.set_name(names, item, data_new, data_updates)
weapon = item['WeaponType']
rarity = item['Rarity']
new_item = {
"id": uid,
"name": name,
"weapon": weapon,
"element": item["ElementalType"],
"rarity": rarity,
}
for k in parse_int:
if item[k].isdigit():
new_item[k] = int(item[k])
inc_LV = {}
inc_Value = {}
for a in [
"Abilities11",
"Abilities12",
"Abilities21",
"Abilities22",
"Abilities31",
"Abilities32",
]:
ability = abilities.get(item[a])
if ability:
# new_item[a.lower()] = ability['Might']
new_item[a.lower()] = ability
level = a[-1]
if "STR" in ability:
inc_LV["STRLV" + level] = ABILITY_LEVEL.get(
rarity, ABILITY_LEVEL["res"])[a]
inc_Value["incSTR" + level] = ability["STR"]
if "def" in ability:
inc_LV["defLV" + level] = ABILITY_LEVEL.get(
rarity, ABILITY_LEVEL["res"])[a]
inc_Value["incDef" + level] = ability["def"]
for s in ["Skill1Name", "Skill2Name"]:
skill = skills.get(item[s], {})
if skill:
new_item[s[0:6].lower()] = skill
new_item.update(inc_LV)
new_item.update(inc_Value)
data_list.append(new_item)
data_dict[uid] = new_item
main.save_file("list", FILE_NAME, data_list)
main.save_file("dict", FILE_NAME, data_dict)
if data_updates[0]:
main.save_file('locales', FILE_NAME, names)
if data_new:
print(data_new)
main.download_images(FILE_NAME, data_new)
def export():
app_id = int(request.args['app_id'])
format = request.args['format']
return jsonify(actors = main.get_actors(app_id), data=main.get_data(app_id), goals=main.get_goals(app_id), disclosures=main.get_disclosures(app_id))
