def normalizeFds():
logging.info('Normalizing Finantial Data')
year = datetime.today().year
for company in getAll(es, index=INDEX_COMPANY):
try:
saveInfos(es, forecast(company['id'], year))
except Exception as error:
logging.error('Exception normalizing: %s', error)
def searchC(alias, term):
list = []
posts = getAll(alias)
for post in posts:
body = post['body']
result = re.findall(term, body, re.IGNORECASE)
if result != []:
list.append(post)
else:
continue
return list
def get_results():
print(title)
## Block of code to fine min cost and min runtime
runtimes = getAll(app, system, datasize, dataset=config["dataset"])
if len(runtimes) == 0:
return [[]], []
df = pd.DataFrame(runtimes, columns=['Runtime', 'Num', 'Type', 'Size'])
df["Cost"] = df.apply(lambda x: (prices[x["Type"] + "." + x["Size"]] /
3600.0) * x["Num"] * x["Runtime"],
axis=1)
min_runtime = df['Runtime'].min()
min_cost = df["Cost"].min()
runtimes = parseLogs(system,
app,
datasize,
configJsonName,
logDir=log_dir,
value_key=value_key)
df = pd.DataFrame(
runtimes,
columns=['Algorithms', 'Budget', 'Best Runtime', 'Experiment'])
# print(df)
df_select = df[df['Budget'].isin(steps)]
df_select['Algorithms'] = df_select['Algorithms'].str.upper()
df_select['Init Samples'] = df.apply(
lambda x: int(x["Algorithms"].split("_")[1]), axis=1)
norm_df = df_select.copy()
if "Exec. Cost" in config["metric"]:
norm_df['Norm. Best Runtime'] = norm_df['Best Runtime'] / min_cost
else:
norm_df['Norm. Best Runtime'] = norm_df['Best Runtime'] / min_runtime
plt.figure(figsize=(20, 10))
ax = sns.barplot(x='Budget',
y='Norm. Best Runtime',
hue='Algorithms',
data=norm_df) # , dodge=0.5)
# ax = sns.boxplot(x='Budget', y='Norm. Best Runtime', hue='Algorithms', data=norm_df)
h, l = ax.get_legend_handles_labels()
if config["legends_outside"]:
ax.legend(bbox_to_anchor=(0, 1.02, 1, 0.2),
loc="lower right",
ncol=config["legend_cols"],
prop={'size': 10})
else:
ax.legend(loc='upper right',
ncol=config["legend_cols"],
prop={'size': 10})
plt.ylim(bottom=1.0)
plt.ylabel("Norm. Best " + config["metric"])
dir = 'plots/norm/' + prefix + "/"
os.makedirs(dir, exist_ok=True)
plt.savefig(dir + title + '.pdf', bbox_inches="tight")
plt.close()
i = 2
for e in params:
df_select[e] = df.apply(lambda x: float(x["Algorithms"].split("_")[i]),
axis=1)
i += 1
X = df_select[["Budget", "Experiment", "Init Samples", *params]].values
Y = df_select["Best Runtime"].values
# calculate_significance(df_select, title)
return X, Y
count = 0
flag = True
if flag:
config = json.load(open(configJsonName, 'r'))
count = 0
violations = pd.DataFrame(
columns=['Algorithms', 'Threshold', "Violations"])
for system in config["systems"]:
for app in config["applications"][system]:
for datasize in config["datasizes"]:
count += 1
# plt.figure()
title = system + "_" + app + "_" + datasize
runtimes = getAll(app,
system,
datasize,
dataset=config["dataset"])
if len(runtimes) == 0:
continue
df = pd.DataFrame(runtimes,
columns=['Runtime', 'Num', 'Type', 'Size'])
df["Cost"] = df.apply(lambda x: (prices[x["Type"] + "." + x[
"Size"]] / 3600.0) * x["Num"] * x["Runtime"],
axis=1)
# df["Cost"] = (prices[df["Type"] + "." + df["Size"]]/3600.0) * df["Num"] * df["Runtime"]
min_runtime = df['Runtime'].min()
min_cost = df["Cost"].min()
if "Runtime" in metric:
min_value = min_runtime
else:
min_value = min_cost
def getPortfolioByType(t):
q = 'type: {0} AND optype: C'.format(t)
for i in getAll(es, index=INDEX_INVESTIMENT, q=q): yield i
def getQuoteHistory(codneg, dt):
q = 'CODNEG: {0} AND DTPREG:[{1:%Y-%m-%d} TO *]'.format(codneg, dt)
for i in getAll(es, index=INDEX_QUOTEHISTORY, q=q): yield i
def getFdByIdAndYear(es, companyid, year, document_type='DFP'):
logging.info('Getting financial data for company id %s and year %s', companyid, year)
query = 'id: {0} AND period: {1}'.format(companyid, year)
ret = [i for i in getAll(es, index=INDEX_FD, doc_type=document_type, q=query)]
return ret[0] if len(ret) > 0 else None
def getQuarterData(companyid, year, q=None):
q = '*' if q is None else '{0:02d}'.format(q*3)
query = 'id: {0} AND period: {1}{2}'.format(companyid, year, q)
return [i for i in getAll(es, index=INDEX_FD, doc_type='ITR', q=query, sort='period:desc')]
def getQuotes():
logging.info('Getting Quotes')
for i in getAll(es, index=INDEX_QUOTE, doc_type=2): yield i
def getFds():
logging.info('Getting Financial Data')
for i in getAll(es, index=INDEX_FD): yield i
def getCompanyById(companyid):
logging.info('Getting company by id %s', companyid)
ret = [i for i in getAll(es, index=INDEX_COMPANYDETAIL, q='id: ' + companyid)]
return ret[0] if len(ret) > 0 else None
def getCompanyByCod(cod):
logging.info('Getting company by trading code %s', cod)
ret = [i for i in getAll(es, index=INDEX_COMPANYDETAIL, q='trading_codes: '+cod)]
return ret[0] if len(ret) > 0 else None
def getCompanyDetailsById(companyid):
logging.info('Getting Company Detail for id %s', companyid)
query = 'id: {0}'.format(companyid)
ret = [i for i in getAll(es, index=INDEX_COMPANYDETAIL, doc_type='companydetails', q=query)]
return ret[0] if len(ret) > 0 else None