def get_price_on_date(self, symbol, date, time="Close"):
"""Gets the price of the given symbol on the given date
Parameters:
symbol : str
date : datetime
time : str
Which column to use to determine price. Valid times are "Open" and "Close"
Returns:
float
The price of the given symbol on the given date
"""
start_time = timer()
if symbol in self.price_files:
df = self.price_files[symbol]
else:
if utils.refresh(utils.get_file_path(config.prices_data_path, prices.price_table_filename, symbol=symbol), refresh=False):
prices.download_data_from_yahoo(symbol, start_date=self.start_date, end_date=self.end_date)
df = pd.read_csv(utils.get_file_path(config.prices_data_path, prices.price_table_filename, symbol=symbol), index_col="Date", parse_dates=["Date"])[self.start_date:self.end_date]
self.price_files[symbol] = df
price = df.loc[date][time] if date in df.index else self.get_price_on_date(symbol, utils.add_business_days(date, -1), time=time)
self.times[get_price_time] = self.times[get_price_time] + timer() - start_time
return price
def get_cgar(symbol, start_date=config.start_date, end_date=config.end_date):
"""Returns the compound annual growth rate of the given symbol
Parameters:
symbol : str
start_date : date, optional
end_date : date, optional
Returns:
float
The compound annual growth rate of the given symbol
"""
if utils.refresh(utils.get_file_path(config.prices_data_path,
prices.price_table_filename,
symbol=symbol),
refresh=False):
prices.download_data_from_yahoo(symbol,
start_date=start_date,
end_date=end_date)
df = pd.read_csv(utils.get_file_path(config.prices_data_path,
prices.price_table_filename,
symbol=symbol),
index_col="Date",
parse_dates=["Date"])[start_date:end_date]
# Formula normally has a -1 at the end
return (df["Close"].add(df["Dividends"].cumsum())[-1] /
df["Close"][0])**(1 / ((df.index[-1] - df.index[0]).days) / 252)
def job():
for func in [ema, sma, macd, rsi, bb]:
buy_list = []
sell_list = []
for symbol in sp.get_sp500():
# TODO: needs a refresh
func.generate_signals(symbol,
start_date=utils.add_business_days(
datetime.date.today(), -200),
end_date=datetime.date.today())
df = pd.read_csv(utils.get_file_path(config.ta_data_path,
func.table_filename,
symbol=symbol),
index_col="Date",
parse_dates=["Date"])
if df.index[-1].date() != datetime.date.today():
print(
"Last date in {} file does not equal todays date. Is today a weekend or a holiday?"
.format(symbol))
if df[func.get_signal_name()][-1] == ta.buy_signal:
buy_list.append(symbol)
if df[func.get_signal_name()][-1] == ta.sell_signal:
sell_list.append(symbol)
df = pd.DataFrame({
"Buy": pd.Series(buy_list),
"Sell": pd.Series(sell_list)
})
df.to_csv(utils.get_file_path(config.ta_data_path,
func.get_signal_name() + table_filename,
dated=True,
start_date="",
end_date=datetime.date.today()),
index=False)
def get_performance(symbol,
start_date=config.start_date,
end_date=config.end_date):
"""Returns the overall performance of the given symbol
Parameters:
symbol : str
start_date : date, optional
end_date : date, optional
Returns:
float
The overall performance of the given symbol
"""
if utils.refresh(utils.get_file_path(config.prices_data_path,
prices.price_table_filename,
symbol=symbol),
refresh=False):
prices.download_data_from_yahoo(symbol,
start_date=start_date,
end_date=end_date)
df = pd.read_csv(utils.get_file_path(config.prices_data_path,
prices.price_table_filename,
symbol=symbol),
index_col="Date",
parse_dates=["Date"])[start_date:end_date]
return df["Close"].add(df["Dividends"].cumsum())[-1] / df["Close"][0]
def generate_signals(symbol,
period=default_period,
thresholds=default_thresholds,
refresh=False,
start_date=config.start_date,
end_date=config.end_date):
"""Calculates the rsi buy/sell signals for the given symbol, saves this data in a .csv file, and plots this data. Only uses the first and last periods
The RSI is a leading momentum indicator.
Parameters:
symbol : str
period : int, optional
thresholds: dict
Must contain keys "Low" and "High", both with a value between 0 and 100
refresh : bool, optional
start_date : date, optional
end_date : date, optional
Returns:
dataframe
A dataframe containing the rsi signals for the given symbol
"""
# Why did I do this differently in plot?
rsi(symbol,
period=period,
refresh=refresh,
start_date=start_date,
end_date=end_date)
df = pd.read_csv(utils.get_file_path(config.ta_data_path,
table_filename,
symbol=symbol),
index_col="Date",
parse_dates=["Date"])[start_date:end_date]
signal_column_name = get_signal_name(period=period)
if signal_column_name not in df.columns:
rsi_column_name = "RSI" + str(period)
conditions = [
(df[rsi_column_name].shift(1) > thresholds["Low"]) &
(df[rsi_column_name] <
thresholds["Low"]), # rsi breaches lower threshold, buy signal
(df[rsi_column_name].shift(1) < thresholds["High"]) &
(df[rsi_column_name] >
thresholds["High"]), # rsi breaches upper threshold, buy signal
False, # (df[rsi_column_name].shift(1) > rsi_thresholds["Low"]) & (df[rsi_column_name] < rsi_thresholds["Low"]), # rsi breaches 50 after a buy signal, soft sell
False # (df[rsi_column_name].shift(1) < rsi_thresholds["High"]) & (df[rsi_column_name] > rsi_thresholds["High"]) # rsi breaches 50 after a sell signal, soft buy
]
df[signal_column_name] = np.select(conditions,
ta.signals,
default=ta.default_signal)
utils.debug(df[signal_column_name])
df.to_csv(
utils.get_file_path(config.ta_data_path,
table_filename,
symbol=symbol))
return df[signal_column_name]
def generate_signals(symbol,
period=default_period,
std=default_std,
refresh=False,
start_date=config.start_date,
end_date=config.end_date):
"""Calculates the bollinger bands buy/sell signals for each period for the given symbol, saves this data in a .csv file, and plots this data. Only uses the first and last periods
The BB is a lagging volatility indicator.
Parameters:
symbol : str
period : int, optional
std : int, optional
refresh : bool, optional
start_date : date, optional
end_date : date, optional
Returns:
dataframe
A dataframe containing the bollinger bands signals for the given symbol
"""
bb(symbol,
period,
std,
refresh=False,
start_date=start_date,
end_date=end_date)
df = pd.read_csv(utils.get_file_path(config.ta_data_path,
table_filename,
symbol=symbol),
index_col="Date",
parse_dates=["Date"])[start_date:end_date]
signal_column_name = get_signal_name(period=period, std=std)
if signal_column_name not in df.columns:
lower_column_name = "Lower"
upper_column_name = "Upper"
conditions = [
((df["Close"].shift(1) > df[lower_column_name].shift(1)) &
(df["Close"] < df[lower_column_name])
), # price crosses lower band; buy signal
((df["Close"].shift(1) < df[upper_column_name].shift(1)) &
(df["Close"] > df[upper_column_name])
), # price crosses upper band; sell signal
False, # ((df["Close"].shift(1) < df["Mid"].shift(1)) & (df["Close"] > df["Mid"])) # bb breaches the mid line after a buy signal, soft sell
False # ((df["Close"].shift(1) > df["Mid"].shift(1)) & (df["Close"] < df["Mid"])) # bb breaches the mid line after a sell signal, soft buy
]
df[signal_column_name] = np.select(conditions,
ta.signals,
default=ta.default_signal)
utils.debug(df[signal_column_name])
df.to_csv(
utils.get_file_path(config.ta_data_path,
table_filename,
symbol=symbol))
return df[signal_column_name]
def plot_signals(symbol=default_symbols, period=default_periods, refresh=False, start_date=config.start_date, end_date=config.end_date):
short_vol_symbol = symbol[0]
long_vol_symbol = symbol[1]
generate_signals(symbol=symbol, period=period, refresh=refresh, start_date=start_date, end_date=end_date)
fig, ax = plot_volforecast(period=period, refresh=refresh, start_date=start_date, end_date=end_date)
df = pd.read_csv(utils.get_file_path(config.ta_data_path, table_filename, symbol=""), index_col="Date", parse_dates=["Date"])[start_date:end_date]
short_vol = pd.read_csv(utils.get_file_path(config.ta_data_path, table_filename, symbol=short_vol_symbol), index_col="Date", parse_dates=["Date"])[start_date:end_date]
long_vol = pd.read_csv(utils.get_file_path(config.ta_data_path, table_filename, symbol=long_vol_symbol), index_col="Date", parse_dates=["Date"])[start_date:end_date]
signal_column_name = get_signal_name()
buy_signals = short_vol.loc[short_vol[signal_column_name] == ta.buy_signal]
ax[0].scatter(buy_signals.index, df.loc[df.index.isin(buy_signals.index)][signal_column_name], label=ta.buy_signal, color=ta.signal_colors[ta.buy_signal], marker=ta.signal_markers[ta.buy_signal], s=config.scatter_size, alpha=config.scatter_alpha)
ax[1].scatter(buy_signals.index, short_vol.loc[short_vol.index.isin(buy_signals.index)]["Close"], label=ta.buy_signal, color=ta.signal_colors[ta.buy_signal], marker=ta.signal_markers[ta.buy_signal], s=config.scatter_size, alpha=config.scatter_alpha)
buy_signals = long_vol.loc[long_vol[signal_column_name] == ta.buy_signal]
ax[0].scatter(buy_signals.index, df.loc[df.index.isin(buy_signals.index)][signal_column_name], label=ta.buy_signal, color=ta.signal_colors[ta.buy_signal], marker=ta.signal_markers[ta.buy_signal], s=config.scatter_size, alpha=config.scatter_alpha)
ax[2].scatter(buy_signals.index, long_vol.loc[long_vol.index.isin(buy_signals.index)]["Close"], label=ta.buy_signal, color=ta.signal_colors[ta.buy_signal], marker=ta.signal_markers[ta.buy_signal], s=config.scatter_size, alpha=config.scatter_alpha)
sell_signals = short_vol.loc[short_vol[signal_column_name] == ta.sell_signal]
ax[0].scatter(sell_signals.index, df.loc[df.index.isin(sell_signals.index)][signal_column_name], label=ta.sell_signal, color=ta.signal_colors[ta.sell_signal], marker=ta.signal_markers[ta.sell_signal], s=config.scatter_size, alpha=config.scatter_alpha)
ax[1].scatter(sell_signals.index, short_vol.loc[short_vol.index.isin(sell_signals.index)]["Close"], label=ta.sell_signal, color=ta.signal_colors[ta.sell_signal], marker=ta.signal_markers[ta.sell_signal], s=config.scatter_size, alpha=config.scatter_alpha)
sell_signals = long_vol.loc[long_vol[signal_column_name] == ta.sell_signal]
ax[0].scatter(sell_signals.index, df.loc[df.index.isin(sell_signals.index)][signal_column_name], label=ta.sell_signal, color=ta.signal_colors[ta.sell_signal], marker=ta.signal_markers[ta.sell_signal], s=config.scatter_size, alpha=config.scatter_alpha)
ax[2].scatter(sell_signals.index, long_vol.loc[long_vol.index.isin(sell_signals.index)]["Close"], label=ta.sell_signal, color=ta.signal_colors[ta.sell_signal], marker=ta.signal_markers[ta.sell_signal], s=config.scatter_size, alpha=config.scatter_alpha)
utils.prettify_ax(ax[0], title=signal_column_name, center=True, start_date=start_date, end_date=end_date)
utils.prettify_ax(ax[1], title=short_vol_symbol + "Price", start_date=start_date, end_date=end_date)
utils.prettify_ax(ax[2], title=long_vol_symbol + "Price", start_date=start_date, end_date=end_date)
utils.prettify_fig(fig)
fig.savefig(utils.get_file_path(config.ta_graphs_path, signal_column_name + graph_filename, symbol=""))
utils.debug(fig)
return fig, ax
def random_walk():
df = pd.read_csv(utils.get_file_path(config.random_walk_data_path,
dates_table_filename),
index_col="Date",
parse_dates=["Date"])
start = 100
positions = [start]
rr = np.random.random(len(df.index) - 1)
downp = rr < prob[0]
upp = rr > prob[1]
for idownp, iupp in zip(downp, upp):
down = idownp and positions[-1] > 1
up = iupp and positions[-1] < 200
positions.append(positions[-1] - down + up)
df["Close"] = positions
df["Dividends"] = 0
df.to_csv(
utils.get_file_path(config.random_walk_data_path,
table_filename,
symbol="RandomWalk"))
fig, ax = plt.subplots(figsize=config.figsize)
ax.plot(df.index, df["Close"], label="Price")
utils.prettify_ax(ax,
title="RandomWalk",
start_date=df.index[0],
end_date=df.index[-1])
fig.savefig(
utils.get_file_path(config.random_walk_graphs_path, graph_filename))
utils.debug(fig)
return df
def build_lsi_model(dictionary, corpus, should_rebuild):
lsi = list()
# DEBUG
should_rebuild = True
if not should_rebuild:
try:
print('Loading LSI Model backup...')
lsi_file = utils.get_file_path(cfg.LDA_BACKUP)
print('LSI file = {}'.format(lsi_file))
lsi = LdaModel.load(lsi_file)
except Exception as exc:
utils.print_exception_details('Building LSI Model', exc)
else:
print('Building LSI Model...')
one_pass = cfg.NUM_PASSES > 1
lsi = LsiModel(corpus,
id2word=dictionary,
num_topics=cfg.NUM_TOPICS,
onepass=one_pass)
print('Done!')
# Save Model Structures
LSI_FILE = utils.get_file_path(cfg.LSI_BACKUP)
lsi.save(LSI_FILE)
return lsi
def get_dividends(self, symbol, date):
"""Adds dividends to the portfolio for the given symbol on the given date
Parameters:
symbol : str
date : datetime
Returns:
float
The dividends added
"""
start_time = timer()
if symbol in self.price_files:
df = self.price_files[symbol]
else:
if utils.refresh(utils.get_file_path(config.prices_data_path, prices.price_table_filename, symbol=symbol), refresh=False):
prices.download_data_from_yahoo(symbol, start_date=self.start_date, end_date=self.end_date)
df = pd.read_csv(utils.get_file_path(config.prices_data_path, prices.price_table_filename, symbol=symbol), index_col="Date", parse_dates=["Date"])[self.start_date:self.end_date]
self.price_files[symbol] = df
dividend = self.portfolio[symbol] * df.loc[date]["Dividends"] if date in df.index and "Dividends" in df.columns else 0
if dividend != 0:
self.cash += dividend
self.total_dividends += dividend
self.log.loc[date][actions_column_name] = self.log.loc[date][actions_column_name] + "Dividend: {} {} Shares totaling {:.2f} ".format(symbol, self.portfolio[symbol], dividend)
# TODO: move this into update_winners_losers
self.cost_basis[symbol] -= df.loc[date]["Dividends"]
self.times[get_dividend_time] = self.times[get_dividend_time] + timer() - start_time
return dividend
def get_annualized_performance(symbol,
start_date=config.start_date,
end_date=config.end_date):
"""Returns the annualized performance of the given symbol
Parameters:
symbol : str
start_date : date, optional
end_date : date, optional
Returns:
float
The annualized performance of the given symbol
"""
if utils.refresh(utils.get_file_path(config.prices_data_path,
prices.price_table_filename,
symbol=symbol),
refresh=False):
prices.download_data_from_yahoo(symbol,
start_date=start_date,
end_date=end_date)
df = pd.read_csv(utils.get_file_path(config.prices_data_path,
prices.price_table_filename,
symbol=symbol),
index_col="Date",
parse_dates=["Date"])[start_date:end_date]
# Not sure about this formula, seems weird. Formula normally has a -1 at the end
# return (1 + (df["Close"].add(df["Dividends"].cumsum())[-1] / df["Close"][0])) ** (365 / (df.index[-1] - df.index[0]).days) # exponent equivalent to (252 / len(df.index))
return (df["Close"].add(df["Dividends"].cumsum())[-1] /
df["Close"][0]) / (len(df.index) / 252) + 1
def get_num_conseq_increase_decrease(symbol,
refresh=False,
start_date=config.start_date,
end_date=config.end_date):
if utils.refresh(utils.get_file_path(config.prices_data_path,
prices.price_table_filename,
symbol=symbol),
refresh=refresh):
prices.download_data_from_yahoo(symbol,
start_date=start_date,
end_date=end_date)
df = pd.read_csv(utils.get_file_path(config.prices_data_path,
prices.price_table_filename,
symbol=symbol),
index_col="Date",
parse_dates=["Date"])[start_date:end_date]
return {
"ConseqIncrease":
((df["Close"].diff() > 0) & (df["Close"].diff().shift(1) > 0)).sum(),
"ConseqDecrease":
((df["Close"].diff() < 0) & (df["Close"].diff().shift(1) < 0)).sum(),
"Reversal":
((df["Close"].diff() < 0) & (df["Close"].diff().shift(1) > 0)).sum() +
((df["Close"].diff() > 0) & (df["Close"].diff().shift(1) < 0)).sum()
}
def get_sharpe_ratio(symbol,
start_date=config.start_date,
end_date=config.end_date):
"""Returns the sharpe ratio of the given symbol
Parameters:
symbol : str
start_date : date, optional
end_date : date, optional
Returns:
float
The sharpe ratio of the given symbol
"""
if utils.refresh(utils.get_file_path(config.prices_data_path,
prices.price_table_filename,
symbol=symbol),
refresh=False):
prices.download_data_from_yahoo(symbol,
start_date=start_date,
end_date=end_date)
df = pd.read_csv(utils.get_file_path(config.prices_data_path,
prices.price_table_filename,
symbol=symbol),
index_col="Date",
parse_dates=["Date"])[start_date:end_date]
# return (df["Close"].add(df["Dividends"].cumsum()) / df["Close"].add(df["Dividends"].cumsum()).shift(1)).mean() / ((df["Close"].add(df["Dividends"].cumsum()) / df["Close"].add(df["Dividends"].cumsum()).shift(1)).std() * np.sqrt(252))
return df["Close"].add(
df["Dividends"].cumsum()).pct_change().mean() / df["Close"].add(
df["Dividends"].cumsum()).pct_change().std() * np.sqrt(252)
def build_lda_model(dictionary, corpus, should_rebuild):
lda = list()
# DEBUG
should_rebuild = True
# debug_print('datapath:LDA', datapath(cfg.LDA_BACKUP))
if not should_rebuild:
try:
print('Loading LDA Model backup...')
lda_file = utils.get_file_path(cfg.LDA_BACKUP)
print('LDA file = {}'.format(lda_file))
lda = LdaModel.load(lda_file)
except Exception as exc:
utils.print_exception_details('Building LDA Model', exc)
else:
print('Building LDA Model...')
lda = LdaModel(corpus,
id2word=dictionary,
random_state=cfg.RANDOM_STATE,
num_topics=cfg.NUM_TOPICS,
passes=cfg.NUM_PASSES)
print('Done!')
# Save Model Structures
LDA_FILE = utils.get_file_path(cfg.LDA_BACKUP)
lda.save(LDA_FILE)
return lda
def build_tfid_model(dictionary, corpus, should_rebuild):
tfid = list()
# DEBUG
should_rebuild = True
if not should_rebuild:
try:
print('Loading TFID Model backup...')
tfid_file = utils.get_file_path(cfg.TFID_BACKUP)
print('TFID file = {}'.format(tfid_file))
tfid = LdaModel.load(tfid_file)
except Exception as exc:
utils.print_exception_details('Building TFID Model', exc)
else:
print('Building TFID Model...')
tfid = TfidfModel(corpus)
print('Done!')
# Save Model Structures
TFID_FILE = utils.get_file_path(cfg.TFID_BACKUP)
tfid.save(TFID_FILE)
return tfid
def search_text(self):
rawpath = get_file_path(self.raw_url)
if rawpath:
return " ".join(rawpath[1:])
rawpath = get_file_path(self.underscore_url)
if rawpath:
return " ".join(rawpath[1:])
return " "
def rsi(symbol,
period=default_period,
refresh=False,
start_date=config.start_date,
end_date=config.end_date):
"""Calculates the relative strength indexe for the given symbol, saves this data in a .csv file, and returns this data
The RSI is a leading momentum indicator.
Parameters:
symbol : str
period : int, optional
refresh : bool, optional
start_date : date, optional
end_date : date, optional
Returns:
dataframe
A dataframe containing the relative strength index for the given symbol
"""
if not utils.refresh(utils.get_file_path(
config.ta_data_path, table_filename, symbol=symbol),
refresh=refresh):
df = pd.read_csv(utils.get_file_path(config.ta_data_path,
table_filename,
symbol=symbol),
index_col="Date",
parse_dates=["Date"])[start_date:end_date]
else:
if utils.refresh(utils.get_file_path(config.prices_data_path,
prices.price_table_filename,
symbol=symbol),
refresh=refresh):
prices.download_data_from_yahoo(symbol,
start_date=start_date,
end_date=end_date)
df = pd.read_csv(utils.get_file_path(config.prices_data_path,
prices.price_table_filename,
symbol=symbol),
usecols=["Date", "Close"],
index_col="Date",
parse_dates=["Date"])[start_date:end_date]
if ("RSI" + str(period)) not in df.columns:
delta = df["Close"].diff()
up, down = delta.copy(), delta.copy()
up[up < 0], down[down > 0] = 0, 0
# df["RSI" + str(period)] = 100 - (100 / (1 + up.rolling(period).mean() / down.abs().rolling(period).mean())) # sma rsi
df["RSI" + str(period)] = 100 - (
100 / (1 + up.ewm(span=period, min_periods=period).mean() /
down.abs().ewm(span=period, min_periods=period).mean())
) # ema rsi
utils.debug(df["RSI" + str(period)])
df.to_csv(
utils.get_file_path(config.ta_data_path,
table_filename,
symbol=symbol))
return df["RSI" + str(period)]
def __init__(self, dir_: str, texture: str, id: str, command, params):
super().__init__(dummy_texture)
self.id = id
self.normal_texture = arcade.load_texture(
utils.get_file_path(dir_, texture + "_normal", "png"))
self.hover_texture = arcade.load_texture(
utils.get_file_path(dir_, texture + "_hover", "png"))
self.press_texture = arcade.load_texture(
utils.get_file_path(dir_, texture + "_pressed", "png"))
self.action = command
self.params = params
def ema(symbol,
period,
refresh=False,
start_date=config.start_date,
end_date=config.end_date):
"""Calculates the exponential moving agerage for the given symbol, saves this data in a .csv file, and returns this data
The EMA is a lagging trend indicator.
Parameters:
symbol : str
period : int
refresh : bool, optional
start_date : date, optional
end_date : date, optional
Returns:
dataframe
A dataframe containing the exponential moving agerage for the given symbol
"""
if not utils.refresh(utils.get_file_path(
config.ta_data_path, table_filename, symbol=symbol),
refresh=refresh):
df = pd.read_csv(utils.get_file_path(config.ta_data_path,
table_filename,
symbol=symbol),
index_col="Date",
parse_dates=["Date"])[start_date:end_date]
else:
if utils.refresh(utils.get_file_path(config.prices_data_path,
prices.price_table_filename,
symbol=symbol),
refresh=refresh):
prices.download_data_from_yahoo(symbol,
start_date=start_date,
end_date=end_date)
df = pd.read_csv(utils.get_file_path(config.prices_data_path,
prices.price_table_filename,
symbol=symbol),
usecols=["Date", "Close"],
index_col="Date",
parse_dates=["Date"])[start_date:end_date]
if ("EMA" + str(period)) not in df.columns:
df["EMA" + str(period)] = df["Close"].ewm(span=period,
min_periods=period,
adjust=False).mean()
utils.debug(df["EMA" + str(period)])
df.to_csv(
utils.get_file_path(config.ta_data_path,
table_filename,
symbol=symbol))
return df["EMA" + str(period)]
def search_text(self):
rawpath = get_file_path(self.raw_url)
if rawpath:
return " ".join(rawpath[1:])
rawpath = get_file_path(self.underscore_url)
if rawpath:
return " ".join(rawpath[1:])
else:
parts = self.raw_url.split('Political File')
url_bits = parts[-1].split("/")
url_bits.reverse()
return "-".join(url_bits)
def search_text(self):
rawpath = get_file_path(self.raw_url)
if rawpath:
return " ".join(rawpath[1:])
rawpath = get_file_path(self.underscore_url)
if rawpath:
return " ".join(rawpath[1:])
else:
parts = self.raw_url.split("Political File")
url_bits = parts[-1].split("/")
url_bits.reverse()
return "-".join(url_bits)
def plot_signals(symbol, period=default_periods, refresh=False, start_date=config.start_date, end_date=config.end_date):
"""Plots the macd buy/sell signals for the given symbol, saves this data in a .csv file, and plots this data. Only uses the first and last periods
The MACD is a lagging trend indicator.
Parameters:
symbol : str
period : int or list of int, optional
Must contain 3 values. First value is signal line, second is fast line, third is slow line.
refresh : bool, optional
start_date : date, optional
end_date : date, optional
Returns:
figure, axes
A figure and axes containing the macd signals for the given symbol
"""
if len(period) != 3:
raise ValueError("MACD requires 3 periods")
generate_signals(symbol, period=period, refresh=refresh, start_date=start_date, end_date=end_date)
fig, ax = plot_macd(symbol, period=period, refresh=refresh, start_date=start_date, end_date=end_date)
df = pd.read_csv(utils.get_file_path(config.ta_data_path, table_filename, symbol=symbol), index_col="Date", parse_dates=["Date"])[start_date:end_date]
macd_column_name = "MACD" + str(period[1]) + "-" + str(period[2])
signal_column_name = "MACD" + str(period[0])
signal_column_name = get_signal_name(period=period)
buy_signals = df.loc[df[signal_column_name] == ta.buy_signal]
ax[0].scatter(buy_signals.index, df.loc[df.index.isin(buy_signals.index)]["Close"], label=ta.buy_signal, color=ta.signal_colors[ta.buy_signal], marker=ta.signal_markers[ta.buy_signal], s=config.scatter_size, alpha=config.scatter_alpha)
ax[1].scatter(buy_signals.index, df.loc[df.index.isin(buy_signals.index)][macd_column_name], label=ta.buy_signal, color=ta.signal_colors[ta.buy_signal], marker=ta.signal_markers[ta.buy_signal], s=config.scatter_size, alpha=config.scatter_alpha)
sell_signals = df.loc[df[signal_column_name] == ta.sell_signal]
ax[0].scatter(sell_signals.index, df.loc[df.index.isin(sell_signals.index)]["Close"], label=ta.sell_signal, color=ta.signal_colors[ta.sell_signal], marker=ta.signal_markers[ta.sell_signal], s=config.scatter_size, alpha=config.scatter_alpha)
ax[1].scatter(sell_signals.index, df.loc[df.index.isin(sell_signals.index)][macd_column_name], label=ta.sell_signal, color=ta.signal_colors[ta.sell_signal], marker=ta.signal_markers[ta.sell_signal], s=config.scatter_size, alpha=config.scatter_alpha)
soft_buy_signals = df.loc[df[signal_column_name] == ta.soft_buy_signal]
ax[0].scatter(soft_buy_signals.index, df.loc[df.index.isin(soft_buy_signals.index)]["Close"], label=ta.soft_buy_signal, color=ta.signal_colors[ta.soft_buy_signal], marker=ta.signal_markers[ta.soft_buy_signal], s=config.scatter_size, alpha=config.scatter_alpha)
ax[1].scatter(soft_buy_signals.index, df.loc[df.index.isin(soft_buy_signals.index)][macd_column_name], label=ta.soft_buy_signal, color=ta.signal_colors[ta.soft_buy_signal], marker=ta.signal_markers[ta.soft_buy_signal], s=config.scatter_size, alpha=config.scatter_alpha)
soft_sell_signals = df.loc[df[signal_column_name] == ta.soft_sell_signal]
ax[0].scatter(soft_sell_signals.index, df.loc[df.index.isin(soft_sell_signals.index)]["Close"], label=ta.soft_sell_signal, color=ta.signal_colors[ta.soft_sell_signal], marker=ta.signal_markers[ta.soft_sell_signal], s=config.scatter_size, alpha=config.scatter_alpha)
ax[1].scatter(soft_sell_signals.index, df.loc[df.index.isin(soft_sell_signals.index)][macd_column_name], label=ta.soft_sell_signal, color=ta.signal_colors[ta.soft_sell_signal], marker=ta.signal_markers[ta.soft_sell_signal], s=config.scatter_size, alpha=config.scatter_alpha)
utils.prettify_ax(ax[0], title=symbol + "Price", start_date=start_date, end_date=end_date)
utils.prettify_ax(ax[1], title=symbol + signal_column_name, center=True, start_date=start_date, end_date=end_date)
utils.prettify_fig(fig)
fig.savefig(utils.get_file_path(config.ta_graphs_path, get_signal_name(period) + graph_filename, symbol=symbol))
utils.debug(fig)
return fig, ax
def plot_percentage_gains(symbol,
refresh=False,
start_date=config.start_date,
end_date=config.end_date):
"""Plots a graph of the percentage gains for the given symbol
Parameters:
symbol : str
refresh : bool, optional
start_date : date, optional
end_date : date, optional
Returns:
figure, axes
A subplot containing the percentage gains for the given symbol
"""
if isinstance(symbol, str):
symbol = [symbol]
symbol.sort()
fig, ax = plt.subplots(figsize=config.figsize)
for s in symbol:
if utils.refresh(utils.get_file_path(config.prices_data_path,
price_table_filename,
symbol=s),
refresh=refresh):
download_data_from_yahoo(s,
start_date=start_date,
end_date=end_date)
df = pd.read_csv(utils.get_file_path(config.prices_data_path,
price_table_filename,
symbol=s),
index_col="Date",
parse_dates=["Date"])[start_date:end_date]
ax.plot(df.index, df["Close"] / df["Close"][0], label=s + "Price")
utils.prettify_ax(ax,
title="".join(str(s) for s in symbol) + "Price",
start_date=start_date,
end_date=end_date)
utils.prettify_fig(fig)
fig.savefig(
utils.get_file_path(config.prices_graphs_path,
price_graph_filename,
symbol=",".join(str(s) for s in symbol)))
utils.debug(fig)
return fig, ax
def macd(symbol, period=default_periods, refresh=False, start_date=config.start_date, end_date=config.end_date):
"""Calculates the exponential moving agerage for the given symbol, saves this data in a .csv file, and returns this data
The EMA is a lagging trend indicator.
Parameters:
symbol : str
period : int\
refresh : bool, optional
start_date : date, optional
end_date : date, optional
Returns:
dataframe
A dataframe containing the exponential moving agerage for the given symbol
"""
if not utils.refresh(utils.get_file_path(config.ta_data_path, table_filename, symbol=symbol), refresh=refresh):
df = pd.read_csv(utils.get_file_path(config.ta_data_path, table_filename, symbol=symbol), index_col="Date", parse_dates=["Date"])[start_date:end_date]
else:
if utils.refresh(utils.get_file_path(config.prices_data_path, prices.price_table_filename, symbol=symbol), refresh=refresh):
prices.download_data_from_yahoo(symbol, start_date=start_date, end_date=end_date)
df = pd.read_csv(utils.get_file_path(config.prices_data_path, prices.price_table_filename, symbol=symbol), usecols=["Date", "Close"], index_col="Date", parse_dates=["Date"])[start_date:end_date]
if len(period) != 3:
raise ValueError("MACD requires 3 periods")
macd_column_name = "MACD" + str(period[1]) + "-" + str(period[2])
signal_column_name = "MACD" + str(period[0])
if macd_column_name not in df.columns or signal_column_name not in df.columns:
if macd_column_name not in df.columns:
'''
# Intermediate steps, can uncomment this part if I want to keep the steps
slow_column_name = "EMA" + str(period[1])
if slow_column_name not in df.columns:
df[slow_column_name] = df["Close"].ewm(span=period[1], min_periods=period[1], adjust=False).mean()
fast_column_name = "EMA" + str(period[2])
if fast_column_name not in df.columns:
df[fast_column_name] = df["Close"].ewm(span=period[2], min_periods=period[2], adjust=False).mean()
df[macd_column_name] = df[slow_column_name] - df[fast_column_name]
'''
df[macd_column_name] = df["Close"].ewm(span=period[1], min_periods=period[1], adjust=False).mean() - df["Close"].ewm(span=period[2], min_periods=period[2], adjust=False).mean()
utils.debug(df[macd_column_name])
if signal_column_name not in df.columns:
df[signal_column_name] = df[macd_column_name].ewm(span=period[0], min_periods=period[0], adjust=False).mean()
utils.debug(df[signal_column_name])
df.to_csv(utils.get_file_path(config.ta_data_path, table_filename, symbol=symbol))
return df[[macd_column_name, signal_column_name]]
def get_beta(symbol_a,
symbol_b,
start_date=config.start_date,
end_date=config.end_date):
"""Returns the beta of symbol_a to symbol_b
Parameters:
symbol_a : str
symbol_b : str
start_date : date, optional
end_date : date, optional
Returns:
float
The beta of symbol_a to symbol_b
"""
if utils.refresh(utils.get_file_path(config.prices_data_path,
prices.price_table_filename,
symbol=symbol_a),
refresh=False):
prices.download_data_from_yahoo(symbol_a,
start_date=start_date,
end_date=end_date)
df_a = pd.read_csv(utils.get_file_path(config.prices_data_path,
prices.price_table_filename,
symbol=symbol_a),
index_col="Date",
parse_dates=["Date"])[start_date:end_date]
a = df_a["Close"].add(df_a["Dividends"].cumsum()).pct_change()[1:]
if utils.refresh(utils.get_file_path(config.prices_data_path,
prices.price_table_filename,
symbol=symbol_b),
refresh=False):
prices.download_data_from_yahoo(symbol_b,
start_date=start_date,
end_date=end_date)
df_b = pd.read_csv(utils.get_file_path(config.prices_data_path,
prices.price_table_filename,
symbol=symbol_b),
index_col="Date",
parse_dates=["Date"])[start_date:end_date]
b = df_b["Close"].add(df_b["Dividends"].cumsum()).pct_change()[1:]
# rolling beta
# df["Beta"] = pd.rolling_cov(df_b["Close"].add(df_a["Dividends"].cumsum()), df_b["Close"].add(df_b["Dividends"].cumsum()), window=window) / pd.rolling_var(df_b["Close"].add(df_b["Dividends"].cumsum()), window=window)
beta = np.cov(a, b)[0][1] / np.var(
b) # Alternately, np.var(b) -> np.cov(a, b)[1][1]
return beta
def plot_macd(symbol, period=default_periods, refresh=False, start_date=config.start_date, end_date=config.end_date):
"""Calculates the macd for the given symbol, saves this data in a .csv file, and plots this data
The MACD is a lagging trend indicator.
Parameters:
symbol : str
period : int or list of int, optional
Must contain 3 values. First value is signal line, second is fast line, third is slow line.\
refresh : bool, optional
start_date : date, optional
end_date : date, optional
Returns:
figure, axes
A figure and axes containing the macd for the given symbol
"""
if not utils.refresh(utils.get_file_path(config.ta_data_path, table_filename, symbol=symbol), refresh=refresh):
df = pd.read_csv(utils.get_file_path(config.ta_data_path, table_filename, symbol=symbol), index_col="Date", parse_dates=["Date"])[start_date:end_date]
else:
if utils.refresh(utils.get_file_path(config.prices_data_path, prices.price_table_filename, symbol=symbol), refresh=refresh):
prices.download_data_from_yahoo(symbol, start_date=start_date, end_date=end_date)
df = pd.read_csv(utils.get_file_path(config.prices_data_path, prices.price_table_filename, symbol=symbol), usecols=["Date", "Close"], index_col="Date", parse_dates=["Date"])[start_date:end_date]
if len(period) != 3:
raise ValueError("MACD requires 3 periods")
if len(df) < period[-1]:
raise ta.InsufficientDataException("Not enough data to compute a period length of " + str(period))
fig, ax = plt.subplots(2, figsize=config.figsize)
ax[0].plot(df.index, df["Close"], label="Price")
utils.prettify_ax(ax[0], title=symbol + "Price", start_date=start_date, end_date=end_date)
macd_column_name = "MACD" + str(period[1]) + "-" + str(period[2])
signal_column_name = "MACD" + str(period[0])
if macd_column_name not in df.columns or signal_column_name not in df.columns:
df = df.join(macd(symbol, period, refresh=False, start_date=start_date, end_date=end_date))
# if len(df) > period[0] and len(df) > period[1] and len(df) > period[2]: # to prevent AttributeError when the column is all None
ax[1].plot(df.index, df[macd_column_name], label="MACD")
ax[1].plot(df.index, df[signal_column_name], label="Signal")
ax[1].plot(df.index, (df[macd_column_name] - df[signal_column_name]), label="Histogram")
# Can't overlay a histogram with line plots so the histogram has to also be a line plot
# ax[1].bar(df.index, np.histogram(np.isfinite(df[signal_column_name] - df[macd_column_name])), normed=True, alpha=config.alpha) # ValueError: incompatible sizes: argument 'height' must be length 3876 or scalar
utils.prettify_ax(ax[1], title=symbol + "MACD", center=True, start_date=start_date, end_date=end_date)
utils.prettify_fig(fig)
fig.savefig(utils.get_file_path(config.ta_graphs_path, get_signal_name(period) + graph_filename, symbol=symbol))
utils.debug(fig)
return fig, ax
def get_average_price(symbol,
columns=["Open", "High", "Low", "Close"],
refresh=False,
start_date=config.start_date,
end_date=config.end_date):
"""Creates a list of the average price by date for the given symbol, adds the data to the existing corresponding .csv file, and returns this data
If no valid columns are provided in method, the data will be all nulls
Parameters:
symbol : str
columns: list of str
The columns to use. The list of expected columns are ["Open", "High", "Low", "Close"],
but fewer columns may be valid as well
refresh : bool, optional
start_date : date, optional
end_date : date, optional
Returns:
dataframe
A dataframe containing average price by date for the given symbol
"""
if utils.refresh(utils.get_file_path(config.prices_data_path,
price_table_filename,
symbol=symbol),
refresh=refresh):
download_data_from_yahoo(symbol,
start_date=start_date,
end_date=end_date)
df = pd.read_csv(utils.get_file_path(config.prices_data_path,
price_table_filename,
symbol=symbol),
index_col="Date",
parse_dates=["Date"])[start_date:end_date]
if avg_name not in df.columns:
df[avg_name] = 0
count = sum(c in df.columns for c in columns)
for c in columns:
if c in df.columns:
df[avg_name] = df[avg_name].add(df[c])
df[avg_name] = df[
avg_name] / count # Will leave null values if no methods were valid
utils.debug(df[avg_name])
df.to_csv(
utils.get_file_path(config.prices_data_path,
price_table_filename,
symbol=symbol))
return df[avg_name]
def create_witness(self, program_file, test_name, test_vector,
nondet_methods):
"""
Creates a witness for the test file produced by crest.
Test files produced by our version of crest specify one test value per line, without
any mention of the variable the value is assigned to.
Because of this, we have to build a fancy witness automaton of the following format:
For each test value specified in the test file, there is one precessor and one
successor state. These two states are connected by one transition for each
call to a CREST_x(..) function. Each of these transitions has the assumption,
that the variable specified in the corresponding CREST_x(..) function has the current
test value.
"""
witness = self.witness_creator.create_witness(
producer=self.get_name(),
program_file=program_file,
test_vector=test_vector,
nondet_methods=nondet_methods,
machine_model=self.machine_model,
error_lines=self.get_error_lines(program_file))
witness_file = test_name + ".witness.graphml"
witness_file = utils.get_file_path(witness_file)
return {'name': witness_file, 'content': witness}
def createFile(request):
if request.method == 'POST':
file_name=request.POST.get("filename","test")
#save file to localhost repo
repo_path=utils.get_repo_path(file_name)
rev_control=utils.MercurialRevisionControl()
rev_control.create_repo(repo_path)
auth_user=request.session['auth_username']
rev_control.hg_rc(repo_path,'ui','username',auth_user)
file_content=request.POST.get("content","")
utils.create_file(repo_path,file_content)
rev_control.add(repo_path)
rev_control.commit(repo_path)
#utils.write_file(dockerfile,request.POST.get("content",""))
#revision_control=utils.MercurialRevisionControl()
#revision_control.create_repo(
#save file to db
file_path=utils.get_file_path(file_name)
file_size=utils.get_file_size(file_path)
created=utils.get_current_datatime()
created_by=request.session.get("auth_username")
modified=created
modified_by=created_by
path=file_path
data=DockerFiles(Name=file_name,Size=file_size,Created=created,CreatedBy=created_by,Modified=modified,ModifiedBy=modified_by,Path=path)
data.save()
return HttpResponseRedirect('/admin/files')
def __init__(self, uri):
file_path = get_file_path(uri)
git_dir = get_real_git_dir(file_path)
if git_dir:
self._dir = git_dir
else:
self._dir = file_path
def __init__(self, dir_: str, texture: str):
super().__init__(dummy_texture)
self.texture = arcade.load_texture(
utils.get_file_path(dir_, texture, "png"))
self.normal_texture = self.texture
self.hover_texture = self.texture
self.pressed_texture = self.texture
def get_info():
page = 1
max_page = 100
f = open(utils.get_file_path(date=utils.current_date()), 'w')
dic = utils.get_output_total_info_dic()
while page <= max_page:
print str(page)
url = "http://yuqing.dz11.com/Home/Nav/getUserFeedbackList?channel=ios&startTime=" + start_time + "%2000%3A00%3A00&endTime=" + end_time + "%2023%3A59%3A59&pageNum=" \
+ str(page) + "&pageSize=20"
print "当前请求URL: " + url
try:
request = urllib2.Request(url)
response = urllib2.urlopen(request)
result = json.loads(response.read().decode('utf-8'))
max_page = int(result['data']['total']) / 20 + 1
page += 1
for record in result['data']['records']:
utils.handle_total_info_data(dic, record)
f.write(record['title'].encode('utf-8') + ' ')
f.write(record['content'].encode('utf-8') + '\n')
except urllib2.URLError, e:
if hasattr(e, "code"):
print e.code
if hasattr(e, "reason"):
print e.reason
def __init__(self, filename="", initial_cash=100000, symbols=["SPY"], benchmark=["SPY", "RSP"],
commission=default_commission, max_portfolio_size=100, soft_signals=False, slippage=0,
short_sell=False, partial_shares=False, stop_loss_limit=1.0,
fail_gracefully=False, refresh=False, start_date=config.start_date, end_date=config.end_date,
signal_func=None, signal_func_args=[], signal_func_kwargs={}):
if len(symbols) == 0:
raise ValueError("Requires at least one symbol")
if signal_func is None:
raise ValueError("Requires a signal function")
if slippage < 0:
raise ValueError("Can't see the future")
self.filename = filename
self.cash = initial_cash
self.initial_cash = initial_cash
self.symbols = symbols
self.benchmark = benchmark
self.commission = commission
self.max_portfolio_size = max_portfolio_size
self.purchase_size = (initial_cash // min(max_portfolio_size, len(symbols) // 5 + 1)) - (self.commission[maximum_commission] if maximum_commission in self.commission else 0)
self.slippage = slippage
self.soft_signals = soft_signals
self.short_sell = short_sell
self.partial_shares = partial_shares
self.stop_loss_limit = stop_loss_limit
self.fail_gracefully = fail_gracefully
self.refresh = refresh
self.start_date = start_date
self.end_date = end_date
self.signal_func = signal_func.generate_signals
self.signal_func_args = signal_func_args
self.signal_func_kwargs = signal_func_kwargs
self.signal_table_filename = signal_func.table_filename # inspect.getmodule(self.signal_func).table_filename
self.signal_name = signal_func.get_signal_name(**signal_func_kwargs) # inspect.getmodule(self.signal_func).get_signal_name(**signal_func_kwargs)
self.portfolio = {}
self.cost_basis = {}
self.winners_losers = {"Winners": 0, "Losers": 0}
self.stop_loss = {}
self.signal_files = {}
self.price_files = {}
self.times = times
self.total_dividends = 0
self.total_commissions = 0
self.total_trades = 0
self.dates = pd.read_csv(utils.get_file_path(config.simulation_data_path, dates_table_filename), index_col="Date", parse_dates=["Date"])[self.start_date:self.end_date].index
self.log = pd.DataFrame(index=self.dates, columns=log_columns)
self.log[actions_column_name] = ""
# TODO make this work for multiple signals
'''
self.signal_names = []
self.signal_table_filename = []
for func in signal_func:
self.signal_name.append(inspect.getmodule(func).signal_name)
self.signal_table_filename.append(inspect.getmodule(func).table_filename)
'''
# self.get_price_on_date.cache_clear()
self.run()
def create_harness(self, program_file, test_name, test_vector, nondet_methods):
harness = self.harness_creator.create_harness(nondet_methods=nondet_methods,
error_method=utils.error_method,
test_vector=test_vector)
harness_file = test_name + '.harness.c'
harness_file = utils.get_file_path(harness_file)
return {'name': harness_file, 'content': harness}
def create_by_old_paste(cls, filehash):
filepath = get_file_path(filehash)
mimetype = magic.from_file(filepath, mime=True)
filestat = os.stat(filepath)
size = filestat.st_size
rst = cls(filehash, mimetype, size, filehash=filehash)
return rst
def create_input_generation_cmds(self, filename):
compiled_file = '.'.join(os.path.basename(filename).split('.')[:-1])
compiled_file = utils.get_file_path(compiled_file, temp_dir=True)
machinem_arg = self.machine_model.compile_parameter
compile_cmd = ['gcc', '-std=gnu11', machinem_arg, '-I', include_dir, '-o', compiled_file, generator_harness, filename, '-lm']
input_generation_cmd = [random_runner, compiled_file]
return [compile_cmd, input_generation_cmd]
def file_name(self):
if self.download_url:
rawpath = get_file_path(self.raw_url)
return rawpath[-1:][0]
else:
parts = self.raw_url.split("Political File")
url_bits = parts[-1].split("/")
url_bits.reverse()
return "-".join(url_bits)
def preview(filehash):
paste_file = PasteFile.get_by_filehash(filehash)
if not paste_file:
filepath = get_file_path(filehash)
if not(os.path.exists(filepath) and (not os.path.islink(filepath))):
return abort(404)
paste_file = PasteFile.create_by_old_paste(filehash)
paste_file.save()
return render_template('success.html', p=paste_file)
def _get_cmd(self, program_file, witness_file):
if not self.executable:
import shutil
self.executable = self.tool.executable()
self.cpa_directory = os.path.join(os.path.dirname(self.executable), '..')
config_copy_dir = utils.get_file_path('config', temp_dir=True)
if not os.path.exists(config_copy_dir):
copy_dir = os.path.join(self.cpa_directory, 'config')
shutil.copytree(copy_dir, config_copy_dir)
return [self.executable] + \
utils.get_cpachecker_options(witness_file) +\
['-witnessValidation', program_file]
def compile(self, program_file, harness_file):
output_file = utils.get_file_path('a.out', temp_dir=True)
compile_cmd = self._get_compile_cmd(program_file, harness_file, output_file)
compile_result = utils.execute(compile_cmd, quiet=True)
if compile_result.returncode != 0:
compile_cmd = self._get_compile_cmd(program_file, harness_file, output_file, 'gnu90')
compile_result = utils.execute(compile_cmd, quiet=True, err_to_output=False)
if compile_result.returncode != 0:
raise utils.CompileError("Compilation failed for harness {}".format(harness_file))
return None
return output_file
def create_input_generation_cmds(self, filename):
import shutil
config_copy_dir = utils.get_file_path('config', temp_dir=True)
if not os.path.exists(config_copy_dir):
copy_dir = os.path.join(base_dir, 'config')
shutil.copytree(copy_dir, config_copy_dir)
input_generation_cmd = [binary]
if self.timelimit > 0:
input_generation_cmd += ['-timelimit', str(self.timelimit)]
input_generation_cmd += ['-tiger-variants',
'-outputpath', tests_dir,
'-spec', utils.spec_file,
filename]
return [input_generation_cmd]
def create_input_generation_cmds(self, filename):
if self.machine_model.is_32:
mm_args = ['-arch', 'i386']
elif self.machine_model.is_64:
mm_args = ['-arch', 'x86_64']
else:
raise AssertionError("Unhandled machine model: " + self.machine_model.name)
compiled_file = '.'.join(os.path.basename(filename).split('.')[:-1] + ['bc'])
compiled_file = utils.get_file_path(compiled_file, temp_dir=True)
compile_cmd = ['clang'] + mm_args + ['-I', include_dir, '-emit-llvm', '-c', '-g', '-o', compiled_file, filename]
input_generation_cmd = ['klee']
if self.timelimit > 0:
input_generation_cmd += ['-max-time', str(self.timelimit)]
input_generation_cmd.append('-only-output-states-covering-new')
input_generation_cmd += ['-search=' + h for h in self.search_heuristic]
input_generation_cmd += ['-output-dir=' + tests_dir]
input_generation_cmd += [compiled_file]
return [compile_cmd, input_generation_cmd]
def create_witness(self, program_file, test_name, test_vector, nondet_methods):
"""
Creates a witness for the test file produced by crest.
Test files produced by our version of crest specify one test value per line, without
any mention of the variable the value is assigned to.
Because of this, we have to build a fancy witness automaton of the following format:
For each test value specified in the test file, there is one precessor and one
successor state. These two states are connected by one transition for each
call to a CREST_x(..) function. Each of these transitions has the assumption,
that the variable specified in the corresponding CREST_x(..) function has the current
test value.
"""
witness = self.witness_creator.create_witness(producer=self.get_name(),
program_file=program_file,
test_vector=test_vector,
nondet_methods=nondet_methods,
machine_model=self.machine_model,
error_lines=self.get_error_lines(program_file))
witness_file = test_name + ".witness.graphml"
witness_file = utils.get_file_path(witness_file)
return {'name': witness_file, 'content': witness}
def retrieve(self, n, issues, dsets):
"""
:param n:
:param issues:
:param dsets:
:return:
"""
logging.info('processing id {}'.format(n))
try:
r = get_ws_call('retrieve', None, n)
self.json = r.json()['issue']
path_to_issue, path_to_dataset = get_file_path(issues, dsets, self.json['uid'])
# Todo find a better fix
for key in fields_to_remove:
del self.json[key]
# Removing the closing date to avoid having null value for currently active issues.
if 'dateClosed' in self.json.keys() and self.json['dateClosed'] is None:
del self.json['dateClosed']
if 'models' not in self.json.keys():
self.json['models'] = []
# Writing dataset file
with open(path_to_dataset, 'w') as dset_file:
if not self.json['datasets']:
logging.info('The issue {} seems to be affecting no datasets.'.format(self.json['uid']))
dset_file.write('No datasets provided with issue.')
for dset in self.json['datasets']:
dset_file.write(dset + '\n')
del self.json['datasets']
# Writing issue file.
with open(path_to_issue, 'w') as data_file:
data_file.write(simplejson.dumps(self.json, indent=4, sort_keys=True))
except Exception as e:
logging.error('An unknown error has occurred, this is the stack {0}, error code: {1}'.format(repr(e), 99))
def _get_file_name(self,file_name):
if file_name is None:
dir_name = os.path.dirname(__file__)
file_name = utils.get_file_path(dir_name,self.default_file)
return file_name
def path(self):
return get_file_path(self.filehash)
def folder_name(self):
rawpath = get_file_path(self.raw_url)
return ":".join(rawpath[1:-1])
def path(self):
return get_file_path(self.raw_url)
#---------------------------------------------------------------------
from PyQt4 import uic
import platform
import traceback
import utils
from github_utils import GitHubApiError, GitHubApi
from PyQt4.QtCore import QSettings, Qt
from PyQt4.QtGui import QStringListModel, QCompleter, QMessageBox
from qgis.core import QGis
from qgis.utils import plugins, pluginMetadata
from qgis.utils import iface
ui_file = utils.get_file_path('main_widget.ui')
uiWidget, qtBaseClass = uic.loadUiType(ui_file)
class MainWidget(qtBaseClass, uiWidget):
def __init__(self, last_exception=None, parent=None):
qtBaseClass.__init__(self, parent)
self.setupUi(self)
self.github = GitHubApi()
self.last_exception = last_exception
self._connect_signals()
self._load_available_trackers()
self._load_settings()
self._load_last_error()
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#---------------------------------------------------------------------
import utils
import json
try:
raise ImportError
import requests
except ImportError:
import os
import sys
request_egg = utils.get_file_path('deps', 'requests-2.10.0-py2.py3-none-any.whl')
sys.path.append(request_egg)
import requests
class GitHubApiError(Exception):
pass
class GitHubApi():
def __init__(self):
self.tracker = None
self.access_token = None
def is_valid(self):
print self.tracker, self.access_token
return (self.tracker != None) and (self.access_token != None)
import os
import utils
from input_generation import BaseInputGenerator
from test_validation import TestValidator
name = "fshell"
fshell_dir = os.path.abspath("./fshell")
bin_dir = os.path.join(fshell_dir, "bin")
fshell_binary = os.path.join(bin_dir, "fshell")
query_file = os.path.join(fshell_dir, "query-block-coverage")
tests_file = utils.get_file_path('testsuite.txt', temp_dir=True)
def get_test_cases(exclude=[]):
if os.path.exists(tests_file):
with open(tests_file, 'r') as inp:
content = [l.strip() for l in inp.readlines()]
if len([l for l in content if "Test Suite" in l]) > 1:
raise AssertionError("More than one test suite exists in " + tests_file)
curr_test = list()
test_cases = list()
count = 1
for line in content:
if line.startswith("IN:"):
test_name = str(count)
if test_name not in exclude:
test_cases.append(utils.TestCase(test_name, tests_file, curr_test))
curr_test = list()
count += 1
if line.startswith("strto"):
def _get_default_file(self):
dir_name = os.path.dirname(__file__)
file_path = utils.get_file_path(dir_name,self.default_file)
return file_path
def save(self):
self.folder, root_path = get_root_path(self.folder_name)
name = self.cleaned_data["file"].name
self.filename, self.path = get_file_path(name, root_path)
handle_uploaded_file(self.cleaned_data["file"], self.path)
from werkzeug import SharedDataMiddleware
from flask import abort, Flask, request, jsonify, redirect, send_file
from ext import db, mako, render_template
from models import PasteFile
from utils import get_file_path, humanize_bytes
from client import create
ONE_MONTH = 60 * 60 * 24 * 30
app = Flask(__name__, template_folder='../../templates/r',
static_folder='../../static')
app.config.from_object('config')
app.wsgi_app = SharedDataMiddleware(app.wsgi_app, {
'/i/': get_file_path()
})
mako.init_app(app)
db.init_app(app)
@app.route('/r/<img_hash>')
def rsize(img_hash):
w = request.args['w']
h = request.args['h']
old_paste = PasteFile.get_by_filehash(img_hash)
new_paste = PasteFile.rsize(old_paste, w, h)
return new_paste.url_i
def file_name(self):
rawpath = get_file_path(self.raw_url)
return (rawpath[-1:][0])
def _git_help_wanted(self):
html = utils.get_file_path('doc', 'github_token.html')
url = QUrl.fromLocalFile(html)
QDesktopServices.openUrl(url)
def search_text(self):
rawpath = get_file_path(self.raw_url)
return " ".join(rawpath[1:])
