def __init__(self):
load_dotenv(find_dotenv('config.env'))
self.writer = FileWriter()
self.reader = FileReader()
self.finder = PathFinder()
self.traveller = TimeTraveller()
self.provider = 'iexcloud'
def __init__(self, usr=None, pwd=None, mfa=None):
# Authentication
load_dotenv()
username = usr or os.environ['RH_USERNAME']
password = pwd or os.environ['RH_PASSWORD']
mfa_code = mfa or pyotp.TOTP(os.environ['RH_2FA']).now()
rh.login(username, password, mfa_code=mfa_code)
self.api = rh
self.writer = FileWriter()
self.reader = FileReader()
self.finder = PathFinder()
def update_iex_ohlc():
for symbol in stock_symbols:
filename = PathFinder().get_ohlc_path(symbol=symbol,
provider=iex.provider)
try:
iex.save_ohlc(symbol=symbol, timeframe=timeframe)
except Exception as e:
print(f'IEX Cloud OHLC update failed for {symbol}.')
print(e)
finally:
if CI and os.path.exists(filename):
os.remove(filename)
def update_poly_dividends():
for symbol in symbols:
filename = PathFinder().get_dividends_path(
symbol=symbol, provider=poly.provider)
try:
poly.save_dividends(symbol=symbol, timeframe='max')
except Exception as e:
print(f'Polygon.io dividend update failed for {symbol}.')
print(e)
finally:
if CI and os.path.exists(filename):
os.remove(filename)
def update_poly_ohlc():
for symbol in all_symbols:
try:
poly.save_ohlc(symbol=symbol, timeframe=FEW_DAYS, retries=1)
except Exception as e:
print(f'Polygon.io OHLC update failed for {symbol}.')
print(e)
finally:
filename = PathFinder().get_ohlc_path(
symbol=symbol, provider=poly.provider)
if C.CI and os.path.exists(filename):
os.remove(filename)
def update_iex_dividends():
for symbol in symbols:
filename = PathFinder().get_dividends_path(
symbol=symbol, provider=iex.provider)
try:
iex.save_dividends(symbol=symbol, timeframe='5y')
except Exception as e:
print(f'IEX Cloud dividend update failed for {symbol}.')
print(e)
finally:
if CI and os.path.exists(filename):
os.remove(filename)
def update_iex_ohlc():
for symbol in stock_symbols:
try:
iex.save_ohlc(symbol=symbol, timeframe='1d',
retries=1 if C.TEST else C.DEFAULT_RETRIES)
except Exception as e:
print(f'IEX Cloud OHLC update failed for {symbol}.')
print(e)
finally:
filename = PathFinder().get_ohlc_path(
symbol=symbol, provider=iex.provider)
if C.CI and os.path.exists(filename):
os.remove(filename)
class MarketData:
def __init__(self, broker=None):
self.writer = FileWriter()
self.reader = FileReader()
self.finder = PathFinder()
def get_symbols(self):
# get cached list of symbols
symbols_path = self.finder.get_symbols_path()
return list(self.reader.load_csv(symbols_path)[C.SYMBOL])
def get_dividends(self, symbol, timeframe=None):
# given a symbol, return a cached dataframe
return self.reader.load_csv(self.finder.get_dividends_path(symbol))
def save_dividends(self, symbol, timeframe=None):
# given a symbol, save its dividend history
if timeframe:
df = self.get_dividends(symbol, timeframe)
else:
df = self.get_dividends(symbol)
self.writer.update_csv(self.finder.get_dividends_path(symbol), df)
def update_poly_splits():
for symbol in symbols:
try:
poly.save_splits(symbol=symbol,
timeframe='3m',
retries=1 if C.TEST else C.DEFAULT_RETRIES)
except Exception as e:
print(f'Polygon.io split update failed for {symbol}.')
print(e)
finally:
filename = PathFinder().get_splits_path(symbol=symbol,
provider=poly.provider)
if C.CI and os.path.exists(filename):
os.remove(filename)
class Robinhood:
# broker operation
def __init__(self, usr=None, pwd=None, mfa=None):
# Authentication
load_dotenv(find_dotenv('config.env'))
username = usr or os.environ['RH_USERNAME']
password = pwd or os.environ['RH_PASSWORD']
mfa_code = mfa or pyotp.TOTP(os.environ['RH_2FA']).now()
rh.login(username, password, mfa_code=mfa_code)
self.api = rh
self.writer = FileWriter()
self.reader = FileReader()
self.finder = PathFinder()
def flatten(self, xxs):
# flattens 2d list into 1d list
return [x for xs in xxs for x in xs]
def get_hists(self, symbols, span='year', interval='day', save=False):
# given a list of symbols,
# return a DataFrame with historical data
hists = [
self.api.get_stock_historicals(symbol, interval, span)
for symbol in symbols
]
clean = [hist for hist in hists if hist != [None]]
df = pd.DataFrame.from_records(self.flatten(clean))
# look into diff b/w tz_localize and tz_convert w param 'US/Eastern'
# ideally store utc time
df['begins_at'] = pd.to_datetime(
df['begins_at']).apply(lambda x: x.tz_localize(None))
# df = df.sort_values('begins_at')
if save is True:
self.writer.save_csv('data/data.csv', df)
return df
def get_names(self, symbols):
# given a list of stock symbols
# return a list of company names
names = []
for symbol in symbols:
if hasattr(self, 'holdings') and symbol in self.holdings:
names.append(self.holdings[symbol]['name'])
else:
names.append(self.api.get_name_by_symbol(symbol))
return names
def save_symbols(self):
# save all the portfolio symbols in a table
symbols = self.get_symbols()
names = self.get_names(symbols)
df = pd.DataFrame({C.SYMBOL: symbols, C.NAME: names})
self.writer.save_csv(self.finder.get_symbols_path(), df)
def get_holdings(self):
if not hasattr(self, 'holdings'):
self.holdings = self.api.build_holdings()
return self.holdings
def get_symbols(self):
if not hasattr(self, 'holdings'):
self.get_holdings()
return [symbol for symbol in self.holdings]
from Constants import PathFinder # noqa autopep8
import Constants as C # noqa autopep8
twit = StockTwits()
symbols = twit.get_symbols()
crypto_symbols = ['BTC-X', 'ETH-X', 'LTC-X', 'XMR-X', 'IOT-X']
if C.TEST:
symbols = crypto_symbols
twit.token = ''
else:
symbols.extend(crypto_symbols)
BATCH = int(os.environ.get('BATCH')) if os.environ.get('BATCH') else 1
# better solution is to dynamically choose 175 most outdated symbols
# First batch
for symbol in symbols[C.TWIT_RATE*(BATCH-1):C.TWIT_RATE*BATCH]:
if symbol in C.SENTIMENT_SYMBOLS_IGNORE:
continue
try:
twit.save_social_sentiment(symbol=symbol, timeframe='1d',
retries=1 if C.TEST else 2)
except Exception as e:
print(f'Stocktwits sentiment update failed for {symbol}.')
print(e)
finally:
filename = PathFinder().get_sentiment_path(
symbol=symbol, provider=twit.provider)
if C.CI and os.path.exists(filename):
os.remove(filename)
class MarketData:
def __init__(self):
load_dotenv(find_dotenv('config.env'))
self.writer = FileWriter()
self.reader = FileReader()
self.finder = PathFinder()
self.traveller = TimeTraveller()
self.provider = 'iexcloud'
def try_again(self, func, **kwargs):
retries = (kwargs['retries']
if 'retries' in kwargs else C.DEFAULT_RETRIES)
delay = (kwargs['delay'] if 'delay' in kwargs else C.DEFAULT_DELAY)
func_args = {
k: v
for k, v in kwargs.items() if k not in {'retries', 'delay'}
}
for retry in range(retries):
try:
return func(**func_args)
except Exception as e:
if retry == retries - 1:
raise e
else:
sleep(delay)
def get_symbols(self):
# get cached list of symbols
symbols_path = self.finder.get_symbols_path()
return list(self.reader.load_csv(symbols_path)[C.SYMBOL])
def get_dividends(self, symbol, timeframe='max'):
# given a symbol, return a cached dataframe
df = self.reader.load_csv(
self.finder.get_dividends_path(symbol, self.provider))
filtered = self.reader.data_in_timeframe(df, C.EX, timeframe)
return filtered
def standardize(self, df, full_mapping, filename, columns, default):
mapping = {k: v for k, v in full_mapping.items() if k in df}
df = df[list(mapping)].rename(columns=mapping)
time_col, val_cols = columns[0], columns[1:]
if time_col in df and set(val_cols).issubset(df.columns):
df = self.reader.update_df(filename, df,
time_col).sort_values(by=[time_col])
# since time col is pd.datetime,
# consider converting to YYYY-MM-DD str format
for val_col in val_cols:
df[val_col] = df[val_col].apply(lambda val: float(val)
if val else default)
return df
def standardize_dividends(self, symbol, df):
full_mapping = dict(
zip(['exDate', 'paymentDate', 'declaredDate', 'amount'],
[C.EX, C.PAY, C.DEC, C.DIV]))
filename = self.finder.get_dividends_path(symbol, self.provider)
return self.standardize(df, full_mapping, filename, [C.EX, C.DIV], 0)
def save_dividends(self, **kwargs):
# given a symbol, save its dividend history
symbol = kwargs['symbol']
filename = self.finder.get_dividends_path(symbol, self.provider)
if os.path.exists(filename):
os.remove(filename)
df = self.reader.update_df(filename, self.get_dividends(**kwargs),
C.EX, C.DATE_FMT)
self.writer.update_csv(filename, df)
if os.path.exists(filename):
return filename
def get_splits(self, symbol, timeframe='max'):
# given a symbol, return a cached dataframe
df = self.reader.load_csv(
self.finder.get_splits_path(symbol, self.provider))
filtered = self.reader.data_in_timeframe(df, C.EX, timeframe)
return filtered
def standardize_splits(self, symbol, df):
full_mapping = dict(
zip(['exDate', 'paymentDate', 'declaredDate', 'ratio'],
[C.EX, C.PAY, C.DEC, C.RATIO]))
filename = self.finder.get_splits_path(symbol, self.provider)
return self.standardize(df, full_mapping, filename, [C.EX, C.RATIO], 1)
def save_splits(self, **kwargs):
# given a symbol, save its splits history
symbol = kwargs['symbol']
filename = self.finder.get_splits_path(symbol, self.provider)
if os.path.exists(filename):
os.remove(filename)
df = self.reader.update_df(filename, self.get_splits(**kwargs), C.EX,
C.DATE_FMT)
self.writer.update_csv(filename, df)
if os.path.exists(filename):
return filename
def standardize_ohlc(self, symbol, df, filename=None):
full_mapping = dict(
zip([
'date', 'open', 'high', 'low', 'close', 'volume', 'average',
'trades'
], [
C.TIME, C.OPEN, C.HIGH, C.LOW, C.CLOSE, C.VOL, C.AVG, C.TRADES
]))
filename = filename or self.finder.get_ohlc_path(symbol, self.provider)
df = self.standardize(df, full_mapping, filename,
[C.TIME, C.OPEN, C.HIGH, C.LOW, C.CLOSE], 0)
for col in [C.VOL, C.TRADES]:
if col in df:
df[col] = df[col].apply(lambda val: 0
if pd.isnull(val) else int(val))
return df
def get_ohlc(self, symbol, timeframe='max'):
df = self.reader.load_csv(
self.finder.get_ohlc_path(symbol, self.provider))
filtered = self.reader.data_in_timeframe(df, C.TIME, timeframe)
return filtered
def save_ohlc(self, **kwargs):
symbol = kwargs['symbol']
filename = self.finder.get_ohlc_path(symbol, self.provider)
if os.path.exists(filename):
os.remove(filename)
df = self.reader.update_df(filename, self.get_ohlc(**kwargs), C.TIME,
C.DATE_FMT)
self.writer.update_csv(filename, df)
if os.path.exists(filename):
return filename
def get_social_sentiment(self, symbol, timeframe='max'):
# given a symbol, return a cached dataframe
df = self.reader.load_csv(self.finder.get_sentiment_path(symbol))
filtered = self.reader.data_in_timeframe(
df, C.TIME, timeframe)[[C.TIME, C.POS, C.NEG]]
return filtered
def get_social_volume(self, symbol, timeframe='max'):
# given a symbol, return a cached dataframe
df = self.reader.load_csv(self.finder.get_sentiment_path(symbol))
filtered = self.reader.data_in_timeframe(
df, C.TIME, timeframe)[[C.TIME, C.VOL, C.DELTA]]
return filtered
def save_social_sentiment(self, **kwargs):
# # given a symbol, save its sentiment data
symbol = kwargs['symbol']
filename = self.finder.get_sentiment_path(symbol)
if os.path.exists(filename):
os.remove(filename)
sen_df = self.reader.update_df(filename,
self.get_social_sentiment(**kwargs),
C.TIME)
sen_df = sen_df[{C.TIME, C.POS, C.NEG}.intersection(sen_df.columns)]
vol_df = self.reader.update_df(filename,
self.get_social_volume(**kwargs),
C.TIME)
vol_df = vol_df[{C.TIME, C.VOL, C.DELTA}.intersection(vol_df.columns)]
if sen_df.empty and not vol_df.empty:
df = vol_df
elif not sen_df.empty and vol_df.empty:
df = sen_df
elif not sen_df.empty and not vol_df.empty:
df = sen_df.merge(vol_df, how="outer", on=C.TIME)
else:
return
self.writer.update_csv(filename, df)
if os.path.exists(filename):
return filename
def standardize_sentiment(self, symbol, df):
full_mapping = dict(
zip(['timestamp', 'bullish', 'bearish'], [C.TIME, C.POS, C.NEG]))
filename = self.finder.get_sentiment_path(symbol, self.provider)
df = self.standardize(df, full_mapping, filename,
[C.TIME, C.POS, C.NEG], 0)
return df[{C.TIME, C.POS, C.NEG}.intersection(df.columns)]
def standardize_volume(self, symbol, df):
full_mapping = dict(
zip(['timestamp', 'volume_score', 'volume_change'],
[C.TIME, C.VOL, C.DELTA]))
filename = self.finder.get_sentiment_path(symbol, self.provider)
df = self.standardize(df, full_mapping, filename,
[C.TIME, C.VOL, C.DELTA], 0)
return df[{C.TIME, C.VOL, C.DELTA}.intersection(df.columns)]
def get_intraday(self, symbol, min=1, timeframe='max', extra_hrs=False):
# implement way to transform 1 min dataset to 5 min data
# or 30 or 60 should be flexible soln
# implement way to only get market hours
# given a symbol, return a cached dataframe
dates = self.traveller.dates_in_range(timeframe)
for date in dates:
df = self.reader.load_csv(
self.finder.get_intraday_path(symbol, date, self.provider))
yield self.reader.data_in_timeframe(df, C.TIME, timeframe)
def save_intraday(self, **kwargs):
symbol = kwargs['symbol']
dfs = self.get_intraday(**kwargs)
filenames = []
for df in dfs:
date = df[C.TIME].iloc[0].strftime(C.DATE_FMT)
filename = self.finder.get_intraday_path(symbol, date,
self.provider)
if os.path.exists(filename):
os.remove(filename)
save_fmt = f'{C.DATE_FMT} {C.TIME_FMT}'
df = self.reader.update_df(filename, df, C.TIME, save_fmt)
self.writer.update_csv(filename, df)
if os.path.exists(filename):
filenames.append(filename)
return filenames
def get_unemployment_rate(self, timeframe='max'):
# given a timeframe, return a cached dataframe
df = self.reader.load_csv(self.finder.get_unemployment_path())
filtered = self.reader.data_in_timeframe(df, C.TIME, timeframe)
return filtered
def standardize_unemployment(self, df):
full_mapping = dict(zip(['time', 'value'], [C.TIME, C.UN_RATE]))
filename = self.finder.get_unemployment_path()
return self.standardize(df, full_mapping, filename,
[C.TIME, C.UN_RATE], 0)
def save_unemployment_rate(self, **kwargs):
# given a symbol, save its dividend history
filename = self.finder.get_unemployment_path()
if os.path.exists(filename):
os.remove(filename)
df = self.reader.update_df(filename,
self.get_unemployment_rate(**kwargs),
C.TIME, '%Y-%m')
self.writer.update_csv(filename, df)
if os.path.exists(filename):
return filename
def standardize_s2f_ratio(self, df):
full_mapping = dict(
zip(['t', 'o.daysTillHalving', 'o.ratio'],
[C.TIME, C.HALVING, C.RATIO]))
filename = self.finder.get_s2f_path()
df = self.standardize(df, full_mapping, filename,
[C.TIME, C.HALVING, C.RATIO], 0)
return df[{C.TIME, C.HALVING, C.RATIO}.intersection(df.columns)]
def get_s2f_ratio(self, timeframe='max'):
# given a symbol, return a cached dataframe
df = self.reader.load_csv(self.finder.get_s2f_path())
filtered = self.reader.data_in_timeframe(
df, C.TIME, timeframe)[[C.TIME, C.HALVING, C.RATIO]]
return filtered
def standardize_s2f_deflection(self, df):
full_mapping = dict(zip(['t', 'v'], [C.TIME, C.VAL]))
filename = self.finder.get_s2f_path()
df = self.standardize(df, full_mapping, filename, [C.TIME, C.VAL], 1)
return df[{C.TIME, C.VAL}.intersection(df.columns)]
def get_s2f_deflection(self, timeframe='max'):
# given a symbol, return a cached dataframe
df = self.reader.load_csv(self.finder.get_s2f_path())
filtered = self.reader.data_in_timeframe(df, C.TIME,
timeframe)[[C.TIME, C.VAL]]
return filtered
def save_s2f(self, **kwargs):
# # given a symbol, save its s2f data
filename = self.finder.get_s2f_path()
if os.path.exists(filename):
os.remove(filename)
rat_df = self.reader.update_df(filename, self.get_s2f_ratio(**kwargs),
C.TIME, C.DATE_FMT)
rat_df = rat_df[{C.TIME, C.HALVING,
C.RATIO}.intersection(rat_df.columns)]
def_df = self.reader.update_df(filename,
self.get_s2f_deflection(**kwargs),
C.TIME, C.DATE_FMT)
def_df = def_df[{C.TIME, C.VAL}.intersection(def_df.columns)]
if rat_df.empty and not def_df.empty:
df = def_df
elif not rat_df.empty and def_df.empty:
df = rat_df
elif not rat_df.empty and not def_df.empty:
df = rat_df.merge(def_df, how="outer", on=C.TIME)
else:
return
self.writer.update_csv(filename, df)
if os.path.exists(filename):
return filename
def __init__(self, broker=None):
self.writer = FileWriter()
self.reader = FileReader()
self.finder = PathFinder()
class Store:
def __init__(self):
load_dotenv()
self.bucket_name = self.get_bucket_name()
self.finder = PathFinder()
def get_bucket_name(self):
return os.environ.get('S3_BUCKET') if not C.DEV else os.environ.get(
'S3_DEV_BUCKET')
def upload_file(self, path):
s3 = boto3.resource('s3')
bucket = s3.Bucket(self.bucket_name)
bucket.upload_file(path, path)
def upload_dir(self, path, truncate=False):
paths = self.finder.get_all_paths(path, truncate=truncate)
with Pool() as p:
p.map(self.upload_file, paths)
def delete_objects(self, keys):
if keys:
objects = [{'Key': key} for key in keys]
s3 = boto3.resource('s3')
bucket = s3.Bucket(self.bucket_name)
bucket.delete_objects(Delete={'Objects': objects})
def get_all_keys(self):
s3 = boto3.resource('s3')
bucket = s3.Bucket(self.bucket_name)
keys = [obj.key for obj in bucket.objects.filter()]
return keys
def key_exists(self, key, download=False):
try:
s3 = boto3.resource('s3')
bucket = s3.Bucket(self.bucket_name)
bucket.Object(key).load()
except ClientError:
return False
else:
return True
def download_file(self, key):
try:
self.finder.make_path(key)
s3 = boto3.resource('s3')
with open(key, 'wb') as file:
bucket = s3.Bucket(self.bucket_name)
bucket.download_fileobj(key, file)
except ClientError as e:
print(f'{key} does not exist in S3.')
os.remove(key)
raise e
def rename_key(self, old_key, new_key):
s3 = boto3.resource('s3')
s3.Object(
self.bucket_name,
new_key).copy_from(CopySource=f'{self.bucket_name}/{old_key}')
s3.Object(self.bucket_name, old_key).delete()
def __init__(self):
load_dotenv(find_dotenv('config.env'))
self.bucket_name = self.get_bucket_name()
self.finder = PathFinder()
class Robinhood:
# broker operation
def __init__(self, usr=None, pwd=None, mfa=None):
# Authentication
load_dotenv()
username = usr or os.environ['RH_USERNAME']
password = pwd or os.environ['RH_PASSWORD']
mfa_code = mfa or pyotp.TOTP(os.environ['RH_2FA']).now()
rh.login(username, password, mfa_code=mfa_code)
self.api = rh
self.writer = FileWriter()
self.reader = FileReader()
self.finder = PathFinder()
def get_symbols_from_instruments(self, instruments):
# given a list of instruments,
# return a list of corresponding symbols
return [
self.api.get_symbol_by_url(instrument)
for instrument in instruments
]
def flatten(self, xxs):
# flattens 2d list into 1d list
return [x for xs in xxs for x in xs]
def get_hists(self, symbols, span='year', interval='day', save=False):
# given a list of symbols,
# return a DataFrame with historical data
hists = [
self.api.get_stock_historicals(symbol, interval, span)
for symbol in symbols
]
clean = [hist for hist in hists if hist != [None]]
df = pd.DataFrame.from_records(self.flatten(clean))
# look into diff b/w tz_localize and tz_convert w param 'US/Eastern'
# ideally store utc time
df['begins_at'] = pd.to_datetime(
df['begins_at']).apply(lambda x: x.tz_localize(None))
# df = df.sort_values('begins_at')
if save is True:
self.writer.save_csv('data/data.csv', df)
return df
def get_names(self, symbols):
# given a list of stock symbols
# return a list of company names
return [self.api.get_name_by_symbol(symbol) for symbol in symbols]
def save_symbols(self):
# save all the portfolio symbols in a table
if not hasattr(self, 'symbols'):
self.load_portfolio()
symbols = list(self.symbols)
names = self.get_names(symbols)
df = pd.DataFrame({C.SYMBOL: symbols, C.NAME: names})
self.writer.update_csv(self.finder.get_symbols_path(), df)
def load_portfolio(self):
start = time.time()
# Data acquisition
self.positions = self.api.get_all_positions()
self.holdings = self.api.build_holdings()
# print(self.holdings)
# Create lookup table instrument -> symbol and vice versa
instruments = [position['instrument'] for position in self.positions]
symbols = self.get_symbols_from_instruments(instruments)
self.instruments = dict(zip(instruments, symbols))
self.symbols = dict(map(reversed, self.instruments.items()))
# Get historical data for all instruments
self.hist = self.get_hists(symbols)
end = time.time()
print(f'Successfully loaded portfolio in {round(end-start, 2)}s.')
import os
import sys
sys.path.append('src')
from DataSource import Glassnode # noqa autopep8
from Constants import PathFinder # noqa autopep8
import Constants as C # noqa autopep8
glass = Glassnode()
try:
glass.save_s2f(timeframe='max', retries=1 if C.TEST else 2)
except Exception as e:
print('Glassnode S2F update failed.')
print(e)
finally:
filename = PathFinder().get_s2f_path()
if C.CI and os.path.exists(filename):
os.remove(filename)
def test_init(self):
assert type(PathFinder()).__name__ == 'PathFinder'
import sys
sys.path.append('src')
from Constants import PathFinder # noqa autopep8
finder = PathFinder()
class TestPathFinder():
def test_init(self):
assert type(PathFinder()).__name__ == 'PathFinder'
def test_get_symbols_path(self):
assert finder.get_symbols_path() == 'data/symbols.csv'
def test_get_dividends_path(self):
assert finder.get_dividends_path(
'aapl') == 'data/dividends/iexcloud/AAPL.csv'
assert finder.get_dividends_path(
'AMD') == 'data/dividends/iexcloud/AMD.csv'
assert finder.get_dividends_path(
'TSLA', 'polygon') == 'data/dividends/polygon/TSLA.csv'
def test_get_splits_path(self):
assert finder.get_splits_path(
'aapl') == 'data/splits/iexcloud/AAPL.csv'
assert finder.get_splits_path('AMD') == 'data/splits/iexcloud/AMD.csv'
assert finder.get_splits_path(
'TSLA', 'polygon') == 'data/splits/polygon/TSLA.csv'
def test_get_sentiment_path(self):
assert finder.get_sentiment_path(
def __init__(self):
load_dotenv()
self.bucket_name = self.get_bucket_name()
self.finder = PathFinder()
class Store:
def __init__(self):
load_dotenv(find_dotenv('config.env'))
self.bucket_name = self.get_bucket_name()
self.finder = PathFinder()
def get_bucket_name(self):
return os.environ.get('S3_BUCKET') if not C.DEV else os.environ.get(
'S3_DEV_BUCKET')
def get_bucket(self):
s3 = boto3.resource('s3')
bucket = s3.Bucket(self.bucket_name)
return bucket
def upload_file(self, path):
bucket = self.get_bucket()
key = path.replace('\\', '/')
bucket.upload_file(path, key)
def upload_dir(self, **kwargs):
paths = self.finder.get_all_paths(**kwargs)
with Pool() as p:
p.map(self.upload_file, paths)
def delete_objects(self, keys):
if keys:
objects = [{'Key': key.replace('\\', '/')} for key in keys]
bucket = self.get_bucket()
bucket.delete_objects(Delete={'Objects': objects})
def get_all_keys(self):
bucket = self.get_bucket()
keys = [obj.key for obj in bucket.objects.filter()]
return keys
def key_exists(self, key, download=False):
key = key.replace('\\', '/')
try:
if download:
self.download_file(key)
else:
bucket = self.get_bucket()
bucket.Object(key).load()
except ClientError:
return False
else:
return True
def download_file(self, key):
try:
self.finder.make_path(key)
with open(key, 'wb') as file:
bucket = self.get_bucket()
s3_key = key.replace('\\', '/')
bucket.download_fileobj(s3_key, file)
except ClientError as e:
print(f'{key} does not exist in S3.')
os.remove(key)
raise e
def copy_object(self, src, dst):
src = src.replace('\\', '/')
dst = dst.replace('\\', '/')
bucket = self.get_bucket()
copy_source = {'Bucket': self.bucket_name, 'Key': src}
bucket.copy(copy_source, dst)
def rename_key(self, old_key, new_key):
old_key = old_key.replace('\\', '/')
new_key = new_key.replace('\\', '/')
self.copy_object(old_key, new_key)
self.delete_objects([old_key])
def last_modified(self, key):
key = key.replace('\\', '/')
bucket = self.get_bucket()
obj = bucket.Object(key)
then = obj.last_modified.replace(tzinfo=None)
return then
def modified_delta(self, key):
key = key.replace('\\', '/')
then = self.last_modified(key)
now = datetime.utcnow()
return now - then
