def make_dataframe_from_csv(ticker):
""" Creates initial datafarme from ticker csv/txt file
Args:
ticker (str): Ticker of the stock
Returns:
csv_df: Dumb Pandas dataframe of the stock. Each column is a string
"""
# Path for the ticker Kibot data
path = os.path.join(PATH_TO_STOCKS_FOLDER, '%s.txt' % ticker)
# path = '/Users/system-void/gdrive/code/data/stocks/5min/%s.txt' % ticker
try:
# Get the size (lines) of file (must open file)
size = sum(1 for l in open(path))
except Exception:
log.error('%7s - Ticker not found in equity database.' % ticker)
raise
# raise ValueError('Invalid path to stock files')
else:
log.info('%7s - Loading history into memory' % ticker)
csv_df = pd.read_csv(path, skiprows=range(
1, size - 7500)) # Read only the last 7500 lines of the csv
return csv_df
def get_hashtag_suggestions(self, query):
params = {
'context': 'hashtag',
'query': query,
'include_reel': 'true',
}
try:
resp = self._make_request(
search_hashtag_endpoint,
params=params,
msg="Request was received")
except RequestException as e:
log.error(e)
else:
data = resp.json()
if data['status'] == 'ok':
return data['hashtags']
else:
log.error('Instagram response data is not ok')
def login(self, username, password):
"""Login to instagram.
A new CSRF token cookie is generated after an authenticated login request.
This token is reused throughout the session. It should be in the request
cookies and in the header `x-csrftoken` for all subsequent POST sensitive requests
Args:
username (str): Your instagram username
password (str): Your instagram password
Raises:
LoginAuthenticationError: Raised when authentication has failed
CheckpointRequired: Raised when instagram has detected an unusual login attempt. This usually happens
when your IP is different or your mid (MACHINE_ID) cookie has changed
"""
self._get_init_csrftoken()
login_data = {'username': username, 'password': password}
try:
log.info("Logging in as {}".format(username))
self._make_request(
login_endpoint, data=login_data, msg="Login request sent")
except requests.exceptions.HTTPError:
resp_data = self.last_resp.json()
if resp_data['message'] == 'checkpoint_required':
self._save_cookies()
checkpoint_url = resp_data['checkpoint_url']
checkpoint_id = checkpoint_url.split("/")[2]
checkpoint_code = checkpoint_url.split("/")[3]
self._make_request(checkpoint_url)
log.debug(
challenge_endpoint.format(
id=checkpoint_id, code=checkpoint_code))
self._make_request(
challenge_endpoint.format(
id=checkpoint_id, code=checkpoint_code),
data={'choice': '1'})
msg = """
Instagram has flagged this login as suspicious. You are either signing in from an unknown IP
or your machine id (mid) has changed.
Please enter the six-digit code that was sent to your instagram registered email to proceed.
"""
log.error(msg)
while True:
verification_code = input(
"Enter the six-digit verification code. Type REPLAY to request another one: "
)
if verification_code == 'REPLAY':
self._make_request(
challenge_replay.format(
id=checkpoint_id, code=checkpoint_code),
post=True)
else:
self._make_request(
challenge_endpoint.format(
id=checkpoint_id, code=checkpoint_code),
data={'security_code': verification_code})
break
if self.is_loggedin:
log.info('Logged in successfully')
# Store cookies
with open('cookies', 'wb+') as file:
log.debug('Saving cookies {}'.format(
self.ses.cookies.get_dict()))
pickle.dump(self.ses.cookies, file)
else:
raise LoginAuthenticationError
else:
resp_data = self.last_resp.json()
if resp_data['authenticated']:
log.info('Logged in successfully')
self.ses.headers.update({
'x-csrftoken': self.last_resp.cookies['csrftoken']
})
assert 'sessionid' in self.ses.cookies.get_dict()
self._save_cookies()
self.username = username
else:
raise LoginAuthenticationError
def _make_request(self,
endpoint,
data=None,
params=None,
msg='',
post=False,
headers=None):
""" Shorthand way to make a request.
Args:
endpoint (str): API endpoint
data (dict, None): Data if using POST request
params (dict, None): Params, if needed
msg (str): Message to log when response is successful
post (bool): True if this is a POST request, FALSE otherwise
Returns:
Response: Requests response object
"""
resp = None
# headers=headers
# Combine persistent headers with new headers temporarily
if headers:
headers = {**self.ses.headers, **headers}
else:
headers = self.ses.headers
try:
if not data and not post:
resp = self.ses.get(
base_endpoint + endpoint, headers=headers, params=params)
resp.raise_for_status()
else:
resp = self.ses.post(
base_endpoint + endpoint,
data=data,
headers=headers,
allow_redirects=False)
resp.raise_for_status()
except RequestException as ex:
if len(resp.text) > 300:
log.error('STATUS: {} - CONTENT: {}'.format(
resp.status_code, resp.text))
else:
log.error('STATUS: {} - CONTENT: {}'.format(
resp.status_code, resp.text))
self.last_resp = resp
self.status = resp.status_code
self.msg = resp.content
raise
except ConnectionResetError as e:
log.error('Server closed the connection')
log.debug("""
STATUS: {}
RESPONSE HEADERS: {}
RESPONSE CONTENT: {}
""".format(self.last_resp.status_code, self.last_resp.headers,
self.last_resp.content))
raise
else:
if len(resp.text) > 300:
log.success('STATUS: {} - CONTENT (truncated): {}'.format(
resp.status_code, resp.text[:300] + "..."))
else:
log.success('STATUS: {} - CONTENT: {}'.format(
resp.status_code, resp.text)) # log.info(msg)
self.last_resp = resp
self.status = resp.status_code
self.msg = resp.content
if msg:
log.info(msg)
return resp
def get_stock_ticker_list(self):
""" Gets a list of stocks given API criteria
Returns:
stocks (list): List of stock tickers matching given criteria as specified in the url instance variable
"""
retries = 4
stocks = []
try:
log.info("Getting request for stock screener")
resp = requests.get(self.url,
auth=(self.username, self.password),
timeout=10)
resp.raise_for_status()
except requests.HTTPError as http_error:
log.error("Server returned the following HTTP Error: {http_error}".
format(http_error=http_error))
except requests.exceptions.Timeout:
log.error('Timeout')
else:
log.info('SUCCESS')
try:
log.info("Trying to read json file . . .")
data1 = resp.json()
except Exception as ex:
print ex
else:
if 'errors' in data1:
print data1['errors'][0]['human']
raise ValueError(data1['errors'][0]['human'])
else:
log.info('SUCCESS')
total_pages = data1['total_pages']
result_count = data1['result_count']
log.info(
"Total pages: {total_pages} \tresult_count: {result_count}"
.format(total_pages=total_pages,
result_count=result_count))
stocks = []
for p in range(1, total_pages + 1):
url_new = self.url + "&page_number=" + str(p)
for r in range(retries):
try:
log.debug(
" Going to page number {p}".format(p=p))
resp = requests.get(url_new,
auth=(self.username,
self.password),
timeout=10)
if resp.status_code == 503:
log.error(
"503 Error: Server is too busy. Retrying again"
)
resp.raise_for_status()
except Exception as ex:
print ex
continue
else:
log.info("SUCCESS")
data = resp.json()
for stock in data['data']:
stocks.append(str(stock['ticker']))
break
return stocks
def test_log_error(capsys):
from log3 import log
log.error("error")
_, stderr = capsys.readouterr()