def make_dataframe_from_csv(ticker): """ Creates initial datafarme from ticker csv/txt file Args: ticker (str): Ticker of the stock Returns: csv_df: Dumb Pandas dataframe of the stock. Each column is a string """ # Path for the ticker Kibot data path = os.path.join(PATH_TO_STOCKS_FOLDER, '%s.txt' % ticker) # path = '/Users/system-void/gdrive/code/data/stocks/5min/%s.txt' % ticker try: # Get the size (lines) of file (must open file) size = sum(1 for l in open(path)) except Exception: log.error('%7s - Ticker not found in equity database.' % ticker) raise # raise ValueError('Invalid path to stock files') else: log.info('%7s - Loading history into memory' % ticker) csv_df = pd.read_csv(path, skiprows=range( 1, size - 7500)) # Read only the last 7500 lines of the csv return csv_df
def get_hashtag_suggestions(self, query): params = { 'context': 'hashtag', 'query': query, 'include_reel': 'true', } try: resp = self._make_request( search_hashtag_endpoint, params=params, msg="Request was received") except RequestException as e: log.error(e) else: data = resp.json() if data['status'] == 'ok': return data['hashtags'] else: log.error('Instagram response data is not ok')
def login(self, username, password): """Login to instagram. A new CSRF token cookie is generated after an authenticated login request. This token is reused throughout the session. It should be in the request cookies and in the header `x-csrftoken` for all subsequent POST sensitive requests Args: username (str): Your instagram username password (str): Your instagram password Raises: LoginAuthenticationError: Raised when authentication has failed CheckpointRequired: Raised when instagram has detected an unusual login attempt. This usually happens when your IP is different or your mid (MACHINE_ID) cookie has changed """ self._get_init_csrftoken() login_data = {'username': username, 'password': password} try: log.info("Logging in as {}".format(username)) self._make_request( login_endpoint, data=login_data, msg="Login request sent") except requests.exceptions.HTTPError: resp_data = self.last_resp.json() if resp_data['message'] == 'checkpoint_required': self._save_cookies() checkpoint_url = resp_data['checkpoint_url'] checkpoint_id = checkpoint_url.split("/")[2] checkpoint_code = checkpoint_url.split("/")[3] self._make_request(checkpoint_url) log.debug( challenge_endpoint.format( id=checkpoint_id, code=checkpoint_code)) self._make_request( challenge_endpoint.format( id=checkpoint_id, code=checkpoint_code), data={'choice': '1'}) msg = """ Instagram has flagged this login as suspicious. You are either signing in from an unknown IP or your machine id (mid) has changed. Please enter the six-digit code that was sent to your instagram registered email to proceed. """ log.error(msg) while True: verification_code = input( "Enter the six-digit verification code. Type REPLAY to request another one: " ) if verification_code == 'REPLAY': self._make_request( challenge_replay.format( id=checkpoint_id, code=checkpoint_code), post=True) else: self._make_request( challenge_endpoint.format( id=checkpoint_id, code=checkpoint_code), data={'security_code': verification_code}) break if self.is_loggedin: log.info('Logged in successfully') # Store cookies with open('cookies', 'wb+') as file: log.debug('Saving cookies {}'.format( self.ses.cookies.get_dict())) pickle.dump(self.ses.cookies, file) else: raise LoginAuthenticationError else: resp_data = self.last_resp.json() if resp_data['authenticated']: log.info('Logged in successfully') self.ses.headers.update({ 'x-csrftoken': self.last_resp.cookies['csrftoken'] }) assert 'sessionid' in self.ses.cookies.get_dict() self._save_cookies() self.username = username else: raise LoginAuthenticationError
def _make_request(self, endpoint, data=None, params=None, msg='', post=False, headers=None): """ Shorthand way to make a request. Args: endpoint (str): API endpoint data (dict, None): Data if using POST request params (dict, None): Params, if needed msg (str): Message to log when response is successful post (bool): True if this is a POST request, FALSE otherwise Returns: Response: Requests response object """ resp = None # headers=headers # Combine persistent headers with new headers temporarily if headers: headers = {**self.ses.headers, **headers} else: headers = self.ses.headers try: if not data and not post: resp = self.ses.get( base_endpoint + endpoint, headers=headers, params=params) resp.raise_for_status() else: resp = self.ses.post( base_endpoint + endpoint, data=data, headers=headers, allow_redirects=False) resp.raise_for_status() except RequestException as ex: if len(resp.text) > 300: log.error('STATUS: {} - CONTENT: {}'.format( resp.status_code, resp.text)) else: log.error('STATUS: {} - CONTENT: {}'.format( resp.status_code, resp.text)) self.last_resp = resp self.status = resp.status_code self.msg = resp.content raise except ConnectionResetError as e: log.error('Server closed the connection') log.debug(""" STATUS: {} RESPONSE HEADERS: {} RESPONSE CONTENT: {} """.format(self.last_resp.status_code, self.last_resp.headers, self.last_resp.content)) raise else: if len(resp.text) > 300: log.success('STATUS: {} - CONTENT (truncated): {}'.format( resp.status_code, resp.text[:300] + "...")) else: log.success('STATUS: {} - CONTENT: {}'.format( resp.status_code, resp.text)) # log.info(msg) self.last_resp = resp self.status = resp.status_code self.msg = resp.content if msg: log.info(msg) return resp
def get_stock_ticker_list(self): """ Gets a list of stocks given API criteria Returns: stocks (list): List of stock tickers matching given criteria as specified in the url instance variable """ retries = 4 stocks = [] try: log.info("Getting request for stock screener") resp = requests.get(self.url, auth=(self.username, self.password), timeout=10) resp.raise_for_status() except requests.HTTPError as http_error: log.error("Server returned the following HTTP Error: {http_error}". format(http_error=http_error)) except requests.exceptions.Timeout: log.error('Timeout') else: log.info('SUCCESS') try: log.info("Trying to read json file . . .") data1 = resp.json() except Exception as ex: print ex else: if 'errors' in data1: print data1['errors'][0]['human'] raise ValueError(data1['errors'][0]['human']) else: log.info('SUCCESS') total_pages = data1['total_pages'] result_count = data1['result_count'] log.info( "Total pages: {total_pages} \tresult_count: {result_count}" .format(total_pages=total_pages, result_count=result_count)) stocks = [] for p in range(1, total_pages + 1): url_new = self.url + "&page_number=" + str(p) for r in range(retries): try: log.debug( " Going to page number {p}".format(p=p)) resp = requests.get(url_new, auth=(self.username, self.password), timeout=10) if resp.status_code == 503: log.error( "503 Error: Server is too busy. Retrying again" ) resp.raise_for_status() except Exception as ex: print ex continue else: log.info("SUCCESS") data = resp.json() for stock in data['data']: stocks.append(str(stock['ticker'])) break return stocks
def test_log_error(capsys): from log3 import log log.error("error") _, stderr = capsys.readouterr()