def check_pass(unhashed_password, hashed_password): if not hashed_password or not unhashed_password.strip(): return False try: passed = compare_passwords(unhashed_password.strip(), hashed_password) except Exception as e: log.debug("some error occurred: {}".format(e)) return False return passed
def __init__(self, size=50, ip_attempt_limit=20, ttl=300, *args, **kwargs): """ :param size int: The number of attempts to remember """ self.size = size self.ip_attempt_limit = ip_attempt_limit self.ip_index = dict() self.auth_index = dict() self.ttl = ttl log.debug("AuthCache: size=%s ip_attempt_limit=%s ttl=%s", size, ip_attempt_limit, ttl)
def on_disconnect(self, notice): """Called when twitter sends a disconnect notice Disconnect codes are listed here: https://dev.twitter.com/docs/streaming-apis/messages#Disconnect_messages_disconnect """ log.warning( "on_disconnect: A disconnect notice was sent: {}".format(notice)) log.debug( "Disconnect codes are listed here: https://dev.twitter.com/docs/streaming-apis/messages#Disconnect_messages_disconnect" ) return
def __cleanup_expired(self): keys = list(self.auth_index.keys()) for key in keys: if (datetime.now() - self.auth_index[key]['most_recent']).total_seconds() >= self.ttl: log.debug("cleanup - removing expired %s", key) del self.auth_index[key] keys = list(self.ip_index.keys()) for key in keys: if (datetime.now() - self.auth_index[key]['most_recent']).total_seconds() >= self.ttl: log.debug("cleanup - removing expired %s", key) del self.ip_index[key]
def check_credentials(self, type, username, password): """ If we've tried to login here at least three times, just reject it. A return of `True` means that we fail. """ key = (type, username, password) if key not in self.auth_index: log.debug("check_credentials -- credentials not yet logged") return False log.debug("check_credentials: %s -> %s", key, self.auth_index[key]) if self.auth_index[key]['attempts'] >= 3: return True return False
def worker(): while True: try: item = infile.readlines_batch() if not item: break start_time = time.time() s = Keywords(client, item) s.data['stream_meta'] = stream_meta s.data['source_language'] = 'auto' response = f(s) outfile.write("{}\n".format(response)) log.debug("classify/keywords with %s posts took %0.3f seconds", len(s.classify_data), (time.time() - start_time)) except: break
def cleanup(self): """ Remove items from this object if we're too big, or if they're expired """ if len(self.ip_index) > self.size: ip_addresses = sorted(self.ip_index, key=lambda k: self.ip_index[k]['most_recent'], reverse=True) for key in ip_addresses[self.size:]: log.debug("cleanup - removing %s", key) del self.ip_index[key] if len(self.auth_index) > self.size: credentials = sorted(self.auth_index, key=lambda k: self.auth_index[k]['most_recent'], reverse=True) for key in credentials[self.size:]: log.debug("cleanup - removing %s", key) del self.auth_index[key] # remove items that are too old... self.__cleanup_expired() return True
def build_access_token(u, a): """ Build an access_token with a set of claims describing the user. :param u: User :param a: Application :return: An access token object """ access_token = None secret = a.jwt_secret.strip() log.debug("building JWT with secret: %s", secret) if secret: now = datetime.datetime.utcnow() tomorrow = now + datetime.timedelta(days=1) iss = 'https://auth.econtext.com/api/authenticate' groups = [g.name for g in u.groups if g.application == a.uid] org_data = {'organization_id': u.organization.uid, 'name': u.organization.name} name = u.name claims = { 'exp': int(tomorrow.timestamp()), # tomorrow 'nbf': int(now.timestamp()), # now 'iat': int(now.timestamp()), # now 'iss': iss, # URL for auth 'sub': u.uid, # user id 'aud': a.uid, # target application, 'groups': groups, 'name': name, 'org': org_data } access_token = jwt.encode( claims, secret, algorithm='HS256' ) return access_token
def check_auth(self, type, username, password, ip_address=None, *args, **kwargs): """ Check whether an authentication attempt has occurred. This should also write a timestamp somewhere so that we know when when this check occurred. We only get rid of attempts that haven't occurred recently. * check whether this specific username+password has been used already * check whether repeated attempts from the same IP have been used THe auth parameter looks like this: { "type": "username" | "apikey", "credential": { "username": "******", "password": "******" }, "application": "APPLICATION" } :return bool False when we detect "abuse" """ log.debug("check_auth") response = True if self.check_credentials(type, username, password): "If we detect too many attempts with the same credentials..." response = False elif self.check_ip_attempts(ip_address): "If we detect too many attempts from the same IP address..." response = False self.cleanup() return response
def check_ip_attempts(self, ip_address): if ip_address is None: log.debug("check_ip_attempts -- no ip_address given") return False if ip_address not in self.ip_index: log.debug("check_ip_attempts -- ip_address not yet logged") return False log.debug("check_ip_attempts: %s -> %s", ip_address, self.ip_index[ip_address]) if self.ip_index[ip_address]['attempts'] >= self.ip_attempt_limit: return True return False
def main(): parser = argparse.ArgumentParser(description=usage) parser.add_argument("-i", "--in", dest="infile", default="stdin", help="Input file", metavar="PATH") parser.add_argument("-o", "--out", dest="outfile", default="stdout", help="Output file", metavar="PATH") parser.add_argument("-u", dest="username", required=True, action="store", default=None, help="API username") parser.add_argument("-p", dest="password", required=True, action="store", default=None, help="API password") parser.add_argument("-w", dest="workers", action="store", default=1, help="How many worker processes to use") parser.add_argument("-v", dest="config_verbose", action="count", default=0, help="Be more or less verbose") parser.add_argument("-m", "--meta", dest="meta", default=None, help="Meta data to be included with each call", metavar="JSON") parser.add_argument("-b", "--base-url", dest="base_url", default="https://api.econtext.com/v2", help="Use a different base-url", metavar="URL") options = parser.parse_args() get_log(options.config_verbose) start = time.time() log.info("Running classification using {} worker processes".format(options.workers)) infile = ropen(options.infile, batch_size=options.chunk_size) # resumable input file if options.outfile == 'stdout': outfile = sys.stdout else: outfile = sopen(options.outfile, 'w') # threadsafe output file stream_meta = None if options.meta: meta_file = open(options.meta) stream_meta = json.load(meta_file) log.debug("stream_meta: %s", json.dumps(stream_meta)) client = Client(options.username, options.password, baseurl=options.base_url) def worker(): while True: try: item = infile.readlines_batch() if not item: break start_time = time.time() s = Keywords(client, item) s.data['stream_meta'] = stream_meta s.data['source_language'] = 'auto' response = f(s) outfile.write("{}\n".format(response)) log.debug("classify/keywords with %s posts took %0.3f seconds", len(s.classify_data), (time.time() - start_time)) except: break threads = [] for i in range(int(options.workers)): t = threading.Thread(target=worker) t.start() threads.append(t) try: for t in threads: t.join() except KeyboardInterrupt: pass finally: infile.close() outfile.close() elapsed = time.time() - start log.info("Total time: {}".format(elapsed)) return True
def on_post(self, req, resp): """ Authenticate a user given a set of credentials. We expect to receive a couple of items here: { "type": "username" | "apikey", "credential": { "username": "******", "password": "******" }, "application": "APPLICATION" } * Passwords are hashed in our DB, so the input should be hashed appropriately as well to match it after receiving it If a user is not a member of the requested application, their authentication request should fail, even if their credentials are otherwise correct. Errors should always be generic. E.g. "Authentication failed" rather than "Authentication failed because no matching username could be found" or "Authentication failed because the provided password did not match" General method here: * Search for the user based on the provided username * If there are no matching users -> Fail * Check the provided password against the hashed password (use ph.verify(user.password, supplied_credential) * If there is no match -> Fail * Success :param req: :param resp: :return: """ mapper = self.meta['mapper'] auth_cache = self.meta['auth_cache'] body = req.context['body'] type = body['type'] username = body['credential']['username'] password = body['credential']['password'] application_id = body['application'] ip_address = body.get("ip_address") try: if auth_cache and not auth_cache.check_auth(type, username, password, ip_address): raise Exception("Failed in auth_cache check") hashed_password = None u = None access_token = None if type == 'username': u = mapper.user.User.get_by_username(username, organization_flag=True, application_flag=True, group_flag=True, apikey_flag=True) hashed_password = u.password elif type == "apikey": a = mapper.apikey.ApiKey.get_by_key(username) # if the ApiKey is not ENABLED ... if not Authenticate.check_status(a): raise Exception("Failed check_status") # else, grab the password and populate the User hashed_password = a.secret u = mapper.user.User.get_by_uid(a.user.uid, organization_flag=True, application_flag=True, group_flag=True, apikey_flag=True) # if user is not ENABLED ... if not Authenticate.check_status(u): raise Exception("Failed check_status") # Check the credentials ... if not Authenticate.check_pass(password, hashed_password): log.debug("check_pass(%s, %s)", password.encode('utf8'), hashed_password.encode('utf8')) log.debug("entered password hash: %s", hash_secret(password.encode('utf8'))) raise Exception("Failed check_pass") applications = {app.uid: app for app in u.applications if app.status != 'DISABLED'} if application_id not in applications: raise Exception("Application not found or not ENABLED") access_token = Authenticate.build_access_token(u, applications.get(application_id)) resp.body = {"authenticated": True, "user": u.to_dict(), "access_token": access_token} except Exception as e: log.exception("Authentication Failure...") log.debug("authentication failed: %s", json.dumps(body)) if auth_cache: auth_cache.add_credential(type, username, password, ip_address) resp.body = {"authenticated": False, "user": None, "access_token": None} return True
def main(): parser = argparse.ArgumentParser( description='Start the Twitter Gardenhose Mapper') parser.add_argument("--config", dest="config_config_file", default="/etc/econtext/twitter.ini", help="Configuration file", metavar="PATH") parser.add_argument("-v", dest="config_verbose", action="count", default=0, help="Be more or less verbose") parser.add_argument("--consumer-key", dest="config_consumer_key", help="Twitter Consumer Key", metavar="STR") parser.add_argument("--consumer-secret", dest="config_consumer_secret", help="Twitter Consumer Secret", metavar="STR") parser.add_argument("--access-token-key", dest="config_access_token_key", help="Twitter Access Token Key", metavar="STR") parser.add_argument("--access-token-secret", dest="config_access_token_secret", help="Twitter Access Token Secret", metavar="STR") parser.add_argument("--econtext-key", dest="config_econtext_key", help="eContext API Key", metavar="STR") parser.add_argument("--econtext-secret", dest="config_econtext_secret", help="eContext API Secret", metavar="STR") parser.add_argument("--econtext-baseurl", dest="config_econtext_baseurl", help="eContext API base URL", metavar="URL") parser.add_argument( "-t", "--threads", dest="config_threads", default=10, help="Number of eContext threads to dedicate to mapping", metavar="INT") parser.add_argument( "--tpc", dest="config_tpc", default=500, help="Number of tweets to include in each eContext call", metavar="INT") parser.add_argument( "-s", "--sentiment", dest="config_sentiment", default=False, action="store_true", help="Include sentiment in results (increases latency)") parser.add_argument("-l", "--languages", dest="config_filter_languages", default=None, help="Language codes to include") options = parser.parse_args() log_add_stream_handler(options.config_verbose) config = load_config(options.config_config_file, default_config) for section in {'config'}.difference(set(config.sections())): config.add_section(section) del options.config_config_file del options.config_verbose config_updates = dict() for k, v in options.__dict__.items(): if v is not None: section, key = k.split("_", 1) if section not in config_updates: config_updates[section] = dict() config_updates[section][key] = str(v) update_config(config, config_updates) try: econtext = get_econtext_api( config_get(config, 'config', 'econtext_key'), config_get(config, 'config', 'econtext_secret'), config_get(config, 'config', 'econtext_baseurl')) q = Queue(maxsize=0) num_threads = int(config_get(config, 'config', 'threads')) tpc = int(config_get(config, 'config', 'tpc')) listener = MyStreamListener(q) sentiment = config_get(config, 'config', 'sentiment') auth = tweepy.OAuthHandler( config_get(config, 'config', 'consumer_key'), config_get(config, 'config', 'consumer_secret')) auth.set_access_token( config_get(config, 'config', 'access_token_key'), config_get(config, 'config', 'access_token_secret')) stream = tweepy.Stream(auth, listener) track = [ x.strip() for x in config_get(config, 'config', 'filter_words').split(",") ] languages = [ x.strip() for x in config_get(config, 'config', 'filter_languages').split(",") ] locations = [ float(x.strip()) for x in config_get(config, 'config', 'filter_locations').split(",") ] workers = [] for i in range(num_threads): log.debug('Starting thread {}'.format(i)) worker = Thread(target=map_threads, args=(q, econtext, tpc, i, sentiment)) worker.setDaemon(True) worker.start() workers.append(worker) if locations: stream.filter(locations=locations, languages=languages) else: stream.filter(track=track, languages=languages) except Exception: log.exception("Caught an Exception in main.py ...") while True: q.put_nowait(None) for i in range(len(workers)): workers[i].join(timeout=2) if not workers[i].is_alive(): log.info("Worker thread {} finished".format( workers[i].ident)) workers[i] = None workers = [w for w in workers if w is not None] if len(workers) == 0: q.join() break q.join() log.info("Finished...")