def check_pass(unhashed_password, hashed_password):
if not hashed_password or not unhashed_password.strip():
return False
try:
passed = compare_passwords(unhashed_password.strip(), hashed_password)
except Exception as e:
log.debug("some error occurred: {}".format(e))
return False
return passed
def __init__(self, size=50, ip_attempt_limit=20, ttl=300, *args, **kwargs):
"""
:param size int: The number of attempts to remember
"""
self.size = size
self.ip_attempt_limit = ip_attempt_limit
self.ip_index = dict()
self.auth_index = dict()
self.ttl = ttl
log.debug("AuthCache: size=%s ip_attempt_limit=%s ttl=%s", size, ip_attempt_limit, ttl)
def on_disconnect(self, notice):
"""Called when twitter sends a disconnect notice
Disconnect codes are listed here:
https://dev.twitter.com/docs/streaming-apis/messages#Disconnect_messages_disconnect
"""
log.warning(
"on_disconnect: A disconnect notice was sent: {}".format(notice))
log.debug(
"Disconnect codes are listed here: https://dev.twitter.com/docs/streaming-apis/messages#Disconnect_messages_disconnect"
)
return
def __cleanup_expired(self):
keys = list(self.auth_index.keys())
for key in keys:
if (datetime.now() - self.auth_index[key]['most_recent']).total_seconds() >= self.ttl:
log.debug("cleanup - removing expired %s", key)
del self.auth_index[key]
keys = list(self.ip_index.keys())
for key in keys:
if (datetime.now() - self.auth_index[key]['most_recent']).total_seconds() >= self.ttl:
log.debug("cleanup - removing expired %s", key)
del self.ip_index[key]
def check_credentials(self, type, username, password):
"""
If we've tried to login here at least three times, just reject
it. A return of `True` means that we fail.
"""
key = (type, username, password)
if key not in self.auth_index:
log.debug("check_credentials -- credentials not yet logged")
return False
log.debug("check_credentials: %s -> %s", key, self.auth_index[key])
if self.auth_index[key]['attempts'] >= 3:
return True
return False
def worker():
while True:
try:
item = infile.readlines_batch()
if not item:
break
start_time = time.time()
s = Keywords(client, item)
s.data['stream_meta'] = stream_meta
s.data['source_language'] = 'auto'
response = f(s)
outfile.write("{}\n".format(response))
log.debug("classify/keywords with %s posts took %0.3f seconds", len(s.classify_data), (time.time() - start_time))
except:
break
def cleanup(self):
"""
Remove items from this object if we're too big, or if they're expired
"""
if len(self.ip_index) > self.size:
ip_addresses = sorted(self.ip_index, key=lambda k: self.ip_index[k]['most_recent'], reverse=True)
for key in ip_addresses[self.size:]:
log.debug("cleanup - removing %s", key)
del self.ip_index[key]
if len(self.auth_index) > self.size:
credentials = sorted(self.auth_index, key=lambda k: self.auth_index[k]['most_recent'], reverse=True)
for key in credentials[self.size:]:
log.debug("cleanup - removing %s", key)
del self.auth_index[key]
# remove items that are too old...
self.__cleanup_expired()
return True
def build_access_token(u, a):
"""
Build an access_token with a set of claims describing the user.
:param u: User
:param a: Application
:return: An access token object
"""
access_token = None
secret = a.jwt_secret.strip()
log.debug("building JWT with secret: %s", secret)
if secret:
now = datetime.datetime.utcnow()
tomorrow = now + datetime.timedelta(days=1)
iss = 'https://auth.econtext.com/api/authenticate'
groups = [g.name for g in u.groups if g.application == a.uid]
org_data = {'organization_id': u.organization.uid, 'name': u.organization.name}
name = u.name
claims = {
'exp': int(tomorrow.timestamp()), # tomorrow
'nbf': int(now.timestamp()), # now
'iat': int(now.timestamp()), # now
'iss': iss, # URL for auth
'sub': u.uid, # user id
'aud': a.uid, # target application,
'groups': groups,
'name': name,
'org': org_data
}
access_token = jwt.encode(
claims,
secret,
algorithm='HS256'
)
return access_token
def check_auth(self, type, username, password, ip_address=None, *args, **kwargs):
"""
Check whether an authentication attempt has occurred.
This should also write a timestamp somewhere so that we know when
when this check occurred. We only get rid of attempts that haven't
occurred recently.
* check whether this specific username+password has been used already
* check whether repeated attempts from the same IP have been used
THe auth parameter looks like this:
{
"type": "username" | "apikey",
"credential": {
"username": "******",
"password": "******"
},
"application": "APPLICATION"
}
:return bool False when we detect "abuse"
"""
log.debug("check_auth")
response = True
if self.check_credentials(type, username, password):
"If we detect too many attempts with the same credentials..."
response = False
elif self.check_ip_attempts(ip_address):
"If we detect too many attempts from the same IP address..."
response = False
self.cleanup()
return response
def check_ip_attempts(self, ip_address):
if ip_address is None:
log.debug("check_ip_attempts -- no ip_address given")
return False
if ip_address not in self.ip_index:
log.debug("check_ip_attempts -- ip_address not yet logged")
return False
log.debug("check_ip_attempts: %s -> %s", ip_address, self.ip_index[ip_address])
if self.ip_index[ip_address]['attempts'] >= self.ip_attempt_limit:
return True
return False
def main():
parser = argparse.ArgumentParser(description=usage)
parser.add_argument("-i", "--in", dest="infile", default="stdin", help="Input file", metavar="PATH")
parser.add_argument("-o", "--out", dest="outfile", default="stdout", help="Output file", metavar="PATH")
parser.add_argument("-u", dest="username", required=True, action="store", default=None, help="API username")
parser.add_argument("-p", dest="password", required=True, action="store", default=None, help="API password")
parser.add_argument("-w", dest="workers", action="store", default=1, help="How many worker processes to use")
parser.add_argument("-v", dest="config_verbose", action="count", default=0, help="Be more or less verbose")
parser.add_argument("-m", "--meta", dest="meta", default=None, help="Meta data to be included with each call", metavar="JSON")
parser.add_argument("-b", "--base-url", dest="base_url", default="https://api.econtext.com/v2", help="Use a different base-url", metavar="URL")
options = parser.parse_args()
get_log(options.config_verbose)
start = time.time()
log.info("Running classification using {} worker processes".format(options.workers))
infile = ropen(options.infile, batch_size=options.chunk_size) # resumable input file
if options.outfile == 'stdout':
outfile = sys.stdout
else:
outfile = sopen(options.outfile, 'w') # threadsafe output file
stream_meta = None
if options.meta:
meta_file = open(options.meta)
stream_meta = json.load(meta_file)
log.debug("stream_meta: %s", json.dumps(stream_meta))
client = Client(options.username, options.password, baseurl=options.base_url)
def worker():
while True:
try:
item = infile.readlines_batch()
if not item:
break
start_time = time.time()
s = Keywords(client, item)
s.data['stream_meta'] = stream_meta
s.data['source_language'] = 'auto'
response = f(s)
outfile.write("{}\n".format(response))
log.debug("classify/keywords with %s posts took %0.3f seconds", len(s.classify_data), (time.time() - start_time))
except:
break
threads = []
for i in range(int(options.workers)):
t = threading.Thread(target=worker)
t.start()
threads.append(t)
try:
for t in threads:
t.join()
except KeyboardInterrupt:
pass
finally:
infile.close()
outfile.close()
elapsed = time.time() - start
log.info("Total time: {}".format(elapsed))
return True
def on_post(self, req, resp):
"""
Authenticate a user given a set of credentials. We expect to
receive a couple of items here:
{
"type": "username" | "apikey",
"credential": {
"username": "******",
"password": "******"
},
"application": "APPLICATION"
}
* Passwords are hashed in our DB, so the input should be hashed
appropriately as well to match it after receiving it
If a user is not a member of the requested application, their
authentication request should fail, even if their credentials
are otherwise correct.
Errors should always be generic. E.g. "Authentication failed"
rather than "Authentication failed because no matching username
could be found" or "Authentication failed because the provided
password did not match"
General method here:
* Search for the user based on the provided username
* If there are no matching users -> Fail
* Check the provided password against the hashed password (use
ph.verify(user.password, supplied_credential)
* If there is no match -> Fail
* Success
:param req:
:param resp:
:return:
"""
mapper = self.meta['mapper']
auth_cache = self.meta['auth_cache']
body = req.context['body']
type = body['type']
username = body['credential']['username']
password = body['credential']['password']
application_id = body['application']
ip_address = body.get("ip_address")
try:
if auth_cache and not auth_cache.check_auth(type, username, password, ip_address):
raise Exception("Failed in auth_cache check")
hashed_password = None
u = None
access_token = None
if type == 'username':
u = mapper.user.User.get_by_username(username, organization_flag=True, application_flag=True, group_flag=True, apikey_flag=True)
hashed_password = u.password
elif type == "apikey":
a = mapper.apikey.ApiKey.get_by_key(username)
# if the ApiKey is not ENABLED ...
if not Authenticate.check_status(a):
raise Exception("Failed check_status")
# else, grab the password and populate the User
hashed_password = a.secret
u = mapper.user.User.get_by_uid(a.user.uid, organization_flag=True, application_flag=True, group_flag=True, apikey_flag=True)
# if user is not ENABLED ...
if not Authenticate.check_status(u):
raise Exception("Failed check_status")
# Check the credentials ...
if not Authenticate.check_pass(password, hashed_password):
log.debug("check_pass(%s, %s)", password.encode('utf8'), hashed_password.encode('utf8'))
log.debug("entered password hash: %s", hash_secret(password.encode('utf8')))
raise Exception("Failed check_pass")
applications = {app.uid: app for app in u.applications if app.status != 'DISABLED'}
if application_id not in applications:
raise Exception("Application not found or not ENABLED")
access_token = Authenticate.build_access_token(u, applications.get(application_id))
resp.body = {"authenticated": True, "user": u.to_dict(), "access_token": access_token}
except Exception as e:
log.exception("Authentication Failure...")
log.debug("authentication failed: %s", json.dumps(body))
if auth_cache:
auth_cache.add_credential(type, username, password, ip_address)
resp.body = {"authenticated": False, "user": None, "access_token": None}
return True
def main():
parser = argparse.ArgumentParser(
description='Start the Twitter Gardenhose Mapper')
parser.add_argument("--config",
dest="config_config_file",
default="/etc/econtext/twitter.ini",
help="Configuration file",
metavar="PATH")
parser.add_argument("-v",
dest="config_verbose",
action="count",
default=0,
help="Be more or less verbose")
parser.add_argument("--consumer-key",
dest="config_consumer_key",
help="Twitter Consumer Key",
metavar="STR")
parser.add_argument("--consumer-secret",
dest="config_consumer_secret",
help="Twitter Consumer Secret",
metavar="STR")
parser.add_argument("--access-token-key",
dest="config_access_token_key",
help="Twitter Access Token Key",
metavar="STR")
parser.add_argument("--access-token-secret",
dest="config_access_token_secret",
help="Twitter Access Token Secret",
metavar="STR")
parser.add_argument("--econtext-key",
dest="config_econtext_key",
help="eContext API Key",
metavar="STR")
parser.add_argument("--econtext-secret",
dest="config_econtext_secret",
help="eContext API Secret",
metavar="STR")
parser.add_argument("--econtext-baseurl",
dest="config_econtext_baseurl",
help="eContext API base URL",
metavar="URL")
parser.add_argument(
"-t",
"--threads",
dest="config_threads",
default=10,
help="Number of eContext threads to dedicate to mapping",
metavar="INT")
parser.add_argument(
"--tpc",
dest="config_tpc",
default=500,
help="Number of tweets to include in each eContext call",
metavar="INT")
parser.add_argument(
"-s",
"--sentiment",
dest="config_sentiment",
default=False,
action="store_true",
help="Include sentiment in results (increases latency)")
parser.add_argument("-l",
"--languages",
dest="config_filter_languages",
default=None,
help="Language codes to include")
options = parser.parse_args()
log_add_stream_handler(options.config_verbose)
config = load_config(options.config_config_file, default_config)
for section in {'config'}.difference(set(config.sections())):
config.add_section(section)
del options.config_config_file
del options.config_verbose
config_updates = dict()
for k, v in options.__dict__.items():
if v is not None:
section, key = k.split("_", 1)
if section not in config_updates:
config_updates[section] = dict()
config_updates[section][key] = str(v)
update_config(config, config_updates)
try:
econtext = get_econtext_api(
config_get(config, 'config', 'econtext_key'),
config_get(config, 'config', 'econtext_secret'),
config_get(config, 'config', 'econtext_baseurl'))
q = Queue(maxsize=0)
num_threads = int(config_get(config, 'config', 'threads'))
tpc = int(config_get(config, 'config', 'tpc'))
listener = MyStreamListener(q)
sentiment = config_get(config, 'config', 'sentiment')
auth = tweepy.OAuthHandler(
config_get(config, 'config', 'consumer_key'),
config_get(config, 'config', 'consumer_secret'))
auth.set_access_token(
config_get(config, 'config', 'access_token_key'),
config_get(config, 'config', 'access_token_secret'))
stream = tweepy.Stream(auth, listener)
track = [
x.strip()
for x in config_get(config, 'config', 'filter_words').split(",")
]
languages = [
x.strip() for x in config_get(config, 'config',
'filter_languages').split(",")
]
locations = [
float(x.strip()) for x in config_get(config, 'config',
'filter_locations').split(",")
]
workers = []
for i in range(num_threads):
log.debug('Starting thread {}'.format(i))
worker = Thread(target=map_threads,
args=(q, econtext, tpc, i, sentiment))
worker.setDaemon(True)
worker.start()
workers.append(worker)
if locations:
stream.filter(locations=locations, languages=languages)
else:
stream.filter(track=track, languages=languages)
except Exception:
log.exception("Caught an Exception in main.py ...")
while True:
q.put_nowait(None)
for i in range(len(workers)):
workers[i].join(timeout=2)
if not workers[i].is_alive():
log.info("Worker thread {} finished".format(
workers[i].ident))
workers[i] = None
workers = [w for w in workers if w is not None]
if len(workers) == 0:
q.join()
break
q.join()
log.info("Finished...")
