def main():
# Set up logger.
FORMAT = '%(asctime)s %(filename)s:%(lineno)s [%(levelname)s] %(message)s'
logging.basicConfig(format=FORMAT, level=logging.DEBUG)
# Initialize Pepper
Pepper.initiate_Pepper()
def verify_password(nickname, password):
"""
Password verification callback.
---
securityDefinitions:
UserSecurity:
type: basic
"""
if not nickname or not password:
return False
api = Pepper('http://master:8080')
try:
auth = api.login(nickname, password, 'pam')
except PepperException:
return False
user = User.query.filter_by(nickname=nickname).first()
if not user:
user = User.create({'nickname': nickname})
current_app.logger.debug('auth')
current_app.logger.debug(auth)
user.token = auth['token']
db.session.add(user)
db.session.commit()
g.current_user = user
return True
def main():
# Set up logger.
FORMAT = '%(asctime)s %(filename)s:%(lineno)s [%(levelname)s] %(message)s'
logging.basicConfig(format=FORMAT, level=logging.DEBUG)
# Initialize Pepper
Pepper.initiate_Pepper()
def main():
# Set up logger.
FORMAT = '%(asctime)s %(filename)s:%(lineno)s [%(levelname)s] %(message)s'
logging.basicConfig(format=FORMAT, level=logging.DEBUG)
# Initialize Pepper
Pepper.speak('Greetings. My name is Pepper. How could I be of service?')
Pepper.initiate_Pepper()
def _connect(self):
api = Pepper(('http://'
+ self.salt_connection_details.salt_master
+ ':'
+ str(self.salt_connection_details.port)))
api.login(self.salt_connection_details.username, self.salt_connection_details.password,
'pam')
return api
def main():
# Set up logger.
FORMAT = '%(asctime)s %(filename)s:%(lineno)s [%(levelname)s] %(message)s'
logging.basicConfig(format=FORMAT, level=logging.DEBUG)
# Initialize Pepper
Pepper.speak('Greetings. My name is Pepper. How could I be of service?')
Pepper.initiate_Pepper()
def get_events():
api = Pepper(url, ignore_ssl_errors=True)
# TODO: find a way.
user = User.objects.first()
api.login(
str(user.username),
user.user_settings.token,
os.environ.get("SALT_AUTH", "alcali"),
)
response = api.req_stream("/events")
return response
def api_connect():
# TODO fix this!
user = get_current_user()
api = Pepper(url, ignore_ssl_errors=True)
login_ret = api.login(
str(user.username),
user.user_settings.token,
os.environ.get("SALT_AUTH", "alcali"),
)
user.user_settings.salt_permissions = json.dumps(login_ret["perms"])
user.save()
return api
def api_connect():
user = get_current_user()
api = Pepper(url, ignore_ssl_errors=True)
try:
login_ret = api.login(
str(user.username),
user.user_settings.token,
os.environ.get("SALT_AUTH", "alcali"),
)
except URLError:
raise PepperException("URL Error")
user.user_settings.salt_permissions = json.dumps(login_ret["perms"])
user.save()
return api
def buildIndex(self, target_url=None, treat_as_root=False, limit=500):
"""Build or add to the Index.
@usage buildIndex [target_url] [treat_as_root] [limit]
@param target_url, optional, a full URL (e.g. http://www.smu.edu)
default is http://lyle.smu.edu/~fmoore/
@param treat_as_root, optional signifier to treat the given URL as a
root URL. e.g. treat a URL like, http://lyle.smu.edu/~fmoore/, as
the root instead of the server instead of http://lyle.smu.edu/
@param limit, optional limit the underlying crawler to N requests
"""
# Use ~fmoore as the default target URL.
if target_url is None:
target_url = "http://lyle.smu.edu/~fmoore/"
treat_as_root = True
# Guard against invalid URLs.
if not urlparse(target_url).scheme:
print "Invalid URL: %s" % target_url
print "HINT: Did you include the scheme? (e.g. http://)"
return
# Ensure boolean value safely
treat_as_root = str(treat_as_root).lower() in ["yes", "y", "true", "t", "1"]
# Ensure and force Integer value
try:
limit = int(limit)
except ValueError:
print "Invalid integer value for limit: '%s'" % limit
print "Using 500 as default"
limit = 500
# Start the run
self.jarvis.run(target_url, treat_as_root, limit)
# After the run, grab all the necessary objects and output some basic stats
fe = self.jarvis.ironman
index = self.jarvis.index
parser = self.jarvis.parser
calculator = self.jarvis.calculator
documents = self.jarvis.documents()
print "\033[95mCrawl Results\033[0m"
for category, results in fe.report.iteritems():
print "\033[94m\t%s:\033[0m" % category
for crawl in results:
print"\t%s" % str(crawl)
print
# Print out number of unique documents encountered
print "Encountered %i unique documents" % len(documents)
print "Removed %i duplicates" % parser.num_duplicates
print
# Initialize Pepper to handle queries
self.pepper = Pepper(documents, calculator, self.stop_words)
#!/usr/bin/python3
from pepper import Pepper
api = Pepper('http://192.168.122.198:8080')
# api = Pepper('http://192.168.178.92:8080')
# print(api)
api.login('test', 'test', 'pam')
# run a command on the minion
#print(api.low([{'client': 'local','tgt': 'salt-master','fun': 'cmd.run','arg': 'salt state.apply utils test=True'}]))
# run connectivy test
# print(api.low([{'client': 'local', 'tgt': 'test-minion', 'fun': 'test.ping'}]))
#
#print(api.low([{'client': 'wheel', 'tgt': '*', 'fun': 'key.list_all'}]))
# accept minion
# print(api.low([{'client': 'wheel', 'tgt': '*', 'fun': 'key.accept', 'match': 'b9840e25adfd'}]))
print(
api.low([{
'client': 'local',
'tgt': 'salt-master',
'fun': 'state.apply',
'arg': ["utils", "test=True"]
}]))
def __init__(self):
self.api = Pepper(settings.get_variable('NUTS_SALT_REST_API_URL'))
import logging
from datetime import datetime
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from model_manager.api.utils.decorators import timeout
from pepper import Pepper
from time import time
LOG = logging.getLogger(__name__)
SALT_API_URL = getattr(settings, 'SALT_API_URL', '127.0.0.1')
try:
SALT_CLIENT = Pepper(SALT_API_URL)
except:
LOG.error('Could not connect to Salt Master API.')
SALT_CLIENT = None
class SaltClientExtension():
@timeout(10)
def _login(self):
client = self.client
if client.auth:
ts = client.auth.get('expire', str(time()))
expires = datetime.fromtimestamp(ts)
now = datetime.now()
if now > expires:
return client
username = getattr(settings, 'SALT_API_USER', 'salt')
password = getattr(settings, 'SALT_API_PASSWORD', 'salt')
# from communicator import Communicator # from games.game import Game from pepper import Pepper import time p = Pepper() p.say_something() time.sleep(10) print(p.get_raw_audio()) # if __name__ == '__main__': # path = os.getcwd() # communicator = Communicator(path=path) # game = Game(path, p) # game.play() # import os # from flask import Flask # # from communicator import Communicator # from games.game import Game #
from pepper import Pepper, BIN_SYMBOLS
c = Pepper().convert
def assert_unchanged(s):
if c(s) != s:
print "original:"
print s
print
print "new: "
print c(s)
assert False
u = assert_unchanged
def test_print():
u("print 1, 2")
u("print >> f, 'a'")
u("print 'a',")
def test_math():
assert c("1 + 3 + 2") == "(1 + 3) + 2"
assert c("1 + 3 * 2") == "1 + (3 * 2)"
assert c("(1 + 2) * 3") == "(1 + 2) * 3"
for op in BIN_SYMBOLS.values():
u("1 {} 2".format(op))
from django.conf import settings
from model_manager.api.utils.decorators import timeout
from pepper import Pepper
@timeout(10)
def _login(client):
user = getattr(settings, 'SALT_API_USER', 'salt')
password = getattr(settings, 'SALT_API_PASSWORD', 'salt')
eauth = getattr(settings, 'SALT_API_EAUTH', 'pam')
client.login(user, password, eauth)
return True
url = getattr(settings, 'SALT_API_URL')
c = Pepper(url)
_login(c)
class SaltApi(object):
def __init__(self):
self.api = Pepper(settings.get_variable('NUTS_SALT_REST_API_URL'))
def connect(self):
'''connects to the salt-api with the defined username and password.
configuration can be done in config.yml in your cwd'''
self.api.login(settings.get_variable('NUTS_SALT_REST_API_USERNAME'),
settings.get_variable('NUTS_SALT_REST_API_PASSWORD'),
settings.get_variable('NUTS_SALT_REST_API_EAUTH'))
def start_task(self, args):
''' starts a the function defined in args.function on targets (args.targets)
with the parameters defined in args.arguments
response format: {u'return': [{u'srv01':True,u'srv020:True}]}
'''
response = self.api.low([{
'client': 'local',
'tgt': args['targets'],
'fun': args['function'],
'arg': args['arguments']
}])
return response
def start_task_async(self, args):
''' starts a the function defined in args.function on targets (args.targets) with
the parameters defined in args.arguments this function is asnyc
and you'll have to collect the results with get_task_result
response format: {u'return': [{u'jid': u'20160718141822884336', u'minions': [u'srv01']}]}
'''
response = self.api.low([{
'client': 'local_async',
'tgt': args['targets'],
'fun': args['function'],
'arg': args['arguments']
}])
return response
def abort_task(self, args):
'''aborts the task with the taskid defined in args.job_id'''
response = self.api.low([{
'client': 'local_async',
'tgt': '*',
'fun': 'saltutil.kill_job',
'arg': [args.job_id]
}])
return response
def get_task_result(self, taskresponse=None, taskid=None):
'''returns the current result of the entered task.
you can either insert the response you got when you started the task OR
you can insert the taskid directly
response format: {u'return': [{u'srv01':True,u'srv020:True}]}
'''
if (taskid is not None) and (taskresponse is not None) and (
taskresponse['return'][0]['jid'] != taskid):
raise ValueError(
'You entered both taskresponse and taskid but they didn\'t match'
)
if taskid is None:
taskid = taskresponse['return'][0]['jid']
return self.api.lookup_jid(taskid)
class UI(object):
def __init__(self):
super(UI, self).__init__()
self.pepper = None
self.stop_words = []
self.top_k = 5
# Initialize Jarvis
self.jarvis = Jarvis(stop_words = self.stop_words)
def run(self):
try:
while(True):
cmd = ""
args = []
raw = raw_input(">")
try:
cmd_and_args = raw.split()
if len(cmd_and_args) == 0: continue
cmd, args = (cmd_and_args[0], cmd_and_args[1:])
if cmd not in self.whiteListedMethods():
print "No command %s" % cmd
cmd = "help"
f = operator.methodcaller(cmd, *args)
except ValueError:
cmd = raw
if cmd not in self.whiteListedMethods():
print "No command %s" % cmd
cmd = "help"
f = operator.methodcaller(cmd)
try:
f(self)
except TypeError as e:
# TODO print help doc string for method in cmd.
print "Wrong arguments for %s" % cmd
self.help(cmd)
except (UIQuitException, EOFError, KeyboardInterrupt) as e:
print
return
def help(self, cmd=None):
"""Print out helpful information about available commands
@usage help [command]
@param command, optional, if present it will output only the help
text for that command; default is to print out all commands' help
text.
"""
if cmd and str(cmd) in self.whiteListedMethods():
print "{}:".format(cmd)
doc_string = getdoc(getattr(self, cmd))
if doc_string:
help_text = doc_string.split('\n')
for block in help_text:
print '\t{:<60}'.format(block.strip())
else:
# if a command was passed
if cmd: print "No command: '%s'" % cmd
for method in self.whiteListedMethods():
print "{}:".format(method)
doc_string = getdoc(getattr(self, method))
if doc_string:
help_text = doc_string.split('\n')
for block in help_text:
print '\t{:<60}'.format(block.strip())
def loadStopWords(self, stop_words_file_path):
"""Load the specified file of stop words
@usage loadStopWords stop_words_file
@param stop_words_file, required, the path to a file of stop words
"""
try:
with open(stop_words_file_path, 'r') as stop_words_file:
self.stop_words = [word.strip() for word in stop_words_file.readlines()]
self.jarvis.stop_words = self.stop_words
except IOError:
print "No file found at '%s'" % stop_words_file_path
def quit(self):
"""Exit the interactive session"""
raise UIQuitException()
def buildIndex(self, target_url=None, treat_as_root=False, limit=500):
"""Build or add to the Index.
@usage buildIndex [target_url] [treat_as_root] [limit]
@param target_url, optional, a full URL (e.g. http://www.smu.edu)
default is http://lyle.smu.edu/~fmoore/
@param treat_as_root, optional signifier to treat the given URL as a
root URL. e.g. treat a URL like, http://lyle.smu.edu/~fmoore/, as
the root instead of the server instead of http://lyle.smu.edu/
@param limit, optional limit the underlying crawler to N requests
"""
# Use ~fmoore as the default target URL.
if target_url is None:
target_url = "http://lyle.smu.edu/~fmoore/"
treat_as_root = True
# Guard against invalid URLs.
if not urlparse(target_url).scheme:
print "Invalid URL: %s" % target_url
print "HINT: Did you include the scheme? (e.g. http://)"
return
# Ensure boolean value safely
treat_as_root = str(treat_as_root).lower() in ["yes", "y", "true", "t", "1"]
# Ensure and force Integer value
try:
limit = int(limit)
except ValueError:
print "Invalid integer value for limit: '%s'" % limit
print "Using 500 as default"
limit = 500
# Start the run
self.jarvis.run(target_url, treat_as_root, limit)
# After the run, grab all the necessary objects and output some basic stats
fe = self.jarvis.ironman
index = self.jarvis.index
parser = self.jarvis.parser
calculator = self.jarvis.calculator
documents = self.jarvis.documents()
print "\033[95mCrawl Results\033[0m"
for category, results in fe.report.iteritems():
print "\033[94m\t%s:\033[0m" % category
for crawl in results:
print"\t%s" % str(crawl)
print
# Print out number of unique documents encountered
print "Encountered %i unique documents" % len(documents)
print "Removed %i duplicates" % parser.num_duplicates
print
# Initialize Pepper to handle queries
self.pepper = Pepper(documents, calculator, self.stop_words)
def query(self, *user_query):
"""Run a query against the index
@usage query query_word [more_query_words...]
@param query_word, a string query of 1 or more words to search
within the index.
@example query foo bar hello world
"""
if self.pepper is None:
print "Index has not been built! Run `buildIndex` first."
return
if len(user_query) == 0:
print "No query given!"
return
else:
user_query = " ".join(user_query)
index = self.jarvis.index
calculator = self.jarvis.calculator
print "| {0:>15} | {1:>15} | {2:>15} | {3:>14} |".format("Term", "DF", "IDF", "F")
for term in self.pepper.p.stemText(user_query, self.stop_words).encode('utf_8', 'ignore').split():
if not index.word_document_frequency.has_key(term): continue
if not calculator.term_idfs.has_key(term): continue
if not index.word_freq.has_key(term): continue
df = index.word_document_frequency[term]
idf = calculator.term_idfs[term]
wf = index.word_freq[term]
print "| {0:>15} | {1:>15d} | {2:>15f} | {3:>14d} |".format(term, df, idf, wf)
print
ranked_docs = self.pepper.handleQuery(user_query)
i = 1
for score, doc in ranked_docs:
if score == 0: break
if i == 1:
print "{0:>15} | {1:>15} | {2:>15} | {3:>14}".format("Rank", "Score", "Document", "ID")
print "{0:>15} | {1:15f} | {2:15s} | {3:14s}".format(i, score, doc.url, doc.id)
i += 1
if i > self.top_k: break
print
if i == 1:
print "No results found for that query :("
else:
if i < self.top_k:
print "NOTE: Only outputted top {:d} documents, as all other "\
"documents had zero scores".format(i-1)
top_doc = ranked_docs[0][1]
print "First 20 words of top ranked document: %s" % top_doc.url
print "%s..." % " ".join(top_doc.full_text.split()[0:20])
def setK(self, k=5):
"""Set the maximum number of K results to show in a query (default is 5)
@usage setK [k]
@param k, optional, an integer value representing the number of
results to show from queries. If left blank, it is set to 5.
"""
# Ensure and force Integer value
try:
k = int(k)
self.top_k = k
except ValueError:
print "Invalid integer value for k: '%s'" % k
def showDoc(self, docID):
"""Show the document with the given ID
@usage showDoc bef194481a024c4cf9178e5e421529b4
@param docID, required, the hash ID of the document (partials are
matched, but may return multiples if not enough of the hash is
given).
"""
id_re = re.compile(re.escape(docID))
matches = []
for doc in self.pepper.documents:
if id_re.match(doc.id):
matches.append(doc)
if len(matches) > 0:
if len(matches) != 1:
print "Warning, more than one document found matching that id."
for doc in matches:
print "Document ID: {}".format(doc.id)
print "Document URL: {}".format(doc.url)
print "Word Frequencies:"
pprint(doc.word_vector)
print "Normalized Document Vector:"
pprint(doc.normalized_tf)
print doc.full_text
else:
print "No document found with ID = {}".format(docID)
def whiteListedMethods(self):
return ["help", "setK", "buildIndex", "query", "loadStopWords", "quit", "showDoc"]
def completer(self, text, state):
matches = []
p = re.compile(re.escape(text))
for m in self.whiteListedMethods():
if p.match(m):
matches.append(m)
try:
return matches[state]
except IndexError:
return None
def _get_pepper():
"""Return a pepper object with auth."""
api = Pepper('http://master:8080', debug_http=True)
api.auth = {'token': g.current_user.token, 'user': g.current_user.nickname, 'eauth': 'pam'}
return api
