def get_rank(team_id, domain_id, trail_id, url):
user = helper.get_user()
url = url.encode('utf-8')
url = urllib.unquote(url)
rank = db.getUrlRank(trail_id, user.get_email(), url)
response = dict(rank=rank)
return json.dumps(response)
def get_user(token):
user = helper.get_user()
if helper.get_token() != token or user is None:
user_auth = factory.get_authentication_object(token,MOCK_AUTH)
user = user_auth.get_user_from_token()
#tangelo.log('session.post verified user: ' + str(user))
return user
def get_user(token):
user = helper.get_user()
if helper.get_token() != token or user is None:
user_auth = factory.get_authentication_object(token)
user = user_auth.get_user_from_token()
#tangelo.log('session.post verified user: ' + str(user))
return user
def get_rank(team_id,domain_id,trail_id, url):
user = helper.get_user()
url = url.encode('utf-8')
url = urllib.unquote(url)
rank = db.getUrlRank(trail_id,user.get_email(),url)
response = dict(rank=rank)
return json.dumps(response)
def invalid_extraction(entity_type, entity_value, domain):
user = session_helper.get_user()
user_name = user.get_user_name()
org = session_helper.get_org()
success = db.mark_invalid_extracted_entity(user_name, entity_type,
entity_value, domain, org) == 0
return json.dumps(dict(success=success))
def get():
"""
Return valid teams for the current user.
[{id: team_id, name:team_name},..]
"""
user = helper.get_user()
teams = db.getTeams(user.get_email())
return json.dumps(teams)
def get():
"""
Return valid teams for the current user.
[{id: team_id, name:team_name},..]
"""
user = helper.get_user()
teams = db.getTeams(user.get_email())
return json.dumps(teams)
def add_trail(trailname, domain, traildescription=u''):
tangelo.log('datawake_trails POST trailname=%s traildescription=%s domain=%s' % (trailname, traildescription, domain))
user = helper.get_user()
org = user.get_org()
invalid = re.match('^[\w]*(?!:)+$', trailname) is None
if invalid:
raise ValueError("Trail names must be alphanumeric and not contain a ':'")
last_row = db.addTrail(org, trailname, traildescription, user.get_email(), domain=domain)
return json.dumps(dict(success=last_row >= 0))
def add_trail(trailname, domain, traildescription=u''):
tangelo.log('datawake_trails POST trailname=%s traildescription=%s domain=%s' % (trailname, traildescription, domain))
user = helper.get_user()
org = user.get_org()
invalid = re.match('^[\w]*(?!:)+$', trailname) is None
if invalid:
raise ValueError("Trail names must be alphanumeric and not contain a ':'")
last_row = db.addTrail(org, trailname, traildescription, user.get_email(), domain=domain)
return json.dumps(dict(success=last_row >= 0))
def get_rank(trailname, url, domain):
user = helper.get_user()
org = user.get_org()
user_id = user.get_user_id()
url = url.encode('utf-8')
url = urllib.unquote(url)
rank = db.getUrlRank(org, user_id, trailname, url, domain=domain)
response = dict(rank=rank)
return json.dumps(response)
def save_page_selection(selection, domain, url):
#tangelo.log('savePageSelection url=' + str(url) + ' selection=' + selection + ' domain=' + domain)
user = helper.get_user()
org = user.get_org()
postId = db.get_post_id(url)
row = db.getBrowsePathData(org, postId, domain)
row_id = -1
if row['org'] == org: # ensure the user is saving a selection to a post from their org
row_id = db.addSelection(postId, selection)
return json.dumps(dict(id=row_id))
def save_page_selection(selection, domain, url):
tangelo.log('savePageSelection url=' + str(url) + ' selection=' + selection + ' domain=' + domain)
user = helper.get_user()
org = user.get_org()
postId = db.get_post_id(url)
row = db.getBrowsePathData(org, postId, domain)
row_id = -1
if row['org'] == org: # ensure the user is saving a selection to a post from their org
row_id = db.addSelection(postId, selection)
return json.dumps(dict(id=row_id))
def post(token=u''):
user = session_helper.get_user()
org = None
if user is not None and session_helper.is_token_in_session() and session_helper.get_token() == token:
#tangelo.log('plugin-sever.session tokens matched using existing session.')
user = session_helper.get_user()
else:
auth_helper = factory.get_authentication_object(token,MOCK_FORENSIC_AUTH)
user = auth_helper.get_user_from_token()
tangelo.log('session.post verified user: ' + str(user))
orgs = datawake_mysql.getOrgLinks(user.get_email())
if len(orgs) == 1:
org = orgs[0]
else:
raise ValueError("No org was found for this user.")
user.set_org(org)
session_helper.set_user(user)
session_helper.set_token(token)
return json.dumps(user.__dict__)
def post(token=u''):
user = session_helper.get_user()
if user is None:
auth_helper = factory.get_authentication_object(token,MOCK_FORENSIC_AUTH)
user = auth_helper.get_user_from_token()
tangelo.log('session.post verified user: ' + str(user))
session_helper.set_user(user)
session_helper.set_token(token)
return json.dumps(user.__dict__)
def get(team_id):
"""
Verify the logged in user has access to the requested team and return that teams list of domains.
:param team_id:
:return: List of domain objects for the team
[{'id':domain_id,'name','name':domainname,'description':domaindescription},..]
"""
user = helper.get_user()
if not db.hasTeamAccess(user.get_email(),team_id):
tangelo.content_type()
tangelo.http_status(401)
tangelo.log("401 Unauthorized domains/?team_id="+str(team_id)+" user: "******"401 Unauthorized"
else:
results = db.get_domains(team_id)
return json.dumps(results)
def get(team_id):
"""
Verify the logged in user has access to the requested team and return that teams list of domains.
:param team_id:
:return: List of domain objects for the team
[{'id':domain_id,'name','name':domainname,'description':domaindescription},..]
"""
user = helper.get_user()
if not db.hasTeamAccess(user.get_email(), team_id):
tangelo.content_type()
tangelo.http_status(401)
tangelo.log("401 Unauthorized domains/?team_id=" + str(team_id) +
" user: "******"401 Unauthorized"
else:
results = db.get_domains(team_id)
return json.dumps(results)
def add_trail(team_id,domain_id,name,description=''):
tangelo.log('datawake_trails POST name=%s description=%s domain=%s team=%s' % (name, description, domain_id,team_id))
#check the trail name
if name is None or len(name) < 1:
raise ValueError("Trail names must have at least one character")
user = helper.get_user()
# create and then return the new trail
try:
newTrailId = db.addTrail(team_id,domain_id,name, description, user.get_email())
newTrail = dict(id=newTrailId,name=name,description=description)
tangelo.log(newTrail)
return json.dumps(newTrail)
except Exception as e:
tangelo.log(e)
tangelo.http_status(501)
return "Failed to create trail. This trail name may already be used."
def add_trail(team_id,domain_id,name,description=''):
tangelo.log('datawake_trails POST name=%s description=%s domain=%s team=%s' % (name, description, domain_id,team_id))
#check the trail name
if name is None or len(name) < 1:
raise ValueError("Trail names must have at least one character")
user = helper.get_user()
# create and then return the new trail
try:
newTrailId = db.addTrail(team_id,domain_id,name, description, user.get_email())
except Exception as e:
tangelo.log(e)
tangelo.http_status(501)
return "Failed to create trail. This trail name may already be used."
newTrail = dict(id=newTrailId,name=name,description=description)
tangelo.log(newTrail)
return json.dumps(newTrail)
def full_page_scrape(team_id, domain_id, trail_id, html, url):
user = helper.get_user()
return scrape_page(team_id, domain_id, trail_id, url, html,
user.get_email())
def set_rank(team_id,domain_id,trail_id,url,rank):
user = helper.get_user()
db.rankUrl(team_id,domain_id,trail_id,user.get_email(),url,rank)
return json.dumps(dict(success=True))
def invalid_extraction(team_id,domain_id,trail_id,feature_type, feature_value):
user = session_helper.get_user()
db.mark_invalid_feature(trail_id,user.get_email(), feature_type, feature_value)
return json.dumps(dict(success=True))
def add_manual_feature(team_id,domain_id,trail_id,url,raw_text,feature_type,feature_value):
user = session_helper.get_user()
db.add_manual_feature(trail_id, user.get_email(),url,raw_text, feature_type, feature_value)
return json.dumps(dict(success=True))
def full_page_scrape(domain, trail, html, url):
user = helper.get_user()
user_id = user.get_user_id()
user_name = user.get_user_name()
org = user.get_org()
return scrape_page(html, url, user_id, user_name, trail, domain, org)
def marked_entities(domain):
user = session_helper.get_user()
user_name = user.get_user_name()
org = session_helper.get_org()
marked_entities_list = db.get_marked_entities(org, domain, user_name)
return json.dumps(dict(marked_entities=marked_entities_list))
def add_team(name, description=''):
user = helper.get_user()
team = db.addTeam(name, description, user.get_email())
db.add_new_domain(team['id'], "Empty",
"An empty domain. Created by default for each team.")
return json.dumps(team)
def full_page_scrape(domain, trail, html, url):
user = helper.get_user()
user_id = user.get_user_id()
user_name = user.get_user_name()
org = user.get_org()
return scrape_page(html, url, user_id, user_name, trail, domain, org)
def set_rank(trailname, url, rank, domain):
user = helper.get_user()
org = user.get_org()
user_id = user.get_user_id()
db.rankUrl(org, user_id, trailname, url, rank, domain=domain)
return json.dumps(dict(success=True))
def invalid_extraction(entity_type, entity_value, domain):
user = session_helper.get_user()
user_name = user.get_user_name()
org = session_helper.get_org()
success = db.mark_invalid_extracted_entity(user_name, entity_type, entity_value, domain, org) == 0
return json.dumps(dict(success=success))
def save_page_selection(team_id,domain_id,trail_id,selection, url):
user = helper.get_user()
row_id = db.addSelection(trail_id,user.get_email(),url, selection)
return json.dumps(dict(id=row_id))
def get():
return json.dumps(dict(user=helper.get_user().__dict__))
def full_page_scrape(team_id,domain_id, trail_id, html, url):
user = helper.get_user()
return scrape_page(team_id,domain_id,trail_id,url,html,user.get_email())
def get():
return json.dumps(dict(user=session_helper.get_user().__dict__, hasSession=True))
def get_marked_features(team_id,domain_id,trail_id):
user = session_helper.get_user()
marked_features_list = db.get_marked_features(trail_id)
return json.dumps(marked_features_list)
def save_page_selection(team_id, domain_id, trail_id, selection, url):
user = helper.get_user()
row_id = db.addSelection(trail_id, user.get_email(), url, selection)
return json.dumps(dict(id=row_id))
def get():
return json.dumps(dict(user=helper.get_user().__dict__))
def add_team(name, description=""):
user = helper.get_user()
team = db.addTeam(name, description, user.get_email())
db.add_new_domain(team["id"], "Empty", "An empty domain. Created by default for each team.")
return json.dumps(team)
def marked_entities(domain):
user = session_helper.get_user()
user_name = user.get_user_name()
org = session_helper.get_org()
marked_entities_list = db.get_marked_entities(org, domain, user_name)
return json.dumps(dict(marked_entities=marked_entities_list))
def set_comments(trail_id, url, comments, crawl_type):
tangelo.log("%i, %s, %s, %s" % (trail_id, url, comments, crawl_type))
user = helper.get_user()
db.setComments(trail_id, url, crawl_type, comments)
return json.dumps(dict(success=True))
def set_rank(team_id, domain_id, trail_id, url, rank):
user = helper.get_user()
db.rankUrl(team_id, domain_id, trail_id, user.get_email(), url, rank)
return json.dumps(dict(success=True))
