def get_reviews(movie_title):
db = mongo.get_db()
reviews = list(db.reviews.find({'itemReviewed.name': movie_title}))
if not reviews:
# Returns an empty dictionary instead of returning None
log.info("No reviews found for " + movie_title)
return reviews
def load_indicator_from_file(file_name):
fh = open(file_name, 'rb')
db = get_db()
reader = csv.DictReader(fh)
for row in reader:
category = CATEGORIES.get(row.get('category'), {})
indicator = {
'id': munge_name(row.get('name')),
'label': row.get('label'),
'question': row.get('question'),
'good': row.get('good').strip()=='1',
'select': row.get('select').strip() == '1',
'description': row.get('description').decode('iso-8859-1'),
'piechart': row.get('piechart'),
'source': row.get('source').decode('iso-8859-1'),
'category': {
'id': row.get('category', '').strip().lower(),
'label': category.get('label'),
'set': category.get('set')
},
'hdi_weight': 0.0}
if row.get('hdi_weight'):
indicator['hdi_weight'] = float(row.get('hdi_weight'))
query = {'id': munge_name(row.get('name'))}
indicator.update(query)
db.indicator.update(query, indicator, upsert=True)
fh.close()
db.indicator.ensure_index('id')
print 'Loaded: %s indicators' % db.indicator.count()
def get_visited():
db = mongo.get_db()
visited = list(db.visited.find())
if visited:
for i, item in enumerate(visited):
visited[i] = item['url']
return visited
def add_to_visited(url):
db = mongo.get_db()
item_id = db.visited.insert_one({'url': url}).inserted_id
if item_id == 0:
log.error("Unable to add URL into visited")
return None
return item_id
def get_movies():
db = mongo.get_db()
movies = list(db.movies.find())
if movies:
for i, item in enumerate(movies):
movies[i] = item['title']
return movies
def add_review(review):
db = mongo.get_db()
# If movie title is not already in db.movies, retrieve movie info and add it
movie_title = review['itemReviewed']['name']
movies = list(db.movies.find({'title': movie_title}))
if not movies:
# Add into db.movies only when movie info is available
# This means that if retrieving movie info is unsuccessful, there might be reviews of the movie in db.reviews, but no record of the movie title and info in db.movies
movie_info = retrieve_movie_info(movie_title)
if movie_info is not None:
add_movie(movie_title, movie_info)
# Check for duplicate reviews for the same movie
existing_reviews = get_reviews(movie_title)
for possible_duplicate in existing_reviews:
if review['url'] == possible_duplicate['url']:
log.info('Review already added into database')
return None
# Add review into db.reviews
review['rank'] = 0 # give initial pageRank
item_id = db.reviews.insert_one(review).inserted_id
if item_id == 0:
log.error("Unable to add review")
return None
return item_id
def get_movie_info(movie_titie):
db = mongo.get_db()
movies = list(db.movies.find({'title': movie_title}))
if movies:
return movies[0]['info']
else:
log.info("No info found for " + movie_title)
return None
def ifExists(self):
"""
Check if given SSS-name, author username and ren-url pair exist in collection or not.
If yes, then return True, else return False
"""
if (get_db(SSS_COLLECTION_NAME, self.filter_obj).count() > 0):
return True
else:
return False
def reparse_table(table_id):
db = mongo.get_db()
table = db.tables.find_one({'code': table_id})
#print table.get('csvdata').encode('utf-8')
print "TABLE", table_id
print table.get('csvdata', '').encode('utf-8')
table.update(table_to_dict(table_id, table.get('csvdata')))
db.tables.update({'code': table_id}, table, upsert=True)
def top_speakers(self, chatroom, **kwargs):
db = mongo.get_db(chatroom)
docs = db.speakers.find(limit=15).sort('count', pymongo.DESCENDING)
result = []
for d in docs:
result.append({'speaker' : d['_id'], 'messages' : d['count'], 'words': d['words'], 'chars' : d['chars']})
return {'top_speakers' : result}
def should_visit_domain(domain):
db = mongo.get_db()
find_domain = db.visitedDomain.find_one({'domain': domain})
if find_domain:
total = find_domain['total']
success = find_domain['success']
if success == 0 and total > 500 or success * 1.0 / total < 0.00001:
return False
return True
def reparse_table(table_id):
db = mongo.get_db()
table = db.tables.find_one({'code': table_id})
#print table.get('csvdata').encode('utf-8')
print "TABLE", table_id
print table.get('csvdata', '').encode('utf-8')
table.update(table_to_dict(table_id, table.get('csvdata')))
db.tables.update({'code': table_id}, table, upsert=True)
def list_speakers(self, chatroom, **kwargs):
db = mongo.get_db(chatroom)
about = db.about
doc = about.find_one({'_id' : 'speakers'})
if not doc:
return {'list_speakers' : []}
else:
return {'list_speakers' : doc['list']}
def queue_push(url, priority=1):
db = mongo.get_db()
item_id = db.queue.insert_one({
'url': url,
'priority': priority
}).inserted_id
if item_id == 0:
log.error("Unable to add URL into queue")
return None
return item_id
def add_movie(movie_title, movie_info):
db = mongo.get_db()
exist = db.movies.find_one({'title': movie_title})
if exist:
return None
else:
item_id = db.movies.insert_one({
'title': movie_title,
'info': movie_info
}).inserted_id
return item_id
def queue_pop():
db = mongo.get_db()
MAX_PRIORITY = 2
priority = 0
item = db.queue.find_one_and_delete({},
sort=[('priority', pymongo.ASCENDING)])
if item is not None:
# db.queue.delete_one({'_id': ObjectId(item['_id'])})
return item['url'], item['priority']
else:
return None, None
def get(json_obj):
"""
wrapper for get_db function to retreive resuts from sss_info collection.
input: json object containing list of conditions to specify projection.
output: return list of matching documents else False if no documents found.
"""
print(json_obj)
print(SSS_COLLECTION_NAME)
results = get_db(SSS_COLLECTION_NAME, json_obj)
print(results.count())
if (results.count() > 0):
return results
return False
def get(code):
db = mongo.get_db()
variable = db.variables.find_one({'code': code})
if (not variable) or config.bool('reload'):
logging.info("Variable %s not found, loading from SOAP...", code)
variable = soap.get_variable(code)
if not variable:
return None
for link in variable.get('links', []):
if u"Tabellen mit Vorkommen des Merkmals" in link.get('titel'):
variable['tables'] = [t for t in load_variable_tables(link.get('href'))]
variable['__ts'] = datetime.utcnow()
db.variables.update({'code': code}, variable, upsert=True)
return variable
def get(table_id):
db = mongo.get_db()
table = db.tables.find_one({'code': table_id})
if (not table) or config.bool('reload'):
logging.info("Table %s not found, loading from SOAP...", table_id)
table_data = soap.get_table(table_id)
if not table_data:
return None
get_data(table_id)
table = {'csvdata': table_data, 'code': table_id}
table.update(table_to_dict(table_id, table_data))
db.tables.update({'code': table_id}, table, upsert=True)
else:
reparse_table(table_id)
return table
def get(table_id):
db = mongo.get_db()
table = db.tables.find_one({'code': table_id})
if (not table) or config.bool('reload'):
logging.info("Table %s not found, loading from SOAP...", table_id)
table_data = soap.get_table(table_id)
if not table_data:
return None
get_data(table_id)
table = {'csvdata': table_data, 'code': table_id}
table.update(table_to_dict(table_id, table_data))
db.tables.update({'code': table_id}, table, upsert=True)
else:
reparse_table(table_id)
return table
def get(code):
db = mongo.get_db()
variable = db.variables.find_one({'code': code})
if (not variable) or config.bool('reload'):
logging.info("Variable %s not found, loading from SOAP...", code)
variable = soap.get_variable(code)
if not variable:
return None
for link in variable.get('links', []):
if u"Tabellen mit Vorkommen des Merkmals" in link.get('titel'):
variable['tables'] = [
t for t in load_variable_tables(link.get('href'))
]
variable['__ts'] = datetime.utcnow()
db.variables.update({'code': code}, variable, upsert=True)
return variable
def load_dataset_from_file(file_name):
fh = open(file_name, 'rb')
db = get_db()
reader = csv.DictReader(fh)
for (count,row) in enumerate(reader):
if not row.get('indicator_name'):
continue
indicator_name = munge_name(row.get('indicator_name'))
dataset = {
'country_name': row.get('country2')}
if row.get('value'):
dataset['value'] = float(row.get('value'))
else:
dataset['value'] = 0.0
if row.get('value_norm'):
dataset['normalized_value'] = float(row.get('value_norm'))
else:
dataset['normalized_value'] = dataset['value']
indicator = db.indicator.find_one({'id': indicator_name})
assert indicator, "Indicator %s could not be found!" % row.get('indicator_name')
if not indicator.get('good'):
dataset['normalized_value'] = 1.0 - dataset['normalized_value']
try:
cc3 = row.get('country')
cc3 = {'ROM': 'ROU',
'ZAR': 'COD',
'TMP': 'TLS'}.get(cc3, cc3)
cc = countries.get(cc3).alpha2
except:
#print row
continue
query = {'indicator': indicator.get('_id'),
'country': cc,
'time': row.get('time')}
dataset.update(query)
dataset['indicator_id'] = indicator.get('id')
# db.datum.update(query, dataset, upsert=True)
db.datum.insert(dataset)
if count % 1000 == 0:
print 'Progress: %s' % count
db.datum.ensure_index('country')
db.datum.ensure_index('indicator')
db.datum.ensure_index('time')
db.datum.ensure_index('indicator_id')
fh.close()
print 'Loaded: %s data points' % db.datum.count()
def update_visited_domain(domain, success):
db = mongo.get_db()
find_domain = db.visitedDomain.find_one({'domain': domain})
success_incremental_value = 1 if success else 0
if find_domain:
db.visitedDomain.update_one(
{'domain': domain},
{'$inc': {
'total': 1,
'success': success_incremental_value
}},
upsert=False)
else:
db.visitedDomain.insert_one({
'domain': domain,
'total': 1,
'success': success_incremental_value
})
def get(stat_id, include_tables=True):
db = mongo.get_db()
statistic = db.statistics.find_one({'code': stat_id})
if (not statistic) or config.bool('reload'):
logging.info("Statistic %s not found, loading from SOAP...", stat_id)
statistic = soap.get_statistic(stat_id)
if not statistic:
return None
tables = []
for table in soap.find_tables_by_statistic(stat_id):
tables.append(table)
statistic['tables'] = tables
statistic['description'] = load_description(stat_id)
statistic['variables'] = [v for v in load_variables(stat_id)]
statistic['__ts'] = datetime.utcnow()
db.statistics.update({'code': stat_id}, statistic, upsert=True)
import table
for _table in statistic['tables']:
table.get(_table.get('code'))
return statistic
def run_scan():
print "[INFO] Starting scans on configured sites"
os.chdir(wpscan_dir)
result = ""
# Scan ----------------------------------------------------------------
p = subprocess.Popen(r'./wpscan.rb --batch --url %s' % wp_site,
stdout=subprocess.PIPE,
shell=True,
universal_newlines=True)
print "[INFO] Scanning '%s'" % wp_site
result = p.communicate()
# Parse the results ---------------------------------------------------
alerts = parse_results(result[0])
for alert in alerts:
print alert
title = re.compile("\[!\] Title:(.+) -").search(alert)
title = title.group(0)[4:-1]
db = mongo.get_db()
# if not it exists, create it and send it
if not mongo.get_vulnerability(db, wp_site, title):
mongo.add_vulnerability(db, wp_site, title)
sendtoHIVE("[WORDPRESS] " + title, alert, wp_site)
def __init__(self, verbose=False):
self.db = get_db()
self.quiz = model.Quiz(u'yourtopia')
self.country_list = self.db.datum.distinct('country')
self.verbose = verbose
def test_mdb():
db = mongo.get_db('test')
db.demo.drop()
assert db.demo.count()==0
db.demo.insert({"test":123})
print(db.demo.count())
import base64
import requests
from flask import Flask, jsonify, request
from flask_cors import CORS
from mongo import get_db
db = get_db()
app = Flask(__name__)
CORS(app, support_credentials=True) # Access-Control-Allow-Origin: *
def get_token():
text = 'admin:Pwd123456'
return "Basic " + base64.b64encode(text.encode()).decode()
headers = {'Content-Type': 'application/json', 'Authorization': get_token()}
@app.route('/', methods=['GET'])
def hello():
return jsonify({'hello': 'hello1', 'msg': 'hahaha'}), 201
@app.route('/user/duty', methods=['GET', 'POST'])
def duty():
info = {'success': False}
if request.method == 'POST':
data = request.json.get('data')
def empty_db():
db = mongo.get_db()
for collection in COLLECTIONS:
db[collection].delete_many({})
def load_terms():
db = get_db()
for term in terms_it():
db.terms.update({"term": term}, {"term": term, "__ts": datetime.utcnow()}, upsert=True)
def add_outgoing_links(url, outgoing_links):
db = mongo.get_db()
item_id = db.links.insert_one({
'key': url,
'links': outgoing_links
}).inserted_id
def updatePageRank(url, rank):
db = mongo.get_db()
db.reviews.update_one({'url': url}, {'$set': {'rank': rank}})
def get_outgoing_links():
db = mongo.get_db()
return list(db.links.find())
def setUp(self):
self.db = mongo.get_db()
#!/usr/bin/python
"""
Super bot created at the Betaworks hackathon. Original author: Tom Germouahauhguh (some kinda euro-name, whatever).
Dependencies:
easy_install dnspython
easy_install xmpppy
"""
import sys
import xmpp
import re
import logging
import mongo
db = mongo.get_db('chatstats')
def handleMessage(conn, message):
body = message.getBody()
m = re.search('\[(.+?)\] (.*)|_(.+?) (.*)_', body)
if not m:
logging.warning("failed to get message from body: %s" % body)
return
user = m.group(1) or m.group(3)
if not user:
return
text = m.group(2) or m.group(4) or ''
words = text.split(' ')
from mongo import get_db, CRUD
treasure_db = get_db("treasure")
treasure_industry = CRUD(treasure_db, "industry")
treasure_industry_detail = CRUD(treasure_db, "industry_detail")
def setUp(self):
self.db = mongo.get_db()
