def get_reviews(movie_title): db = mongo.get_db() reviews = list(db.reviews.find({'itemReviewed.name': movie_title})) if not reviews: # Returns an empty dictionary instead of returning None log.info("No reviews found for " + movie_title) return reviews
def load_indicator_from_file(file_name): fh = open(file_name, 'rb') db = get_db() reader = csv.DictReader(fh) for row in reader: category = CATEGORIES.get(row.get('category'), {}) indicator = { 'id': munge_name(row.get('name')), 'label': row.get('label'), 'question': row.get('question'), 'good': row.get('good').strip()=='1', 'select': row.get('select').strip() == '1', 'description': row.get('description').decode('iso-8859-1'), 'piechart': row.get('piechart'), 'source': row.get('source').decode('iso-8859-1'), 'category': { 'id': row.get('category', '').strip().lower(), 'label': category.get('label'), 'set': category.get('set') }, 'hdi_weight': 0.0} if row.get('hdi_weight'): indicator['hdi_weight'] = float(row.get('hdi_weight')) query = {'id': munge_name(row.get('name'))} indicator.update(query) db.indicator.update(query, indicator, upsert=True) fh.close() db.indicator.ensure_index('id') print 'Loaded: %s indicators' % db.indicator.count()
def get_visited(): db = mongo.get_db() visited = list(db.visited.find()) if visited: for i, item in enumerate(visited): visited[i] = item['url'] return visited
def add_to_visited(url): db = mongo.get_db() item_id = db.visited.insert_one({'url': url}).inserted_id if item_id == 0: log.error("Unable to add URL into visited") return None return item_id
def get_movies(): db = mongo.get_db() movies = list(db.movies.find()) if movies: for i, item in enumerate(movies): movies[i] = item['title'] return movies
def add_review(review): db = mongo.get_db() # If movie title is not already in db.movies, retrieve movie info and add it movie_title = review['itemReviewed']['name'] movies = list(db.movies.find({'title': movie_title})) if not movies: # Add into db.movies only when movie info is available # This means that if retrieving movie info is unsuccessful, there might be reviews of the movie in db.reviews, but no record of the movie title and info in db.movies movie_info = retrieve_movie_info(movie_title) if movie_info is not None: add_movie(movie_title, movie_info) # Check for duplicate reviews for the same movie existing_reviews = get_reviews(movie_title) for possible_duplicate in existing_reviews: if review['url'] == possible_duplicate['url']: log.info('Review already added into database') return None # Add review into db.reviews review['rank'] = 0 # give initial pageRank item_id = db.reviews.insert_one(review).inserted_id if item_id == 0: log.error("Unable to add review") return None return item_id
def get_movie_info(movie_titie): db = mongo.get_db() movies = list(db.movies.find({'title': movie_title})) if movies: return movies[0]['info'] else: log.info("No info found for " + movie_title) return None
def ifExists(self): """ Check if given SSS-name, author username and ren-url pair exist in collection or not. If yes, then return True, else return False """ if (get_db(SSS_COLLECTION_NAME, self.filter_obj).count() > 0): return True else: return False
def reparse_table(table_id): db = mongo.get_db() table = db.tables.find_one({'code': table_id}) #print table.get('csvdata').encode('utf-8') print "TABLE", table_id print table.get('csvdata', '').encode('utf-8') table.update(table_to_dict(table_id, table.get('csvdata'))) db.tables.update({'code': table_id}, table, upsert=True)
def top_speakers(self, chatroom, **kwargs): db = mongo.get_db(chatroom) docs = db.speakers.find(limit=15).sort('count', pymongo.DESCENDING) result = [] for d in docs: result.append({'speaker' : d['_id'], 'messages' : d['count'], 'words': d['words'], 'chars' : d['chars']}) return {'top_speakers' : result}
def should_visit_domain(domain): db = mongo.get_db() find_domain = db.visitedDomain.find_one({'domain': domain}) if find_domain: total = find_domain['total'] success = find_domain['success'] if success == 0 and total > 500 or success * 1.0 / total < 0.00001: return False return True
def list_speakers(self, chatroom, **kwargs): db = mongo.get_db(chatroom) about = db.about doc = about.find_one({'_id' : 'speakers'}) if not doc: return {'list_speakers' : []} else: return {'list_speakers' : doc['list']}
def queue_push(url, priority=1): db = mongo.get_db() item_id = db.queue.insert_one({ 'url': url, 'priority': priority }).inserted_id if item_id == 0: log.error("Unable to add URL into queue") return None return item_id
def add_movie(movie_title, movie_info): db = mongo.get_db() exist = db.movies.find_one({'title': movie_title}) if exist: return None else: item_id = db.movies.insert_one({ 'title': movie_title, 'info': movie_info }).inserted_id return item_id
def queue_pop(): db = mongo.get_db() MAX_PRIORITY = 2 priority = 0 item = db.queue.find_one_and_delete({}, sort=[('priority', pymongo.ASCENDING)]) if item is not None: # db.queue.delete_one({'_id': ObjectId(item['_id'])}) return item['url'], item['priority'] else: return None, None
def get(json_obj): """ wrapper for get_db function to retreive resuts from sss_info collection. input: json object containing list of conditions to specify projection. output: return list of matching documents else False if no documents found. """ print(json_obj) print(SSS_COLLECTION_NAME) results = get_db(SSS_COLLECTION_NAME, json_obj) print(results.count()) if (results.count() > 0): return results return False
def get(code): db = mongo.get_db() variable = db.variables.find_one({'code': code}) if (not variable) or config.bool('reload'): logging.info("Variable %s not found, loading from SOAP...", code) variable = soap.get_variable(code) if not variable: return None for link in variable.get('links', []): if u"Tabellen mit Vorkommen des Merkmals" in link.get('titel'): variable['tables'] = [t for t in load_variable_tables(link.get('href'))] variable['__ts'] = datetime.utcnow() db.variables.update({'code': code}, variable, upsert=True) return variable
def get(table_id): db = mongo.get_db() table = db.tables.find_one({'code': table_id}) if (not table) or config.bool('reload'): logging.info("Table %s not found, loading from SOAP...", table_id) table_data = soap.get_table(table_id) if not table_data: return None get_data(table_id) table = {'csvdata': table_data, 'code': table_id} table.update(table_to_dict(table_id, table_data)) db.tables.update({'code': table_id}, table, upsert=True) else: reparse_table(table_id) return table
def get(code): db = mongo.get_db() variable = db.variables.find_one({'code': code}) if (not variable) or config.bool('reload'): logging.info("Variable %s not found, loading from SOAP...", code) variable = soap.get_variable(code) if not variable: return None for link in variable.get('links', []): if u"Tabellen mit Vorkommen des Merkmals" in link.get('titel'): variable['tables'] = [ t for t in load_variable_tables(link.get('href')) ] variable['__ts'] = datetime.utcnow() db.variables.update({'code': code}, variable, upsert=True) return variable
def load_dataset_from_file(file_name): fh = open(file_name, 'rb') db = get_db() reader = csv.DictReader(fh) for (count,row) in enumerate(reader): if not row.get('indicator_name'): continue indicator_name = munge_name(row.get('indicator_name')) dataset = { 'country_name': row.get('country2')} if row.get('value'): dataset['value'] = float(row.get('value')) else: dataset['value'] = 0.0 if row.get('value_norm'): dataset['normalized_value'] = float(row.get('value_norm')) else: dataset['normalized_value'] = dataset['value'] indicator = db.indicator.find_one({'id': indicator_name}) assert indicator, "Indicator %s could not be found!" % row.get('indicator_name') if not indicator.get('good'): dataset['normalized_value'] = 1.0 - dataset['normalized_value'] try: cc3 = row.get('country') cc3 = {'ROM': 'ROU', 'ZAR': 'COD', 'TMP': 'TLS'}.get(cc3, cc3) cc = countries.get(cc3).alpha2 except: #print row continue query = {'indicator': indicator.get('_id'), 'country': cc, 'time': row.get('time')} dataset.update(query) dataset['indicator_id'] = indicator.get('id') # db.datum.update(query, dataset, upsert=True) db.datum.insert(dataset) if count % 1000 == 0: print 'Progress: %s' % count db.datum.ensure_index('country') db.datum.ensure_index('indicator') db.datum.ensure_index('time') db.datum.ensure_index('indicator_id') fh.close() print 'Loaded: %s data points' % db.datum.count()
def update_visited_domain(domain, success): db = mongo.get_db() find_domain = db.visitedDomain.find_one({'domain': domain}) success_incremental_value = 1 if success else 0 if find_domain: db.visitedDomain.update_one( {'domain': domain}, {'$inc': { 'total': 1, 'success': success_incremental_value }}, upsert=False) else: db.visitedDomain.insert_one({ 'domain': domain, 'total': 1, 'success': success_incremental_value })
def get(stat_id, include_tables=True): db = mongo.get_db() statistic = db.statistics.find_one({'code': stat_id}) if (not statistic) or config.bool('reload'): logging.info("Statistic %s not found, loading from SOAP...", stat_id) statistic = soap.get_statistic(stat_id) if not statistic: return None tables = [] for table in soap.find_tables_by_statistic(stat_id): tables.append(table) statistic['tables'] = tables statistic['description'] = load_description(stat_id) statistic['variables'] = [v for v in load_variables(stat_id)] statistic['__ts'] = datetime.utcnow() db.statistics.update({'code': stat_id}, statistic, upsert=True) import table for _table in statistic['tables']: table.get(_table.get('code')) return statistic
def run_scan(): print "[INFO] Starting scans on configured sites" os.chdir(wpscan_dir) result = "" # Scan ---------------------------------------------------------------- p = subprocess.Popen(r'./wpscan.rb --batch --url %s' % wp_site, stdout=subprocess.PIPE, shell=True, universal_newlines=True) print "[INFO] Scanning '%s'" % wp_site result = p.communicate() # Parse the results --------------------------------------------------- alerts = parse_results(result[0]) for alert in alerts: print alert title = re.compile("\[!\] Title:(.+) -").search(alert) title = title.group(0)[4:-1] db = mongo.get_db() # if not it exists, create it and send it if not mongo.get_vulnerability(db, wp_site, title): mongo.add_vulnerability(db, wp_site, title) sendtoHIVE("[WORDPRESS] " + title, alert, wp_site)
def __init__(self, verbose=False): self.db = get_db() self.quiz = model.Quiz(u'yourtopia') self.country_list = self.db.datum.distinct('country') self.verbose = verbose
def test_mdb(): db = mongo.get_db('test') db.demo.drop() assert db.demo.count()==0 db.demo.insert({"test":123}) print(db.demo.count())
import base64 import requests from flask import Flask, jsonify, request from flask_cors import CORS from mongo import get_db db = get_db() app = Flask(__name__) CORS(app, support_credentials=True) # Access-Control-Allow-Origin: * def get_token(): text = 'admin:Pwd123456' return "Basic " + base64.b64encode(text.encode()).decode() headers = {'Content-Type': 'application/json', 'Authorization': get_token()} @app.route('/', methods=['GET']) def hello(): return jsonify({'hello': 'hello1', 'msg': 'hahaha'}), 201 @app.route('/user/duty', methods=['GET', 'POST']) def duty(): info = {'success': False} if request.method == 'POST': data = request.json.get('data')
def empty_db(): db = mongo.get_db() for collection in COLLECTIONS: db[collection].delete_many({})
def load_terms(): db = get_db() for term in terms_it(): db.terms.update({"term": term}, {"term": term, "__ts": datetime.utcnow()}, upsert=True)
def add_outgoing_links(url, outgoing_links): db = mongo.get_db() item_id = db.links.insert_one({ 'key': url, 'links': outgoing_links }).inserted_id
def updatePageRank(url, rank): db = mongo.get_db() db.reviews.update_one({'url': url}, {'$set': {'rank': rank}})
def get_outgoing_links(): db = mongo.get_db() return list(db.links.find())
def setUp(self): self.db = mongo.get_db()
#!/usr/bin/python """ Super bot created at the Betaworks hackathon. Original author: Tom Germouahauhguh (some kinda euro-name, whatever). Dependencies: easy_install dnspython easy_install xmpppy """ import sys import xmpp import re import logging import mongo db = mongo.get_db('chatstats') def handleMessage(conn, message): body = message.getBody() m = re.search('\[(.+?)\] (.*)|_(.+?) (.*)_', body) if not m: logging.warning("failed to get message from body: %s" % body) return user = m.group(1) or m.group(3) if not user: return text = m.group(2) or m.group(4) or '' words = text.split(' ')
from mongo import get_db, CRUD treasure_db = get_db("treasure") treasure_industry = CRUD(treasure_db, "industry") treasure_industry_detail = CRUD(treasure_db, "industry_detail")