def loadumls():
    engine = RedisEngine(prefix='umls')
    data = pd.read_csv("../umls/[email protected]")
    df = pd.DataFrame(data)
    df_dictarr = df.to_dict('records')
    for item in df_dictarr:
        # print item['value']
        # id, search phrase, data
        engine.store_json(item['value'], item['label'], {
            'label': item['label'],
            'value': item['value']
        })
Beispiel #2
0
def load_redis_engine():
    redis_url = urlparse.urlparse(settings.REDISTOGO_URL)
    if redis_url.scheme == "redis":
        engine = RedisEngine(host=redis_url.hostname,
                             port=redis_url.port,
                             password=redis_url.password)
        try:
            info = engine.client.info()
            if "db0" in info:
                nb_keys = info["db0"]["keys"]
            else:
                nb_keys = 0
            print "Conn. Redis server, %s keys stored." % nb_keys
            return engine
        except ConnectionError:
            if settings.DEBUG:
                raise ConnectionError("Redis Server is not reachable.")
            else:
                return None
    else:
        if settings.DEBUG:
            raise RedisError("Redis Server '%s' URL is not valid." %
                             settings.REDISTOGO_URL)
        else:
            return None
class LoadRedisWithLocationPrefixes(object):
    def __init__(self, config_name, locations_json_file, locations_geocoded_json_file):
        self.locations_json_file = locations_json_file
        self.locations_geocoded_json_file = locations_geocoded_json_file

        #   The RedisEngine is instantiated that will perform 'autocompletion'
        #   critical: No stop words are specified, as it was discovered via
        #   testing that running with the default 'stop_words=None' still
        #   associates a default set of stopwords to be utilized, one entry
        #   of which is 'a' which precludes autocompletion when a simple 'a'
        #   is provided.
        self.engine = RedisEngine(
            prefix=config[config_name].REDIS_AUTOCOMPLETE_SORTED_SET,
            stop_words=set(),
            cache_timeout=300,
            host=config[config_name].REDIS_HOSTNAME,
            port=config[config_name].REDIS_PORT,
            db=config[config_name].REDIS_DB,
            password=config[config_name].REDIS_PASSWORD,
        )
        # self.engine = RedisEngine(prefix=config[config_name].REDIS_AUTOCOMPLETE_SORTED_SET, stop_words=None, cache_timeout=300, host=config[config_name].REDIS_HOSTNAME, port=config[config_name].REDIS_PORT, db=config[config_name].REDIS_DB, password=config[config_name].REDIS_PASSWORD)
        self.engine.flush()

    def load_locations_prefixes_into_redis(self):
        basepath = os.path.dirname(__file__)
        filepath = os.path.abspath(os.path.join(basepath, "..", "data", self.locations_json_file))
        json_file = open(filepath, "r")

        #   Iterate over all filmlocation json entries and extract the 'Locations' entry from the
        #   json object, strip the phrase of all non-alphanumeric characters before inserting into
        #   Redis.
        for json_string in json_file:
            json_object = json.loads(json_string)
            location = json_object["Locations"]
            if location and location.strip() != "None":
                location = re.sub(r"([^\s\w]|_|-)+", " ", location)
                # Store the filmid (unique), search phrase, full metadata for filmlocation
                self.engine.store_json(json_object["filmid"], location, json_object)
    def __init__(self, config_name, locations_json_file, locations_geocoded_json_file):
        self.locations_json_file = locations_json_file
        self.locations_geocoded_json_file = locations_geocoded_json_file

        #   The RedisEngine is instantiated that will perform 'autocompletion'
        #   critical: No stop words are specified, as it was discovered via
        #   testing that running with the default 'stop_words=None' still
        #   associates a default set of stopwords to be utilized, one entry
        #   of which is 'a' which precludes autocompletion when a simple 'a'
        #   is provided.
        self.engine = RedisEngine(
            prefix=config[config_name].REDIS_AUTOCOMPLETE_SORTED_SET,
            stop_words=set(),
            cache_timeout=300,
            host=config[config_name].REDIS_HOSTNAME,
            port=config[config_name].REDIS_PORT,
            db=config[config_name].REDIS_DB,
            password=config[config_name].REDIS_PASSWORD,
        )
        # self.engine = RedisEngine(prefix=config[config_name].REDIS_AUTOCOMPLETE_SORTED_SET, stop_words=None, cache_timeout=300, host=config[config_name].REDIS_HOSTNAME, port=config[config_name].REDIS_PORT, db=config[config_name].REDIS_DB, password=config[config_name].REDIS_PASSWORD)
        self.engine.flush()
Beispiel #5
0
import os
import json
import bson
import pymongo
from pymongo import Connection
import urllib2
from redis_completion import RedisEngine
import urlparse
import redis

url = urlparse.urlparse(
    'redis://:[email protected]:6777')
r = redis.Redis(host=url.hostname, port=url.port, password=url.password)

engine = RedisEngine(host=url.hostname, port=url.port, password=url.password)

connection = Connection(
    'mongodb://*****:*****@alex.mongohq.com:10013/app8222672')
jobs_collection = connection.app8222672.jobs

jobs = []
for job in jobs_collection.find():
    a = job['title']
    a = a.lower()
    for word in a.split():
        jobs.append(word)

print jobs
map(engine.store, jobs)
#print engine.search(query.decode('cp1252'))
Beispiel #6
0
import urllib2
from redis_completion import RedisEngine
import urlparse
import redis

engine = RedisEngine()
titles = [
    'python programming', 'programming c', 'unit testing python',
    'testing software', 'software design'
]
map(engine.store, titles)

print engine.search('test')
Beispiel #7
0
# -*- coding:utf-8 -*-

import mmseg 
from pinyin import Pinyin
from redis_completion import RedisEngine

pinyin = Pinyin()
engine = RedisEngine()

def store_movie(movie):
    phrase = movie["title"]
    seg_phrase = " ".join(mmseg.seg_txt(phrase))
    _pinyin_phrase = pinyin.get_pinyin(phrase)
    py_phrase = "".join([p[0] for p in _pinyin_phrase]).encode("utf-8")
    pinyin_phrase = "".join(_pinyin_phrase).encode("utf-8")
    phrase = "%s %s %s %s" % (phrase, seg_phrase, pinyin_phrase, py_phrase)
    engine.store_json(movie["id"], phrase, movie)

def load_datas():
    movies = [
        {"id":20513051, "title": "被偷走的那五年", "director":"黄真真"}
      , {"id":20513052, "title": "十面埋伏", "director":"张艺谋"}
      , {"id":20513053, "title": "龙门镖局", "director":"王勇"}
      , {"id":20513054, "title": "致我们终将逝去的青春", "director":"赵薇"}
      , {"id":20513055, "title": "金枝欲孽2", "director":"戚其义"}
    ]
    for movie in movies:
        store_movie(movie)

load_datas()
for phrase in ["偷", "lmb", "jinzhi"]:
Beispiel #8
0
import urllib2
from redis_completion import RedisEngine
import csv
import redis

engine = RedisEngine()

# UPDATING CACHE

txtfile = open('BestBuy.txt', "r") 
productList = txtfile.read().split("\n")
setProductList = set(productList)
productList = list(setProductList)

map(engine.store, productList)

Options = engine.search('Ultra HD TV Silver')
for item in Options:
	print item




$$$$$$$$$$$$$$$$$$$$$

>>> mystring = "alfa"
>>> myfile = open("/home/gustacaste/FileA", 'w')
>>> myfile.write(mystring)
>>> myfile.close()
Beispiel #9
0
import urllib2
from redis_completion import RedisEngine

engine = RedisEngine(prefix='stocks')

def load_data():
    url = 'http://media.charlesleifer.com/downloads/misc/NYSE.txt'
    contents = urllib2.urlopen(url).read()
    for row in contents.splitlines()[1:]:
        ticker, company = row.split('\t')
        engine.store_json(ticker, company, {'ticker': ticker, 'company': company}) # id, search phrase, data

def search(p, **kwargs):
    return engine.search_json(p, **kwargs)

if __name__ == '__main__':
    engine.flush()
    print 'Loading data (may take a few seconds...)'
    load_data()

    print 'Search data by typing a partial phrase, like "uni sta"'
    print 'Type "q" at any time to quit'

    while 1:
        cmd = raw_input('? ')
        if cmd == 'q':
            break
        results = search(cmd)
        print 'Found %s matches' % len(results)
        for result in results:
            print '%s: %s' % (result['ticker'], result['company'])
Beispiel #10
0
# -*- coding: utf8

import urllib2
from redis_completion import RedisEngine
import urlparse
import redis

engine = RedisEngine()
titles = [
    'python programming', 'programming c', 'unit testing python',
    u'программирование'.decode('utf-8')
]
map(engine.store, titles)
from redis_completion import RedisEngine
import pandas as pd
import json
engine = RedisEngine(prefix='umls')


def loadumls():
    engine = RedisEngine(prefix='umls')
    data = pd.read_csv("../umls/[email protected]")
    df = pd.DataFrame(data)
    df_dictarr = df.to_dict('records')
    for item in df_dictarr:
        # print item['value']
        # id, search phrase, data
        engine.store_json(item['value'], item['label'], {
            'label': item['label'],
            'value': item['value']
        })


def search(p):
    return engine.search_json(p)
Beispiel #12
0
 def get_engine(self):
     return RedisEngine(prefix='testac', db=15)
Beispiel #13
0
# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

########NEW FILE########
__FILENAME__ = stocks
import urllib2
from redis_completion import RedisEngine

engine = RedisEngine(prefix='stocks')


def load_data():
    url = 'http://media.charlesleifer.com/downloads/misc/NYSE.txt'
    contents = urllib2.urlopen(url).read()
    for row in contents.splitlines()[1:]:
        ticker, company = row.split('\t')
        engine.store_json(ticker, company, {
            'ticker': ticker,
            'company': company
        })  # id, search phrase, data


def search(p, **kwargs):
    return engine.search_json(p, **kwargs)
Beispiel #14
0
def create_app(env='debug'):
    """ TODO:
    if config is None:
        config = os.path.join(app.root_path, 'production.cfg')

    app.config.from_pyfile(config)
    """
    app = Flask(__name__)
    app.config.from_object(__name__)

    app.config.update(SECRET_KEY=os.urandom(20))
    if (env == 'debug'):
        app.debug = True
    if (env == 'prod'):
        app.debug = False

    if app.debug:
        from flaskext.lesscss import lesscss
        lesscss(app)
    app.static_path = '/static'

    # connect to the database
    connection = Connection(
        'mongodb://*****:*****@alex.mongohq.com:10013/app8222672')
    jobs_collection = connection.app8222672.jobs
    users_collection = connection.app8222672.users
    url = urlparse.urlparse(
        'redis://:[email protected]:6777')
    autocomplete_engine = RedisEngine(host=url.hostname,
                                      port=url.port,
                                      password=url.password)

    @app.route('/index/')
    @app.route('/home/')
    @app.route('/')
    def home():
        return render_template('home.html')

    @app.route('/settings/')
    def settings():
        return render_template('settings.html')

    @app.route('/list')
    def list():
        jobs = []
        for job in jobs_collection.find():
            job['id'] = str(job['_id'])
            jobs.append(job)
        return render_template('list.html', jobs=jobs)

    @app.route('/delete/<jobid>')
    def delete(jobid):
        response = jobs_collection.remove({'_id': bson.ObjectId(jobid)})
        if (response == None):
            flash('Job erased')
        else:
            flash('Error occured')
        return redirect(url_for('list'))

    @app.route('/jobs/<jobid>')
    def inside(jobid):
        found_job = jobs_collection.find_one(
            {'_id': bson.ObjectId(oid=str(jobid))})
        found_job['id'] = str(found_job['_id'])
        found_job['objectid'] = str(found_job['_id'])
        found_job['_id'] = str(found_job['_id'])
        return render_template('inside.html', job=found_job)

    @app.route('/edit/<jobid>')
    def edit(jobid):
        return render_template('add.html', job=jobid)

    @app.errorhandler(404)
    def page_not_found(e):
        return render_template('404.html'), 404

    # ==================== REGISTRATION ==================

    @app.route('/logout')
    def logout():
        # remove the username from the session if it's there
        session.pop('logged_in', None)
        return redirect(url_for('home'))

    @app.route('/registration')
    def registration():
        return render_template('registration.html')

    def get_user_id(email):
        return users_collection.find_one({'email': email})

    @app.route('/login', methods=['GET', 'POST'])
    @app.route('/register', methods=['GET', 'POST'])
    def register():
        """Registers the user."""
        error = None
        if request.method == 'POST':
            logged_user = get_user_id(request.form['email'])
            if (logged_user):
                if (request.form['password'] == logged_user['password']):
                    flash('Logged in as ' + logged_user['email'])
                    session['logged_in'] = logged_user['email']
                    return redirect(url_for('home', session=session))
                else:
                    flash('Wrong password!')
                    return redirect(url_for('register'))
            if not request.form['email'] or \
                     '@' not in request.form['email']:
                error = 'You have to enter a valid email address'
            elif not request.form['password']:
                error = 'You have to enter a password'
            elif get_user_id(request.form['email']) is not None:
                error = 'The username is already taken'
            else:
                new_user_id = users_collection.save({
                    'email':
                    request.form['email'],
                    'password':
                    request.form['password'],
                    'status':
                    'awaiting confirm'
                })
                payload = {
                    'from':
                    'Excited User <*****@*****.**>',
                    'to':
                    request.form['email'],
                    'subject':
                    'Quick Hunt account confirmation',
                    'text':
                    'http://quickhunt.herokuapp.com/activate_user/' +
                    str(new_user_id)
                }
                r = requests.post(
                    "https://api.mailgun.net/v2/app8222672.mailgun.org/messages",
                    auth=HTTPBasicAuth('api',
                                       'key-9m9vuzkafbyjqhm9ieq71n0lu9dgf9b9'),
                    data=payload)
                flash(
                    'You were successfully registered. Confirm registration and login.'
                )
                session['logged_in'] = request.form['email']
                flash('logged in successfuly')
                return redirect(url_for('home'))
        #flash('no luck ((' + request.method + error)
        flash('error:' + str(error))
        return render_template('login.html', error=error)

    @app.route('/activate_user/<user_id>')
    def activate_user(user_id):
        """
        Activate user function.
        """
        found_user = users_collection.find_one(
            {'_id': bson.ObjectId(oid=str(user_id))})
        if not found_user:
            return abort(404)
        else:
            if found_user['status'] == 'awaiting_confirm':
                ### Setting the user status active here ###*
                confirm_mail = {
                    'from': 'Quick Hunt <*****@*****.**>',
                    'to': found_user['email'],
                    'subject': 'Quick Hunt account confirmation',
                    'text': 'Subscription confirmed.'
                }
                r = requests.post(
                    "https://api.mailgun.net/v2/app8222672.mailgun.org/messages",
                    auth=HTTPBasicAuth('api',
                                       'key-9m9vuzkafbyjqhm9ieq71n0lu9dgf9b9'),
                    data=confirm_mail)
                flash('user has been activated', 'info')
            elif found_user['status'] == 'active':
                flash('user already activated', 'info')
            return redirect(url_for('content'))

    """ This is the API part of the equation """
    """
    @app.errorhandler(404)
    def not_found(error=None):
        message = {
                'status': 404,
                'message': 'Not Found: ' + request.url,
        }
        resp = jsonify(message)
        resp.status_code = 404

        return resp
    """

    @app.route('/api/search/', methods=['GET'])
    @app.route('/api/search/<query>', methods=['GET'])
    def search(query=None):
        jobs = []
        for job in jobs_collection.find():
            job['_id'] = str(job['_id'])
            jobs.append(job)
        return jsonify({'result': jobs})

    @app.route('/api/jobs/<jobid>', methods=['GET'])
    def get_job(jobid):
        found_job = jobs_collection.find_one(
            {'_id': bson.ObjectId(oid=str(jobid))})
        found_job['id'] = str(found_job['_id'])
        found_job['objectid'] = str(found_job['_id'])
        found_job['_id'] = str(found_job['_id'])
        #if userid in users:
        return jsonify(found_job)
        #else:
        #    return not_found()
        #return undef

    @app.route('/api/jobs/new', methods=['POST'])
    def create_job():
        js = json.dumps(request.data)
        json_data = json.loads(request.data)
        jobs_collection.save(json_data)
        resp = Response(js, status=200, mimetype='application/json')
        return resp

    @app.route('/api/jobs/<jobid>', methods=['PUT'])
    def update_job(jobid):
        js = json.dumps(request.data)
        print 'js:' + str(js)
        json_data = json.loads(request.data)
        json_data['_id'] = bson.ObjectId(json_data['objectid'])
        jobs_collection.save(json_data)
        resp = Response(js, status=200, mimetype='application/json')
        return resp

    @app.route('/api/jobs/<jobid>', methods=['DELETE'])
    def delete_job(jobid):
        response = jobs_collection.remove({'_id': bson.ObjectId(jobid)})
        if (response == None):
            return jsonify({success: 'Success'})
        else:
            return jsonify({error: 'Error'})

    @app.route('/api/autocomplete/', methods=['GET'])
    def autocomplete():
        js = {}
        searchword = request.args.get('q', '')
        if searchword:
            js = json.dumps({'result': autocomplete_engine.search(searchword)})
        else:
            js = {'error': 'invalid argument'}
        return Response(js, status=200, mimetype='application/json')

    @app.route('/api/favorites/add/<jobid>',
               methods=['GET'])  # FIX IT (i mean get)
    def add_to_favorites(jobid):
        me = users_collection.find_one({'email': session['logged_in']})
        if 'favorites' in me:
            me['favorites'].append(jobid)
        else:
            me['favorites'] = [jobid]
        js = users_collection.save(me)
        print js
        resp = Response({'reply': js}, status=200, mimetype='application/json')
        return resp

    @app.route('/api/favorites/delete/<jobid>',
               methods=['DELETE'])  # FIX IT (i mean get)
    def remove_from_favorites(jobid):
        me = users_collection.find_one({'email': session['logged_in']})
        if 'favorites' in me:
            favorites = me['favorites']
            favoretes.remove(jobid)
            me['favorites'] = favorites
        js = users_collection.save(me)
        print js
        resp = Response({'reply': js}, status=200, mimetype='application/json')
        return resp

    return app
Beispiel #15
0
import urllib2
from redis_completion import RedisEngine
import urlparse
import redis

engine = RedisEngine()
titles = ['python programming', 'programming c', 'unit testing python',
                  'testing software', 'software design']
map(engine.store, titles)


print engine.search('test')
Beispiel #16
0
REDIS_DB = app.config['REDIS_DB']
AWS_KEY = app.config['AWS_KEY']
AWS_SECRET_KEY = app.config['AWS_SECRET_KEY']
BUCKET_NAME = app.config['BUCKET_NAME']
LOGOS_BUCKET_NAME = app.config['LOGOS_BUCKET_NAME']
SENDGRID_USERNAME = os.environ.get('SENDGRID_USERNAME', '')
SENDGRID_PASSWORD = os.environ.get('SENDGRID_PASSWORD', '')

redis_url = os.environ.get('REDISTOGO_URL', None)


if redis_url:
    redis_url = urlparse.urlparse(redis_url)
    redisco.connection_setup(host=redis_url.hostname, port=redis_url.port, db=0, password=redis_url.password)
        
    autocomplete_engine = RedisEngine(host=redis_url.hostname, port=redis_url.port, db=0, password=redis_url.password)
    
    redisco.connection_setup(host=redis_url.hostname, port=redis_url.port, db=0, password=redis_url.password)
    autocomplete_engine = RedisEngine(host=redis_url.hostname, port=redis_url.port, db=0, password=redis_url.password)       
else:
    redisco.connection_setup(host='localhost', port=6379, db=0)
    
    autocomplete_redis_client = redis.Redis(host='localhost', port=6379, db=0)
    autocomplete_engine = RedisEngine(autocomplete_redis_client)    

     



ALLOWED_EXTENSIONS = set(('txt', 'pdf', 'ppt', 'pptx', 'zip', 'tar', 'rar'))
ALLOWED_IMAGE_EXTENSIONS = set(('png', 'jpg'))