def setup():
    """
    Make sure we're working with a fresh database. Build a client for
    interacting with that database and save it as a global.
    """
    global ROOT_CLIENT, PROJECT, USERNAME, RELOGIN_CLIENT
    user_info_str = subprocess.check_output('tellme lumi-test', shell=True)
    user_info = eval(user_info_str)
    USERNAME = user_info['username']

    ROOT_CLIENT = LuminosoClient.connect(ROOT_URL,
                                         username=USERNAME,
                                         password=user_info['password'])
    RELOGIN_CLIENT = LuminosoClient.connect(ROOT_URL,
                                            username=USERNAME,
                                            password=user_info['password'],
                                            auto_login=True)

    # check to see if the project exists; also create the client we'll use
    projects = ROOT_CLIENT.get(USERNAME + '/projects')
    projlist = [proj['name'] for proj in projects]
    PROJECT = ROOT_CLIENT.change_path(USERNAME + '/projects/' + PROJECT_NAME)

    if PROJECT_NAME in projlist:
        logger.warn('The test database existed already. '
                    'We have to clean it up.')
        ROOT_CLIENT.delete(USERNAME + '/projects', project=PROJECT_NAME)

    # create the project
    logger.info("Creating project: " + PROJECT_NAME)
    logger.info("Existing projects: %r" % projlist)
    ROOT_CLIENT.post(USERNAME + '/projects', project=PROJECT_NAME)
    PROJECT.get()
def post_set_topics():
    app.logger.info('Set the topics')
    # see API documentation: https://api.luminoso.com/v4/
    # use account # and username provisioned
    client = LuminosoClient.connect(request.args.get('luminoso_account'),
                                    username=request.args.get('luminoso_user_name'), 
                                    password=request.args.get('luminoso_password'))
    # Get project id from project name:
    project = client.get(name=request.args.get('project_name'))[0]
    project = client.change_path(project['project_id'])
    # delete and then recreate topics
    topics = project.get('topics/')
    for topic in topics:
        project.delete('topics/id/'+topic['_id'])
        new_topic = project.post('topics/', name="Oceans", text="ocean marine")
        new_topic = project.post('topics/', name="Whales", text="whales")
        new_topic = project.post('topics/', name="Seals", text="seals")
        new_topic = project.post('topics/', name="Japan", text="Japan")
        new_topic = project.post('topics/', name="Overfishing", text="fishing overfishing")
        new_topic = project.post('topics/', name="Forests", text="forests")
        new_topic = project.post('topics/', name="Climate & Energy", text="climate fuel pollution")
        new_topic = project.post('topics/', name="Polar", text="polar ice")
        topics = project.get('topics/')

    return 'true' 
def post_set_topics():
    app.logger.info('Set the topics')
    # see API documentation: https://api.luminoso.com/v4/
    # use account # and username provisioned
    client = LuminosoClient.connect(
        request.args.get('luminoso_account'),
        username=request.args.get('luminoso_user_name'),
        password=request.args.get('luminoso_password'))
    # Get project id from project name:
    project = client.get(name=request.args.get('project_name'))[0]
    project = client.change_path(project['project_id'])
    # delete and then recreate topics
    topics = project.get('topics/')
    for topic in topics:
        project.delete('topics/id/' + topic['_id'])
        new_topic = project.post('topics/', name="Oceans", text="ocean marine")
        new_topic = project.post('topics/', name="Whales", text="whales")
        new_topic = project.post('topics/', name="Seals", text="seals")
        new_topic = project.post('topics/', name="Japan", text="Japan")
        new_topic = project.post('topics/',
                                 name="Overfishing",
                                 text="fishing overfishing")
        new_topic = project.post('topics/', name="Forests", text="forests")
        new_topic = project.post('topics/',
                                 name="Climate & Energy",
                                 text="climate fuel pollution")
        new_topic = project.post('topics/', name="Polar", text="polar ice")
        topics = project.get('topics/')

    return 'true'
Пример #4
0
def push(request):
	correlations = {}  # dictionary of correlations

# handle lack of request body
	if len(request.body) == 0:
		return HttpResponseRedirect("/")

	json_data = json.loads(request.body)  # load text from body of request
	try:
		# connect to Luminoso and our english corpus
		project_name = "GP english sample"
		client = LuminosoClient.connect('/projects/u64t648d/', 
				 username='******', password=pw)  
		project = client.get(name=project_name)[0]
		project = client.change_path(project['project_id'])

		# get correlations to each topic
		result = project.put('topics/text_correlation', text=json_data['text'])
		topics = project.get('topics/')
		for key, value in result.iteritems():
			correlations.update({(t['name'], value) for t in topics if t['_id']==key})

	except KeyError:
		HttpResponseServerError('Malformed data')
	return HttpResponse(json.dumps(correlations))
def get_luminoso():
    app.logger.info('Get from luminoso')
    
    client = LuminosoClient.connect(request.args.get('luminoso_account'),
                                    username=request.args.get('luminoso_user_name'), 
                                    password=request.args.get('luminoso_password'))
    project_info_list = client.get()
    
    return json.dumps(project_info_list)
def get_luminosodelete():
    app.logger.info('Delete a project on Luminoso')
    client = LuminosoClient.connect(request.args.get('luminoso_account'),
                                    username=request.args.get('luminoso_user_name'), 
                                    password=request.args.get('luminoso_password'))
   # Delete new project by POSTing its name
    client.delete(request.args.get('project_id'))
    
    return "Project Deleted "
def get_luminosodelete():
    app.logger.info('Delete a project on Luminoso')
    client = LuminosoClient.connect(
        request.args.get('luminoso_account'),
        username=request.args.get('luminoso_user_name'),
        password=request.args.get('luminoso_password'))
    # Delete new project by POSTing its name
    client.delete(request.args.get('project_id'))

    return "Project Deleted "
def get_luminoso():
    app.logger.info('Get from luminoso')

    client = LuminosoClient.connect(
        request.args.get('luminoso_account'),
        username=request.args.get('luminoso_user_name'),
        password=request.args.get('luminoso_password'))
    project_info_list = client.get()

    return json.dumps(project_info_list)
def get_luminosocreate():
    
    app.logger.info('Create a project on Luminoso')
    
    client = LuminosoClient.connect(request.args.get('luminoso_account'),
                                    username=request.args.get('luminoso_user_name'), 
                                    password=request.args.get('luminoso_password'))
    # Create a new project by POSTing its name
    project_id = client.post(name=request.args.get('project_name'))['project_id']
    # use that project from here on
    project = client.change_path(project_id)
    return 'true'
Пример #10
0
def upload_stream(stream,
                  server,
                  account,
                  projname,
                  reader_dict,
                  username=None,
                  password=None,
                  append=False,
                  stage=False):
    """
    Given a file-like object containing a JSON stream, upload it to
    Luminoso with the given account name and project name.
    """
    client = LuminosoClient.connect(server,
                                    username=username,
                                    password=password)
    if not append:
        # If we're not appending to an existing project, create new project.
        info = client.post('/projects/' + account, name=projname)
        project_id = info['project_id']
    else:
        projects = client.get('/projects/' + account, name=projname)
        if len(projects) == 0:
            print 'No such project exists!'
            return
        if len(projects) > 1:
            print 'Warning: Multiple projects with name "%s".  ' % projname,
        project_id = projects[0]['project_id']
        print 'Using existing project with id %s.' % project_id

    project = client.change_path('/projects/' + account + '/' + project_id)

    if stage:
        url = 'docs'
    else:
        url = 'docs/preload'

    counter = 0
    final_job_id = None
    for batch in batches(stream, 1000):
        counter += 1
        documents = list(batch)
        job_id = project.upload(url, documents, readers=reader_dict)
        print 'Uploaded batch #%d' % (counter)
        final_job_id = job_id

    if not stage:
        # Calculate the docs into the assoc space.
        print 'Calculating.'
        final_job_id = project.post('docs/recalculate', readers=reader_dict)

    if final_job_id is not None:
        project.wait_for(final_job_id)
def get_luminosocreate():

    app.logger.info('Create a project on Luminoso')

    client = LuminosoClient.connect(
        request.args.get('luminoso_account'),
        username=request.args.get('luminoso_user_name'),
        password=request.args.get('luminoso_password'))
    # Create a new project by POSTing its name
    project_id = client.post(
        name=request.args.get('project_name'))['project_id']
    # use that project from here on
    project = client.change_path(project_id)
    return 'true'
def get_update_docs():
    
    app.logger.info('Upload documents Luminoso')
    
    client = LuminosoClient.connect(request.args.get('luminoso_account'),
                                    username=request.args.get('luminoso_user_name'), 
                                    password=request.args.get('luminoso_password'))
    # Create a new project by POSTing its name
    project = client.get(name=request.args.get('project_name'))[0]
    project = client.change_path(project['project_id'])

    try:
        json_object = json.loads(request.args.get('project_doc'))
    except ValueError, e:
        return False
def get_update_docs():

    app.logger.info('Upload documents Luminoso')

    client = LuminosoClient.connect(
        request.args.get('luminoso_account'),
        username=request.args.get('luminoso_user_name'),
        password=request.args.get('luminoso_password'))
    # Create a new project by POSTing its name
    project = client.get(name=request.args.get('project_name'))[0]
    project = client.change_path(project['project_id'])

    try:
        json_object = json.loads(request.args.get('project_doc'))
    except ValueError, e:
        return False
Пример #14
0
def upload_stream(stream, server, account, projname, reader_dict,
                  username=None, password=None,
                  append=False, stage=False):
    """
    Given a file-like object containing a JSON stream, upload it to
    Luminoso with the given account name and project name.
    """
    client = LuminosoClient.connect(server,
                                    username=username, password=password)
    if not append:
        # If we're not appending to an existing project, create new project.
        info = client.post('/projects/' + account, name=projname)
        project_id = info['project_id']
    else:
        projects = client.get('/projects/' + account, name=projname)
        if len(projects) == 0:
            print 'No such project exists!'
            return
        if len(projects) > 1:
            print 'Warning: Multiple projects with name "%s".  ' % projname,
        project_id = projects[0]['project_id']
        print 'Using existing project with id %s.' % project_id

    project = client.change_path('/projects/' + account + '/' + project_id)

    if stage:
        url = 'docs'
    else:
        url = 'docs/preload'

    counter = 0
    final_job_id = None
    for batch in batches(stream, 1000):
        counter += 1
        documents = list(batch)
        job_id = project.upload(url, documents, readers=reader_dict)
        print 'Uploaded batch #%d' % (counter)
        final_job_id = job_id

    if not stage:
        # Calculate the docs into the assoc space.
        print 'Calculating.'
        final_job_id = project.post('docs/recalculate', readers=reader_dict)

    if final_job_id is not None:
        project.wait_for(final_job_id)
def upload_stream(stream, server, account, projname, language=None,
                  username=None, password=None,
                  append=False, stage=False):
    """
    Given a file-like object containing a JSON stream, upload it to
    Luminoso with the given account name and project name.
    """
    client = LuminosoClient.connect(server,
                                    username=username, password=password)
    if not append:
        # If we're not appending to an existing project, create new project.
        info = client.post('/projects/' + account, name=projname)
        project_id = info['project_id']
        print('New project ID:', project_id)
    else:
        projects = client.get('/projects/' + account, name=projname)
        if len(projects) == 0:
            print('No such project exists!')
            return
        if len(projects) > 1:
            print('Warning: Multiple projects with name "%s".  ' % projname,
                  end='')
        project_id = projects[0]['project_id']
        print('Using existing project with id %s.' % project_id)

    project = client.change_path('/projects/' + account + '/' + project_id)

    counter = 0
    for batch in batches(stream, 1000):
        counter += 1
        documents = list(batch)
        project.upload('docs', documents)
        print('Uploaded batch #%d' % (counter))

    if not stage:
        # Calculate the docs into the assoc space.
        print('Calculating.')
        kwargs = {}
        if language is not None:
            kwargs = {'language': language}
        job_id = project.post('docs/recalculate', **kwargs)
        project.wait_for(job_id)
def get_correlation():
    
    app.logger.info('Get Correlation')
    
    client = LuminosoClient.connect(request.args.get('luminoso_account'),
                                    username=request.args.get('luminoso_user_name'), 
                                    password=request.args.get('luminoso_password'))
    
    project = client.get(name=request.args.get('project_name'))[0]
    project = client.change_path(project['project_id'])
    
    # get list of topics
    topics = project.get('topics/')
    result = project.put('topics/text_correlation', text=request.args.get('article_text'))

    correlations = []
    for key, value in result.iteritems():
        correlations.append([(t['name'], value) for t in topics if t['_id']==key])

    return json.dumps(correlations) 
Пример #17
0
def upload_stream(stream, server, account, projname, reader_dict):
    """
    Given a file-like object containing a JSON stream, upload it to
    Luminoso with the given account name and project name.
    """
    client = LuminosoClient.connect(server)
    try:
        client.post(account + '/projects/', project=projname)
    except LuminosoAPIError as e:
        pass
    project = client.change_path(account + '/projects/' + projname)

    counter = 0
    for batch in batches(stream, 100):
        counter += 1
        documents = list(batch)
        job_id = project.upload('docs', documents, width=4, readers=reader_dict)
        print 'Uploaded batch #%d into job %s' % (counter, job_id)

    print 'Committing.'
    final_job_id = project.post('docs/calculate', width=4)
    project.wait_for(final_job_id)
def get_correlation():

    app.logger.info('Get Correlation')

    client = LuminosoClient.connect(
        request.args.get('luminoso_account'),
        username=request.args.get('luminoso_user_name'),
        password=request.args.get('luminoso_password'))

    project = client.get(name=request.args.get('project_name'))[0]
    project = client.change_path(project['project_id'])

    # get list of topics
    topics = project.get('topics/')
    result = project.put('topics/text_correlation',
                         text=request.args.get('article_text'))

    correlations = []
    for key, value in result.iteritems():
        correlations.append([(t['name'], value) for t in topics
                             if t['_id'] == key])

    return json.dumps(correlations)
                    if soup.find(id="postingbody"):
                        post = soup.find(id="postingbody").get_text().replace("\n",'').replace("\t",'')
                        document.update({'title' : headline})
                        document.update({'text' :  post})
                        document.update({'date' : time.time()})
                        document.update({'source' : {'url' : site}})
                        document.update({'queries' : [locations[i]]})
                        documents.append(document)          # add the document to the documents list
                except Exception, err:
                    print "Error:", sys.exc_info()[0]
        except Exception, err:
            print "Error:", sys.exc_info()[0]
    
print("Total length of documents: %s" % len(documents))

client = LuminosoClient.connect('/projects/e26y767s/', username='******')
print("_________________Creating new project: %s" % project_name)
project_info = client.post(name=project_name)
project = client.change_path(project_info['project_id'])
project.upload('docs', documents)
print("_________________Documents uploaded")

sys.exit()  # exit here
job_id = project.post('docs/recalculate')
print job_id
status = project.get('/jobs/id/'+str(job_id)+'/')
print status
project.wait_for(job_id)
print("_________________Calculations complete")

Пример #20
0
def connect_client(project_name):
#   client connection
    client = LuminosoClient.connect('/projects/t55y685c/', username='******', password=pw)
    project = client.get(name=project_name)[0]
    project = client.change_path(project['project_id'])
    return project