def __init__(self, sources):
possible_sources = ['cnbc', 'reddit']
self.discoverers = []
for source in sources:
assert source in possible_sources, 'invalid source specified for data pool'
if source == 'cnbc':
authenticator = IAMAuthenticator(
"EBkvmVslhKY36GZBRJ44attJ4zYkSfKIfmlUG2B0_8p6")
discovery = DiscoveryV1(version="2019-04-30",
authenticator=authenticator)
discovery.set_service_url(
'https://api.us-east.discovery.watson.cloud.ibm.com/instances/622978c2-cc19-4abd-bc99-ef72da6c53fd'
)
env_id = 'b098ddfa-f993-407c-bd74-b20a2c6fc54f'
collection_id = '02d700e6-69c2-4d51-9d2f-b09e8e15ce8d'
elif source == 'reddit':
authenticator = IAMAuthenticator(
"6j66m1ZJdM6KYMYj8J074qXjWE6FLqsLG4XpA8Bm5Wog")
discovery = DiscoveryV1(version="2019-04-30",
authenticator=authenticator)
discovery.set_service_url(
'https://api.us-south.discovery.watson.cloud.ibm.com/instances/e3a97512-2f2e-4ff6-88c3-2255a1e6f56d'
)
env_id = '6afed5c1-e09a-4407-bd3a-2659124616af'
collection_id = 'afc340a2-57dc-486c-b471-9b687e2505c4'
self.discoverers.append({
'discovery': discovery,
'env_id': env_id,
'collection_id': collection_id
})
def main(args):
query = args.get("query","K9")
count = args.get("count",3)
authenticator = IAMAuthenticator('xxxx')
discovery = DiscoveryV1(
version='2018-08-01',
authenticator=authenticator)
discovery.set_service_url('xxxx')
query_results = discovery.query(
environment_id='xxxx',
collection_id='xxxx',
query = query,
passages= True,
passages_count= 100).get_result()
result = {"entities": [],"passages": []}
try:
id = query_results['results'][0]['id']
passage_list = query_results['passages']
entities = query_results['results'][0]['enriched_text']['entities']
for entity in entities:
if entity['count'] > count:
result["entities"].append(entity)
for passage in passage_list:
if passage['document_id']==id:
result["passages"].append(passage)
result["body"] = passage['passage_text']
break
except IndexError:
return result
return result
def watson_auth():
authenticator = IAMAuthenticator(API_KEY)
discovery = DiscoveryV1(version='2019-04-30', authenticator=authenticator)
discovery.set_service_url(
'https://gateway.watsonplatform.net/discovery/api')
return discovery
def lookInCloud(symbol):
# API: IRQ4Zx1CVhAIYrC-O6JKhW74GWP_9AaMWvv8bGSScZrL
authenticator = IAMAuthenticator('cpuKkVNrkO0AAyCxJ0U8Y0OQNxQtR_a16CT_kqHffwYL')
discovery = DiscoveryV1(
version='2020-11-26',
authenticator=authenticator
)
#discovery.set_service_url('https://api.eu-gb.discovery.watson.cloud.ibm.com/instances/aa8947c6-0ecb-45d8-8335-15a09c1a77a9')
#discovery.set_service_url('https://api.eu-gb.discovery.watson.cloud.ibm.com/instances/aa8947c6-0ecb-45d8-8335-15a09c1a77a9/v1/environments/system/collections/news-en/query?version=2020-11-26&aggregation=filter%28enriched_title.entities.type%3A%3ACompany%29.term%28enriched_title.entities.text%29.timeslice%28crawl_date%2C1day%29.term%28enriched_text.sentiment.document.label%29&filter=IBM&deduplicate=false&highlight=true&passages=true&passages.count=12&query=')
#discovery.set_service_url('https://api.eu-gb.discovery.watson.cloud.ibm.com/instances/3e86f8dd-e512-446f-8c24-e887f47c1ab9/v1/environments/system/collections/news-en/query?version=2018-12-03&count=1&aggregation=filter%28enriched_title.entities.type%3A%3ACompany%29.term%28enriched_title.entities.text%29.timeslice%28crawl_date%2C1day%29.term%28enriched_text.sentiment.document.label%29&filter=IBM&deduplicate=false&highlight=true&passages=true&passages.count=12&query=')
discovery.set_service_url('https://api.eu-gb.discovery.watson.cloud.ibm.com/instances/b9e8228a-afed-4556-97f0-80073219bd22/v1/environments/system/collections/news-en/query?version=2018-12-03&aggregation=filter%28enriched_title.entities.type%3A%3ACompany%29.term%28enriched_title.entities.text%29.timeslice%28crawl_date%2C1day%29.term%28enriched_text.sentiment.document.label%29&filter=IBM&deduplicate=false&highlight=true&passages=true&passages.count=12&query=')
# We are going to access a system collection with English news
# You could change the language to news-de or news-es...
news_environment_id = 'system'
collection_id = 'news-en'
response = discovery.query(
news_environment_id,
collection_id,
query=symbol,
count=6,
deduplicate=True).get_result()
return response
def setup(self):
#Authenticate
authenticator = IAMAuthenticator(self.api_key)
self.discovery = DiscoveryV1(version='2019-04-30',
authenticator=authenticator)
self.discovery.set_service_url(self.url)
# Get environment info
environments = self.discovery.list_environments().get_result()
system_environments = [
x for x in environments['environments']
if x['name'] == self.environment_name
]
self.environment_id = system_environments[0]['environment_id']
# Get collection info
collections = self.discovery.list_collections(
self.environment_id).get_result()
#print(collections)
system_collections = [
x for x in collections['collections']
if x['name'] == self.collection_name
]
self.collection_id = system_collections[0]['collection_id']
def uploadDocs(configs):
documentIds = []
authenticator = IAMAuthenticator(configs["apiKey"])
discovery = DiscoveryV1(version='2019-04-30', authenticator=authenticator)
discovery.set_service_url(configs["serviceUrl"])
files = os.listdir(configs["outputDir"])
for i, file in enumerate(files):
filename = file.split('.')
if filename[1] == 'docx':
print("\n\033[92m[uploadDoc] Uploading {}{}\033[0m".format(
configs["outputDir"], file))
with open(os.path.join(configs["outputDir"], file),
'rb') as fileinfo:
add_doc = discovery.add_document(configs["envId"],
configs["collectionId"],
file=fileinfo).get_result()
print(json.dumps(add_doc, indent=2))
documentIds.append({
'document_name': file,
'document_id': add_doc["document_id"],
"timestamp": str(datetime.datetime.now()),
"status": add_doc["status"]
})
with open('uploadLog.json', 'w') as outfile:
json.dump(documentIds, outfile, indent=4)
def inst():
authenticator = IAMAuthenticator(API_KEY)
discovery = DiscoveryV1(version=SERVICE_VERSION,
authenticator=authenticator)
discovery.set_service_url(URL)
return discovery
def getScore(name, age, address):
with open('auth.json') as f:
data = json.load(f)
authenticator = IAMAuthenticator(data['apikey'])
environmentId = data['environment_id']
collectionId = data['collection_id']
discovery = DiscoveryV1(
version='2019-04-30',
authenticator=authenticator
)
country,state,city = address[0],address[1],address[2]
discovery.set_service_url(data['url'])
filter = '(enriched_text.entities.type::"Person",enriched_text.keywords.text:"'+name+'"),' + \
'(cps_enriched_text.entities.type::Age,cps_enriched_text.entities.text::"'+str(age)+'"),' +\
'(cps_enriched_text.entities.type:Location,cps_enriched_text.entities.text:"' + country + '"),' +\
'(cps_enriched_text.entities.type:Location,cps_enriched_text.entities.text:"' + state + '")'
# '(cps_enriched_text.entities.type:Location,cps_enriched_text.entities.text:"' + city + '")'
filter = '(enriched_text.entities.type::"Person",enriched_text.keywords.text::"'+name+'"),' + \
'(cps_enriched_text.entities.type::Age),' + \
'(cps_enriched_text.entities.type::Location)'
csvWriter = csv.writer(open('results.csv', 'a+'))
#labels
chunksize=100
csvWriter.writerow(['docID','Name','Location/address','Age','Name Match', 'Age Match', 'Location Match', 'Relevancy'])
response = discovery.query(environmentId,collectionId,
filter=filter)
#query through respons and get outputwriter = csv.writer(open('output.csv', 'a+'))
currentdate = datetime.now()
# print(response)
for article in response.result['results']:
score = 0 # relevancy
name_found = 0
age_found = 0
addr_found = 0
#check age
#2018-02-01T07:55:00.000+02:00
start_date = datetime.strptime(article['published'][:-10], '%Y-%m-%dT%H:%M:%S')
difference_in_years = relativedelta(currentdate, start_date).years
article_age = age - difference_in_years
# print(article_age)
for entity in article['cps_enriched_text']['entities']:
if entity['type'] == "Person" and entity['text'].lower() == name.lower(): #found name
name_found = 1
elif entity['type'] == "Location" and (entity['text'].lower() == state.lower() or entity['text'].lower() == city.lower()):
addr_found = 1
#check date of article
elif entity['type']== "Age" and str(article_age) in entity['text']:
age_found = 1
score = name_found and age_found and addr_found
csvWriter.writerow([article['id'], name, state + ',' + city, age, name_found, age_found, addr_found, score])
def writable_environment_id(discovery: DiscoveryV1) -> str:
"""
Return the environment id of the writable environment, if any.
"""
for environment in discovery.list_environments().get_result(
)["environments"]:
if not environment["read_only"]:
return environment["environment_id"]
print("Error: no writable environment found. "
"Please create a bring your own data environment.")
exit(1)
return "" # pylint complains! Ha!
def uploadFiles(config):
authenticator = IAMAuthenticator(config["discovery_api_key"])
discovery = DiscoveryV1(version='2019-04-30', authenticator=authenticator)
discovery.set_service_url(config["discovery_url"])
for file in os.listdir(config["conversionDir"]):
if file.endswith(".pdf"):
fileName = os.path.join(config["conversionDir"], file)
with open(fileName, "rb") as fileinfo:
print("[File Upload] uploading {0}".format(fileName))
add_doc = discovery.add_document(
config["discovery_environment_id"],
config["discovery_collection_id"],
file=fileinfo).get_result()
print(json.dumps(add_doc, indent=2))
def main(args):
authenticator = IAMAuthenticator('fEimjQ_OMKVR0RmWgu3cr4W-Q6jo4VaJ-m2oLwYwTLzR')
discovery_pdf = DiscoveryV1(
version='2019-04-30',
authenticator=authenticator
)
discovery_pdf.set_service_url('https://api.eu-gb.discovery.watson.cloud.ibm.com/instances/66c464a5-af86-427e-92dd-35a40b092969')
myqry = args["mydata"]
try:
mydata = discovery_pdf.query('bbda9998-e223-4758-af3c-99e3fe520676', '451e8d81-8ddb-4e44-9214-d910a8fe0a82', filter=None, query=None, natural_language_query=myqry, passages=None, aggregation=None, count=1, return_='text', offset=None, sort=None, highlight=None, passages_fields=None, passages_count=None, passages_characters=None, deduplicate=None, deduplicate_field=None, similar=None, similar_document_ids=None, similar_fields=None, bias=None, spelling_suggestions=None, x_watson_logging_opt_out=None)
myresult = mydata.get_result()['results'][0]['text']
print(json.dumps(myresult, indent=2))
return {"msg": json.dumps(myresult, indent=2)}
except ApiException as ex:
print("Method failed with status code " + str(ex.code) + ": " + ex.message)
def getDocDetails(configs):
authenticator = IAMAuthenticator(configs["apiKey"])
discovery = DiscoveryV1(version='2019-04-30',authenticator=authenticator)
discovery.set_service_url(configs["serviceUrl"])
uploadLog,fileLog = {},[]
with open('uploadLog.json') as f:
uploadLog = json.load(f)
for document in uploadLog:
doc_info = discovery.get_document_status(
configs["envId"],
configs["collectionId"],
document["document_id"]).get_result()
print("update for "+document["document_id"])
doc_info["timestamp"] = str(datetime.datetime.now())
fileLog.append(doc_info)
with open('availabilityLog.json','w') as outfile:
json.dump(fileLog,outfile,indent = 4)
def getDiscoveryChartOne():
discovery = DiscoveryV1(version=discovery_version,
iam_apikey=discovery_iam_apikey,
url=discovery_url)
# discovery.set_detailed_response(True)
response = discovery.query(collection_id=discovery_collection_id,
environment_id=discovery_environment_id,
filter=None,
query="",
natural_language_query=None,
passages=None,
aggregation="timeslice(発生日,1day)",
count="2",
return_fields=None,
offset=None,
sort=None,
highlight=None,
passages_fields=None,
passages_count=None,
passages_characters=None,
deduplicate=None,
deduplicate_field=None,
similar=None,
similar_document_ids=None,
similar_fields=None,
logging_opt_out=None,
collection_ids=None,
bias=None)
json_data = json.dumps(response.get_result(), indent=2, ensure_ascii=False)
p_obj = json.loads(json_data)
responseDemo = {}
for resultD in p_obj['aggregations'][0]['results']:
if resultD['matching_results'] > 0:
responseDemo[resultD['key_as_string'][5:10].replace(
'-', ' ')] = resultD['matching_results']
# response = Response(json.dumps(responseDemo))
# response['Access-Control-Allow-Origin'] = '*'
# response['Access-Control-Allow-Methods'] = 'POST, GET, OPTIONS'
# response['Access-Control-Max-Age'] = '1000'
# response['Access-Control-Allow-Headers'] = '*'
return jsonify(results=responseDemo)
def main(args: Args) -> None:
"""
Main program. Reads files from disk and optionally sends each file to Discovery
"""
discovery = DiscoveryV1(
args.version,
url=args.url,
username=args.username,
password=args.password,
iam_apikey=args.iam_api_key,
)
args.environment_id = writable_environment_id(discovery)
collections = discovery.list_collections(
args.environment_id).get_result()["collections"]
if len(collections) == 1:
args.collection_id = collections[0]["collection_id"]
if not args.collection_id:
if collections:
print(
"Error: multiple collections found. Please specify which one to use."
)
else:
print(
"Error: no target collection found. Please create a collection."
)
exit(1)
target = Target(discovery, args.environment_id, args.collection_id)
index_list = existing_sha1s(target)
work_q: queue.Queue = queue.Queue(CONCURRENCY)
with concurrent.futures.ThreadPoolExecutor(max_workers=CONCURRENCY +
1) as executor:
executor.submit(walk_paths, args.paths, index_list, work_q,
args.dry_run)
futures = set()
item = work_q.get()
while item:
futures.add(executor.submit(send_file, target, *item))
while len(futures) >= CONCURRENCY:
# We're at our desired concurrency, wait for something to complete.
_, futures = concurrent.futures.wait(
futures, return_when=concurrent.futures.FIRST_COMPLETED)
item = work_q.get()
def initialize_discovery(collection_name=config.DISCOVERY_COLLECTION_NAME):
# In the constructor, letting the SDK manage the token
authenticator = IAMAuthenticator(
apikey='LTk2AGh0Mjsu3G9bGGSkU92xeCBAYdfa4lN4fjaA-CR4')
discovery = DiscoveryV1(version='2018-08-01', authenticator=authenticator)
# discovery.set_service_url('<url_as_per_region>')
# discovery = DiscoveryV1(
# version='2018-08-01',
# url='https://gateway.watsonplatform.net/discovery/api',
# iam_apikey='LTk2AGh0Mjsu3G9bGGSkU92xeCBAYdfa4lN4fjaA-CR4'
# )
# get the environment id
environments = json.dumps(discovery.list_environments().get_result())
environments = json.loads(environments)
environment_id = environments['environments'][1]['environment_id']
# get the collections
collections = json.dumps(
discovery.list_collections(environment_id).get_result(), indent=2)
collections = json.loads(collections)
# iterate through collections in Discovery
print(
"======================= AVAILABLE COLLECTIONS IN CORPUS ======================="
)
for collection in collections['collections']:
print('COLLECTION ID:', collection['collection_id'],
'COLLECTION NAME:', collection['name'])
# find the collection the user wants, and extract the collection_id
if str(collection['name']) == collection_name:
collection_id = collection['collection_id']
else:
# raise Exception('ERROR: COLLECTION NAME %s DOES NOT EXIST.' % collection_name)
pass
print("--- USING COLLECTION ID: %s " % collection_id)
return discovery, environment_id, collection_id
def get_answer(iam_apikey, url, environment_id, collection_id, question):
authenticator = IAMAuthenticator(iam_apikey)
discovery = DiscoveryV1(
version="2018-12-03",
authenticator=authenticator
)
discovery.set_service_url(url)
query_result = discovery.query(
environment_id,
collection_id,
natural_language_query=question,
passages=True,
count=3, passages_count=3).get_result()['results'][0]
answer = multiple_line_answer(query_result['answer'])
#TODO: handler when question returns no asnwer
return { 'message': answer}
def discovery_init(COLLECTION_NAME=config.DISCOVERY_COLLECTION_NAME,
default=all_params['discovery']['default']):
response = {}
discovery_param = all_params['discovery']
# default =
authenticator = IAMAuthenticator(apikey=discovery_param[default]['apikey'])
endurl = "https://gateway.watsonplatform.net/discovery/api"
discovery = DiscoveryV1(
version=discovery_param[default]['version'],
authenticator=authenticator
# iam_apikey=discovery_param[default]['apikey'],
# url=discovery_param[default]['url']
)
if default == "a1":
discovery.set_service_url(discovery_param[default]['url'])
environments = discovery.list_environments().get_result()
# print(json.dumps(environments, indent=2))
print("Discovery instance loaded " + default)
js = json.dumps(environments)
jres = json.loads(js)
# print(jres['environments'][1]['environment_id'])
env_id = jres['environments'][1]['environment_id']
collections = discovery.list_collections(env_id).get_result()
cols = json.dumps(collections, indent=2)
colres = json.loads(cols)
#print(colres['collections'])
for item in colres['collections']:
if item['name'] == COLLECTION_NAME:
print('COLLECTION ID:', item['collection_id'], 'COLLECTION NAME:',
item['name'])
col_id = item['collection_id']
response['env_id'] = env_id
response['col_id'] = col_id
response['discovery'] = discovery
return response
def scrape_cnbc(day):
authenticator = IAMAuthenticator("EBkvmVslhKY36GZBRJ44attJ4zYkSfKIfmlUG2B0_8p6")
discovery = DiscoveryV1(version="2019-04-30", authenticator=authenticator)
discovery.set_service_url('https://api.us-east.discovery.watson.cloud.ibm.com/instances/622978c2-cc19-4abd-bc99-ef72da6c53fd')
env_id = 'b098ddfa-f993-407c-bd74-b20a2c6fc54f'
collection_id = '02d700e6-69c2-4d51-9d2f-b09e8e15ce8d'
content = requests.get(day.strftime("https://www.cnbc.com/site-map/%Y/%B/%d"), allow_redirects=True).content
soup = BeautifulSoup(content, features="html.parser")
articles_list = soup.find('div', {'class': 'SiteMapArticleList-articleData'})
links = [article['href'] for article in articles_list.findChildren('a', recursive=True)]
for link in links:
print(link)
try:
content = requests.get(link, allow_redirects=True).content
soup = BeautifulSoup(content, features="html.parser")
article_title = soup.find('h1', {'class': lambda value: value and value.startswith("ArticleHeader")}).text
article_body = ' '.join(x.text for x in soup.find_all('div',{'class':'group'}))
article_date = date(*[int(x) for x in link.split('/')[-4:-1]])
document = {
'title': article_title,
'body': article_body,
'url': link,
'year': article_date.year,
'month': article_date.month,
'day': article_date.day
}
discovery.add_document(env_id, collection_id, file=json.dumps(document), filename='-'.join(article_title.split(' ')), file_content_type='application/json')
except Exception as e:
print('Failed to parse or upload document')
print(e)
'''
def main():
output = {}
# Load the classifier
authenticator = IAMAuthenticator(
'd61RIxMA4RwhewIoThcevX0xJqAo80mMyAnkjwNb8ePy')
discovery = DiscoveryV1(version='2018-08-01', authenticator=authenticator)
discovery.set_service_url(
'https://gateway.watsonplatform.net/natural-language-classifier/api')
classifier = NLClassifier(authenticator)
# Take Input, can input multiple lines
with open('../data/test_data.csv', newline='') as csvfile:
reader = csv.reader(csvfile)
for row in reader:
text = row[0]
# Clean the text
for mark in del_marks:
text = text.replace(mark, '')
text = text.replace(''', '\'')
# Classify
result = classifier.classify('90dbdex665-nlc-266',
text).get_result()
confidence = result['classes'][0]['confidence']
normalized_confidence = (confidence - min_confidence) / (
max_confidence - min_confidence)
output[text] = normalized_confidence
csvfile.close()
# Write output
file = open('output.txt', 'w+')
for out in output:
string = out + ', ' + str(output[out])
file.write(string)
file.close()
# coding: utf-8
from __future__ import print_function
import json
from ibm_watson import DiscoveryV1
# If service instance provides API key authentication
discovery = DiscoveryV1(
version='2018-08-01',
## url is optional, and defaults to the URL below. Use the correct URL for your region.
url='https://gateway.watsonplatform.net/discovery/api',
iam_apikey='DBxOesEcwYTQK9-dvcaxTwBICWk0s3RwwEW6m-2eppDn')
# discovery = DiscoveryV1(
# version='2018-08-01',
# ## url is optional, and defaults to the URL below. Use the correct URL for your region.
# # url='https://gateway.watsonplatform.net/discovery/api',
# username='******',
# password='******')
environments = discovery.list_environments().get_result()
print(json.dumps(environments, indent=2))
news_environment_id = 'system'
print(json.dumps(news_environment_id, indent=2))
collections = discovery.list_collections(news_environment_id).get_result()
news_collections = [x for x in collections['collections']]
print(json.dumps(collections, indent=2))
configurations = discovery.list_configurations(
environment_id=news_environment_id).get_result()
def get_disco():
return DiscoveryV1(
version="2019-08-20",
iam_apikey='0xn8K3fpjG6WKz3SGuXuZQvsmnV1OVhjMAUxnMbx0MUV',
url='https://gateway-wdc.watsonplatform.net/discovery/api')
from ibm_watson import DiscoveryV1
from ibm_cloud_sdk_core.authenticators import IAMAuthenticator
api_key = '<your api key>'
environment_id = '<environment_id>'
collection_id = '<collection_id>'
authenticator = IAMAuthenticator(api_key)
discovery = DiscoveryV1(version='2018-08-01', authenticator=authenticator)
discovery.set_service_url('https://gateway.watsonplatform.net/discovery/api')
query_results = discovery.query(environment_id,
collection_id,
text='copyright').get_result()
docs = query_results['results']
for doc in docs:
discovery.delete_document(environment_id, collection_id, doc["id"])
def get_watson_online_store():
load_dotenv(os.path.join(os.path.dirname(__file__), ".env"))
# Use these env vars first if set
bot_id = os.environ.get("BOT_ID")
slack_bot_token = os.environ.get('SLACK_BOT_TOKEN')
authenticator = (get_authenticator_from_environment('assistant') or
get_authenticator_from_environment('conversation'))
cloudant_account = os.environ.get("CLOUDANT_USERNAME")
cloudant_iam_apikey = os.environ.get("CLOUDANT_IAM_APIKEY")
cloudant_db_name = os.environ.get(
"CLOUDANT_DB_NAME") or 'watson_online_store'
# If the CLOUDANT_USERNAME env var was not set then use
# VCAP_SERVICES like a WatsonService would.
if not cloudant_iam_apikey:
vcap_services = os.environ.get("VCAP_SERVICES")
vcap_env = json.loads(vcap_services) if vcap_services else None
if vcap_env:
cloudant_creds = WatsonEnv.get_vcap_credentials(
vcap_env, 'cloudantNoSQLDB')
if cloudant_creds:
if 'apikey' in cloudant_creds:
cloudant_iam_apikey = cloudant_creds['apikey']
if 'username' in cloudant_creds:
cloudant_account = cloudant_creds['username']
# Instantiate Watson Assistant client.
# - only give a url if we have one (don't override the default)
assistant_client = AssistantV1(
version='2018-09-20',
authenticator=authenticator)
# Instantiate Cloudant DB.
cloudant_online_store = CloudantOnlineStore(
Cloudant.iam(
cloudant_account,
cloudant_iam_apikey,
connect=True
),
cloudant_db_name
)
# Instantiate Watson Discovery client.
# - only give a url if we have one (don't override the default)
discovery_client = DiscoveryV1(
version='2019-11-22',
)
# Instantiate Slack chatbot.
if not slack_bot_token or 'placeholder' in slack_bot_token:
print("SLACK_BOT_TOKEN needs to be set correctly. "
"It is currently set to '%s'." % slack_bot_token)
print("Only the web UI will be available.")
slack_client = None
else:
slack_client = SlackClient(slack_bot_token)
# If BOT_ID wasn't set, we can get it using SLACK_BOT_USER.
if not bot_id:
bot_id = WatsonEnv.get_slack_user_id(slack_client)
if not bot_id:
print("Error: Missing BOT_ID or invalid SLACK_BOT_USER.")
return None
# Start Watson Online Store app.
watsononlinestore = WatsonOnlineStore(bot_id,
slack_client,
assistant_client,
discovery_client,
cloudant_online_store)
return watsononlinestore
from app import app
from flask import render_template, request
from ibm_watson import DiscoveryV1
from ibm_cloud_sdk_core.authenticators import IAMAuthenticator
import string
apikey = '7dOTS6YQInAIFTNG9n_kQbp-PGVhyzJ0Q0KECjzjkqnD'
watsonversion = '2019-04-30'
url = 'https://api.eu-gb.discovery.watson.cloud.ibm.com'
authenticator = IAMAuthenticator(apikey)
discovery = DiscoveryV1(version=watsonversion, authenticator=authenticator)
discovery.set_service_url(url)
@app.route('/')
def index():
return render_template('public/main.html')
@app.route('/search-page', methods=['GET', 'POST'])
def search():
searchquery = request.args.get('query')
print(searchquery)
environment_id = '200a459f-203a-4f47-944b-d328f64f11fd'
collection_id = 'e5347b2a-5ba7-4288-911b-083d0dddd20c'
ibm_api_key_id=COS_API_KEY_ID,
ibm_auth_endpoint=COS_AUTH_ENDPOINT,
config=Config(signature_version="oauth"),
endpoint_url=COS_ENDPOINT,
ibm_service_instance_id=COS_RESOURCE_CRN,
)
# Personality Insight Authenticator
authenticator = IAMAuthenticator(PERSONALITY_INSIGHT_AUTHENTICATOR)
personality_insights = PersonalityInsightsV3(version='2017-10-13',
authenticator=authenticator)
personality_insights.set_service_url(PERSONALITY_INSIGHT_SERVICE_URL)
# Discovery Authenticator
authenticator = IAMAuthenticator(DISCOVERY_AUTHENTICATOR)
DiscoveryService = DiscoveryV1(version='2019-11-22',
authenticator=authenticator)
DiscoveryService.set_service_url(DISCOVERY_SERVICE_URL)
def get_url_content(url):
global counter
headers = {'User-Agent': USER_AGENT}
session = requests.Session()
adapter = requests.adapters.HTTPAdapter(max_retries=20)
session.mount('https://', adapter)
session.mount('http://', adapter)
# Get content from source. If fail, skip.
import json
from datetime import datetime
from ibm_watson import DiscoveryV1
from ibm_cloud_sdk_core.authenticators import IAMAuthenticator
from secrets import DISCOVERY_API_KEY
# define the IDS and URLS from Watson Discovery Service
DISCOVERY_URL = 'https://api.eu-gb.discovery.watson.cloud.ibm.com/instances/738047c5-5416-4845-8d23-c7ebd4d0c7e7'
DISCOVERY_VERSION = '2019-04-30'
ENVIRONMENT_ID = 'f3055480-a528-4b3d-bc9a-7e45e428b615'
COLLECTION_ID = '304d4569-672f-47ad-9445-0becdd0cd381'
# create an authenticator object
authenticator = IAMAuthenticator(DISCOVERY_API_KEY)
discovery = DiscoveryV1(version=DISCOVERY_VERSION, authenticator=authenticator)
# connect to the discovery endpont
discovery.set_service_url(DISCOVERY_URL)
discovery.set_default_headers({'x-watson-learning-opt-out': "true"})
discovery.set_disable_ssl_verification(True)
def set_relevancy(query, document_id, relevance):
"""
Set the relevance of a document for a given query.
The relevance can be an integer between 0 and 10.
"""
return discovery.add_training_data(environment_id=ENVIRONMENT_ID,
collection_id=COLLECTION_ID,
natural_language_query=query,
examples=[{
"document_id": document_id,
import praw
import json
from ibm_watson import DiscoveryV1
from ibm_cloud_sdk_core.authenticators import IAMAuthenticator
from datetime import datetime
def remove_non_ascii(text):
return ''.join(i for i in text if ord(i)<128)
authenticator = IAMAuthenticator("6j66m1ZJdM6KYMYj8J074qXjWE6FLqsLG4XpA8Bm5Wog")
discovery = DiscoveryV1(version="2019-04-30", authenticator=authenticator)
discovery.set_service_url('https://api.us-south.discovery.watson.cloud.ibm.com/instances/e3a97512-2f2e-4ff6-88c3-2255a1e6f56d')
env_id = '6afed5c1-e09a-4407-bd3a-2659124616af'
collection_id = 'afc340a2-57dc-486c-b471-9b687e2505c4'
reddit = praw.Reddit(client_id='JwuaAJFEkxPH6A',
client_secret='I-OFNj8ZuWSWWnG2gBCk9cG9-2k',
password='******',
user_agent='scraper by /u/hivebot5914',
username='******')
reset = True
if reset:
for i in range(int(1000/50)):
my_query = discovery.query(env_id, collection_id, query='*.*', count=50).get_result()
if len(my_query) == 0:
break
for doc in my_query['results']:
print('reset', doc['title'])
discovery.delete_document(env_id, collection_id, doc['id'])
def get_watson_online_store():
load_dotenv(os.path.join(os.path.dirname(__file__), ".env"))
# Use these env vars first if set
bot_id = os.environ.get("BOT_ID")
slack_bot_token = os.environ.get('SLACK_BOT_TOKEN')
assistant_iam_apikey = os.environ.get("ASSISTANT_IAM_APIKEY")
assistant_url = os.environ.get("ASSISTANT_URL")
if not assistant_url:
# Direct access to VCAP to workaround SDK problems
vcap_services = os.environ.get("VCAP_SERVICES")
vcap_env = json.loads(vcap_services) if vcap_services else None
if vcap_env:
assistant_creds = WatsonEnv.get_vcap_credentials(
vcap_env, 'conversation')
if assistant_creds:
assistant_url = assistant_creds['url'] # overrides default
assistant_iam_apikey = (assistant_iam_apikey
or assistant_creds.get('apikey'))
cloudant_account = os.environ.get("CLOUDANT_USERNAME")
cloudant_iam_apikey = os.environ.get("CLOUDANT_IAM_APIKEY")
cloudant_db_name = os.environ.get(
"CLOUDANT_DB_NAME") or 'watson_online_store'
discovery_url = os.environ.get('DISCOVERY_URL')
discovery_iam_apikey = os.environ.get("DISCOVERY_IAM_APIKEY")
# If the CLOUDANT_USERNAME env var was not set then use
# VCAP_SERVICES like a WatsonService would.
if not cloudant_iam_apikey:
vcap_services = os.environ.get("VCAP_SERVICES")
vcap_env = json.loads(vcap_services) if vcap_services else None
if vcap_env:
cloudant_creds = WatsonEnv.get_vcap_credentials(
vcap_env, 'cloudantNoSQLDB')
if cloudant_creds:
if 'apikey' in cloudant_creds:
cloudant_iam_apikey = cloudant_creds['apikey']
if 'username' in cloudant_creds:
cloudant_account = cloudant_creds['username']
# Instantiate Watson Assistant client.
# - only give a url if we have one (don't override the default)
assistant_kwargs = {
'version': '2019-02-28',
'iam_apikey': assistant_iam_apikey
}
if assistant_url:
assistant_kwargs['url'] = assistant_url
assistant_client = AssistantV1(**assistant_kwargs)
# Instantiate Cloudant DB.
cloudant_online_store = CloudantOnlineStore(
Cloudant.iam(cloudant_account, cloudant_iam_apikey, connect=True),
cloudant_db_name)
# Instantiate Watson Discovery client.
# - only give a url if we have one (don't override the default)
discovery_kwargs = {
'version': '2019-04-30',
'iam_apikey': discovery_iam_apikey
}
if discovery_url:
discovery_kwargs['url'] = discovery_url
discovery_client = DiscoveryV1(**discovery_kwargs)
# Instantiate Slack chatbot.
if not slack_bot_token or 'placeholder' in slack_bot_token:
print("SLACK_BOT_TOKEN needs to be set correctly. "
"It is currently set to '%s'." % slack_bot_token)
print("Only the web UI will be available.")
slack_client = None
else:
slack_client = SlackClient(slack_bot_token)
# If BOT_ID wasn't set, we can get it using SLACK_BOT_USER.
if not bot_id:
bot_id = WatsonEnv.get_slack_user_id(slack_client)
if not bot_id:
print("Error: Missing BOT_ID or invalid SLACK_BOT_USER.")
return None
# Start Watson Online Store app.
watsononlinestore = WatsonOnlineStore(bot_id, slack_client,
assistant_client,
discovery_client,
cloudant_online_store)
return watsononlinestore
import json
from ibm_watson import DiscoveryV1
from ibm_cloud_sdk_core.authenticators import IAMAuthenticator
authenticator = IAMAuthenticator('o2AHBjAYfudK0mXXt7dFmQFx8oaldIae5oG3du2iIf9K')
discovery = DiscoveryV1(
version='2019-04-30',
authenticator=authenticator
)
discovery.set_service_url('https://api.us-south.discovery.watson.cloud.ibm.com/instances/beff1375-93ea-4b19-b90a-10fbf38f45fd')
envID = '35ef0ced-f8c5-4f16-a57c-098c66505472'
colID = 'c7bf0198-9e14-40db-9e96-2b4d348585c1'
def getNLQ(tweet: str):
areaFilter = '(latitude>' + str(eval("40 - .5")) + ',latitude<' + str(eval("40 - (.5) * -1")) + ',longitude>' + str(eval("(83 * 1) - .5")) + ',longitude<' + str(eval("(83 * 1) - (.5 * -1)")) + ')'
DetailedResponse = discovery.query(environment_id = envID, collection_id = colID, filter = areaFilter, natural_language_query = tweet, count = 10)
response = json.dumps(DetailedResponse.get_result(), indent = 2)
response = json.loads(response)
return response
import discovery
import pandas as pd
import visual_recognition
app = Flask(__name__)
CORS(app)
assistant = AssistantV1(
version='2019-06-13',
url=<url>,
iam_apikey=<api_key>
)
discovery = DiscoveryV1(
version="2019-06-12",
url=<url>,
iam_apikey=<api_key>
)
@app.route('/chatlog')
def chat_log():
response = assistant.list_logs(workspace_id='ace10bc5-2a14-440a-97c1-835faf5c46df').get_result()
return json.dumps(response)
@app.route('/upload', methods=["POST"])
def send_img():
ss = visual_recognition.image_to_upload_and_classify()
return ss
@app.route('/sendmessage', methods={"POST"})
