def getDBDtls():
if request.method == 'POST' and request.form['dbdtls'] == 'False':
dbusername = request.form['username']
dbpassword = request.form['password']
dburl = request.form['url']
global db_user
global db_pass
global db_url
db_user = dbusername
db_pass = dbpassword
db_url = dburl
client = Cloudant(db_user, db_pass, url=db_url)
client.connect()
try:
myDatabase = client.create_database("user_db")
global db_svc
db_svc = {"dbsvc": myDatabase}
except CloudantException as err:
print err
client = Cloudant(db_user, db_pass, url=db_url)
client.connect()
client.delete_database("user_db")
myDatabase = client.create_database("user_db")
global db_svc
db_svc = {"dbsvc": myDatabase}
return render_template('index.html', getUsrDtls=True)
class CouchDB():
def __init__(self, cfg):
self._cfg = cfg
self._client = Cloudant(self._cfg['user'],
self._cfg['password'],
url=self._cfg['host'])
self._client.connect()
self._db = self._get_db()
def _get_db(self):
databases = self._client.all_dbs()
db_name = self._cfg[self._name]
if not db_name in databases:
self._client.create_database(db_name)
return self._client[db_name]
def update_document(self, document_id, attributes_dict):
if attributes_dict is None:
return False
document = self._db[document_id]
for key in attributes_dict.keys():
document[key] = attributes_dict[key]
document.save()
return True
def reconnect(self):
try:
self._client.disconnect()
self._client.connect()
except Exceptions as e:
print('Eror: {}'.format(e))
def main(dict):
serviceUsername = dict['CLOUDANT_ACCOUNT']
servicePassword = dict['CLOUDANT_PASSWORD']
serviceURL = dict['CLOUDANT_URL']
# Use the IBM Cloudant library to create an IBM Cloudant client.
client = Cloudant(serviceUsername, servicePassword, url=serviceURL)
# Connect to the server
client.connect()
databaseName = dict['CLOUDANT_DBNAME']
# Create an instance of the database.
myDatabaseDemo = client.create_database(databaseName)
# Check that the database now exists.
if myDatabaseDemo.exists():
print("'{0}' successfully created.\n".format(databaseName))
service = AssistantV1(iam_apikey=dict['IAM_APIKEY'],
version=dict['WDC_VERSION'],
url=dict['WDC_URL'])
response = service.get_workspace(workspace_id=dict['WDC_WORKSPACEID'],
export='true').get_result()
newDocument = myDatabaseDemo.create_document(response)
return {'status': 200, 'message': 'Backup realizado com sucesso!'}
def init_db():
global my_database
client = Cloudant(username, password, url=url, connect=True)
if hasattr(client, 'users'):
my_database = client['user']
else:
my_database = client.create_database('my_database')
def main(dict):
user = dict.get("username", "<username from credentials>")
passwd = dict.get("password", "<password from credentials>")
host = dict.get("host", "<host from credentials>")
db_name = "test_cloud_functions_python_3_ibm_runtime"
client = Cloudant(user,
passwd,
url='https://{0}'.format(host),
connect=True)
try:
client.delete_database(db_name)
except CloudantClientException as ex:
if ex.status_code != 404:
raise CloudantClientException(ex.status_code, db_name)
my_database = client.create_database(db_name)
if my_database.exists():
print('SUCCESS DB exists!!')
"""Sample document"""
_id = str(uuid.uuid4())
data = {"agentId": "Suzzy", "type": "User"}
update_data = {"lastname": "Queue"}
"""Run tests"""
test_create_doc(my_database, _id, data)
test_update_doc(my_database, _id, update_data)
test_get_doc_by_id(my_database, _id)
doc = test_fetch_doc_by_id(my_database, _id)
test_delete_doc_by_id(my_database, doc['_id'])
return doc
def main():
with open('dbuploadcred.json') as f:
creds = json.load(f)
serviceUsername = creds['username']
servicePassword = creds['password']
serviceURL = creds['url']
client = Cloudant(serviceUsername, servicePassword, url=serviceURL)
client.connect()
databaseName = "population"
popDatabase = client.create_database(databaseName)
if popDatabase.exists():
print("'{0}' successfully created.\n".format(databaseName))
# Retrieve the fields in each row.
for key in population_dict:
city_name = key
popSize = population_dict[key]
# Create a JSON document that represents all the data in the row.
jsonDocument = {
"cityField": city_name,
"populationField": popSize,
}
# Create a document by using the Database API.
newDocument = popDatabase.create_document(jsonDocument)
# Check that the documents exist in the database.
if newDocument.exists():
print("Document successfully created.")
def main(dict):
user = dict.get("username", "<username from credentials>")
passwd = dict.get("password", "<password from credentials>")
host = dict.get("host", "<host from credentials>")
db_name = "test_cloud_functions_python_3_ibm_runtime"
client = Cloudant(user,
passwd,
url='https://{0}'.format(host),
connect=True)
try:
client.delete_database(db_name)
except CloudantClientException as ex:
if ex.status_code != 404:
raise CloudantClientException(ex.status_code, db_name)
my_database = client.create_database('my_database')
if my_database.exists():
print('SUCCESS DB exists!!')
data = {"_id": "p3", "firstname": "Suzzy", "lastname": "Queue"}
my_document = my_database.create_document(data)
if my_document.exists():
print('SUCCESS DOC exist!!')
my_document_read = my_database['p3']
return my_document_read
def kriging_plot():
#date = str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M"))
data = dataset.data()
data.sort_values(by=['component'])
#data.set_index(keys=['component'], drop=False,inplace=True)
names = data['component'].unique().tolist()
for component in names:
compdata = data.loc[data.component == component]
compdata = compdata.reset_index(drop=True)
if (4 > len(compdata.index)):
continue
krige_data = krige_task(compdata[[
'latitude', 'longitude', 'value', 'unit', 'component', 'x', 'y'
]])
#define documentDataFrame(compdata, columns=['latitude','longitude', 'value', 'unit', 'component'] )
document = {
'date': str(compdata['toTime'].iloc[0]),
'component': str(compdata['component'].iloc[0]),
'data':
data.loc[data.component == component].to_json(orient='index'),
'krige_data': simplejson.dumps(krige_data.tolist())
}
# 'data' : compdata.to_json(orient='index'),
#connect to db
client = Cloudant(user, password, url=url, connect=True)
db = client.create_database(db_name, throw_on_exists=False)
#store document
document = db.create_document(document)
#disconnect from db
client.disconnect()
return data.to_html()
def getItems(dictUse):
from cloudant.client import Cloudant
from cloudant.error import CloudantException
from cloudant.result import Result, ResultByKey
import json
returningList = []
numberList = []
serviceUsername = "******"
servicePassword = "******"
serviceURL = "https://c2c7ebdf-134b-4f53-9805-d37f520c6933-bluemix.cloudant.com"
client = Cloudant(serviceUsername, servicePassword, url=serviceURL)
client.connect()
#name storage_unit
databaseName = "storage_unit"
myDatabaseDemo = client.create_database(databaseName)
result_collection = Result(myDatabaseDemo.all_docs, include_docs=True)
for doc in result_collection:
if(doc["doc"]["Blood Type"] == dictUse["Blood Type"] and (doc["doc"]["Location"]).lower() == dictUse["Location"].lower()):
returningList.append(doc["doc"]["Name"])
numberList.append(doc["doc"]["Number"])
client.disconnect()
returnString = ""
for i in range (len(returningList)):
if(i == len(returningList) - 1):
returnString += returningList[i]
else:
returnString += returningList[i] + ', '
return returnString, numberList
def get_img(_id):
#connect to the db
client = Cloudant(user, password, url=url, connect=True)
db = client.create_database(db_name, throw_on_exists=False)
#get the document from the db
doc = db[_id]
#load the krige data
H = simplejson.loads(doc['krige_data'])
buffr = io.BytesIO()
#min max values for the color
maxvalue = 100
minvalue = 0
#set the min max values according to whats unhealthy
#max will be unhealthy value, and in turn be red on the generated image
#we set values based on https://uk-air.defra.gov.uk/air-pollution/daqi?view=more-info&pollutant=no2 , Accsessed 11.05.2018
if (doc['component'] == 'PM2.5'):
maxvalue = 60
minvalue = 0
if (doc['component'] == 'PM10'):
maxvalue = 85
minvalue = 0
if (doc['component'] == 'NO2'):
maxvalue = 420
minvalue = 0
#plot the figure
fig, ax = subplots()
ax.imshow(H,
cmap=my_cmap,
vmin=minvalue,
vmax=maxvalue,
origin='lower',
interpolation='gaussian',
alpha=0.7,
extent=[X0, X1, Y0, Y1])
ax.axis('off')
#set proper image size and extent to plot
fig.set_size_inches(5.95, 5)
imgextent = ax.get_window_extent().transformed(
fig.dpi_scale_trans.inverted())
fig.savefig(buffr,
format='svg',
bbox_inches=imgextent,
transparent=True,
pad_inches=0,
frameon=None)
#go to start of file
buffr.seek(0)
#disconnect from db
client.disconnect()
return send_file(buffr, mimetype='image/svg+xml')
class DBCloudant:
def __init__(self):
self.client = ""
self.session = ""
self.dataBase = ""
self.document = ""
self.query = ""
def connect(self):
self.client = Cloudant(conf.username,
conf.password,
url=conf.url,
connect=True)
self.session = self.client.session()
if self.session['userCtx']['name']:
print("Coneccion realizada correctamente")
else:
print("Error al conectar con la BD")
def disconnect(self):
self.client.disconnect()
def createDB(self, name):
try:
self.dataBase = self.client.create_database(name)
if self.dataBase.exists():
print("DB creada correctamente")
except Exception as error:
print("DB ya existente")
def getDataBases(self):
return self.client.all_dbs()
def deleteDataBase(self, name):
return self.client.delete_database(name)
def openDB(self, name):
self.dataBase = self.client[name]
def createDocument(self, data):
self.document = self.dataBase.create_document(data)
if self.document.exists():
print("Se creo el documento correctamente")
def getDocument(self, name):
self.document = self.dataBase[name]
return self.document
def getAllDocuments(self, name):
return self.dataBase
def getDocumentBySensor(self, sensor):
self.query = cl.query.Query(self.dataBase)
with self.query.custom_result(selector={"sensor": sensor}) as rst:
return rst
def createDatabase():
client = Cloudant(
"863e5b40-95ef-474f-86dd-848b379f3fbe-bluemix",
"90ca5b2be3996997179a453cc936f7d7b4e5b456e03502a9317219c917ae5a46",
url=
"https://*****:*****@863e5b40-95ef-474f-86dd-848b379f3fbe-bluemix.cloudantnosqldb.appdomain.cloud"
)
client.connect()
database_name = "bene-db"
my_database = client.create_database(database_name)
return my_database
def add_signal(status, load):
client = Cloudant(
"39a4348e-3ce1-40cd-b016-1f85569d409e-bluemix",
"48e26645f504209f85b4c44d74a4cb14bc0d059a22b361534b78f406a513f8ff",
url=
"https://*****:*****@39a4348e-3ce1-40cd-b016-1f85569d409e-bluemix.cloudant.com"
)
client.connect()
databaseName = load
myDatabase = client.create_database(databaseName)
client.delete_database(databaseName)
# if myDatabase.exists():
# print("'{0}' successfully created.\n".format(databaseName))
# client.delete_database(databaseName)
myDatabase = client.create_database(databaseName)
sampleData = [[status, "kitchen"]]
# Create documents using the sample data.
# Go through each row in the array
for document in sampleData:
# Retrieve the fields in each row.
number = document[0]
name = document[1]
# Create a JSON document that represents
# all the data in the row.
jsonDocument = {"status": number, "circuit": name}
# Create a document using the Database API.
newDocument = myDatabase.create_document(jsonDocument)
# Check that the document exists in the database.
if newDocument.exists():
print("Document '{0}' successfully created.".format(number))
client.disconnect()
def looping():
while True:
time.sleep(5)
ser = serial.Serial("/dev/ttyACM0", 9600)
ser.baudrate = 9600
temp1 = ser.readline()
# temp1 = 20
ser = serial.Serial("/dev/ttyACM1", 9600)
ser.baudrate = 9600
temp2 = ser.readline()
# temp2 = 20
print temp1, temp2
if int(temp1[0:2]) < 35:
# if temp1 < 35:
l1['color'] = "#2ecc71"
l1['Safe'] = "true"
else:
l1['color'] = "#c0392b"
l1['Safe'] = "false"
# if temp2 < 35:
if int(temp2[0:2]) < 35:
l2['color'] = "#2ecc71"
l2['Safe'] = "true"
else:
l2['color'] = "#c0392b"
l2['Safe'] = "false"
serviceUsername = "******"
servicePassword = "******"
serviceURL = "https://*****:*****@ac99bfe8-c64e-48a5-9f62-e3b7c44ba586-bluemix.cloudant.com"
client = Cloudant(serviceUsername, servicePassword, url=serviceURL)
client.connect()
databaseName = "sensordbs"
try:
client.delete_database(databaseName)
except CloudantException:
print "There was a problem deleting '{0}'.\n".format(databaseName)
else:
print "'{0}' successfully deleted.\n".format(databaseName)
myDatabaseDemo = client.create_database(databaseName)
if myDatabaseDemo.exists():
print "'{0}' successfully created.\n".format(databaseName)
newDocument = myDatabaseDemo.create_document(l1)
newDocument = myDatabaseDemo.create_document(l2)
# Check that the document exists in the database.
if newDocument.exists():
print "Document"
def main():
USERNAME = "******"
PASSWORD = "******"
cloudant_url = "https://*****:*****@ba62d85b-fc08-4706-8a9e-ba91bfbf6174-bluemix.cloudant.com"
my_cl_db_name = 'my_cl_db'
count_doc = 0
client = Cloudant(USERNAME, PASSWORD, url=cloudant_url)
client.connect() # Connect to the server
my_cl_db = client.create_database(my_cl_db_name) # Create db
if my_cl_db.exists():
print("Database Created.")
else:
print("Error# 1: Databse Creation Failed.")
exit(True)
# Read Data and Send to Database
data_file = open('Data.txt', 'r')
fieldnames = ("member_id", "product id", "date",
"number of helpful feedbacks", "number of feedbacks",
"rating", "title", "body")
reader = csv.DictReader(
data_file, fieldnames,
delimiter='\t') # Create a dictionary object using the field names
for row in reader: # Take each row from dictionary and convert as needed and prepare another dictionary object
my_data_doc = {
'member_id': row['member_id'],
'product id': row['product id'],
'date': row['date'],
'number of helpful feedbacks':
int(row['number of helpful feedbacks']),
'number of feedbacks': int(row['number of feedbacks']),
'rating': round(float(row['rating']), 1),
'title': row['title'],
'body': row['body']
}
my_cl_db.create_document(my_data_doc) # Send the data to database
count_doc += 1
print("Total Document sent: ", count_doc)
time.sleep(0.5) # Otherwise server raises Too many requests error
print("Data Store Complete.")
client.disconnect()
def init_cloudant_client(account, database_name, password):
"""
:param account:
:param database_name:
:param password:
:return: Cloudant client and database
"""
client = Cloudant(account, password, account=account, connect=True)
try:
db = client[database_name]
except KeyError as ke:
print("Database not found [{}]. Creating database".format(ke))
db = client.create_database(database_name)
return client, db
def cloudantstorage():
client = Cloudant(
"65eb56e4-a72d-4b13-b226-1626961c5389-bluemix",
"899a62a1c1864b9ebf46c8b51ee287b2ffd3baf9314dee8400d1d1e1b33c137c",
url=
"https://*****:*****@65eb56e4-a72d-4b13-b226-1626961c5389-bluemix.cloudantnosqldb.appdomain.cloud"
)
client.connect()
#Provide your database name
database_name = "insidegarage"
my_database = client.create_database(database_name)
if my_database.exists():
print(f"'{database_name}' successfully created.")
return my_database
def test_sendtocloudant(self):
###########################################################
############## my credentials #############################
###########################################################
serviceUsername = "******"
servicePassword = "******"
serviceURL = "https://68019c5d-fdfa-44f5-abee-707e5488207b-bluemix.cloudant.com"
###########################################################
parent_url = 'https://www.ibm.com'
list_of_urls = ['https://www.google.com', 'https://www.google.com']
jsonList = {"parent_url": parent_url, "list_of_urls": list_of_urls}
databaseName = 'valid_urls'
client = Cloudant(serviceUsername, servicePassword, url=serviceURL)
client.connect()
database = client.create_database(databaseName)
newDocument = database.create_document(jsonList)
self.assertEqual(database[newDocument['_id']]['list_of_urls'],
list_of_urls)
def main():
# For more info on how to get the credentials:
# https://console.bluemix.net/docs/services/Cloudant/getting-started.html#getting-started-with-cloudant
cClient = Cloudant(
"<USERNAME>",
"<PASSWORD>",
url="https://<USERNAME>:<PASSWORD>@<USERNAME>.cloudant.com")
cClient.connect()
# just an example
db_name = "finanza"
database = cClient.create_database(db_name)
if database.exists():
msg = "database '{0}' successfully created.\n"
print(msg.format(db_name))
csv_inflator(database)
# sample_inflator(database)
cClient.disconnect()
def all_entries():
#connect to the db
client = Cloudant(user, password, url=url, connect=True)
db = client.create_database(db_name, throw_on_exists=False)
#get all docs
docs = list(map(lambda doc: doc, db))
#put them into a dataframe
fdocs = json_normalize(docs)
fdocs = DataFrame(fdocs, columns=['date', 'component', 'data', '_id'])
fdocs['date'] = to_datetime(fdocs['date'])
fdocs = fdocs.reset_index(drop=True)
fdocs.sort_values(['date', 'component'])
#get the components
components = fdocs['component'].unique().tolist()
data = [None] * len(fdocs)
for i, row in fdocs.iterrows():
tmp = read_json(fdocs.loc[i, 'data'], orient='index')
tmp = tmp.reset_index()
data[i] = tmp
fdocs.loc[i, 'data'] = i
#make a list of same size as components
complist = [None] * len(components)
for i in range(len(components)):
#drop everything but relevant info
tmp = fdocs.drop(fdocs[fdocs.component != components[i]].index)
#drop duplicates
tmp = tmp.drop_duplicates(subset=['date'], keep='first', inplace=False)
#sort them
tmp = tmp.sort_values(['date'], ascending=[False])
#re index the dataframe
tmp = tmp.reset_index(drop=True)
#put the dataframe into the list
complist[i] = tmp
#disconnect from db
client.disconnect()
return render_template('entries.html', entries=complist, data=data)
def ConectDataBase():
try:
client = Cloudant(
"4637db36-4b7c-47d7-b006-6f846e72a22a-bluemix",
"ac21a13b101baabea9f1ea2594b622c54e7f2799b2f96fefb80173b37209ee12",
url=
"https://*****:*****@4637db36-4b7c-47d7-b006-6f846e72a22a-bluemix.cloudant.com"
)
client.connect()
databaseName = "oguiadomochileirodasantarticas3"
myDatabase = client.create_database(databaseName)
if myDatabase.exists():
print('{0} criado com sucesso'.format(databaseName))
else:
print('{0} não foi criado'.format(databaseName))
client.disconnect()
except Exception as ex:
print(ex.message)
from cloudant.client import Cloudant
from cloudant.error import CloudantException
from cloudant.result import Result, ResultByKey
CLOUDANT_USERNAME = "******"
CLOUDANT_PASSWORD = "******"
CLOUDANT_URL = "{0}.cloudantnosqldb.appdomain.cloud".format(CLOUDANT_USERNAME)
client = Cloudant(CLOUDANT_USERNAME,
CLOUDANT_PASSWORD,
url="https://{0}:{1}@{2}".format(CLOUDANT_USERNAME,
CLOUDANT_PASSWORD,
CLOUDANT_URL))
client.connect()
my_database = client.create_database("database3")
data_json = {
'cat': "meow",
}
@app.route('/')
def hello_world():
return app.send_static_file('index.html')
@app.errorhandler(404)
@app.route("/error404")
def page_not_found(error):
return app.send_static_file('404.html')
client = Cloudant(cred['credentials']['username'],
cred['credentials']['password'],
url=cred['credentials']['url'])
client.connect()
# DBが存在していたら削除
print "既存データベースの削除"
try:
db = client[dbname]
if db.exists():
client.delete_database(dbname)
except:
print "新規データベースの作成"
# DBを新規作成
db = client.create_database(dbname)
# 失敗したら終了
if db.exists():
print 'SUCCESS!! Database creation on cloudant'
else:
sys.exit()
# エクセルシートからのCSVデータを読んで
# NLCトレーニングデータを作成する
# リアクションデータをクラウダントに登録する
#
reader = codecs.open(cnf['input_csv'],'r','shift_jis')
lines = reader.readlines()
writer = codecs.open(cnf['training_csv'],'w','utf_8')
class QueryCardzDatabase:
database_name = ""
database = None
client = None
def __init__(self, database_name, ibm_credentials_path):
self.database_name = database_name
with open(ibm_credentials_path) as json_file:
ibm_credentials = json.load(json_file)
self.client = Cloudant(ibm_credentials['username'],
ibm_credentials["password"],
url=ibm_credentials["url"])
def connect(self):
self.client.connect()
self.database = self.client.create_database(databaseName)
def getCards(self, user_email, topic):
user_record = self.getUserRecord(user_email)
if not user_record:
return []
all_res = user_record[QUERIES]
print(all_res)
return list(filter(lambda obj: topic in obj[TOPICS], all_res))
def getUserRecord(self, user_email):
if user_email is None:
return None
if user_email in self.database:
return self.database[user_email]
return None
def getTopics(self, user_email):
user_record = self.getUserRecord(user_email)
if not user_record:
return []
doc = user_record[TOPICS]
return doc
def search_query(self, user_email, search, answer, search_topic_array):
doc_exists = user_email in self.database
search_data_object = {
QUERY: search,
TOPICS: search_topic_array,
ANSWER: answer
}
if doc_exists:
doc = self.getUserRecord(user_email)
for i in doc[QUERIES]:
if i[QUERY] == search:
return
doc[QUERIES].append(search_data_object)
doc[TOPICS] += search_topic_array
doc[TOPICS] = list(set(doc[TOPICS]))
doc.save()
else:
data = {
"_id": user_email,
QUERIES: [search_data_object],
TOPICS: search_topic_array
}
doc = self.database.create_document(data)
servicePassword = credentialsData['password']
target.write("Got password: "******"\n")
# ... and the URL of the service within Bluemix.
serviceURL = credentialsData['url']
target.write("Got URL: ")
target.write(serviceURL)
target.write("\n")
# We now have all the details we need to work with the Cloudant NoSQL DB service instance.
# Connect to the service instance.
client = Cloudant(serviceUsername, servicePassword, url=serviceURL)
client.connect()
# Create a database within the instance.
myDatabaseDemo = client.create_database(databaseName)
if myDatabaseDemo.exists():
target.write("'{0}' successfully created.\n".format(databaseName))
# Create a very simple JSON document with the current date and time.
jsonDocument = {"rightNow": strftime("%Y-%m-%d %H:%M:%S", gmtime())}
# Store the JSON document in the database.
newDocument = myDatabaseDemo.create_document(jsonDocument)
if newDocument.exists():
target.write("Document successfully created.\n")
# All done - disconnect from the service instance.
client.disconnect()
# Put another clear indication of the current date and time at the bottom of the page.
target.write("\n====\n")
target.write(strftime("%Y-%m-%d %H:%M:%S", gmtime()))
target.write("\n====\n")
url=cred['services']['cloudantNoSQLDB'][0]['credentials']['url'])
client.connect()
# DBが存在していたら削除
print "既存データベース ", output_db, " の作成"
try:
db = client[output_db]
if db.exists():
client.delete_database(output_db)
except:
pass
# DBを新規作成
print "新規データベース ", output_db, " の作成"
try:
db = client.create_database(output_db)
print "データベース作成成功"
except:
print "データベース作成失敗"
sys.exit()
# 文字コード変換など前処理結果を中間ファイルへ出力
fin = codecs.open(input_file, "r", "shift_jis")
fout = codecs.open(output_file, "w", "utf-8")
line_no = 0
while True:
try:
line = fin.readline()
if line == "":
break
line = line.rstrip('\r\n') + "\n"
def html():
from cloudant.client import Cloudant
from cloudant.error import CloudantException
from cloudant.result import Result, ResultByKey
import time as t
client = Cloudant("6656429c-2491-40a6-b026-31dd597c43de-bluemix", "3847820cff823b729fdc0863eeac335529dc19ccab353aa401f6de6b57c38e57", url="https://*****:*****@6656429c-2491-40a6-b026-31dd597c43de-bluemix.cloudantnosqldb.appdomain.cloud")
client.connect()
my_database = client.create_database("url_history")
#dtt=[]
result_collection = Result(my_database.all_docs, descending=True,include_docs=True)
liss=['URL','Property','Domain','Registrar','Organisation','Alexa Rank','Address','City','State','Zipcode','Country','E-mails','time']
html_p="""<!DOCTYPE html><html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<!-- Font Awesome -->
<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.11.2/css/all.css">
<!-- Google Fonts Roboto -->
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700&display=swap">
<!-- Bootstrap core CSS -->
<link rel="stylesheet" href="static/css/bootstrap.min.css">
<!-- Material Design Bootstrap -->
<link rel="stylesheet" href="static/css/mdb.min.css'">
<!-- Your custom styles (optional) -->
<link rel="stylesheet" href="static/css/style.css">
<!-- MDBootstrap Datatables -->
<link href="static/css/addons/datatables2.min.css" rel="stylesheet">
<title>Malicious URL Detector</title>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<!-- Bootstrap CSS -->
<!-- Font Awesome -->
<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.8.2/css/all.css">
<!-- Bootstrap core CSS -->
<link href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/4.3.1/css/bootstrap.min.css" rel="stylesheet">
<!-- Material Design Bootstrap -->
<link href="https://cdnjs.cloudflare.com/ajax/libs/mdbootstrap/4.10.1/css/mdb.min.css" rel="stylesheet">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/limonte-sweetalert2/8.11.8/sweetalert2.css">
<script src="https://cdnjs.cloudflare.com/ajax/libs/limonte-sweetalert2/8.11.8/sweetalert2.all.js"></script>
<script src="https://cdn.jsdelivr.net/npm/promise-polyfill"></script>
<link rel="icon" href="/static/46-512.png" type="image/png">
<title>{% block title %}Malicious URL Detector{% endblock %}</title>
</head>
<body>
<!--Navbar-->
<nav class="navbar navbar-expand-lg navbar-dark primary-color fixed-top scrolling-navbar" >
<!-- Navbar brand -->
<!-- Collapse button -->
<button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#basicExampleNav"
aria-controls="basicExampleNav" aria-expanded="false" aria-label="Toggle navigation">
<span class="navbar-toggler-icon"></span>
</button>
<!-- Collapsible content -->
<div class="collapse navbar-collapse" id="basicExampleNav">
<!-- Links -->
<ul class="navbar-nav mr-auto">
<li class="nav-item">
<a class="nav-link" href="/">Home
<span class="sr-only">(current)</span>
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="https://mudvfinal.eu-gb.cf.appdomain.cloud/geturlhistory">URL History Database</a>
</li>
<li class="nav-item">
<a class="nav-link" href="https://mudvfinal.eu-gb.cf.appdomain.cloud/fetchanalysis">Live Data Analysis</a>
</li>
<li class="nav-item">
<a class="nav-link" href="https://mudvfinal.eu-gb.cf.appdomain.cloud/testresults">Test Results</a>
</li>
<li class="nav-item">
<a class="nav-link" href="https://mudvfinal.eu-gb.cf.appdomain.cloud/getuserfeedbackform">User FeedBack</a>
</li>
<li class="nav-item">
<a class="nav-link" href="https://mudvfinal.eu-gb.cf.appdomain.cloud/discuss">Discuss Channel</a>
</li>
<li class="nav-item">
<a class="nav-link" href="https://mudvfinal.eu-gb.cf.appdomain.cloud/getdataset">Download Dataset(URL History)</a>
</li>
<li class="nav-item">
<a class="nav-link" href="https://mudvfinal.eu-gb.cf.appdomain.cloud/cloudantcsv">CloudantDataset</a>
</li>
<li class="nav-item">
<a class="nav-link" href="https://mudvfinal.eu-gb.cf.appdomain.cloud/about">About Application</a>
</li>
<li class="nav-item">
<a class="nav-link" href="https://mudvfinal.eu-gb.cf.appdomain.cloud/static/chrome-ext.zip" download>Download Chrome Extension</a>
</li>
</ul>
<!-- Links -->
</div>
<!-- Collapsible content -->
</nav>
<!--/.Navbar-->
</br></br>
</br>
</br>
</br>
<h2 class='mb-3' style="text-align:center;color:black">Dataset (Blacklisting purpose accumulated with Chrome Extension)</h2>
<div class="alert alert-primary alert-dismissible fade show" role="alert" style="margin-right: 25%;
margin-left: 25%;
margin-top: 2%;">
<strong>Recent 2000 URLs are shown</strong>
<button type="button" class="close" data-dismiss="alert" aria-label="Close">
<span aria-hidden="true">×</span>
</button>
</div>
<table id="dtBasicExample" class="table" width="100%" >
<thead>
<tr>
<th>Index Number</th>
<th>URL</th>
<th>Status</th>
<th>Domain</th>
<th>Registrar</th>
<th>Organisation</th>
<th>Alexa Rank</th>
<th>Address</th>
<th>City</th>
<th>State</th>
<th>Zip Code</th>
<th>Country</th>
<th>Email</th>
<th>Time</th>
</tr>
</thead>
<tbody>"""
# for result in result_collection:
# url=str(result['doc']['URL'])
# status=str(result['doc']['Property'])
# name=str(result['doc']['Name'])
# org=str(result['doc']['Organisation'])
# add=str(result['doc']['Address'])
# city=str(result['doc']['City'])
# state=str(result['doc']['State'])
# ziip=str(result['doc']['Zipcode'])
# country=str(result['doc']['Country'])
# email=str(result['doc']['E-mails'])
# dom=str(result['doc']['Domain'])
# rank=str(result['doc']['Alexa Rank'])
# reg=str(result['doc']['Registrar'])
# time=str(result['doc']['time'])
# a=[url,status,name,org,add,city,state,ziip,country,email,dom,rank,reg,time]
# dtt.append(a)
# t.sleep(0.0001)
ii=0
res=result_collection[:]
cot=len(res)
#print ("Length",cot)
if cot>2000:
cott=cot-2000
res=result_collection[cott:]
else:
cott=cot
res=result_collection[0:]
for result in res:
ii=ii+1
html_p=html_p+"<tr>"
html_p=html_p+"<td>"+str(ii)+"</td>"
html_p=html_p+"<td>"+str(result['doc']['URL'])+"</td>"
html_p=html_p+"<td>"+str(result['doc']['Property'])+"</td>"
html_p=html_p+"<td>"+str(result['doc']['Domain'])+"</td>"
html_p=html_p+"<td>"+str(result['doc']['Registrar'])+"</td>"
html_p=html_p+"<td>"+str(result['doc']['Organisation'])+"</td>"
html_p=html_p+"<td>"+str(result['doc']['Alexa Rank'])+"</td>"
html_p=html_p+"<td>"+str(result['doc']['Address'])+"</td>"
html_p=html_p+"<td>"+str(result['doc']['City'])+"</td>"
html_p=html_p+"<td>"+str(result['doc']['State'])+"</td>"
html_p=html_p+"<td>"+str(result['doc']['Zipcode'])+"</td>"
html_p=html_p+"<td>"+str(result['doc']['Country'])+"</td>"
html_p=html_p+"<td>"+str(result['doc']['E-mails'])+"</td>"
html_p=html_p+"<td>"+str(result['doc']['time'])+"</td>"
html_p=html_p+"</tr>"
t.sleep(0.0000001)
html_p=html_p+"""</tbody></table></br></br>
<!-- End your project here-->
<script>
Swal.fire(
'Most recent 2000 URLs from Cloudant Database',
'Loaded successfully!',
'success'
)
</script>
<div class="container-fluid">
<!-- Footer -->
<footer class="page-footer font-small blue fixed-bottom">
<!-- Copyright -->
<div class="footer-copyright text-center py-3"
style="background: black;color: white;font-weight:bold;font-size:0.5rem;">
<span>© 2020 Copyright: <a href="https://abhisheksportfolio.herokuapp.com/">Abhishek Saxena
<a class="ins-ic" href="https://github.com/abhisheksaxena1998"><i class="fab fa-github fa-3x"></i></a></span>
</div>
<!-- Copyright -->
</footer>
<!-- Footer -->
</div>
<!-- Optional JavaScript -->
<!-- jQuery first, then Popper.js, then Bootstrap JS -->
<!-- JQuery -->
<!-- JQuery -->
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js"></script>
<!-- Bootstrap tooltips -->
<script type="text/javascript"
src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.4/umd/popper.min.js"></script>
<!-- Bootstrap core JavaScript -->
<script type="text/javascript"
src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/4.3.1/js/bootstrap.min.js"></script>
<!-- MDB core JavaScript -->
<script type="text/javascript"
src="https://cdnjs.cloudflare.com/ajax/libs/mdbootstrap/4.10.1/js/mdb.min.js"></script>
</body>
</html>
<!-- jQuery -->
<script type="text/javascript" src="static/js/jquery.min.js"></script>
<!-- Bootstrap tooltips -->
<script type="text/javascript" src="static/js/popper.min.js"></script>
<!-- Bootstrap core JavaScript -->
<script type="text/javascript" src="static/js/bootstrap.min.js"></script>
<!-- MDB core JavaScript -->
<script type="text/javascript" src="static/js/mdb.min.js"></script>
<!-- MDBootstrap Datatables -->
<script type="text/javascript" src="static/js/addons/datatables2.min.js"></script>
<script>
$(document).ready(function () {
$('#dtBasicExample').DataTable();
$('.dataTables_length').addClass('bs-select');
});
</script>
</body>
</html>"""
return html_p
# Cloudant認証情報の取得
f = open('cloudant_credentials_id.json', 'r')
cred = json.load(f)
f.close()
print cred
# データベース名の取得
f = open('database_name.json', 'r')
dbn = json.load(f)
f.close()
print dbn['name']
client = Cloudant(cred['credentials']['username'],
cred['credentials']['password'],
url=cred['credentials']['url'])
# Connect to the server
client.connect()
# Create a database using an initialized client
# The result is a new CloudantDatabase or CouchDatabase based on the client
my_database = client.create_database(dbn['name'])
# You can check that the database exists
if my_database.exists():
print 'SUCCESS!!'
# Disconnect from the server
client.disconnect()
db_name = 'mydb'
applications_db = 'applications'
client = None
db = None
appi_db = None
if 'VCAP_SERVICES' in os.environ:
vcap = json.loads(os.getenv('VCAP_SERVICES'))
print('Found VCAP_SERVICES')
if 'cloudantNoSQLDB' in vcap:
creds = vcap['cloudantNoSQLDB'][0]['credentials']
user = creds['username']
password = creds['password']
url = 'https://' + creds['host']
client = Cloudant(user, password, url=url, connect=True)
db = client.create_database(db_name, throw_on_exists=False)
appi_db = client.create_database(applications_db,
throw_on_exists=False)
elif "CLOUDANT_URL" in os.environ:
client = Cloudant(os.environ['CLOUDANT_USERNAME'],
os.environ['CLOUDANT_PASSWORD'],
url=os.environ['CLOUDANT_URL'],
connect=True)
db = client.create_database(db_name, throw_on_exists=False)
appi_db = client.create_database(applications_db, throw_on_exists=False)
elif os.path.isfile('vcap-local.json'):
with open('vcap-local.json') as f:
vcap = json.load(f)
print('Found local VCAP_SERVICES')
creds = vcap['services']['cloudantNoSQLDB'][0]['credentials']
client.connect()
dbs= client.all_dbs()
print dbs
for db in dbs:
print str(db)
if str(db) == 'my_database':
temp = 1
break
else:
temp =0
if temp==1:
my_database = client['my_database']
else:
my_database = client.create_database('my_database')
app = Flask(__name__)
@app.route('/')
def Welcome():
return render_template('index.html')
@app.route('/upload/', methods=['POST','GET'])
def upload():
version_number=1
if request.method=='POST':
file = request.files['myFile']
de=request.form['desc']
if 'VCAP_SERVICES' in os.environ:
vcap = json.loads(os.getenv('VCAP_SERVICES'))
print('Found VCAP_SERVICES')
if 'cloudantNoSQLDB' in vcap:
creds = vcap['cloudantNoSQLDB'][0]['credentials']
user = creds['username']
#password = creds['password']
api_key = creds['apikey']
#url = 'https://' + creds['host']
url = creds['url']
client = Cloudant.iam(user, api_key, connect=True)
#db = client.create_database(db_name, throw_on_exists=False)
elif "CLOUDANT_URL" in os.environ:
client = Cloudant(os.environ['CLOUDANT_USERNAME'], os.environ['CLOUDANT_PASSWORD'], url=os.environ['CLOUDANT_URL'], connect=True)
db = client.create_database(db_name, throw_on_exists=False)
elif os.path.isfile('vcap-local.json'):
with open('vcap-local.json') as f:
vcap = json.load(f)
print('Found local VCAP_SERVICES')
creds = vcap['services']['cloudantNoSQLDB'][0]['credentials']
user = creds['username']
#password = creds['password']
api_key = creds['apikey']
#url = 'https://' + creds['host']
url = creds['url']
client = Cloudant.iam(user, api_key, connect=True)
#db = client.create_database(db_name, throw_on_exists=False)
# On IBM Cloud Cloud Foundry, get the port number from the environment variable PORT
# When running this app on the local machine, default the port to 8000
ibm_service_instance_id=COS_RESOURCE_CRN,
ibm_auth_endpoint=COS_AUTH_ENDPOINT,
config=Config(signature_version="oauth"),
endpoint_url=COS_ENDPOINT)
client = Cloudant(
"e110f3bc-b2e8-483b-9c09-7ebd72c72c1b-bluemix",
"962ef67bd0897965b5ce8d4964fee0cf992c757adb5a35a6c185163dda592dbf",
url=
"https://*****:*****@e110f3bc-b2e8-483b-9c09-7ebd72c72c1b-bluemix.cloudantnosqldb.appdomain.cloud"
)
client.connect()
#Provide your database name
database_name = "sample"
my_database = client.create_database(database_name)
if my_database.exists():
print(f"'{database_name}' successfully created.")
def audiofile(a):
with open(join(dirname(__file__), './.', a), 'rb') as audio_file:
results = speech_to_text.recognize(
audio=audio_file,
content_type='audio/mp3',
).get_result()
b = results['results'][0]['alternatives'][0]['transcript']
print(b)
if b[0:4] == "help":
cam()
elif b[0:5] == "hello":
class CloudantDB(threading.Thread):
def _readConfig(self):
update = False
if not os.path.isdir(self._homeDir):
print "Creating homeDir"
os.makedirs(self._homeDir)
if os.path.isfile(self._configFileName):
self._config.read(self._configFileName)
else:
print "Config file not found"
update = True
if not self._config.has_section('REDIS'):
print "Adding Redis part"
update = True
self._config.add_section("REDIS")
if not self._config.has_option("REDIS", "ServerAddress"):
print "No Server Address"
update = True
self._config.set("REDIS", "ServerAddress", "<ServerAddress>")
if not self._config.has_option("REDIS", "ServerPort"):
print "No Server Port"
update = True
self._config.set("REDIS", "ServerPort", "6379")
if not self._config.has_section('CLOUDANT'):
print "Adding Cloudant part"
update = True
self._config.add_section("CLOUDANT")
if not self._config.has_option("CLOUDANT", "ServerAddress"):
print "No Server Address"
update = True
self._config.set("CLOUDANT", "ServerAddress", "<ServerAddress>")
if not self._config.has_option("CLOUDANT", "Username"):
print "No Username"
update = True
self._config.set("CLOUDANT", "Username", "Didditulle")
if not self._config.has_option("CLOUDANT", "Password"):
print "No Password"
update = True
self._config.set("CLOUDANT", "Password", "geheim")
if update:
with open(self._configFileName, 'w') as f:
self._config.write(f)
def __init__(self):
threading.Thread.__init__(self)
self.setDaemon(True)
self._homeDir = os.path.expanduser("~/.sensomatic")
self._configFileName = self._homeDir + '/config.ini'
self._config = ConfigParser.ConfigParser()
self._readConfig()
self._redis = redis.StrictRedis(host=self._config.get("REDIS", "ServerAddress"), port=self._config.get("REDIS", "ServerPort"), db=0)
self._cloudant = Cloudant(self._config.get("CLOUDANT", "Username"), self._config.get("CLOUDANT", "Password"), url=self._config.get("CLOUDANT", "ServerAddress"))
self.checkDB()
def checkDB(self):
self._cloudant.connect()
if "usshorizon" in self._cloudant.all_dbs():
self._database = self._cloudant['usshorizon']
return True
else:
print "Create DB"
self._database = self._cloudant.create_database('usshorizon')
if self._database.exists():
print "Success"
return True
else:
print "Error"
return False
def getData(self):
data = {'timestamp':time.time()}
for key in self._redis.keys():
k = key.split('/')
l = len(k)
w = data
for i in range(l):
if k[i] not in w:
w[k[i]] = {}
w = w[k[i]]
if i == l - 1:
w['value'] = self._redis.get(key)
return data
def run(self):
while True:
d = self.getData()
# print d
try:
if self.checkDB():
self._database.create_document(d)
except Exception as e:
print e
time.sleep(60)
time.sleep(10)
