def add_node(cls, user_id, cmp_token, page_id, page_token):
"""
Create a new document where
:param str user_id: User's facebook id
:param str cmp_token: Token that represent the installation
:param str page_id: page id to link feeds
:param str page_token: page valid token
"""
document = cls.getcompanytoken(page_id)
if document is None:
node = {
"user_id": user_id,
"dbname": cmp_token,
"page_id": page_id,
"page_token": page_token
}
with cloudant(USERNAME, PASSWORD, url=URL) as client:
my_database = client[DBNAME]
my_document = my_database.create_document(node)
node["_id"] = my_document["_id"]
else:
with cloudant(USERNAME, PASSWORD, url=URL) as client:
my_database = client[DBNAME]
node = my_database[document["_id"]]
node["user_id"] = user_id
node["dbname"] = cmp_token
node["page_token"] = page_token
node.save()
return node
def extract_app_details(top_apps):
""" Grab all desired content from each specific app page within top 100 free
apps: id, name, description, image, rating, total_reviews
"""
for app in top_apps:
app_id = app["id"]
name = to_ascii(app["title"])
# Go to app's iTunes page.
app_url = app["url"]
r = requests.get(app_url)
# Extract information App Insights requires.
soup = BeautifulSoup(r.text, 'html.parser')
description = find_description(soup)
image = find_image(soup)
category = find_category(soup)
(rating, total_reviews) = find_rating_and_num_reviews(soup)
upload_to_db(app_id, name, category, description, image, rating,
total_reviews, app["title"])
# Check all documents uploaded into cloudant.
with cloudant(Settings.CLOUDANT_USERNAME,
Settings.CLOUDANT_PASSWORD,
url=Settings.CLOUDANT_URL) as client:
session = client.session()
documents = []
for document in client[Settings.DATABASE_NAME]:
documents.append(document)
# Don't overload cloudant
time.sleep(.300)
return {'results': documents}
def test_database_with_two_docs(self):
"""
_test_database_with_two_docs_
Test to make sure that our iterator works in the case where
there are fewer docs to retrieve than it retrieves in one
chunk.
"""
dbname = "cloudant-itertest-twodocs-{0}".format(unicode(uuid.uuid4()))
self.last_db = dbname
with cloudant(self.user, self.password, account=self.user) as c:
session = c.session()
db = c.create_database(dbname)
doc1 = db.create_document(
{"_id": "doc1", "testing": "doc1"}
)
doc2 = db.create_document(
{"_id": "doc2", "testing": "doc2"}
)
docs = []
# Make sure that iterator fetches docs
for doc in db:
docs.append(doc)
self.assertEqual(len(docs), 2)
def test_database_with_many_docs(self):
"""
_test_database_with_many_docs_
Test to make sure that we can iterator through stuff
"""
dbname = "cloudant-itertest-manydocs-{0}".format(unicode_(uuid.uuid4()))
self.last_db = dbname
with cloudant(self.user, self.password, account=self.user) as c:
session = c.session()
db = c.create_database(dbname)
for i in range(0,300):
db.create_document({
"_id": "doc{0}".format(i),
"testing": "document {0}".format(i)
})
docs = []
for doc in db:
docs.append(doc)
self.assertEqual(len(docs), 300)
unique_ids = set([doc['id'] for doc in docs])
self.assertEqual(len(unique_ids), 300)
def test_database_with_two_docs(self):
"""
_test_database_with_two_docs_
Test to make sure that our iterator works in the case where
there are fewer docs to retrieve than it retrieves in one
chunk.
"""
dbname = "cloudant-itertest-twodocs-{0}".format(unicode_(uuid.uuid4()))
self.last_db = dbname
with cloudant(self.user, self.password, account=self.user) as c:
session = c.session()
db = c.create_database(dbname)
doc1 = db.create_document(
{"_id": "doc1", "testing": "doc1"}
)
doc2 = db.create_document(
{"_id": "doc2", "testing": "doc2"}
)
docs = []
# Make sure that iterator fetches docs
for doc in db:
docs.append(doc)
self.assertEqual(len(docs), 2)
def test_follow_replication(self):
"""
_test_follow_replication_
Test to make sure that we can follow a replication.
"""
dbsource = u"test_follow_replication_source_{}".format(unicode(uuid.uuid4()))
dbtarget = u"test_follow_replication_target_{}".format(unicode(uuid.uuid4()))
self.dbs = [dbsource, dbtarget]
with cloudant(self.user, self.passwd, account=self.user) as c:
dbs = c.create_database(dbsource)
dbt = c.create_database(dbtarget)
doc1 = dbs.create_document({"_id": "doc1", "testing": "document 1"})
doc2 = dbs.create_document({"_id": "doc2", "testing": "document 1"})
doc3 = dbs.create_document({"_id": "doc3", "testing": "document 1"})
replicator = ReplicatorDatabase(c)
repl_id = u"test_follow_replication_{}".format(unicode(uuid.uuid4()))
self.replication_ids.append(repl_id)
ret = replicator.create_replication(source_db=dbs, target_db=dbt, repl_id=repl_id, continuous=False)
updates = [update for update in replicator.follow_replication(repl_id)]
self.assertTrue(len(updates) > 0)
self.assertEqual(updates[-1]["_replication_state"], "completed")
def load_config(config_file):
config_json = json.load(config_file)
config_file.close()
config['cloudant_auth'] = config_json['cloudant_auth']
config['cloudant_user'] = config_json['cloudant_user']
config['cloudant_account'] = config_json['cloudant_account']
config['relationship'] = config_json['relationship']
config['host_id'] = config_json['host_id']
config['doc_threshold'] = config_json['threshold']
# Connect to database
with cloudant(config['cloudant_user'],
config['cloudant_auth'],
account=config['cloudant_user']) as client:
db = client[config['main_db_name']]
#db = CloudantDatabase(client, config['main_db_name'])
# Read in configuration of relationship from database
with Document(db, config['relationship']) as relationshipdoc:
config['rsync_flags'] = relationshipdoc['rsyncflags']
# config['rsync_excluded'] = relationshipdoc['excludedfiles']
config['rsync_source'] = relationshipdoc['sourcehost']
config['rsync_target'] = relationshipdoc['targethost']
config['rsync_source_dir'] = relationshipdoc['sourcedir']
config['rsync_target_dir'] = relationshipdoc['targetdir']
# Get hosts' IP addresses and names
with Document(db, config['rsync_source']) as doc:
config['source_ip'] = doc['ip4']
config['source_name'] = doc['hostname']
with Document(db, config['rsync_target']) as doc:
config['target_name'] = doc['hostname']
config['target_ip'] = doc['ip4']
def test_database_with_many_docs(self):
"""
_test_database_with_many_docs_
Test to make sure that we can iterator through stuff
"""
dbname = "cloudant-itertest-manydocs-{0}".format(unicode(uuid.uuid4()))
self.last_db = dbname
with cloudant(self.user, self.password, account=self.user) as c:
session = c.session()
db = c.create_database(dbname)
for i in range(0,300):
db.create_document({
"_id": "doc{0}".format(i),
"testing": "document {0}".format(i)
})
docs = []
for doc in db:
docs.append(doc)
self.assertTrue(len(docs) == 300)
unique_ids = set([doc['id'] for doc in docs])
self.assertTrue(len(unique_ids) == 300)
def test_replication_state(self):
"""
_test_replication_state_
Verify that we can get the replication state.
"""
dbsource = u"test_replication_state_source_{}".format(
unicode(uuid.uuid4()))
dbtarget = u"test_replication_state_target_{}".format(
unicode(uuid.uuid4()))
self.dbs = [dbsource, dbtarget]
with cloudant(self.user, self.passwd) as c:
dbs = c.create_database(dbsource)
dbt = c.create_database(dbtarget)
doc1 = dbs.create_document(
{"_id": "doc1", "testing": "document 1"}
)
doc2 = dbs.create_document(
{"_id": "doc2", "testing": "document 1"}
)
doc3 = dbs.create_document(
{"_id": "doc3", "testing": "document 1"}
)
replicator = ReplicatorDatabase(c)
repl_id = u"test_replication_state_{}".format(
unicode(uuid.uuid4()))
self.replication_ids.append(repl_id)
ret = replicator.create_replication(
source_db=dbs,
target_db=dbt,
repl_id=repl_id,
continuous=False,
)
replication_state = "not_yet_set"
while True:
# Verify that replication_state returns either None
# (if the field doesn't exist yet), or a valid
# replication state.
replication_state = replicator.replication_state(repl_id)
if replication_state is not None:
self.assertTrue(
replication_state in [
'completed',
'error',
'triggered'
]
)
if replication_state in ('error', 'completed'):
break
LOG.debug("got replication state: {}".format(
replication_state))
time.sleep(1)
def tearDown(self):
with cloudant(self.user, self.passwd) as c:
replicator_db = ReplicatorDatabase(c)
while self.replication_ids:
replicator_db.stop_replication(self.replication_ids.pop())
while self.dbs:
c.delete_database(self.dbs.pop())
def tearDown(self):
with cloudant(self.user, self.passwd, account=self.user) as c:
replicator = Replicator(c)
while self.replication_ids:
replicator.stop_replication(self.replication_ids.pop())
while self.dbs:
c.delete_database(self.dbs.pop())
def tearDown(self):
with cloudant(self.user, self.passwd, account=self.user) as c:
replicator = Replicator(c)
while self.replication_ids:
replicator.stop_replication(self.replication_ids.pop())
while self.dbs:
c.delete_database(self.dbs.pop())
def delete_database():
""" Delete the cloudant database created.
"""
with cloudant(Settings.CLOUDANT_USERNAME,
Settings.CLOUDANT_PASSWORD,
url=Settings.CLOUDANT_URL) as client:
session = client.session()
client.delete_database(Settings.DATABASE_NAME)
return
def test_replication_state(self):
"""
_test_replication_state_
Verify that we can get the replication state.
"""
dbsource = unicode_("test_replication_state_source_{}".format(
unicode_(uuid.uuid4())))
dbtarget = unicode_("test_replication_state_target_{}".format(
unicode_(uuid.uuid4())))
self.dbs = [dbsource, dbtarget]
with cloudant(self.user, self.passwd, account=self.user) as c:
dbs = c.create_database(dbsource)
dbt = c.create_database(dbtarget)
doc1 = dbs.create_document({
"_id": "doc1",
"testing": "document 1"
})
doc2 = dbs.create_document({
"_id": "doc2",
"testing": "document 1"
})
doc3 = dbs.create_document({
"_id": "doc3",
"testing": "document 1"
})
replicator = Replicator(c)
repl_id = unicode_("test_replication_state_{}".format(
unicode_(uuid.uuid4())))
self.replication_ids.append(repl_id)
ret = replicator.create_replication(
source_db=dbs,
target_db=dbt,
repl_id=repl_id,
continuous=False,
)
replication_state = "not_yet_set"
while True:
# Verify that replication_state returns either None
# (if the field doesn't exist yet), or a valid
# replication state.
replication_state = replicator.replication_state(repl_id)
if replication_state is not None:
self.assertTrue(replication_state in
['completed', 'error', 'triggered'])
if replication_state in ('error', 'completed'):
break
LOG.debug(
"got replication state: {}".format(replication_state))
time.sleep(1)
def test_init(self):
"""
_test_init_
Verify that we can init our database object.
"""
with cloudant(self.user, self.passwd, account=self.user) as c:
replicator = Replicator(c)
replicator.all_docs()
def del_node(cls, page_id):
"""
Delete a node with page_id
"""
document = cls.getcompanytoken(page_id)
if document is not None:
with cloudant(USERNAME, PASSWORD, url=URL) as client:
database = client[DBNAME]
todelete = database[document["_id"]]
todelete.delete()
def test_cloudant_context_helper(self):
"""
Test that the cloudant context helper works as expected.
"""
try:
with cloudant(self.user, self.pwd, account=self.account) as c:
self.assertIsInstance(c, Cloudant)
self.assertIsInstance(c.r_session, requests.Session)
except Exception as err:
self.fail('Exception {0} was raised.'.format(str(err)))
def test_init(self):
"""
_test_init_
Verify that we can init our database object.
"""
with cloudant(self.user, self.passwd) as c:
replicator = ReplicatorDatabase(c)
replicator.all_docs()
def test_cloudant_context_helper(self):
"""
Test that the cloudant context helper works as expected.
"""
try:
with cloudant(self.user, self.pwd, account=self.account) as c:
self.assertIsInstance(c, Cloudant)
self.assertIsInstance(c.r_session, requests.Session)
except Exception as err:
self.fail('Exception {0} was raised.'.format(str(err)))
def getcompanytoken(cls, page_id):
"""
Get the document that represent the facebook page.
If not exists return None
"""
with cloudant(USERNAME, PASSWORD, url=URL) as client:
database = client[DBNAME]
query = Query(database, selector={"page_id": {"$eq": page_id}})
for node in query(limit=1)['docs']:
return node
return None
def test_delete(self):
with cloudant(self.user, self.passwd) as c:
db = c.create_database(self.dbname)
doc1 = db.create_document({"_id": "doc1", "testing": "document 1"})
doc1.save()
doc1.fetch()
doc1.delete()
self.assertRaises(requests.HTTPError, doc1.fetch)
def create_index(cls):
"""
Create an Indexs for the "page_id" attribute
"""
with cloudant(USERNAME, PASSWORD, url=URL) as client:
clouddb = CloudantDatabase(client, DBNAME)
clouddb.create_query_index(index_name='pageid-index',
index_type='text',
fields=[{
"name": "page_id",
"type": "string"
}])
def test_delete(self):
with cloudant(self.user, self.passwd, account=self.user) as c:
db = c.create_database(self.dbname)
doc1 = db.create_document({"_id": "doc1", "testing": "document 1"})
doc1.save()
doc1.fetch()
doc1.delete()
self.assertRaises(requests.HTTPError, doc1.fetch)
def add_document(cls, dbname, document):
"""
Create a new document where
:param str dbname: The database name to insert the new document
:param json document: Document with feed info
"""
with cloudant(USERNAME, PASSWORD, url=URL) as client:
my_database = client[dbname]
my_document = my_database.create_document(document)
document["_id"] = my_document["_id"]
return document
def scanning_errors_new(
scan_db,
scan_id): #Still broken due to cloudant-python "group_level" bug
errors = 0
ddoc = scandb_views['problem_files'][0]
view = scandb_views['problem_files'][1]
with cloudant(config['cloudant_user'],
config['cloudant_auth'],
account=config['cloudant_user']) as client:
db = client[scan_db]
result = db.get_view_result(ddoc, view, reduce=True, group_level=1)
errors = result[[scan_id, None, None]:[scan_id, {}, {}]]['value']
return errors
def test_end_to_end(self):
"""
End to end database and document crud tests
"""
with cloudant(self.user, self.passwd, account=self.user) as c:
session = c.session()
self.assertEqual(session['userCtx']['name'], self.user)
db = c.create_database(self.dbname)
try:
self.assertTrue(self.dbname in c)
self.assertTrue(db.exists())
# creating docs
doc1 = db.new_document()
doc2 = db.create_document({'_id': 'womp',
"testing": "document2"})
doc3 = db.create_document({"testing": "document3"})
self.assertTrue('_id' in doc1)
self.assertTrue('_rev' in doc1)
self.assertTrue('_id' in doc2)
self.assertTrue('_rev' in doc2)
self.assertTrue('_id' in doc3)
self.assertTrue('_rev' in doc3)
# verifying access via dict api
self.assertTrue(doc1['_id'] in db)
self.assertTrue(doc2['_id'] in db)
self.assertTrue(doc3['_id'] in db)
self.assertTrue(db[doc1['_id']] == doc1)
self.assertTrue(db[doc2['_id']] == doc2)
self.assertTrue(db[doc3['_id']] == doc3)
# test working context for updating docs
with doc2 as working_doc:
working_doc['field1'] = [1, 2, 3]
working_doc['field2'] = {'a': 'b'}
self.assertEqual(
c[self.dbname]['womp']['field2'],
{'a': 'b'}
)
finally:
# remove test database
c.delete_database(self.dbname)
def test_end_to_end(self):
"""
End to end database and document crud tests
"""
with cloudant(self.user, self.passwd, account=self.user) as c:
session = c.session()
self.assertEqual(session['userCtx']['name'], self.user)
db = c.create_database(self.dbname)
try:
self.assertIn(self.dbname, c)
self.assertTrue(db.exists())
# creating docs
doc1 = db.new_document()
doc2 = db.create_document({'_id': 'womp',
"testing": "document2"})
doc3 = db.create_document({"testing": "document3"})
self.assertIn('_id', doc1)
self.assertIn('_rev', doc1)
self.assertIn('_id', doc2)
self.assertIn('_rev', doc2)
self.assertIn('_id', doc3)
self.assertIn('_rev', doc3)
# verifying access via dict api
self.assertIn(doc1['_id'], db)
self.assertIn(doc2['_id'], db)
self.assertIn(doc3['_id'], db)
self.assertEqual(db[doc1['_id']], doc1)
self.assertEqual(db[doc2['_id']], doc2)
self.assertEqual(db[doc3['_id']], doc3)
# test working context for updating docs
with doc2 as working_doc:
working_doc['field1'] = [1, 2, 3]
working_doc['field2'] = {'a': 'b'}
self.assertEqual(
c[self.dbname]['womp']['field2'],
{'a': 'b'}
)
finally:
# remove test database
c.delete_database(self.dbname)
def test_follow_replication_with_errors(self):
"""
_test_follow_replication_with_errors_
Test to make sure that we exit the follow loop when we submit
a bad replication.
"""
dbsource = unicode_("test_follow_replication_source_error_{}".format(
unicode_(uuid.uuid4())))
dbtarget = unicode_("test_follow_replication_target_error_{}".format(
unicode_(uuid.uuid4())))
self.dbs = [dbsource, dbtarget]
with cloudant(self.user, self.passwd, account=self.user) as c:
dbs = c.create_database(dbsource)
dbt = c.create_database(dbtarget)
doc1 = dbs.create_document({
"_id": "doc1",
"testing": "document 1"
})
doc2 = dbs.create_document({
"_id": "doc2",
"testing": "document 1"
})
doc3 = dbs.create_document({
"_id": "doc3",
"testing": "document 1"
})
replicator = Replicator(c)
repl_id = unicode_("test_follow_replication_{}".format(
unicode_(uuid.uuid4())))
self.replication_ids.append(repl_id)
ret = replicator.create_replication(
source_db=dbs,
target_db=dbt,
# Deliberately override these good params with bad params
source=dbsource + "foo",
target=dbtarget + "foo",
repl_id=repl_id,
continuous=False,
)
updates = [
update for update in replicator.follow_replication(repl_id)
]
self.assertTrue(len(updates) > 0)
self.assertEqual(updates[-1]['_replication_state'], 'error')
def dashboard():
data = config_app()
with cloudant(data.config.get('USER_NAME'), data.config.get('PASSWORD'), account=data.config.get('ACCOUNT')) as client:
my_db = client.all_dbs()
try:
my_database = client[data.config.get('DATABASE_NAME')]
except Exception as error:
print("Error opening the Database: " + error)
try:
for document in my_database:
flash(document)
except Exception as err:
print("Error retrieving data" + err)
return ', '.join(my_db)
def test_create_replication(self):
"""
_test_create_replication_
Make a couple of test databases, and confirm that docs from
one get transferred to t'other.
"""
dbsource = u"test_create_replication_source_{}".format(unicode(uuid.uuid4()))
dbtarget = u"test_create_replication_target_{}".format(unicode(uuid.uuid4()))
self.dbs = [dbsource, dbtarget]
with cloudant(self.user, self.passwd, account=self.user) as c:
dbs = c.create_database(dbsource)
dbt = c.create_database(dbtarget)
doc1 = dbs.create_document({"_id": "doc1", "testing": "document 1"})
doc2 = dbs.create_document({"_id": "doc2", "testing": "document 1"})
doc3 = dbs.create_document({"_id": "doc3", "testing": "document 1"})
replicator = ReplicatorDatabase(c)
repl_id = u"test_create_replication_{}".format(unicode(uuid.uuid4()))
self.replication_ids.append(repl_id)
ret = replicator.create_replication(source_db=dbs, target_db=dbt, repl_id=repl_id, continuous=False)
try:
repl_doc = replicator[repl_id]
except KeyError:
repl_doc = None
if not repl_doc or not (repl_doc.get("_replication_state", "none") in ("completed, error")):
for change in replicator.changes():
if change.get("id") == repl_id:
try:
repl_doc = replicator[repl_id]
repl_doc.fetch()
except KeyError:
pass
if repl_doc and (repl_doc.get("_replication_state", "none") in ("completed", "error")):
break
else:
LOG.debug(u"Waiting for replication to complete " u"(repl_doc: {})".format(repl_doc))
self.assertTrue(repl_doc)
self.assertEqual(repl_doc.get("_replication_state"), "completed")
for d in ["doc1", "doc2", "doc3"]:
self.assertTrue(dbt[d])
self.assertEqual(dbt[d]["testing"], dbs[d]["testing"])
def test_follow_replication_with_errors(self):
"""
_test_follow_replication_with_errors_
Test to make sure that we exit the follow loop when we submit
a bad replication.
"""
dbsource = u"test_follow_replication_source_error_{}".format(
unicode(uuid.uuid4()))
dbtarget = u"test_follow_replication_target_error_{}".format(
unicode(uuid.uuid4()))
self.dbs = [dbsource, dbtarget]
with cloudant(self.user, self.passwd) as c:
dbs = c.create_database(dbsource)
dbt = c.create_database(dbtarget)
doc1 = dbs.create_document(
{"_id": "doc1", "testing": "document 1"}
)
doc2 = dbs.create_document(
{"_id": "doc2", "testing": "document 1"}
)
doc3 = dbs.create_document(
{"_id": "doc3", "testing": "document 1"}
)
replicator = ReplicatorDatabase(c)
repl_id = u"test_follow_replication_{}".format(
unicode(uuid.uuid4()))
self.replication_ids.append(repl_id)
ret = replicator.create_replication(
source_db=dbs,
target_db=dbt,
# Deliberately override these good params with bad params
source=dbsource + "foo",
target=dbtarget + "foo",
repl_id=repl_id,
continuous=False,
)
updates = [
update for update in replicator.follow_replication(repl_id)
]
self.assertTrue(len(updates) > 0)
self.assertEqual(updates[-1]['_replication_state'], 'error')
def test_follow_replication(self):
"""
_test_follow_replication_
Test to make sure that we can follow a replication.
"""
dbsource = unicode_("test_follow_replication_source_{}".format(
unicode_(uuid.uuid4())))
dbtarget = unicode_("test_follow_replication_target_{}".format(
unicode_(uuid.uuid4())))
self.dbs = [dbsource, dbtarget]
with cloudant(self.user, self.passwd, account=self.user) as c:
dbs = c.create_database(dbsource)
dbt = c.create_database(dbtarget)
doc1 = dbs.create_document({
"_id": "doc1",
"testing": "document 1"
})
doc2 = dbs.create_document({
"_id": "doc2",
"testing": "document 1"
})
doc3 = dbs.create_document({
"_id": "doc3",
"testing": "document 1"
})
replicator = Replicator(c)
repl_id = unicode_("test_follow_replication_{}".format(
unicode_(uuid.uuid4())))
self.replication_ids.append(repl_id)
ret = replicator.create_replication(
source_db=dbs,
target_db=dbt,
repl_id=repl_id,
continuous=False,
)
updates = [
update for update in replicator.follow_replication(repl_id)
]
self.assertTrue(len(updates) > 0)
self.assertEqual(updates[-1]['_replication_state'], 'completed')
def files_scanned_new(
scan_db, scan_id,
host_id): #Still broken due to cloudant-python "group_level" bug
ddoc = scandb_views['file_types'][0]
view = scandb_views['file_types'][1]
stats = dict()
with cloudant(config['cloudant_user'],
config['cloudant_auth'],
account=config['cloudant_user']) as client:
db = client[scan_db]
result = db.get_view_result(ddoc, view, group_level=2, reduce=True)
stats = result[[host_id, scan_id, None]:[host_id, scan_id, {}]]
if len(stats) > 0:
return stats
else:
zeroes = dict(sum=0, count=0, min=0, max=0, sumsqr=0)
return zeroes
def delete():
USERNAME = ''
PASSWORD = ''
with cloudant(USERNAME, PASSWORD, url='') as client:
my_db = client['my_database']
file_name = request.form['filename']
for document in my_db:
if document['file_name'] == file_name:
#f2 = file_name.split('.')
#f3 = f2[0] + '.' + f2[1]
document.delete()
return ("File found and deleted")
#document.delete_attachment(file_name)
else:
return ('File not found')
return 'File deleted'
def listing():
result = []
result1 = []
result2 = []
USERNAME = '******'
PASSWORD = ''
with cloudant(USERNAME, PASSWORD, url='') as client:
my_db = client['my_database']
for document in my_db:
result.append(document['file_name'])
result1.append(document['version no'])
result2.append(document['date and time'])
print ''
return render_template("R:/result.html",
result=result,
result1=result1,
result2=result2)
def test_list_replications(self):
"""
_test_list_replications_
Verify that we get a list of replications documents back when
we got to list replications.
"""
with cloudant(self.user, self.passwd) as c:
replicator = ReplicatorDatabase(c)
repl_ids = []
num_reps = 3
for i in range(0, num_reps):
tag = "{0}_{1}".format(i, unicode(uuid.uuid4()))
dbsource = u"test_list_repl_src_{}".format(tag)
dbtarget = u"test_list_repl_tgt_{}".format(tag)
self.dbs.append(dbsource)
self.dbs.append(dbtarget)
dbs = c.create_database(dbsource)
dbt = c.create_database(dbtarget)
doc1 = dbs.create_document(
{"_id": "doc1", "testing": "document 1"}
)
repl_id = u"test_create_replication_{}".format(tag)
self.replication_ids.append(repl_id)
repl_ids.append(repl_id)
ret = replicator.create_replication(
source_db=dbs,
target_db=dbt,
repl_id=repl_id,
continuous=False
)
replications = replicator.list_replications()
ids = [doc['_id'] for doc in replications]
found_ids = [i for i in ids if i in repl_ids]
self.assertEqual(num_reps, len(found_ids))
def test_list_replications(self):
"""
_test_list_replications_
Verify that we get a list of replications documents back when
we got to list replications.
"""
with cloudant(self.user, self.passwd, account=self.user) as c:
replicator = Replicator(c)
repl_ids = []
num_reps = 3
for i in range(0, num_reps):
tag = "{0}_{1}".format(i, unicode_(uuid.uuid4()))
dbsource = unicode_("test_list_repl_src_{}".format(tag))
dbtarget = unicode_("test_list_repl_tgt_{}".format(tag))
self.dbs.append(dbsource)
self.dbs.append(dbtarget)
dbs = c.create_database(dbsource)
dbt = c.create_database(dbtarget)
doc1 = dbs.create_document({
"_id": "doc1",
"testing": "document 1"
})
repl_id = unicode_("test_create_replication_{}".format(tag))
self.replication_ids.append(repl_id)
repl_ids.append(repl_id)
ret = replicator.create_replication(source_db=dbs,
target_db=dbt,
repl_id=repl_id,
continuous=False)
replications = replicator.list_replications()
ids = [doc['_id'] for doc in replications]
found_ids = [i for i in ids if i in repl_ids]
self.assertEqual(num_reps, len(found_ids))
def test_changes_include_docs(self):
"""
_test_changes_include_docs
Test to verify that we can pass 'include_docs' successfully
through the changes pipeline.
"""
dbname = "cloudant-changes-test-with-docs{0}".format(
unicode_(uuid.uuid4()))
self.last_db = dbname
with cloudant(self.user, self.password, account=self.user) as c:
session = c.session()
db = c.create_database(dbname)
n = 0
def make_doc(n):
doc = db.create_document({
"_id": "doc{}".format(n),
"testing": "doc{}".format(n)
})
return doc
doc = make_doc(n)
for change in db.changes(include_docs=True):
LOG.debug(unicode_(change))
if change is not None:
self.assertEqual(change['id'], doc['_id'])
self.assertEqual(
# Verify that doc is included, and looks like
# the right doc.
change.get('doc', {}).get('testing', {}),
'doc{}'.format(n))
n += 1
doc = make_doc(n)
if n > 10:
break
self.assertTrue(n > 10)
def test_changes_include_docs(self):
"""
_test_changes_include_docs
Test to verify that we can pass 'include_docs' successfully
through the changes pipeline.
"""
dbname = "cloudant-changes-test-with-docs{0}".format(
unicode_(uuid.uuid4()))
self.last_db = dbname
with cloudant(self.user, self.password, account=self.user) as c:
session = c.session()
db = c.create_database(dbname)
n = 0
def make_doc(n):
doc = db.create_document(
{"_id": "doc{}".format(n), "testing": "doc{}".format(n)}
)
return doc
doc = make_doc(n)
for change in db.changes(include_docs=True):
LOG.debug(unicode_(change))
if change is not None:
self.assertEqual(change['id'], doc['_id'])
self.assertEqual(
# Verify that doc is included, and looks like
# the right doc.
change.get('doc', {}).get('testing', {}),
'doc{}'.format(n)
)
n += 1
doc = make_doc(n)
if n > 10:
break
self.assertTrue(n > 10)
def get_conn(self):
"""
Opens a connection to the cloudant service and closes it automatically if used as context manager.
.. note::
In the connection form:
- 'host' equals the 'Account' (optional)
- 'login' equals the 'Username (or API Key)' (required)
- 'password' equals the 'Password' (required)
:return: an authorized cloudant session context manager object.
:rtype: cloudant
"""
conn = self.get_connection(self.cloudant_conn_id)
self._validate_connection(conn)
cloudant_session = cloudant(user=conn.login, passwd=conn.password, account=conn.host)
return cloudant_session
def download():
USERNAME = ''
PASSWORD = ''
with cloudant(USERNAME, PASSWORD, url='') as client:
my_db = client['my_database']
file_name = request.form['filename']
for document in my_db:
if (document['file_name'] == file_name):
f2 = file_name.split('.')
f3 = f2[0] + '.' + f2[1]
file = document.get_attachment(f3, attachment_type='binary')
print file
response = make_response(file)
response.headers[
"Content-Disposition"] = "attachment; filename=%s" % file_name
return response
else:
response = 'File not found'
return response
def get_conn(self) -> cloudant:
"""
Opens a connection to the cloudant service and closes it automatically if used as context manager.
.. note::
In the connection form:
- 'host' equals the 'Account' (optional)
- 'login' equals the 'Username (or API Key)' (required)
- 'password' equals the 'Password' (required)
:return: an authorized cloudant session context manager object.
:rtype: cloudant
"""
conn = self.get_connection(self.cloudant_conn_id)
self._validate_connection(conn)
cloudant_session = cloudant(user=conn.login, passwd=conn.password, account=conn.host)
return cloudant_session
def upload_to_db(app_id, name, category, description, image, rating,
total_reviews, raw_name):
""" Uploads app details to cloudant database where id is the key to
the document in cloudant. The document includes the app name, description,
url, rating and the total number of reviews.
"""
# Connect to cloudant. Exits cloudant when block is exited.
with cloudant(Settings.CLOUDANT_USERNAME,
Settings.CLOUDANT_PASSWORD,
url=Settings.CLOUDANT_URL) as client:
session = client.session()
# Create or check for db:
try:
# Create database.
app_db = client.create_database(Settings.DATABASE_NAME)
except:
# Database already exists.
app_db = client[Settings.DATABASE_NAME]
# Check database exists
if not app_db.exists():
raise NonexistentDatabase("The database %s does not exist." %
Settings.DATABASE_NAME)
# Create or update document and perform fetch/save after exiting this block
with Document(app_db, app_id) as doc:
doc['name'] = name.encode('utf-8')
doc['description'] = description.encode('utf-8')
doc['image'] = image
doc['category'] = category
doc['rating'] = rating
doc['total_reviews'] = total_reviews
doc['keyword'] = query_for_top_keywords(raw_name, category)
doc['turnarounds'] = int(query_num_turnarounds(raw_name))
doc['sentiment'] = float(query_average_app_sentiment(raw_name))
# Display document for error checking
print(app_db[app_id])
return
def go():
with cloudant(args.username, args.password, url=args.host) as client:
session = client.session()
source_db = client[args.source]
target_db = client[args.target]
print_metadata("Source", source_db.metadata())
print_metadata("Target", target_db.metadata())
since_seq = get_checkpoint(target_db)
batch = []
for change in source_db.changes(since=since_seq,
continuous=not args.batch,
include_docs=True):
if 'deleted' in change:
batch.append({'_id': change['id'], '_deleted': True})
elif 'doc' in change:
# created or updated
doc = change['doc']
# we don't know whether the target contains this so assume create
del doc['_rev']
batch.append(doc)
# attachments not supported - bail if we find one
if '_attachments' in doc:
raise Exception("Cannot proceed - document has _attachments:\n" + json.dumps(doc))
# If we reach the batch size, upload to the target
if len(batch) == args.batchsize:
upload_batch(change, batch, target_db)
batch = []
# Drain the batch
upload_batch(change, batch, target_db)
def test_changes(self):
"""
_test_changes_
Test to verify that we can connect to a live changes
feed. Verify that we are actually staying connected by
creating new docs while reading from the _changes feed.
"""
dbname = "cloudant-changes-test-{0}".format(unicode_(uuid.uuid4()))
self.last_db = dbname
with cloudant(self.user, self.password, account=self.user) as c:
session = c.session()
db = c.create_database(dbname)
n = 0
def make_doc(n):
doc = db.create_document({
"_id": "doc{}".format(n),
"testing": "doc{}".format(n)
})
return doc
doc = make_doc(n)
for change in db.changes():
LOG.debug(unicode_(change))
if change is not None:
self.assertEqual(change['id'], doc['_id'])
n += 1
doc = make_doc(n)
if n > 10:
break
self.assertTrue(n > 10)
def test_changes(self):
"""
_test_changes_
Test to verify that we can connect to a live changes
feed. Verify that we are actually staying connected by
creating new docs while reading from the _changes feed.
"""
dbname = "cloudant-changes-test-{0}".format(unicode_(uuid.uuid4()))
self.last_db = dbname
with cloudant(self.user, self.password, account=self.user) as c:
session = c.session()
db = c.create_database(dbname)
n = 0
def make_doc(n):
doc = db.create_document(
{"_id": "doc{}".format(n), "testing": "doc{}".format(n)}
)
return doc
doc = make_doc(n)
for change in db.changes():
LOG.debug(unicode_(change))
if change is not None:
self.assertEqual(change['id'], doc['_id'])
n += 1
doc = make_doc(n)
if n > 10:
break
self.assertTrue(n > 10)
def tearDown(self):
with cloudant(self.user, self.passwd) as c:
c.delete_database(self.dbname)
def tearDown(self):
if self.last_db is not None:
with cloudant(self.user, self.password, account=self.user) as c:
c.delete_database(self.last_db)
__author__ = "Ayushi Dalmia"
__email__ = "*****@*****.**"
from cloudant import cloudant
import json
from cloudant.result import Result,ResultByKey
import random
from cloudant.design_document import DesignDocument
from cloudant.query import Query
with open('credentials.json') as f:
cred = json.load(f)
with cloudant(str(cred['credentials']['username']),str(cred['credentials']['password']),url=str(cred['credentials']['url'])) as client:
my_database = client['test']
index = my_database.create_query_index(fields=[{'name': 'description','type':'string'}],index_type='text')
selector = {'$text': "happiest "}
docs = my_database.get_query_result(selector,use_index = index.name)
for d in docs:
print d
'''
#create design doc
ddoc = DesignDocument(my_database,document_id="_design/description")
config_file = os.path.expanduser('~/.config')
config = ConfigParser.RawConfigParser()
config.read(config_file)
acct = config.get('cloudant', 'account')
user = config.get('cloudant', 'user')
pwd = config.get('cloudant', 'pwd')
if not acct or not user or not pwd:
raise Exception('Cannot connect with Cloudant client.')
kafka = config.get('kafka', 'server')
topic = config.get('kafka', 'topic')
if not kafka or not topic:
raise Exception('Cannot connect with Kafka.')
kafka_client = KafkaClient(kafka)
producer = SimpleProducer(kafka_client, async=False)
count = 0
with cloudant(user, pwd, account=acct) as client:
db = client['sdp_kafka_test0000']
if not db.exists():
raise Exception('Database does not exists!!')
for change in db.changes():
producer.send_messages(topic, json.dumps(change))
count = count + 1
print '{0} changes processed'.format(count)
kafka_client.close()
