def add_node(cls, user_id, cmp_token, page_id, page_token): """ Create a new document where :param str user_id: User's facebook id :param str cmp_token: Token that represent the installation :param str page_id: page id to link feeds :param str page_token: page valid token """ document = cls.getcompanytoken(page_id) if document is None: node = { "user_id": user_id, "dbname": cmp_token, "page_id": page_id, "page_token": page_token } with cloudant(USERNAME, PASSWORD, url=URL) as client: my_database = client[DBNAME] my_document = my_database.create_document(node) node["_id"] = my_document["_id"] else: with cloudant(USERNAME, PASSWORD, url=URL) as client: my_database = client[DBNAME] node = my_database[document["_id"]] node["user_id"] = user_id node["dbname"] = cmp_token node["page_token"] = page_token node.save() return node
def extract_app_details(top_apps): """ Grab all desired content from each specific app page within top 100 free apps: id, name, description, image, rating, total_reviews """ for app in top_apps: app_id = app["id"] name = to_ascii(app["title"]) # Go to app's iTunes page. app_url = app["url"] r = requests.get(app_url) # Extract information App Insights requires. soup = BeautifulSoup(r.text, 'html.parser') description = find_description(soup) image = find_image(soup) category = find_category(soup) (rating, total_reviews) = find_rating_and_num_reviews(soup) upload_to_db(app_id, name, category, description, image, rating, total_reviews, app["title"]) # Check all documents uploaded into cloudant. with cloudant(Settings.CLOUDANT_USERNAME, Settings.CLOUDANT_PASSWORD, url=Settings.CLOUDANT_URL) as client: session = client.session() documents = [] for document in client[Settings.DATABASE_NAME]: documents.append(document) # Don't overload cloudant time.sleep(.300) return {'results': documents}
def test_database_with_two_docs(self): """ _test_database_with_two_docs_ Test to make sure that our iterator works in the case where there are fewer docs to retrieve than it retrieves in one chunk. """ dbname = "cloudant-itertest-twodocs-{0}".format(unicode(uuid.uuid4())) self.last_db = dbname with cloudant(self.user, self.password, account=self.user) as c: session = c.session() db = c.create_database(dbname) doc1 = db.create_document( {"_id": "doc1", "testing": "doc1"} ) doc2 = db.create_document( {"_id": "doc2", "testing": "doc2"} ) docs = [] # Make sure that iterator fetches docs for doc in db: docs.append(doc) self.assertEqual(len(docs), 2)
def test_database_with_many_docs(self): """ _test_database_with_many_docs_ Test to make sure that we can iterator through stuff """ dbname = "cloudant-itertest-manydocs-{0}".format(unicode_(uuid.uuid4())) self.last_db = dbname with cloudant(self.user, self.password, account=self.user) as c: session = c.session() db = c.create_database(dbname) for i in range(0,300): db.create_document({ "_id": "doc{0}".format(i), "testing": "document {0}".format(i) }) docs = [] for doc in db: docs.append(doc) self.assertEqual(len(docs), 300) unique_ids = set([doc['id'] for doc in docs]) self.assertEqual(len(unique_ids), 300)
def test_database_with_two_docs(self): """ _test_database_with_two_docs_ Test to make sure that our iterator works in the case where there are fewer docs to retrieve than it retrieves in one chunk. """ dbname = "cloudant-itertest-twodocs-{0}".format(unicode_(uuid.uuid4())) self.last_db = dbname with cloudant(self.user, self.password, account=self.user) as c: session = c.session() db = c.create_database(dbname) doc1 = db.create_document( {"_id": "doc1", "testing": "doc1"} ) doc2 = db.create_document( {"_id": "doc2", "testing": "doc2"} ) docs = [] # Make sure that iterator fetches docs for doc in db: docs.append(doc) self.assertEqual(len(docs), 2)
def test_follow_replication(self): """ _test_follow_replication_ Test to make sure that we can follow a replication. """ dbsource = u"test_follow_replication_source_{}".format(unicode(uuid.uuid4())) dbtarget = u"test_follow_replication_target_{}".format(unicode(uuid.uuid4())) self.dbs = [dbsource, dbtarget] with cloudant(self.user, self.passwd, account=self.user) as c: dbs = c.create_database(dbsource) dbt = c.create_database(dbtarget) doc1 = dbs.create_document({"_id": "doc1", "testing": "document 1"}) doc2 = dbs.create_document({"_id": "doc2", "testing": "document 1"}) doc3 = dbs.create_document({"_id": "doc3", "testing": "document 1"}) replicator = ReplicatorDatabase(c) repl_id = u"test_follow_replication_{}".format(unicode(uuid.uuid4())) self.replication_ids.append(repl_id) ret = replicator.create_replication(source_db=dbs, target_db=dbt, repl_id=repl_id, continuous=False) updates = [update for update in replicator.follow_replication(repl_id)] self.assertTrue(len(updates) > 0) self.assertEqual(updates[-1]["_replication_state"], "completed")
def load_config(config_file): config_json = json.load(config_file) config_file.close() config['cloudant_auth'] = config_json['cloudant_auth'] config['cloudant_user'] = config_json['cloudant_user'] config['cloudant_account'] = config_json['cloudant_account'] config['relationship'] = config_json['relationship'] config['host_id'] = config_json['host_id'] config['doc_threshold'] = config_json['threshold'] # Connect to database with cloudant(config['cloudant_user'], config['cloudant_auth'], account=config['cloudant_user']) as client: db = client[config['main_db_name']] #db = CloudantDatabase(client, config['main_db_name']) # Read in configuration of relationship from database with Document(db, config['relationship']) as relationshipdoc: config['rsync_flags'] = relationshipdoc['rsyncflags'] # config['rsync_excluded'] = relationshipdoc['excludedfiles'] config['rsync_source'] = relationshipdoc['sourcehost'] config['rsync_target'] = relationshipdoc['targethost'] config['rsync_source_dir'] = relationshipdoc['sourcedir'] config['rsync_target_dir'] = relationshipdoc['targetdir'] # Get hosts' IP addresses and names with Document(db, config['rsync_source']) as doc: config['source_ip'] = doc['ip4'] config['source_name'] = doc['hostname'] with Document(db, config['rsync_target']) as doc: config['target_name'] = doc['hostname'] config['target_ip'] = doc['ip4']
def test_database_with_many_docs(self): """ _test_database_with_many_docs_ Test to make sure that we can iterator through stuff """ dbname = "cloudant-itertest-manydocs-{0}".format(unicode(uuid.uuid4())) self.last_db = dbname with cloudant(self.user, self.password, account=self.user) as c: session = c.session() db = c.create_database(dbname) for i in range(0,300): db.create_document({ "_id": "doc{0}".format(i), "testing": "document {0}".format(i) }) docs = [] for doc in db: docs.append(doc) self.assertTrue(len(docs) == 300) unique_ids = set([doc['id'] for doc in docs]) self.assertTrue(len(unique_ids) == 300)
def test_replication_state(self): """ _test_replication_state_ Verify that we can get the replication state. """ dbsource = u"test_replication_state_source_{}".format( unicode(uuid.uuid4())) dbtarget = u"test_replication_state_target_{}".format( unicode(uuid.uuid4())) self.dbs = [dbsource, dbtarget] with cloudant(self.user, self.passwd) as c: dbs = c.create_database(dbsource) dbt = c.create_database(dbtarget) doc1 = dbs.create_document( {"_id": "doc1", "testing": "document 1"} ) doc2 = dbs.create_document( {"_id": "doc2", "testing": "document 1"} ) doc3 = dbs.create_document( {"_id": "doc3", "testing": "document 1"} ) replicator = ReplicatorDatabase(c) repl_id = u"test_replication_state_{}".format( unicode(uuid.uuid4())) self.replication_ids.append(repl_id) ret = replicator.create_replication( source_db=dbs, target_db=dbt, repl_id=repl_id, continuous=False, ) replication_state = "not_yet_set" while True: # Verify that replication_state returns either None # (if the field doesn't exist yet), or a valid # replication state. replication_state = replicator.replication_state(repl_id) if replication_state is not None: self.assertTrue( replication_state in [ 'completed', 'error', 'triggered' ] ) if replication_state in ('error', 'completed'): break LOG.debug("got replication state: {}".format( replication_state)) time.sleep(1)
def tearDown(self): with cloudant(self.user, self.passwd) as c: replicator_db = ReplicatorDatabase(c) while self.replication_ids: replicator_db.stop_replication(self.replication_ids.pop()) while self.dbs: c.delete_database(self.dbs.pop())
def tearDown(self): with cloudant(self.user, self.passwd, account=self.user) as c: replicator = Replicator(c) while self.replication_ids: replicator.stop_replication(self.replication_ids.pop()) while self.dbs: c.delete_database(self.dbs.pop())
def delete_database(): """ Delete the cloudant database created. """ with cloudant(Settings.CLOUDANT_USERNAME, Settings.CLOUDANT_PASSWORD, url=Settings.CLOUDANT_URL) as client: session = client.session() client.delete_database(Settings.DATABASE_NAME) return
def test_replication_state(self): """ _test_replication_state_ Verify that we can get the replication state. """ dbsource = unicode_("test_replication_state_source_{}".format( unicode_(uuid.uuid4()))) dbtarget = unicode_("test_replication_state_target_{}".format( unicode_(uuid.uuid4()))) self.dbs = [dbsource, dbtarget] with cloudant(self.user, self.passwd, account=self.user) as c: dbs = c.create_database(dbsource) dbt = c.create_database(dbtarget) doc1 = dbs.create_document({ "_id": "doc1", "testing": "document 1" }) doc2 = dbs.create_document({ "_id": "doc2", "testing": "document 1" }) doc3 = dbs.create_document({ "_id": "doc3", "testing": "document 1" }) replicator = Replicator(c) repl_id = unicode_("test_replication_state_{}".format( unicode_(uuid.uuid4()))) self.replication_ids.append(repl_id) ret = replicator.create_replication( source_db=dbs, target_db=dbt, repl_id=repl_id, continuous=False, ) replication_state = "not_yet_set" while True: # Verify that replication_state returns either None # (if the field doesn't exist yet), or a valid # replication state. replication_state = replicator.replication_state(repl_id) if replication_state is not None: self.assertTrue(replication_state in ['completed', 'error', 'triggered']) if replication_state in ('error', 'completed'): break LOG.debug( "got replication state: {}".format(replication_state)) time.sleep(1)
def test_init(self): """ _test_init_ Verify that we can init our database object. """ with cloudant(self.user, self.passwd, account=self.user) as c: replicator = Replicator(c) replicator.all_docs()
def del_node(cls, page_id): """ Delete a node with page_id """ document = cls.getcompanytoken(page_id) if document is not None: with cloudant(USERNAME, PASSWORD, url=URL) as client: database = client[DBNAME] todelete = database[document["_id"]] todelete.delete()
def test_cloudant_context_helper(self): """ Test that the cloudant context helper works as expected. """ try: with cloudant(self.user, self.pwd, account=self.account) as c: self.assertIsInstance(c, Cloudant) self.assertIsInstance(c.r_session, requests.Session) except Exception as err: self.fail('Exception {0} was raised.'.format(str(err)))
def test_init(self): """ _test_init_ Verify that we can init our database object. """ with cloudant(self.user, self.passwd) as c: replicator = ReplicatorDatabase(c) replicator.all_docs()
def getcompanytoken(cls, page_id): """ Get the document that represent the facebook page. If not exists return None """ with cloudant(USERNAME, PASSWORD, url=URL) as client: database = client[DBNAME] query = Query(database, selector={"page_id": {"$eq": page_id}}) for node in query(limit=1)['docs']: return node return None
def test_delete(self): with cloudant(self.user, self.passwd) as c: db = c.create_database(self.dbname) doc1 = db.create_document({"_id": "doc1", "testing": "document 1"}) doc1.save() doc1.fetch() doc1.delete() self.assertRaises(requests.HTTPError, doc1.fetch)
def create_index(cls): """ Create an Indexs for the "page_id" attribute """ with cloudant(USERNAME, PASSWORD, url=URL) as client: clouddb = CloudantDatabase(client, DBNAME) clouddb.create_query_index(index_name='pageid-index', index_type='text', fields=[{ "name": "page_id", "type": "string" }])
def test_delete(self): with cloudant(self.user, self.passwd, account=self.user) as c: db = c.create_database(self.dbname) doc1 = db.create_document({"_id": "doc1", "testing": "document 1"}) doc1.save() doc1.fetch() doc1.delete() self.assertRaises(requests.HTTPError, doc1.fetch)
def add_document(cls, dbname, document): """ Create a new document where :param str dbname: The database name to insert the new document :param json document: Document with feed info """ with cloudant(USERNAME, PASSWORD, url=URL) as client: my_database = client[dbname] my_document = my_database.create_document(document) document["_id"] = my_document["_id"] return document
def scanning_errors_new( scan_db, scan_id): #Still broken due to cloudant-python "group_level" bug errors = 0 ddoc = scandb_views['problem_files'][0] view = scandb_views['problem_files'][1] with cloudant(config['cloudant_user'], config['cloudant_auth'], account=config['cloudant_user']) as client: db = client[scan_db] result = db.get_view_result(ddoc, view, reduce=True, group_level=1) errors = result[[scan_id, None, None]:[scan_id, {}, {}]]['value'] return errors
def test_end_to_end(self): """ End to end database and document crud tests """ with cloudant(self.user, self.passwd, account=self.user) as c: session = c.session() self.assertEqual(session['userCtx']['name'], self.user) db = c.create_database(self.dbname) try: self.assertTrue(self.dbname in c) self.assertTrue(db.exists()) # creating docs doc1 = db.new_document() doc2 = db.create_document({'_id': 'womp', "testing": "document2"}) doc3 = db.create_document({"testing": "document3"}) self.assertTrue('_id' in doc1) self.assertTrue('_rev' in doc1) self.assertTrue('_id' in doc2) self.assertTrue('_rev' in doc2) self.assertTrue('_id' in doc3) self.assertTrue('_rev' in doc3) # verifying access via dict api self.assertTrue(doc1['_id'] in db) self.assertTrue(doc2['_id'] in db) self.assertTrue(doc3['_id'] in db) self.assertTrue(db[doc1['_id']] == doc1) self.assertTrue(db[doc2['_id']] == doc2) self.assertTrue(db[doc3['_id']] == doc3) # test working context for updating docs with doc2 as working_doc: working_doc['field1'] = [1, 2, 3] working_doc['field2'] = {'a': 'b'} self.assertEqual( c[self.dbname]['womp']['field2'], {'a': 'b'} ) finally: # remove test database c.delete_database(self.dbname)
def test_end_to_end(self): """ End to end database and document crud tests """ with cloudant(self.user, self.passwd, account=self.user) as c: session = c.session() self.assertEqual(session['userCtx']['name'], self.user) db = c.create_database(self.dbname) try: self.assertIn(self.dbname, c) self.assertTrue(db.exists()) # creating docs doc1 = db.new_document() doc2 = db.create_document({'_id': 'womp', "testing": "document2"}) doc3 = db.create_document({"testing": "document3"}) self.assertIn('_id', doc1) self.assertIn('_rev', doc1) self.assertIn('_id', doc2) self.assertIn('_rev', doc2) self.assertIn('_id', doc3) self.assertIn('_rev', doc3) # verifying access via dict api self.assertIn(doc1['_id'], db) self.assertIn(doc2['_id'], db) self.assertIn(doc3['_id'], db) self.assertEqual(db[doc1['_id']], doc1) self.assertEqual(db[doc2['_id']], doc2) self.assertEqual(db[doc3['_id']], doc3) # test working context for updating docs with doc2 as working_doc: working_doc['field1'] = [1, 2, 3] working_doc['field2'] = {'a': 'b'} self.assertEqual( c[self.dbname]['womp']['field2'], {'a': 'b'} ) finally: # remove test database c.delete_database(self.dbname)
def test_follow_replication_with_errors(self): """ _test_follow_replication_with_errors_ Test to make sure that we exit the follow loop when we submit a bad replication. """ dbsource = unicode_("test_follow_replication_source_error_{}".format( unicode_(uuid.uuid4()))) dbtarget = unicode_("test_follow_replication_target_error_{}".format( unicode_(uuid.uuid4()))) self.dbs = [dbsource, dbtarget] with cloudant(self.user, self.passwd, account=self.user) as c: dbs = c.create_database(dbsource) dbt = c.create_database(dbtarget) doc1 = dbs.create_document({ "_id": "doc1", "testing": "document 1" }) doc2 = dbs.create_document({ "_id": "doc2", "testing": "document 1" }) doc3 = dbs.create_document({ "_id": "doc3", "testing": "document 1" }) replicator = Replicator(c) repl_id = unicode_("test_follow_replication_{}".format( unicode_(uuid.uuid4()))) self.replication_ids.append(repl_id) ret = replicator.create_replication( source_db=dbs, target_db=dbt, # Deliberately override these good params with bad params source=dbsource + "foo", target=dbtarget + "foo", repl_id=repl_id, continuous=False, ) updates = [ update for update in replicator.follow_replication(repl_id) ] self.assertTrue(len(updates) > 0) self.assertEqual(updates[-1]['_replication_state'], 'error')
def dashboard(): data = config_app() with cloudant(data.config.get('USER_NAME'), data.config.get('PASSWORD'), account=data.config.get('ACCOUNT')) as client: my_db = client.all_dbs() try: my_database = client[data.config.get('DATABASE_NAME')] except Exception as error: print("Error opening the Database: " + error) try: for document in my_database: flash(document) except Exception as err: print("Error retrieving data" + err) return ', '.join(my_db)
def test_create_replication(self): """ _test_create_replication_ Make a couple of test databases, and confirm that docs from one get transferred to t'other. """ dbsource = u"test_create_replication_source_{}".format(unicode(uuid.uuid4())) dbtarget = u"test_create_replication_target_{}".format(unicode(uuid.uuid4())) self.dbs = [dbsource, dbtarget] with cloudant(self.user, self.passwd, account=self.user) as c: dbs = c.create_database(dbsource) dbt = c.create_database(dbtarget) doc1 = dbs.create_document({"_id": "doc1", "testing": "document 1"}) doc2 = dbs.create_document({"_id": "doc2", "testing": "document 1"}) doc3 = dbs.create_document({"_id": "doc3", "testing": "document 1"}) replicator = ReplicatorDatabase(c) repl_id = u"test_create_replication_{}".format(unicode(uuid.uuid4())) self.replication_ids.append(repl_id) ret = replicator.create_replication(source_db=dbs, target_db=dbt, repl_id=repl_id, continuous=False) try: repl_doc = replicator[repl_id] except KeyError: repl_doc = None if not repl_doc or not (repl_doc.get("_replication_state", "none") in ("completed, error")): for change in replicator.changes(): if change.get("id") == repl_id: try: repl_doc = replicator[repl_id] repl_doc.fetch() except KeyError: pass if repl_doc and (repl_doc.get("_replication_state", "none") in ("completed", "error")): break else: LOG.debug(u"Waiting for replication to complete " u"(repl_doc: {})".format(repl_doc)) self.assertTrue(repl_doc) self.assertEqual(repl_doc.get("_replication_state"), "completed") for d in ["doc1", "doc2", "doc3"]: self.assertTrue(dbt[d]) self.assertEqual(dbt[d]["testing"], dbs[d]["testing"])
def test_follow_replication_with_errors(self): """ _test_follow_replication_with_errors_ Test to make sure that we exit the follow loop when we submit a bad replication. """ dbsource = u"test_follow_replication_source_error_{}".format( unicode(uuid.uuid4())) dbtarget = u"test_follow_replication_target_error_{}".format( unicode(uuid.uuid4())) self.dbs = [dbsource, dbtarget] with cloudant(self.user, self.passwd) as c: dbs = c.create_database(dbsource) dbt = c.create_database(dbtarget) doc1 = dbs.create_document( {"_id": "doc1", "testing": "document 1"} ) doc2 = dbs.create_document( {"_id": "doc2", "testing": "document 1"} ) doc3 = dbs.create_document( {"_id": "doc3", "testing": "document 1"} ) replicator = ReplicatorDatabase(c) repl_id = u"test_follow_replication_{}".format( unicode(uuid.uuid4())) self.replication_ids.append(repl_id) ret = replicator.create_replication( source_db=dbs, target_db=dbt, # Deliberately override these good params with bad params source=dbsource + "foo", target=dbtarget + "foo", repl_id=repl_id, continuous=False, ) updates = [ update for update in replicator.follow_replication(repl_id) ] self.assertTrue(len(updates) > 0) self.assertEqual(updates[-1]['_replication_state'], 'error')
def test_follow_replication(self): """ _test_follow_replication_ Test to make sure that we can follow a replication. """ dbsource = unicode_("test_follow_replication_source_{}".format( unicode_(uuid.uuid4()))) dbtarget = unicode_("test_follow_replication_target_{}".format( unicode_(uuid.uuid4()))) self.dbs = [dbsource, dbtarget] with cloudant(self.user, self.passwd, account=self.user) as c: dbs = c.create_database(dbsource) dbt = c.create_database(dbtarget) doc1 = dbs.create_document({ "_id": "doc1", "testing": "document 1" }) doc2 = dbs.create_document({ "_id": "doc2", "testing": "document 1" }) doc3 = dbs.create_document({ "_id": "doc3", "testing": "document 1" }) replicator = Replicator(c) repl_id = unicode_("test_follow_replication_{}".format( unicode_(uuid.uuid4()))) self.replication_ids.append(repl_id) ret = replicator.create_replication( source_db=dbs, target_db=dbt, repl_id=repl_id, continuous=False, ) updates = [ update for update in replicator.follow_replication(repl_id) ] self.assertTrue(len(updates) > 0) self.assertEqual(updates[-1]['_replication_state'], 'completed')
def files_scanned_new( scan_db, scan_id, host_id): #Still broken due to cloudant-python "group_level" bug ddoc = scandb_views['file_types'][0] view = scandb_views['file_types'][1] stats = dict() with cloudant(config['cloudant_user'], config['cloudant_auth'], account=config['cloudant_user']) as client: db = client[scan_db] result = db.get_view_result(ddoc, view, group_level=2, reduce=True) stats = result[[host_id, scan_id, None]:[host_id, scan_id, {}]] if len(stats) > 0: return stats else: zeroes = dict(sum=0, count=0, min=0, max=0, sumsqr=0) return zeroes
def delete(): USERNAME = '' PASSWORD = '' with cloudant(USERNAME, PASSWORD, url='') as client: my_db = client['my_database'] file_name = request.form['filename'] for document in my_db: if document['file_name'] == file_name: #f2 = file_name.split('.') #f3 = f2[0] + '.' + f2[1] document.delete() return ("File found and deleted") #document.delete_attachment(file_name) else: return ('File not found') return 'File deleted'
def listing(): result = [] result1 = [] result2 = [] USERNAME = '******' PASSWORD = '' with cloudant(USERNAME, PASSWORD, url='') as client: my_db = client['my_database'] for document in my_db: result.append(document['file_name']) result1.append(document['version no']) result2.append(document['date and time']) print '' return render_template("R:/result.html", result=result, result1=result1, result2=result2)
def test_list_replications(self): """ _test_list_replications_ Verify that we get a list of replications documents back when we got to list replications. """ with cloudant(self.user, self.passwd) as c: replicator = ReplicatorDatabase(c) repl_ids = [] num_reps = 3 for i in range(0, num_reps): tag = "{0}_{1}".format(i, unicode(uuid.uuid4())) dbsource = u"test_list_repl_src_{}".format(tag) dbtarget = u"test_list_repl_tgt_{}".format(tag) self.dbs.append(dbsource) self.dbs.append(dbtarget) dbs = c.create_database(dbsource) dbt = c.create_database(dbtarget) doc1 = dbs.create_document( {"_id": "doc1", "testing": "document 1"} ) repl_id = u"test_create_replication_{}".format(tag) self.replication_ids.append(repl_id) repl_ids.append(repl_id) ret = replicator.create_replication( source_db=dbs, target_db=dbt, repl_id=repl_id, continuous=False ) replications = replicator.list_replications() ids = [doc['_id'] for doc in replications] found_ids = [i for i in ids if i in repl_ids] self.assertEqual(num_reps, len(found_ids))
def test_list_replications(self): """ _test_list_replications_ Verify that we get a list of replications documents back when we got to list replications. """ with cloudant(self.user, self.passwd, account=self.user) as c: replicator = Replicator(c) repl_ids = [] num_reps = 3 for i in range(0, num_reps): tag = "{0}_{1}".format(i, unicode_(uuid.uuid4())) dbsource = unicode_("test_list_repl_src_{}".format(tag)) dbtarget = unicode_("test_list_repl_tgt_{}".format(tag)) self.dbs.append(dbsource) self.dbs.append(dbtarget) dbs = c.create_database(dbsource) dbt = c.create_database(dbtarget) doc1 = dbs.create_document({ "_id": "doc1", "testing": "document 1" }) repl_id = unicode_("test_create_replication_{}".format(tag)) self.replication_ids.append(repl_id) repl_ids.append(repl_id) ret = replicator.create_replication(source_db=dbs, target_db=dbt, repl_id=repl_id, continuous=False) replications = replicator.list_replications() ids = [doc['_id'] for doc in replications] found_ids = [i for i in ids if i in repl_ids] self.assertEqual(num_reps, len(found_ids))
def test_changes_include_docs(self): """ _test_changes_include_docs Test to verify that we can pass 'include_docs' successfully through the changes pipeline. """ dbname = "cloudant-changes-test-with-docs{0}".format( unicode_(uuid.uuid4())) self.last_db = dbname with cloudant(self.user, self.password, account=self.user) as c: session = c.session() db = c.create_database(dbname) n = 0 def make_doc(n): doc = db.create_document({ "_id": "doc{}".format(n), "testing": "doc{}".format(n) }) return doc doc = make_doc(n) for change in db.changes(include_docs=True): LOG.debug(unicode_(change)) if change is not None: self.assertEqual(change['id'], doc['_id']) self.assertEqual( # Verify that doc is included, and looks like # the right doc. change.get('doc', {}).get('testing', {}), 'doc{}'.format(n)) n += 1 doc = make_doc(n) if n > 10: break self.assertTrue(n > 10)
def test_changes_include_docs(self): """ _test_changes_include_docs Test to verify that we can pass 'include_docs' successfully through the changes pipeline. """ dbname = "cloudant-changes-test-with-docs{0}".format( unicode_(uuid.uuid4())) self.last_db = dbname with cloudant(self.user, self.password, account=self.user) as c: session = c.session() db = c.create_database(dbname) n = 0 def make_doc(n): doc = db.create_document( {"_id": "doc{}".format(n), "testing": "doc{}".format(n)} ) return doc doc = make_doc(n) for change in db.changes(include_docs=True): LOG.debug(unicode_(change)) if change is not None: self.assertEqual(change['id'], doc['_id']) self.assertEqual( # Verify that doc is included, and looks like # the right doc. change.get('doc', {}).get('testing', {}), 'doc{}'.format(n) ) n += 1 doc = make_doc(n) if n > 10: break self.assertTrue(n > 10)
def get_conn(self): """ Opens a connection to the cloudant service and closes it automatically if used as context manager. .. note:: In the connection form: - 'host' equals the 'Account' (optional) - 'login' equals the 'Username (or API Key)' (required) - 'password' equals the 'Password' (required) :return: an authorized cloudant session context manager object. :rtype: cloudant """ conn = self.get_connection(self.cloudant_conn_id) self._validate_connection(conn) cloudant_session = cloudant(user=conn.login, passwd=conn.password, account=conn.host) return cloudant_session
def download(): USERNAME = '' PASSWORD = '' with cloudant(USERNAME, PASSWORD, url='') as client: my_db = client['my_database'] file_name = request.form['filename'] for document in my_db: if (document['file_name'] == file_name): f2 = file_name.split('.') f3 = f2[0] + '.' + f2[1] file = document.get_attachment(f3, attachment_type='binary') print file response = make_response(file) response.headers[ "Content-Disposition"] = "attachment; filename=%s" % file_name return response else: response = 'File not found' return response
def get_conn(self) -> cloudant: """ Opens a connection to the cloudant service and closes it automatically if used as context manager. .. note:: In the connection form: - 'host' equals the 'Account' (optional) - 'login' equals the 'Username (or API Key)' (required) - 'password' equals the 'Password' (required) :return: an authorized cloudant session context manager object. :rtype: cloudant """ conn = self.get_connection(self.cloudant_conn_id) self._validate_connection(conn) cloudant_session = cloudant(user=conn.login, passwd=conn.password, account=conn.host) return cloudant_session
def upload_to_db(app_id, name, category, description, image, rating, total_reviews, raw_name): """ Uploads app details to cloudant database where id is the key to the document in cloudant. The document includes the app name, description, url, rating and the total number of reviews. """ # Connect to cloudant. Exits cloudant when block is exited. with cloudant(Settings.CLOUDANT_USERNAME, Settings.CLOUDANT_PASSWORD, url=Settings.CLOUDANT_URL) as client: session = client.session() # Create or check for db: try: # Create database. app_db = client.create_database(Settings.DATABASE_NAME) except: # Database already exists. app_db = client[Settings.DATABASE_NAME] # Check database exists if not app_db.exists(): raise NonexistentDatabase("The database %s does not exist." % Settings.DATABASE_NAME) # Create or update document and perform fetch/save after exiting this block with Document(app_db, app_id) as doc: doc['name'] = name.encode('utf-8') doc['description'] = description.encode('utf-8') doc['image'] = image doc['category'] = category doc['rating'] = rating doc['total_reviews'] = total_reviews doc['keyword'] = query_for_top_keywords(raw_name, category) doc['turnarounds'] = int(query_num_turnarounds(raw_name)) doc['sentiment'] = float(query_average_app_sentiment(raw_name)) # Display document for error checking print(app_db[app_id]) return
def go(): with cloudant(args.username, args.password, url=args.host) as client: session = client.session() source_db = client[args.source] target_db = client[args.target] print_metadata("Source", source_db.metadata()) print_metadata("Target", target_db.metadata()) since_seq = get_checkpoint(target_db) batch = [] for change in source_db.changes(since=since_seq, continuous=not args.batch, include_docs=True): if 'deleted' in change: batch.append({'_id': change['id'], '_deleted': True}) elif 'doc' in change: # created or updated doc = change['doc'] # we don't know whether the target contains this so assume create del doc['_rev'] batch.append(doc) # attachments not supported - bail if we find one if '_attachments' in doc: raise Exception("Cannot proceed - document has _attachments:\n" + json.dumps(doc)) # If we reach the batch size, upload to the target if len(batch) == args.batchsize: upload_batch(change, batch, target_db) batch = [] # Drain the batch upload_batch(change, batch, target_db)
def test_changes(self): """ _test_changes_ Test to verify that we can connect to a live changes feed. Verify that we are actually staying connected by creating new docs while reading from the _changes feed. """ dbname = "cloudant-changes-test-{0}".format(unicode_(uuid.uuid4())) self.last_db = dbname with cloudant(self.user, self.password, account=self.user) as c: session = c.session() db = c.create_database(dbname) n = 0 def make_doc(n): doc = db.create_document({ "_id": "doc{}".format(n), "testing": "doc{}".format(n) }) return doc doc = make_doc(n) for change in db.changes(): LOG.debug(unicode_(change)) if change is not None: self.assertEqual(change['id'], doc['_id']) n += 1 doc = make_doc(n) if n > 10: break self.assertTrue(n > 10)
def test_changes(self): """ _test_changes_ Test to verify that we can connect to a live changes feed. Verify that we are actually staying connected by creating new docs while reading from the _changes feed. """ dbname = "cloudant-changes-test-{0}".format(unicode_(uuid.uuid4())) self.last_db = dbname with cloudant(self.user, self.password, account=self.user) as c: session = c.session() db = c.create_database(dbname) n = 0 def make_doc(n): doc = db.create_document( {"_id": "doc{}".format(n), "testing": "doc{}".format(n)} ) return doc doc = make_doc(n) for change in db.changes(): LOG.debug(unicode_(change)) if change is not None: self.assertEqual(change['id'], doc['_id']) n += 1 doc = make_doc(n) if n > 10: break self.assertTrue(n > 10)
def tearDown(self): with cloudant(self.user, self.passwd) as c: c.delete_database(self.dbname)
def tearDown(self): if self.last_db is not None: with cloudant(self.user, self.password, account=self.user) as c: c.delete_database(self.last_db)
__author__ = "Ayushi Dalmia" __email__ = "*****@*****.**" from cloudant import cloudant import json from cloudant.result import Result,ResultByKey import random from cloudant.design_document import DesignDocument from cloudant.query import Query with open('credentials.json') as f: cred = json.load(f) with cloudant(str(cred['credentials']['username']),str(cred['credentials']['password']),url=str(cred['credentials']['url'])) as client: my_database = client['test'] index = my_database.create_query_index(fields=[{'name': 'description','type':'string'}],index_type='text') selector = {'$text': "happiest "} docs = my_database.get_query_result(selector,use_index = index.name) for d in docs: print d ''' #create design doc ddoc = DesignDocument(my_database,document_id="_design/description")
config_file = os.path.expanduser('~/.config') config = ConfigParser.RawConfigParser() config.read(config_file) acct = config.get('cloudant', 'account') user = config.get('cloudant', 'user') pwd = config.get('cloudant', 'pwd') if not acct or not user or not pwd: raise Exception('Cannot connect with Cloudant client.') kafka = config.get('kafka', 'server') topic = config.get('kafka', 'topic') if not kafka or not topic: raise Exception('Cannot connect with Kafka.') kafka_client = KafkaClient(kafka) producer = SimpleProducer(kafka_client, async=False) count = 0 with cloudant(user, pwd, account=acct) as client: db = client['sdp_kafka_test0000'] if not db.exists(): raise Exception('Database does not exists!!') for change in db.changes(): producer.send_messages(topic, json.dumps(change)) count = count + 1 print '{0} changes processed'.format(count) kafka_client.close()