def _get_connection(self):
"""Connect to the CouchDB server."""
if self.username and self.password:
conn_string = f'{self.scheme}://{self.username}:{self.password}@{self.host}:{self.port}'
server = pycouchdb.Server(conn_string, authmethod='basic')
else:
conn_string = f'{self.scheme}://{self.host}:{self.port}'
server = pycouchdb.Server(conn_string)
try:
return server.database(self.container)
except pycouchdb.exceptions.NotFound:
return server.create(self.container)
def _get_connection(self):
"""Connect to the CouchDB server."""
if self.username and self.password:
conn_string = '%s://%s:%s@%s:%s' % (
self.scheme, self.username, self.password,
self.host, str(self.port))
server = pycouchdb.Server(conn_string, authmethod='basic')
else:
conn_string = '%s://%s:%s' % (
self.scheme, self.host, str(self.port))
server = pycouchdb.Server(conn_string)
try:
return server.database(self.container)
except pycouchdb.exceptions.NotFound:
return server.create(self.container)
def server():
server = pycouchdb.Server(SERVER_URL)
for db in server:
server.delete(db)
if "_users" not in server:
server.create("_users")
return server
def setUpClass(cls):
cls.s = couchdb.Server(SERVER_URL)
try:
cls.db = cls.s.create('testing3')
except Conflict:
cls.s.delete("testing3")
cls.db = cls.s.create('testing3')
docs = cls.db.save_bulk([
{
"_id": "kk1",
"name": "Andrey"
},
{
"_id": "kk2",
"name": "Pepe"
},
{
"_id": "kk3",
"name": "Alex"
},
])
querydoc = {
"_id": "_design/testing",
"views": {
"names": {
"map": "function(doc) { emit(doc.name, 1); }",
"reduce":
"function(keys, values) { return sum(values); }",
}
}
}
cls.db.save(querydoc)
def dbCreateIfNeeded():
couchDB = None
couchDBServer = pycouchdb.Server()
try:
# create the couchdb instance - if it doesn't exist yet
couchDB = couchDBServer.database("leopardmon-testtargets")
logging.info("leopardmon-testargets DB successfully opened")
except pycouchdb.exceptions.NotFound as e:
couchDB = couchDBServer.create("leopardmon-testtargets")
logging.info("leopardmon-testargets does not exist - create the DB")
try:
# create the couchdb instance - if it doesn't exist yet
couchDB = couchDBServer.database("leopardmon-testresults")
logging.info("leopardmon-testresults DB successfully opened")
except pycouchdb.exceptions.NotFound as e:
couchDB = couchDBServer.create("leopardmon-testresults")
logging.info("leopardmon-testresults does not exist - create the DB")
try:
# create the couchdb instance - if it doesn't exist yet
couchDB = couchDBServer.database("leopardmon-users")
logging.info("leopardmon-users DB successfully opened")
except pycouchdb.exceptions.NotFound as e:
couchDB = couchDBServer.create("leopardmon-users")
logging.info("leopardmon-users does not exist - create the DB")
def loadFromDB(self) -> object:
# get the database
couchDBServer = pycouchdb.Server()
couchDB = couchDBServer.database("leopardmon-testresults")
_viewDoc = None
try:
_viewDoc = couchDB.get("_design/listview")
except pycouchdb.exceptions.NotFound as e:
# view does exist - create it
_viewdoc = {
"_id": "_design/listview",
"views": {
"names": {
"map": "function(doc) { emit(doc.name, 1); }",
"reduce": "function(k, v) { return sum(v); }",
}
}
}
viewdoc = couchDB.save(_viewdoc)
# use the view to list all documents
docnames = couchDB.query("listview/names", group='true')
# load the targetinfo objects
for docname in docnames:
doc = couchDB.get(docname["key"])
testResults = doc
print(doc)
self.searchResults.append(doc)
def getPaper():
server = couchdb.Server('http://*****:*****@localhost:5984/')
db = server.database('tag_paper')
l = []
paper_list = list(db.query("testing/AllPaper"))
for paper in paper_list:
l.append(paper['key'])
return l
def __init__(self):
config = configparser.ConfigParser()
config.read('/var/lib/app/couch.ini')
password = config.get('admins', 'admin')
self.dbserver = pycouchdb.Server(f'http://*****:*****@db:5984/')
try:
self.scoresdb = self.dbserver.database("scores")
except pycouchdb.exceptions.NotFound:
self.scoresdb = self.dbserver.create("scores")
def dbGetUser(name):
couchDBServer = pycouchdb.Server()
couchDB = couchDBServer.database("leopardmon-users")
doc = None
try:
doc = couchDB.get(name)
except pycouchdb.exceptions.NotFound as e:
doc = None
return doc
def setUpClass(cls):
cls.s = couchdb.Server(SERVER_URL)
try:
cls.db = cls.s.create('testing4')
except Conflict:
cls.s.delete("testing4")
cls.db = cls.s.create('testing4')
cls.db.save_bulk([
{"_id": "kk1", "name": "Andrey"},
])
def conn_couchDB():
# CouchDB configurations
config = ConfigParser()
config.read('configuration.ini')
# print(config.sections())
user = config.get('couchdb_config', 'couchdb_user')
password = config.get('couchdb_config', 'couchdb_password')
server = pycouchdb.Server("http://%s:%[email protected]:5984/" %
(user, password))
return server
def _open(self):
conninfo = self.connection.client
dbname = conninfo.virtual_host
proto = conninfo.ssl and 'https' or 'http'
if not dbname or dbname == '/':
dbname = DEFAULT_DATABASE
port = conninfo.port or DEFAULT_PORT
if conninfo.userid and conninfo.password:
server = pycouchdb.Server('%s://%s:%s@%s:%s/' % (
proto, conninfo.userid, conninfo.password,
conninfo.hostname, port),
authmethod='basic')
else:
server = pycouchdb.Server('%s://%s:%s/' % (
proto, conninfo.hostname, port))
try:
return server.database(dbname)
except exceptions.NotFound:
return server.create(dbname)
def dbPutUser(name, email, organization, password):
couchDBServer = pycouchdb.Server()
couchDB = couchDBServer.database("leopardmon-users")
doc = {}
doc["_id"] = name
doc["name"] = name
doc["email"] = email
doc["organization"] = organization
doc["password"] = password
couchDB.save(doc)
def __init__(self, target_couchdb_uri, target_couchdb_name, **kwargs):
"""
:param target_couchdb_uri: the uri of the couchdb server to forward
changes to.
:param target_couchdb_name: the name of the target database.
"""
super(BaseCouchdbChangesProcessor, self).__init__(**kwargs)
target_server = pycouchdb.Server(target_couchdb_uri)
self._target_couchdb = target_server.database(target_couchdb_name)
def db_insert():
server = pycouchdb.Server(DB_HOST)
info = server.info()
db = server.database("app")
doc = db.get("3e2c5875dbfe6131b63a4ab1935094df")
new_doc = db.save({"name": "FOO", "extra": "HOO"})
list = [
{
'server': info,
'database': db.config()
},
doc,
new_doc,
]
return jsonify(results=list)
def __init__(self,
couchdb_uri,
couchdb_name,
feed_kwargs=None,
limit=1000,
flush_interval=10,
processor=None,
seqtracker=None):
"""Initialises the consumer.
:param couchdb_uri: the uri of your couchdb server.
:param couchdb_name: the name of the database you want changes from.
:param feed_kwargs: the arguments to be passed to the feed url.
:param limit: maximum number of changes to be processed in a batch.
:param flush_interval: the maximum time, in seconds, to wait before
processing a batch of changes.
:param processor: a subclass of
`cchain.processors.base.BaseChangesProcessor`.
:param seqtracker: a subclass of
`cchain.seqtrackers.base.BaseSeqTracker`.
"""
server = pycouchdb.Server(couchdb_uri)
self._couchdb = server.database(couchdb_name)
self._limit = limit
self._flush_interval = datetime.timedelta(seconds=flush_interval)
self._processor = processor
self._seqtracker = seqtracker
default_feed_kwargs = {
'include_docs': 'true',
}
feed_kwargs = feed_kwargs or {}
default_feed_kwargs.update(feed_kwargs)
self._feed_kwargs = default_feed_kwargs
self._buffer = []
self._feed_reader = self.feed_reader_class(
limit=self._limit,
flush_interval=self._flush_interval,
processor=self._processor,
seqtracker=self._seqtracker)
def __init__(self, server, db, designdocs=None):
log.debug("setting up couchdb {} on server {}".format(server, db))
self.server = server
self.db = db
self.couch_server = pycouchdb.Server(self.server)
self.couchdb = None
try:
self.couchdb = self.couch_server.database(db)
log.debug("db exists")
except NotFound:
log.debug("db not found, creating it")
self.couch_server.create(db)
self.couchdb = self.couch_server.database(db)
self.designdocs = designdocs # this is a pyramid asset spec
if self.designdocs:
log.debug("updating design docs")
self.update_designdocs()
def _connect(self):
"""
Get a new connection object to the database
Raises
------
Cannot Connect to CouchDB
If the connection to the conn_url fails
this will be raised
Returns
-------
connection : pycouchdb object
client for couchdb
"""
try:
return pycouchdb.Server(self._conn_url)
except:
raise ValueError("Cannot Connect to CouchDB")
def __init__(self, couchdb_uri, couchdb_name, seq_doc_id):
"""Reads the current version of the sequence documents from
the database and stores it on the instance.
"""
server = pycouchdb.Server(couchdb_uri)
self._couchdb = server.database(couchdb_name)
try:
seq_doc = self._couchdb.get(seq_doc_id)
except pycouchdb.exceptions.NotFound:
seq_doc = {
'_id': seq_doc_id,
}
self._seq_doc = seq_doc
super(CouchDBSeqTracker, self).__init__()
def store(self):
# get the couchDB server and instance
# initialize global variable couch DB
# create the couchdb instance - if it doesn't exist yet
couchDBServer = pycouchdb.Server()
couchDB = couchDBServer.database("leopardmon-testresults")
try:
doc = None
try:
doc = couchDB.get(self.name)
self._rev = doc["_rev"]
except:
pass
except pycouchdb.exceptions.NotFound as e:
pass
# update the date/time stamps
now = datetime.datetime.now()
nowDateTimeStr = now.strftime("%Y-%m-%d %H:%M:%S")
self.lastTested = nowDateTimeStr
if (self.lastStatus > 0):
self.lastFailure = nowDateTimeStr
else:
self.lastSuccess = nowDateTimeStr
resultTuple = {}
resultTuple["datetime"] = nowDateTimeStr
resultTuple["status"] = self.lastStatus
resultTuple["timeelapsed"] = self.lastTimeElapsed
results = list()
results.append(resultTuple)
if (doc != None):
results = results + doc["results"][0:63]
self.results = results
doc = couchDB.save(self.__dict__)
def main():
# log start up message
logging.info(
"***************************************************************")
logging.info("networkscan Data Collector has started")
logging.info("Working directory is %s", os.getcwd())
couchDBServer = pycouchdb.Server(couchDBServerUrl)
logging.info("CouchDB Server is %s", couchDBServerUrl)
logging.info("CouchDB Server version is %s",
couchDBServer.info()['version'])
# initialize global variable couch DB
global couchDB
couchDB = couchDBServer.database("networkscan")
readSSHUsernamePassword()
try:
hostname = socket.gethostname()
externalip = get('https://api.ipify.org').text
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('8.8.8.8', 1)) # connect() for UDP doesn't send packets
localipaddress = s.getsockname()[0]
logging.info("Hostname is %s", hostname)
logging.info("Local IP is %s and external IP is %s", localipaddress,
externalip)
except Exception as e:
logging.exception("Exception occurred")
logging.error("Unable to get network information")
printIfInteractive("Scanning subnet " + baseAddress)
# create an address list and then iterate through each address and probe for network services
addresses = createAddressList()
for address in addresses:
prober = networkProber(address)
prober.probe()
printIfInteractive(
"Scan complete! ")
def store(self):
# get the couchDB server and instance
# initialize global variable couch DB
# create the couchdb instance - if it doesn't exist yet
couchDBServer = pycouchdb.Server()
couchDB = couchDBServer.database("leopardmon-testtargets")
try:
doc = None
try:
doc = couchDB.get(self.name)
self._rev = doc["_rev"]
except:
pass
except pycouchdb.exceptions.Conflict as e:
pass
doc = couchDB.save(self.__dict__)
def deduplicate(server, db):
#pass http://username:password@ip_address:5984/ to server constructor:
server = pycouchdb.Server(server)
couchdb = server.database(db)
##twitterNmCount : count the number of twitter of each id_str, query as a list
## twitters : a list of all twitters in the database
## twitter_toDelete : a list of twitter id_str to delete. if count of a id_str is more than 1,
## put this id_str and the number of twitter with this id_str that we need to delete
twitter_toDelete = []
twitterNumCount = list(couchdb.query("id_count/id_str", group='true'))
twitters = list(couchdb.query("id/id_str"))
#print("twitters: "+str(twitters))
for item in twitterNumCount:
if item['value'] > 1:
twitter_toDelete.append({
'id': item['key'],
'count': item['value'] - 1
})
#print("to_del: "+str(twitter_toDelete))
while (len(twitter_toDelete) > 0):
##while we have something to delete
for twitter in twitters:
uuid = twitter['id']
#print(uuid)
id = twitter['key']
for item in twitter_toDelete:
if id == item['id']:
couchdb.delete(uuid)
if item['count'] > 1:
item['count'] -= 1
else:
##remove the id_str if we have delete all duplicate twitter of this id_str
twitter_toDelete.remove(item)
def checkDuplicaton(server, db, file):
#pass http://username:password@ip_address:5984/ to server constructor:
server = pycouchdb.Server(server)
db = server.database(db)
f = open(file, "r")
if f.mode == 'r':
contents = f.read()
data = json.loads(contents)
#data = json.loads(json.loads(n))
twitid = str(data['id'])
#print(type(twitid))
not_duplicated = True
idmap = list(db.query("id/id_str"))
for item in idmap:
#print(type(item["key"]))
if twitid == item["key"]:
not_duplicated = False
return not_duplicated
except:
pass
# Save to db
try:
db.save(dic_tweet)
print("success")
except:
print("Duplicated tweet or system error")
pass
i += 1
line = f.readline()
if __name__ == "__main__":
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
suburbs_melb = "melbourne.json"
suburbs_syd = "sydney.json"
sub_dic = suburbs_shapely_processor.read_json(suburbs_melb, suburbs_syd)
# Database
# server = connection.conn_couchDB()
server = pycouchdb.Server("http://%s:%[email protected]:5984/" % ('admin', 'admin*230'))
dbname = 'processed_data'
db = server.database(dbname)
# need to change file name
data_dic = read_data("Tweets-100.json", rank, size, sub_dic, db)
import socket
import tweepy
import pycouchdb
import preprocess
host = []
fname = 'hosts'
config = open(fname, 'r')
for line in config:
host.append(line.strip())
config.close()
# Connect to the CouchDB database
while True:
try:
couch = pycouchdb.Server('http://*****:*****@' + host[1] + ':5984/')
# Connect to instance 1 according to IP address and port number
obj = socket.socket()
obj.connect((host[4], 5985))
if obj is not None and couch is not None:
break
except:
couch = None
obj = None
try:
raw_db = couch.database('ccc_ass_2')
except:
raw_db = couch.create('ccc_ass_2')
try:
db = couch.database('final_result')
except:
return "pos"
elif polarity < -0.1:
return "neg"
else:
return "neu"
auth = tweepy.OAuthHandler(
"lnpDYWDIWShVKmEez297bOfec",
"0eYwwH3IRU8qkt5IceBrvQF0aHUaPdpEfNE7ldkEJezzgVE1ry")
auth.set_access_token("974163114433306624-fSrCQPL7HCM33RxE76V2dvFmsuQ1v5n",
"i2eBRy83UvrFqhEbIWQ2u29L2ivD2TSVMJYgubPJXEhNR")
api = tweepy.API(auth)
server = pycouchdb.Server("http://*****:*****@172.26.132.72:5984/")
db = server.database("twitter")
time.sleep(5)
result = api.search(geocode="-37.999250,144.997395,20km",
count=100,
result_type='mixed')
for item in result:
data = item._json
suburb = "None"
if data["place"] and data['place']['place_type'] == 'neighborhood':
suburb = find_suburb_place(data['place']['name'])
elif data['coordinates']:
suburb = find_suburb([
data["coordinates"]["coordinates"][0],
def setUpClass(cls):
cls.s = couchdb.Server(SERVER_URL)
cls.db = cls.s.create('testing5')
def setUpClass(cls):
cls.s = couchdb.Server(SERVER_URL, authmethod="basic")
# Created: Sept 6, 2020
#
# Purpose:
#
# Demonstrate the use of Kafka Python streaming APIs.
# In this example, demonstrate Kafka streaming API to build a consumer.
#
import os # need this for popen
import time # for sleep
import json
from kafka import KafkaConsumer # consumer of events
import pycouchdb
# Set up server/db
couch = pycouchdb.Server('http://*****:*****@localhost:5984')
db = couch.create('assignment1')
# We can make this more sophisticated/elegant but for now it is just
# hardcoded to the setup I have on my local VMs
# acquire the consumer
# (you will need to change this to your bootstrap server's IP addr)
consumer = KafkaConsumer (bootstrap_servers="129.114.25.52:9092")
# subscribe to topic
consumer.subscribe (topics=["utilization1", "utilization2"])
# we keep reading and printing
for msg in consumer:
