def getCampaignWordsPerDay(campaignID):
# print "Get Campaign Words Per Day"
connection = Connection(DB.getDBHost(), DB.getDBPort())
try:
db = connection.meteor
campaign = db.campaigns.find_one({"_id":campaignID})
connection.close()
if (campaign == ""):
return "1"
wordsPerDay = campaign['sendcount']
if (wordsPerDay == ""):
return "1"
return wordsPerDay
except:
print "**** getCampaignWordsPerDay Exception ****"
return "1"
def getActiveWordList(campaignID):
print "ACTIVE Word List Order"
connection = Connection(DB.getDBHost(), DB.getDBPort())
# print "Connecting to Campaigns"
db = connection.meteor
# print "Connected to Words"
activeWordList = []
# print "Active Word List Order"
campaign = db.campaigns.find_one({"_id":campaignID})
connection.close()
if (campaign == ""):
return ""
wordOrder = campaign['cwordorder']
if (wordOrder == ""):
return ""
wordOrderArray = wordOrder.split(",")
for word in wordOrderArray:
if (len(word) > 0):
activeWordList.append(word)
# print "activeWordList add- > " + word
return activeWordList
def getCampaignName(campaignID):
# print "Get Campaign Name"
connection = Connection(DB.getDBHost(), DB.getDBPort())
try:
db = connection.meteor
campaign = db.campaigns.find_one({"_id":campaignID})
connection.close()
if (campaign == ""):
return "N/A"
name = campaign['campaign']
if (name == ""):
return "N/A"
return name
except:
print "**** getCampaignName Exception ****"
return "N/A"
def init_apns():
mongodb = None
while not mongodb:
try:
mongodb = Connection(options.mongohost, options.mongoport)
except:
pass
# wait 5 secs to reconnect
time.sleep(5)
masterdb = mongodb[options.masterdb]
apps = masterdb.applications.find({'enableapns': 1})
apnsconns = {}
for app in apps:
apnsconns[app['shortname']] = []
conns = int(app['connections'])
if conns > 5:
conns = 5
if conns < 1:
conns = 1
if not app.has_key('environment'):
app['environment'] = 'sandbox'
for instanceid in range(0, conns):
apn = APNClient(app['environment'], app['certfile'],
app['keyfile'], app['shortname'], instanceid)
apnsconns[app['shortname']].append(apn)
mongodb.close()
return apnsconns
class BaseMongoDB(object):
def __init__(self, db, host=DEFAULT_HOST, user=None, password=None,
admin_db=None, ssl=DEFAULT_SSL, port=DEFAULT_PORT,
timeout=DEFAULT_TIMEOUT,
write_concern=DEFAULT_WRITE_CONCERN):
self._host = host
self._password = password
self._user = user
self._db = db
self._admin_db = admin_db
self._ssl = ssl
self._port = port
self._timeout = timeout
self._write_concern = write_concern
@property
def db(self):
# FIXME: memoize
if not hasattr(self, '_db_proxy'):
try:
self._proxy = Connection(self._host, self._port,
ssl=self._ssl,
tz_aware=True,
network_timeout=self._timeout)
except errors.ConnectionFailure as e:
raise errors.ConnectionFailure("MongoDB Failed to connect (%s): %s" % (self._host, e))
self._db_proxy = self._auth_db()
# return the connected, authenticated database object
return self._db_proxy
def close(self):
if hasattr(self, '_proxy'):
self._proxy.close()
def _auth_db(self):
'''
by default the default user only has read-only access to db
'''
if not self._password:
pass
elif self._admin_db:
admin_db = self._proxy[self._admin_db]
if not admin_db.authenticate(self._user, self._password):
raise RuntimeError(
"MongoDB failed to authenticate with user (%s)" % self._user)
else:
if not self._proxy[self._db].authenticate(self._user, self._password):
raise RuntimeError(
"MongoDB failed to authenticate with user (%s)" % self._user)
return self._proxy[self._db]
def set_collection(self, collection):
self.collection = collection
def __getitem__(self, collection):
return self.db[collection]
def getActiveStudents(campaignID):
# print "ACTIVE Students"
connection = Connection(DB.getDBHost(), DB.getDBPort())
# print "Connecting to Students"
try:
db = connection.meteor
print "Connected to Students"
activeStudentList = []
print "Active Student List for Campaign " + campaignID
for students in db.students.find({'campaign':campaignID}):
# print students['cell'] + " " + students['campaign']
# print " IS Active"
activeStudentList.append(students['cell'])
connection.close()
# print "ACTIVE Students List EXIT"
return activeStudentList
except:
print "GET ACTIVE STUDENT EXCEPTION"
def run(observationNumber=None):
# Create an empty response object.
global nodeCount
response = {}
# open a link to the backing database
connection = Connection('localhost', 27017)
db = connection["FeederWatch"]
data_coll = db['interactions']
# split the single JSON object up into a hierarchy of objects by traversing the nested dictionaries and writing
# documents in mongo during the traversal. Store in a collection according to the name of the file dropped.
try:
query = {'observation': observationNumber}
interactionCursor = data_coll.find(query)
except:
response['error'] = "Could not find this observation"
return bson.json_util.dumps(response)
connection.close()
response['data'] = []
for x in interactionCursor:
response['data'].append(x)
# Pack the results into the response object, and return it.
#response['result'] = jsonoutput
response['result'] = 'OK'
# Return the response object.
#tangelo.log(str(response))
return bson.json_util.dumps(response)
def getActiveWordAnswerTF(thisWord, ansNumber):
connection = Connection(getDBHost(), getDBPort())
question = ""
print "Find This Word Answer TF " + thisWord + " Answer Number " + str(ansNumber)
try:
db = connection.meteor
word = db.words.find_one({'seqnum':thisWord})
connection.close()
print "FOUND ANSWER"
answer = "F"
if ansNumber > 5:
answer = "Answer Number Error 2"
if ansNumber < 1:
answer = "Answer Number Error 1"
ansIndex = "active" + str(ansNumber)
if (word[ansIndex] == True):
answer = "T"
except:
print "EXCEPTION - TF ANSWER QUESTION"
return answer
def getActiveWordRemediation(thisWord, rightWrong):
connection = Connection(getDBHost(), getDBPort())
question = ""
print "Find Correct Remediation for word " + thisWord + " Right/Wrong " + rightWrong
try:
db = connection.meteor
word = db.words.find_one({'seqnum':thisWord})
connection.close()
print "Found Word"
if (rightWrong == "Correct"):
remediation = word['remedifcorrect']
else:
remediation = word['remedifwrong']
return remediation
except:
print "EXCEPTION - In getActiveWordRemediation"
return ""
def ec2():
aws_access_key_id = ''
aws_secret_access_key = ''
kw_params = {}
kw_params.update({'aws_access_key_id':aws_access_key_id})
kw_params.update({'aws_secret_access_key':aws_secret_access_key})
conn = boto.ec2.connect_to_region('ap-northeast-1', **kw_params)
#res = conn.get_all_instances(filters={'tag-value': 'mongodb-query-test'})
res = conn.get_all_instances()
con = Connection(host='localhost')
db = con['mongodb-test']
a = ['ec2-xxxxxx.ap-northeast-1.compute.amazonaws.com']
for x in a:
print x
db.public_dns_name.insert({
'public_dns_name': x
})
#if res:
# for inst in res:
# name = inst.instances[0].tags['Name']
# logging.warning(name)
# if name == 'ClientMaster2':
# if inst.instances[0].public_dns_name != None:
# logging.warning(name)
# print inst.instances[0].public_dns_name
# db.public_dns_name.insert({
# 'public_dns_name': inst.instances[0].public_dns_name
# })
con.close()
def getActiveWordAnswer(thisWord, ansNumber):
connection = Connection(getDBHost(), getDBPort())
question = ""
print "Find This Word Answer " + thisWord + " Answer Number " + str(ansNumber)
try:
db = connection.meteor
collection = db.words
word = db.words.find_one({'seqnum':thisWord})
connection.close()
print "FOUND ANSWER"
answer = ""
if ansNumber > 5:
answer = "Answer Number Error 2"
if ansNumber < 1:
answer = "Answer Number Error 1"
ansIndex = "ans" + str(ansNumber)
print " Answer index " + ansIndex
answer = word[ansIndex]
except:
print "EXCEPTION - ANSWER QUESTION"
answer = ""
return answer
def ec2():
aws_access_key_id = ''
aws_secret_access_key = ''
kw_params = {}
kw_params.update({'aws_access_key_id': aws_access_key_id})
kw_params.update({'aws_secret_access_key': aws_secret_access_key})
conn = boto.ec2.connect_to_region('ap-northeast-1', **kw_params)
#res = conn.get_all_instances(filters={'tag-value': 'mongodb-query-test'})
res = conn.get_all_instances()
con = Connection(host='localhost')
db = con['mongodb-test']
a = ['ec2-xxxxxx.ap-northeast-1.compute.amazonaws.com']
for x in a:
print x
db.public_dns_name.insert({'public_dns_name': x})
#if res:
# for inst in res:
# name = inst.instances[0].tags['Name']
# logging.warning(name)
# if name == 'ClientMaster2':
# if inst.instances[0].public_dns_name != None:
# logging.warning(name)
# print inst.instances[0].public_dns_name
# db.public_dns_name.insert({
# 'public_dns_name': inst.instances[0].public_dns_name
# })
con.close()
def load_json(filename='', db_name='', collection_name='', clean=False):
"""
-------------------------
load data into Mongo db
-------------------------
"""
connection = Connection() ##('localhost', 27017) #! get connection
db = connection[db_name] #! get database
collection = db[collection_name] #! get collection
if clean:
collection.remove() #
count = 0
with open(filename, 'rb') as input_file:
for idx, line in enumerate(input_file):
line = line.strip('\r\n').strip('\n')
json_obj = json.loads(line, encoding="utf-8")
try:
collection.insert(json_obj)
count += 1
except:
print traceback.format_exc()
break
connection.close()
print '------------- total json objects found = %s --- ' % (idx+1)
print '------------- total json objects loaded = %s --- ' % (count)
class Mongo(Output):
def init(self):
addr = "localhost"
port = 27017
db_name = "twitter"
self.connection = Connection(addr, port)
self.db = self.connection[db_name]
def throw(self, packet):
data = packet["data"]
if data.get("event") is None:
self.db.tweet.insert(data)
elif "favorite" in data["event"]:
self.db.favorite.insert(data)
elif "list" in data["event"]:
self.db.list.insert(data)
elif "retweet" == data["event"]:
self.db.retweet.insert(data)
elif "follow" == data["event"]:
self.db.follow.insert(data)
elif "dm" == data["event"]:
self.db.dm.insert(data)
elif "delete" == data["event"]:
self.db.delete.insert(data)
def cleanup(self):
self.connection.close()
def execute(self, arborapi=None):
print self.name + " executing"
# setup mongo connection and look through all input collections, copying
connection = Connection('localhost', 27017)
if len(self.databaseName) > 0:
db = connection[self.databaseName]
else:
db = connection['arbor']
outputcoll = db[self.outputInformation.collectionName]
# clear out the output to prepare for running an algorithm
outputcoll.drop()
# loop through all inputs and process all objects in the inputs
for thisInput in self.inputs:
inputcoll = db[thisInput.collectionName]
# find all documents in this input (repeated for each input)
queryResults = inputcoll.find()
print "found that ", thisInput.collectionName, " has ", queryResults.count(
), " records"
# write the documents into the output collection and indicate the output time changed
for result in queryResults:
outputcoll.insert(result)
# pause the writer enough to allow the write to complete? Found example code here:
# http://sourceforge.net/u/rick446/random/ci/master/tree/lib.py
db.last_status()
# rest the filter's modified time and assign it to the output object
self.outputInformation.modifiedTime = self.modifiedTime = time.time()
connection.close()
def query_log():
connection = Connection('localhost', 27017)
db = connection.mydb
collection = db.logdata
for doc in collection.find():
print(doc)
connection.close()
def displaydb():
"""
displays all data inside db called file name
something similar to read db
"""
global host
connection = Connection(host, 27017)
db = connection[dbname]
collection = db[coll]
posts = db.coll
if posts.count():
for post in posts.find():
for key in post:
if key != '_id':
print '%22s' % (str(key)),
print
break
for post in posts.find():
for key, val in post.iteritems():
if key != '_id':
print '%22s' % (str(val)),
print
else:
return 'no data'
connection.close()
class MongoHandler():
__metaclass__ = Singleton
def __init__(self):
try:
self.db_connection = None
self.init_config()
self.init_db()
except Exception as e:
raise Exception('In MongoHandler init: ' + str(e))
def init_config(self):
self.db_connection_str = 'localhost:27017'
#self.db_user = config_handler.get_key_from_config('db_user', 'admin')
#self.db_password = config_handler.get_key_from_config('db_password', 'int311af33d')
self.db_name = 'intellafeed'
def init_db(self):
self.db_connection = Connection(self.db_connection_str)
self.db = self.db_connection[self.db_name]
#self.db.authenticate(self.db_user, self.db_password)
def close_connection(self):
if self.db_connection:
self.db_connection.close()
def execute(self):
print self.name + " executing"
# setup mongo connection and look through all input collections, copying
connection = Connection('localhost', 27017)
if len(self.databaseName) > 0:
db = connection[self.databaseName]
else:
db = connection['arbor']
outputcoll = db[self.outputInformation.collectionName]
# clear out the output to prepare for running an algorithm
outputcoll.drop()
# loop through all parameters on this filter and output a line in the collection
# for each parameter as a key/value pair
for thisparam in self.parameters:
outdict = dict()
outdict['key'] = thisparam
outdict['value'] = self.parameters[thisparam]
# find all documents in this input (repeated for each input)
outputcoll.insert(outdict)
# pause the writer enough to allow the write to complete? Found example code here:
# http://sourceforge.net/u/rick446/random/ci/master/tree/lib.py
db.last_status()
# rest the filter's modified time and assign it to the output object
self.outputInformation.modifiedTime = self.modifiedTime = time.time()
connection.close()
class MongoDataSource(DataSource): # define parent class
connection = None
database = None
def open_connection(self, options):
print 'Opening mongo connection'
if not options["host"] or not options["port"] or not options[
"database"]:
return False
# Create a connection to mongodb
self.connection = Connection(options["host"], options["port"])
self.database = self.connection[options["database"]]
def get_collection(self, collection):
if collection in self.database.collection_names():
return self.database[collection]
else:
return None
def database():
return self.database
def index(self, object, collection):
print "Indexing into MongoDB"
self.database[collection].save(object)
def close_connection(self):
self.connection.close()
class DB(BaseDB):
def __init__(self):
self.connection = Connection()
self.db = self.connection['oauth']
#make sure the Tranform class which is a SONManipulator is used
self.db.add_son_manipulator(Transform())
self.models = self.db.models
def get(self, key):
document = self.models.find_one({'key':key})
return document['class']
def put(self, key, data):
#if we don't do this we end up with mutiple copies of the same data
if not self.contains(key):
document = {'key':key, 'class': data}
self.models.insert(document)
else:
self.update(key, data)
def update(self, key, data):
#import pdb; pdb.set_trace()
#seems you can't use a SONManipulator on updates,
#manipulate=True also adds a new _id which causes an
#'OperationFailure: Modifiers and non-modifiers cannot be mixed'
#Solution: do the transform directly and use the result to
#update the entry with and don't set manipulate=True
data = json.loads(jsonpickle.encode(data))
self.models.update({'key':key}, {"$set": {'class':data}}, multi=False, safe=True)
def delete(self, key):
self.models.remove({'key':key})
def commit(self):
#we don't need this for MongoDB
pass
def abort(self):
#we could do this I think,
#but am currently debating whether to use it or not
pass
def contains(self, key):
#returns None if no entry with that key filter exists
#so wrapping it in bool will return False if None,
#True on anything else
return bool(self.models.find_one({'key':key}))
def close(self):
self.connection.close()
def run():
# Create an empty response object.
response = {}
collectionNames = []
# look through the collections in the ivaan database and return the name of all collections
# that match the naming profile for tables. This is matching to see if the collection name
# begins with "table_cardiac" since it is only returning cardiac studies from the IVAaN database
connection = Connection('localhost', 27017)
db = connection['ivaan']
# get a list of all collections (excluding system collections)
collection_list = db.collection_names(False)
for coll in collection_list:
# if it is a table, then add it to the response
if (str(coll[:14]) == 'table_cardiac_'):
print "found table:", coll
# don't return the prefix in the project name. Users don't have to know the
# cardiac project collection names are prepended
collectionNames.append(coll[14:])
connection.close()
# if no projects found at all, return a default name
if len(collectionNames) == 0:
collectionNames.append("default")
# Pack the results into the response object, and return it.
response['result'] = collectionNames
# Return the response object.
tangelo.log(str(response))
return bson.json_util.dumps(response)
def CalculateSpeciationBySeparateConnection(system, database, port,
tree_collection_name,
matrix_collection_name, character,
verbose):
connection = Connection(system, port)
db = connection[database]
tree_coll = db[str(tree_collection_name)]
matrix_coll = db[str(matrix_collection_name)]
# startup up an R interpreter to do the processing. We will be converting a tree, so create a tree handler
robjects.r("library('geiger')")
r = robjects.r
r('source("/Users/clisle/Projects/Arbor/code/python-R-integration/arbor2apeTreeHandler.R")'
)
r('treeHandler = new("arbor2apeTreeHandler")')
# load function definitions used for speciation
r('source("speciationDefines.R")')
result = InvokeSpeciation(tree_collection_name, tree_coll,
matrix_collection_name, matrix_coll, character,
verbose)
if (connection):
connection.close()
return result
def run(obsid,source,target,interaction,name,lat,lng,datetime):
# Create an empty response object.
global nodeCount
response = {}
# open a link to the backing database
connection = Connection('localhost', 27017)
db = connection["FeederWatch"]
data_coll = db['interactions']
currenttime = arrow.now().timestamp
# Write out a single record for the observation. The location is saved
# as a tuple [longitude,latitude] to be compatible with standard mongoDB geo practices
try:
location_point = [float(lng),float(lat)]
record = {'observation':obsid,'source':source,'target':target,'interaction':interaction,'location_name':name,'location': location_point,'date':datetime,'recorded':currenttime}
interactionCursor = data_coll.insert(record)
except:
response['error'] = "Could not write"
return bson.json_util.dumps(response)
connection.close()
response['data'] = []
# Pack the results into the response object, and return it.
#response['result'] = jsonoutput
response['result'] = 'OK'
# Return the response object.
#tangelo.log(str(response))
return bson.json_util.dumps(response)
def connection(self, kind=0, q=None):
'''
Access MongoDB and load topology or events data.
:param kink: specify which request should be running in the DataBase.
:return: a cursor of topology or events.
:rtype: Cursor of elements dictionnary or NoneType.
'''
if self.username is None:
connection = Connection()
db = connection['canopsis']
else:
connection = MongoClient(self.MONGO_URL, self.MONGO_PORT)
db = connection["canopsis"]
# Do the authentication
db.authenticate(self.username, self.password)
if kind == 0:
query = self.qr
# Format string
json_acceptable = query.replace("'", "\"")
query = json.loads(json_acceptable)
cursor = db.objectv1.find(query)
else:
query = q
# Format string
json_acceptable = query.replace("'", "\"")
query = json.loads(json_acceptable)
cursor = db.eventsv1.find(query)
connection.close()
return cursor
def init_apns():
mongodb = None
while not mongodb:
try:
mongodb = Connection(options.mongohost, options.mongoport)
except:
pass
# wait 5 secs to reconnect
time.sleep(5)
masterdb = mongodb[options.masterdb]
apps = masterdb.applications.find({'enableapns': 1})
apnsconns = {}
for app in apps:
apnsconns[app['shortname']] = []
conns = int(app['connections'])
if conns > 5:
conns = 5
if conns < 1:
conns = 1
if 'environment' not in app:
app['environment'] = 'sandbox'
if 'certfile' in app and 'keyfile' in app and 'shortname' in app:
for instanceid in range(0, conns):
try:
apn = APNClient(app['environment'], app['certfile'], app['keyfile'], app['shortname'], instanceid)
except Exception as ex:
logging.error(ex)
continue
apnsconns[app['shortname']].append(apn)
mongodb.close()
return apnsconns
def setData(self, serialData):
try:
print serialData
decoded = json.loads(serialData, object_hook=json_util.object_hook)
id_no = decoded['id_no']
sensor = decoded['sensor']
porta = decoded['porta']
valor = str(decoded['valor'])
if sensor == "2":
valor = valor + '.00'
datahora = decoded['datahora']
lab = decoded['lab']
#p = 1000
#salva no banco de dados
con = Connection('127.0.0.1', 27017, safe=True)
db = con['inoshare']
pacote = {'id_no':id_no, 'sensor':sensor, 'porta':porta, 'valor':valor, 'datahora':datahora, 'lab': lab}
inoshare = db.inoshare
inoshare.insert(pacote)
print pacote
con.close()
except Exception as e:
print "Erro no filter data: ", e.message, e.args
def displaydb():
"""
displays all data inside db called file name
something similar to read db
"""
global host
connection = Connection(host, 27017)
db = connection[dbname]
collection = db[coll]
posts = db.coll
if posts.count():
for post in posts.find():
for key in post:
if key != "_id":
print "%22s" % (str(key)),
print
break
for post in posts.find():
for key, val in post.iteritems():
if key != "_id":
print "%22s" % (str(val)),
print
else:
return "no data"
connection.close()
class Deleted(Filter):
def init(self):
self.watch = [] # 監視対象のスクリーンネームを入れる. 空なら全部通知
self.exclude = []
addr = "localhost"
port = 27017
db_name = "twitter"
self.connection = Connection(addr, port)
self.col = self.connection[db_name].tweet
def filter(self, packet):
data = packet["data"]
if not isinstance(data, dict):
return None
if data.get("event") == "delete":
try:
tweet = self.col.find({u"id": data["id"]})[0]
except IndexError:
return None
detail = {"user": tweet[u"user"][u"screen_name"], "post": tweet[u"text"]}
if str(detail["user"]) in self.exclude or (self.watch != [] and str(detail["user"]) not in self.watch):
return None
self.send(u"%(user)s deleted: %(post)s" % detail, exclude=["favbot"])
def cleanup(self):
self.connection.close()
def execute(self, arborapi=None):
print self.name+" executing"
# setup mongo connection and look through all input collections, copying
connection = Connection('localhost', 27017)
if len(self.databaseName)>0:
db = connection[self.databaseName]
else:
db = connection['arbor']
outputcoll = db[self.outputInformation.collectionName]
# clear out the output to prepare for running an algorithm
outputcoll.drop()
# loop through all inputs and process all objects in the inputs
for thisInput in self.inputs:
inputcoll = db[thisInput.collectionName]
# find all documents in this input (repeated for each input)
queryResults = inputcoll.find()
print "found that ", thisInput.collectionName," has ",queryResults.count(), " records"
# write the documents into the output collection and indicate the output time changed
for result in queryResults:
outputcoll.insert(result)
# pause the writer enough to allow the write to complete? Found example code here:
# http://sourceforge.net/u/rick446/random/ci/master/tree/lib.py
db.last_status()
# rest the filter's modified time and assign it to the output object
self.outputInformation.modifiedTime = self.modifiedTime = time.time()
connection.close()
def execute(self):
print self.name+" executing"
# setup mongo connection and look through all input collections, copying
connection = Connection('localhost', 27017)
if len(self.databaseName)>0:
db = connection[self.databaseName]
else:
db = connection['arbor']
outputcoll = db[self.outputInformation.collectionName]
# clear out the output to prepare for running an algorithm
outputcoll.drop()
# loop through all parameters on this filter and output a line in the collection
# for each parameter as a key/value pair
for thisparam in self.parameters:
outdict = dict()
outdict['key'] = thisparam
outdict['value'] = self.parameters[thisparam]
# find all documents in this input (repeated for each input)
outputcoll.insert(outdict)
# pause the writer enough to allow the write to complete? Found example code here:
# http://sourceforge.net/u/rick446/random/ci/master/tree/lib.py
db.last_status()
# rest the filter's modified time and assign it to the output object
self.outputInformation.modifiedTime = self.modifiedTime = time.time()
connection.close()
def run(tablename):
# Create an empty response object.
response = {}
print "using collection:",tablename
# first find out the types of the attributes in this collection. Create a dictionary with the names and types
connection = Connection('localhost', 27017)
db = connection['ivaan']
dataset_collection = db[tablename]
#tablerecord = dataset_collection.find()[0]
# Now that we have the proper types in the table collection stored in a handy local dictionary "attributes", lets
# build a query for mongoDB depending on how many filters are enabled.
querystring = {}
print "query to perform:", querystring
# now that we have the query build, execute it and return the matching records from the collection
connection = Connection('localhost', 27017)
db = connection['polar']
dataset_collection = db[tablename]
# Do a find operation with the passed arguments.
it = dataset_collection.find(querystring)
results = [x for x in it]
connection.close()
print results
# convert from the local json to a geojson multipoint result
#latitudes = [30, 30, 30]
#longitudes = [10, 20, 30]
#define multipoint geometry
multipoint = ogr.Geometry(ogr.wkbMultiPoint)
#create point geometry and add to multipoint geometry
for i in range(len(results)):
point = ogr.Geometry(ogr.wkbPoint)
fixedlocation = convertStringToFloatPoint(results[i]['stationLng'], results[i]['stationLat'])
point.AddPoint(fixedlocation['lng'],fixedlocation['lat'])
multipoint.AddGeometry(point)
#convert geometry to GeoJSON format
geojson_multipoint = multipoint.ExportToJson()
# Pack the results into the response object, and return it.
response['count'] = it.count()
response['data'] = geojson_multipoint
response['result'] = 'OK'
# Return the response object.
tangelo.log(str(response))
return bson.json_util.dumps(response)
def _drop_database(self, database_name):
c = Connection()
try:
if database_name in c.database_names():
self.log("Dropping database: %s" % database_name)
c.drop_database(database_name)
finally:
c.close()
def to_db(infile, db_name):
conn = Connection('localhost', 27017)
db = conn[db_name]
locations_info = decode_locations(infile)
for i in locations_info:
locations_info[i]['_id'] = i
db['location_info'].insert(locations_info[i])
conn.close()
def AssignHierarchicalNamesSeparateConnection(system, database, port, tree_collection_name, verbose):
connection = Connection(system, port)
db = connection[database]
data_coll = db[str(tree_collection_name)]
AssignHierarchicalNames(data_coll)
if connection:
connection.close()
def get_full_code(object_id):
"""get the full code document from the database."""
connection = Connection()
database = connection.codingbooth
collection = database.codes
result = collection.find_one({'_id': ObjectId(object_id)})
connection.close()
return result
def load_user(object_id):
"""Load the user based on the given id."""
connection = Connection()
database = connection.codingbooth
collection = database.users
result = collection.find_one({'_id': ObjectId(object_id)})
connection.close()
return result
def get_code_from_name(name):
"""Retrieve code from the database given a name."""
connection = Connection()
database = connection.codingbooth
collection = database.codes
result = collection.find_one({'name': name})
connection.close()
return result
def get_code(object_id):
"""Retrieve the code from the database."""
connection = Connection()
database = connection.codingbooth
collection = database.codes
result = collection.find_one({'_id': ObjectId(object_id)})
connection.close()
return result['code']
def load_user_by_email(email):
"""Load the user based on the given email"""
connection = Connection()
database = connection.codingbooth
collection = database.users
result = collection.find_one({'email': email})
connection.close()
return result
def _drop_database(self, database_name):
c = Connection()
try:
if database_name in c.database_names():
self.log("Dropping database: %s" % database_name)
c.drop_database(database_name)
finally:
c.close()
def set_compile_results(object_id, results):
"""Put the compilation results into the database."""
connection = Connection()
database = connection.codingbooth
collection = database.codes
db_id = collection.update({'_id': ObjectId(object_id)},
{'$set': {'compilation': results}})
connection.close()
return db_id
def AssignHierarchicalNamesSeparateConnection(system, database, port,
tree_collection_name, verbose):
connection = Connection(system, port)
db = connection[database]
data_coll = db[str(tree_collection_name)]
AssignHierarchicalNames(data_coll)
if (connection):
connection.close()
def set_run_results(object_id, results):
"""Put the results of running the compiled executable into the database."""
connection = Connection()
database = connection.codingbooth
collection = database.codes
db_id = collection.update({'_id': ObjectId(object_id)},
{'$set': {'run': results}})
connection.close()
return db_id
def execute(self, arborapi=None):
print self.name + " executing"
# setup mongo connection and look through all input collections, copying
connection = Connection('localhost', 27017)
if len(self.databaseName) > 0:
db = connection[self.databaseName]
else:
db = connection['arbor']
outputcoll = db[self.outputInformation.collectionName]
# clear out the output to prepare for running an algorithm
outputcoll.drop()
# loop through all inputs and process all objects in the inputs
for thisInput in self.inputs:
inputcoll = db[thisInput.collectionName]
# find all documents in this input (repeated for each input) that match the
# test criteria. If no criteria is specified, pass records through
for case in switch(self.parameters['filterOperation']):
if case('GreaterThan'):
query = {
self.parameters['filterAttribute']: {
'$gt': self.parameters['filterValue']
}
}
break
if case('LessThan'):
query = {
self.parameters['filterAttribute']: {
'$lt': self.parameters['filterValue']
}
}
break
if case('NotEqual'):
query = {
self.parameters['filterAttribute']: {
'$ne': self.parameters['filterValue']
}
}
break
if case('Equal') or case('EqualTo'):
query = {
self.parameters['filterAttribute']:
self.parameters['filterValue']
}
break
print "query used was: ", query
queryResults = inputcoll.find(query)
# write the documents into the output collection and indicate the output time changed
for result in queryResults:
outputcoll.insert(result)
# rest the filter's modified time and assign it to the output object
self.outputInformation.modifiedTime = self.modifiedTime = time.time()
print self.name, " passed ", outputcoll.count(), " records"
connection.close()
def create_user(email, password):
"""Create a user in the database with a given name, email, and password."""
connection = Connection()
database = connection.codingbooth
collection = database.users
db_id = collection.insert({
'email': email,
'password': password})
connection.close()
return db_id
def save_user(object_id, email, password):
"""Save the user to the database."""
connection = Connection()
database = connection.codingbooth
collection = database.users
collection.update({'_id': ObjectId(object_id)},
{'$set': {
'email': email,
'password': password}})
connection.close()
def mongodb_parser(db, collection, args=None):
conn = Connection()
db = conn[db]
collection = db[collection]
res = collection.find(args)
reslist = list(res)
for r in reslist:
r.pop('_id')
jsonlist = json.dumps({'result': reslist})
conn.close()
return jsonlist
def mongodb_write(new_dict):
'''
Write register to the Monto Database
'''
try:
mongo_conn = Connection(MONGODB_SERVER, 27017)
except Exception:
sys.exit("Failure to connect to the mongo database")
db = mongo_conn[MONGODB_DB]
db2ins = db.db2ins
db2ins.insert(new_dict)
mongo_conn.close()
def set_code(object_id=None, code=""):
"""Store the code block into the database."""
connection = Connection()
database = connection.codingbooth
collection = database.codes
if object_id:
collection.update({'_id': ObjectId(object_id)},
{'$set': {'code': code}})
db_id = object_id
else:
db_id = collection.insert({'code': code})
connection.close()
return str(db_id)
def run(obsid, source, target, interaction):
# Create an empty response object.
global nodeCount
response = {}
# open a link to the backing database
connection = Connection('localhost', 27017)
db = connection["FeederWatch"]
data_coll = db['interactions']
# split the single JSON object up into a hierarchy of objects by traversing the nested dictionaries and writing
# documents in mongo during the traversal. Store in a collection according to the name of the file dropped. The location is saved
# as a tuple [longitude,latitude] to be compatible with standard mongoDB geo practices
if source == 'null' and target == 'null':
# this is the case where the last record of this observation is being deleted
try:
record = {'observation': obsid}
data_coll.remove(record)
# return the remaining records, so the UI can re-render
response['data'] = data_coll.find({'observation': obsid})
except:
response['error'] = "Could not delete"
return bson.json_util.dumps(response)
else:
# this is the ordinary case where one record out of many is being deleted
try:
record = {
'observation': obsid,
'source': source,
'target': target,
'interaction': interaction
}
data_coll.remove(record)
# return the remaining records, so the UI can re-render
response['data'] = data_coll.find({'observation': obsid})
except:
response['error'] = "Could not delete"
return bson.json_util.dumps(response)
connection.close()
# Pack the results into the response object, and return it.
#response['result'] = jsonoutput
response['result'] = 'OK'
# Return the response object.
#tangelo.log(str(response))
return bson.json_util.dumps(response)
def process(id):
stuff = majdoor.fetch_koji_build(id)
if stuff and len(stuff) == 3:
package, nvr, urls = stuff
else:
print "??? majdoor skipped / failed", id
return "OK"
if not urls:
return
for arch, url in urls:
basename = url.split('/')[-1]
path = os.path.join("cache", nvr, basename)
if path.endswith(".rpm") and not \
path.endswith(".src.rpm") and \
not "-debuginfo-" in path:
output = scanner.analyze(path)
print output
connection = Connection()
db = connection.test_database
analysis = db.analysis
analysis.insert(json.loads(output))
connection.close()
# do rpmgrill stuff, spawn as we don't know how rpmgrill affets our env.
basepath = os.path.join(os.path.realpath("cache"), nvr)
print "Running rpmgrill on", basepath
p = subprocess.Popen("./invoke_rpmgrill.sh %s" % basepath,
stderr=subprocess.PIPE,
shell=True)
_, err = p.communicate()
output = os.path.join(os.path.realpath("cache"), nvr, "rpmgrill.json")
if not os.path.exists(output):
print "!!! rpmgrill failed for", basepath
print err
else:
with open(output) as f:
data = f.read()
# we store rpmgrill.json in a database
connection = Connection()
db = connection.test_database
rpmgrill = db.rpmgrill
entry = {"nvr": nvr, "output": data, "package": package}
rpmgrill.insert(entry)
print "!!!", err
return "OK"
def run(study=None, projectname='first'):
# Create an empty response object.
global nodeCount
response = {}
# first decode the argument from being passed through a URL
jsonObj = bson.json_util.loads(study)
# Append the project name to the "table_" prefix, so this dataset
# will be read as a IVAaN tabular dataset. Then any dashes are replaced with
# underscores to avoid dashes in mongo collection names
print "using project name: ", projectname
# build name from "table" + date + dropped filename
collectionName = 'table_cardiac_' + projectname
collectionNameNoDashes = string.replace(collectionName, '-', '_')
print "modified collection name: ", collectionNameNoDashes
# open a database connection
connection = Connection('localhost', 27017)
db = connection["ivaan"]
data_coll = db[collectionNameNoDashes]
# we have a single JSON object here with all the fields to store, but some of them are floating point numbers which
# are coming across as strings (e.g. "4.555") instead of a number 4.555. To fix this, we will iterate through the fields
# and convert values into numbers if they are convertable
storageDict = {}
for attrib in jsonObj.keys():
if isinstance(jsonObj[attrib], list):
storageDict[attrib] = jsonObj[attrib]
else:
storageDict[attrib] = convertIfNumber(jsonObj[attrib])
try:
data_coll.insert(storageDict)
except ValueError:
response['error'] = "Could not convert to JSON"
return bson.json_util.dumps(response)
connection.close()
# Pack the results into the response object, and return it.
#response['result'] = jsonoutput
response['result'] = 'OK'
# Return the response object.
tangelo.log(str(response))
return bson.json_util.dumps(response)
def insertdb(data):
"""
data is a list of dict's which ll be inserted into db called dbname and a collection called dbname
"""
global host
connection = Connection(host, 27017)
db = connection[dbname]
collection = db[coll]
post_id = []
posts = db.coll
post_id.append(posts.insert(data))
if verbose:
print 'inserted:', data
connection.close()
return post_id
class BooleanSearch:
'''
This class accepts a word, queries MongoDB and returns all the postings list as a Python set
Also, given two sets of postings list this class performs boolean operations of AND, OR and NOT using
python set object methods of Intersection, Union and Difference.
Additionally, given a set of postings list, it queries the chunk files to ouput the log entries
'''
def __init__(self):
self.con = Connection()
self.db = self.con.indexer
self.idf = self.db.idf
self.outputPath = "Processed_Data/"
def __del__(self):
self.con.close()
def getPostingsListSet(self, word): # Given a word returns a set of postings list from MongoDB
mySet = set()
for row in self.idf.find({'Term':word}):
pList = row['postingsList']
mystr = '.'
mystr = mystr.join([str(x) for x in pList.values()])
mySet.add(mystr)
return mySet
def printLogs(self, pListSet): # Give a set of postings list, queries the chunk files to output the logs
for oneList in pListSet:
items = oneList.split('.')
fn = self.outputPath + items[0]
fhndl = open(fn, 'r')
fhndl.seek(int(items[1]))
print fhndl.read(int(items[2])).strip()
fhndl.close()
def termAND(self, set1, set2): # Given 2 sets of postings list, performs boolean AND
setAND = set1.intersection(set2)
return setAND
def termOR(self, set1, set2): # Given 2 sets of postings list, performs boolean OR
setOR = set1.union(set2)
return setOR
def termNOT(self, set1, set2): # Given 2 sets of postings list, performs boolean NOT
setNOT = set2.difference(set1)
return setNOT
def singleTerm(self, word): # returns postings list for a single term when no boolean operations is required
return self.getPostingsListSet(word)
