async def _search(self, request, search_params):
is_single = search_params.get('single')
order_by_list = search_params.get('order_by', [])
# paginate
# num_results = 20
results_per_page = self._compute_results_per_page(request)
page_num = 1
if results_per_page > 0:
page_num = int(request.args.get('page', 1))
start = (page_num - 1) * results_per_page
# end = start + results_per_page
# total_pages = int(math.ceil(num_results / results_per_page))
else:
start = 0
# total_pages = 1
query = None
if 'filters' in search_params:
filters = search_params['filters']
filters["doc_type"] = self.collection_name
query = Query(self.db.db, selector=filters)
else:
query = Query(self.db.db,
selector={"doc_type": self.collection_name})
if is_single:
for document in query(limit=1)['docs']:
result = to_dict(document,
exclude=self.exclude_columns,
include=self.include_columns)
break
else:
query_result = None
if (results_per_page is not None) and (results_per_page > 0):
if (start is not None) and (start > 0):
query_result = query.result[start:start + results_per_page]
else:
query_result = query.result[:results_per_page]
else:
if (start is not None) and (start > 0):
query_result = query.result[start:]
else:
query_result = query.result
objects = []
for document in query_result:
objects.append(
to_dict(document,
exclude=self.exclude_columns,
include=self.include_columns))
new_page = page_num + 1
result = {
"page": page_num,
"next_page": new_page,
"objects": objects
}
return result
def getRecvEmail(my_database, receiverID, emailLimit=0):
# Test if database exists
try:
my_database.exists()
except Exception:
return -1
# If zero return all email
if int(emailLimit) == 0:
query = Query(my_database,
selector={
'receiver': receiverID,
'date': {
'$gt': 0
}
})
else:
query = Query(my_database,
selector={
'receiver': receiverID,
'date': {
'$gt': 0
}
},
limit=int(emailLimit))
return query(sort=[{'date': 'desc'}])['docs']
def query(self, database=None, selectorField=None, value=None):
if self.dbclient is None:
self.dbclient.connect()
db = self.dbclient[database]
query = Query(db)
if query is not None:
with query.custom_result(selector={selectorField: value}) as res:
if res is not None:
return res[0]
else:
return None
else:
return None
def get_from_db_by_params(user_name, region_name, place_name, db_name, object_example, asdocs=False, dates=None, word_list=None, logic='or_logic', exact=False):
print("get_from_db_by_params")
print(user_name + ";" + region_name + ";" + place_name + ";" + db_name + ";" + str(type(object_example)) + ";" +
str(asdocs))
get_db_client = Cloudant(serviceUsername, servicePassword, url=serviceURL, adapter=Replay429Adapter(retries=10, initialBackoff=0.01))
try:
our_list = []
get_db_client.connect()
db = get_db_client.get(db_name, remote=True)
print("get_from_db_by_params logic is:" + logic)
selector = generate_single_place_selector(user_name, region_name, place_name, dates, word_list, logic, exact)
print("the selector is: " + str(selector))
query_result = QueryResult(Query(db, selector=selector))
if asdocs is True:
for doc in query_result:
our_list.append(doc)
else:
for doc in query_result:
our_list.append(object_example.build_from_document(doc))
except CloudantException as exc:
print("CloudantException in get_from_db_by_params")
print(exc)
return exc
except Exception as exc:
print("non CloudantException exception in get_from_db_by_params")
print(exc)
return exc
finally:
get_db_client.disconnect()
print("get_from_db_by_params id: " + str(db_name) + " len: " + str(len(our_list)))
return our_list
def get_from_db(id_list, db_name, object_example=None):
result_list = []
client_get_from_db = Cloudant(serviceUsername, servicePassword, url=serviceURL, adapter=httpAdapter2)
try:
client_get_from_db.connect()
db = client_get_from_db.get(db_name, remote=True)
if db is None:
print("error: no tweet db ready yet")
return ""
query_result = QueryResult(Query(db, selector={'_id': {'$in': id_list}}))
index = 0
for doc in query_result:
if object_example is "asdocs":
result_list.append(doc)
elif object_example is not None:
result_list.append(object_example.build_from_document(doc))
index = + 1
else:
dec = jsonpickle.decode(str(doc['value']))
result_list.append(dec)
index =+ 1
print("query_result len: " + str(index))
except CloudantException as exc:
print("CloudantException in get_from_db")
print(exc)
result_list = []
except Exception as exc:
print("non CloudantException exception in get_from_db")
print(exc)
result_list = []
finally:
client_get_from_db.disconnect()
print("get_from_db id: " + str(db_name) + " len: " + str(len(result_list)))
return result_list
def get4Years(db, currentYear):
query = Query(db)
resp = query(
selector={"$and": [
{
"type": {
"$eq": "years"
}
},
{
"year": {
"$in": [
currentYear-3,
currentYear-2,
currentYear-1
]
}
},
{
"crop": {
"$ne": "תפוא"
}
}
]},
fields=["plot"],
)
return resp
def test_callable_with_invalid_value_types(self):
"""
Test Query __call__ by passing in invalid selector
"""
test_data = [
{'selector': 'blah'}, # Should be a dict
{'limit': 'blah'}, # Should be an int
{'skip': 'blah'}, # Should be an int
{'sort': 'blah'}, # Should be a list
{'fields': 'blah'}, # Should be a list
{'r': 'blah'}, # Should be an int
{'bookmark': 1}, # Should be a basestring
{'use_index': 1} # Should be a basestring
]
for argument in test_data:
query = Query(self.db)
try:
query(**argument)
self.fail('Above statement should raise an Exception')
except CloudantArgumentError as err:
self.assertTrue(str(err).startswith(
'Argument {0} is not an instance of expected type:'.format(
list(argument.keys())[0]
)
))
def find(cls, type, selector={}, fields=[], limit=100):
"""List documents filtered by fields."""
results = Query(cls.db(type),
selector=selector,
fields=fields,
limit=limit)
return results()['docs']
def find_doc(self, doc_type, property_name, property_value):
"""
Finds a doc based on the specified doc_type, property_name, and property_value.
Parameters
----------
doc_type - The type value of the document stored in Cloudant
property_name - The property name to search for
property_value - The value that should match for the specified property name
"""
try:
self.client.connect()
db = self.client[self.db_name]
selector = {
'_id': {
'$gt': 0
},
'type': doc_type,
property_name: property_value
}
query = Query(db, selector=selector)
for doc in query()['docs']:
return doc
return None
finally:
self.client.disconnect()
def get_user(name):
if name not in user_dictionary.keys(
): # we need to handle the creation of a new user
user = User(my_name=name)
user_dictionary[name] = user
user.save_me_to_db()
else:
myclient = Cloudant(serviceUsername,
servicePassword,
url=serviceURL,
adapter=httpAdapter)
myclient.connect()
users_db = myclient.get(user_database_name, remote=True)
#if users_db is not None:
users_returned = QueryResult(
Query(users_db, selector={'_id': {
'$eq': name
}}))
if users_returned is not None:
list = users_returned.all()
for u in list:
t1 = u['_id']
t2 = str(u['value'])
# print(t2)
dec = jsonpickle.decode(t2)
user_dictionary[t1] = dec
myclient.disconnect()
return user_dictionary[name]
def find_doc(self, doc_type, property_name, property_value):
"""Finds a doc in Cloudant DB
:param str doc_type: type value of the document stored in Cloudant
:param str property_name: property name to search for
:param str property_value: value that should match for the specified
property name
:returns: doc from query or None
:rtype: dict, None
"""
try:
self.client.connect()
db = self.client[self.db_name]
selector = {
'_id': {
'$gt': 0
},
'type': doc_type,
property_name: property_value
}
query = Query(db, selector=selector)
for doc in query()['docs']:
return doc
return None
except Exception:
LOG.exception("Cloudant DB exception:")
finally:
self.client.disconnect()
def put_best_model_in_production(model_metrics, db_name):
"""
Función para poner el mejor modelo en producción.
Args:
model_metrics (dict): Info del modelo.
db_name (str): Nombre de la base de datos.
"""
# conexión a la base de datos elegida
db = client.get_database(db_name)
# consulta para traer el documento con la info del modelo en producción
query = Query(db, selector={'status': {'$eq': 'in_production'}})
res = query()['docs']
# id del modelo en producción
best_model_id = model_metrics['_id']
# en caso de que SÍ haya un modelo en producción
if len(res) != 0:
# se realiza una comparación entre el modelo entrenado y el modelo en producción
best_model_id, worse_model_id = get_best_model(model_metrics, res[0])
# se marca el peor modelo (entre ambos) como "NO en producción"
worse_model_doc = db[worse_model_id]
worse_model_doc['status'] = 'none'
# se actualiza el marcado en la BDD
worse_model_doc.save()
else:
# primer modelo entrenado va a automáticamente a producción
print('------> FIRST model going in production')
# se marca el mejor modelo como "SÍ en producción"
best_model_doc = db[best_model_id]
best_model_doc['status'] = 'in_production'
# se actualiza el marcado en la BDD
best_model_doc.save()
def getCloudantData(id):
USERNAME = os.environ['cloudant_username']
PASSWORD = os.environ['cloudant_password']
URL = os.environ['cloudant_url']
APIKEY = os.environ['cloudant_apikey']
DATABASENAME = os.environ['cloudant_databasename']
# IBM Cloudant Legacy authentication
client = Cloudant(USERNAME, PASSWORD, url=URL)
client.connect()
# IAM Authentication (uncomment if needed, and comment out IBM Cloudant Legacy authentication section above)
# client = Cloudant.iam(USERNAME, APIKEY)
# client.connect()
myDatabase = client[DATABASENAME]
# query = Query(myDatabase, skip=10, limit=100)
# result = QueryResult(query, skip=10, limit=100)
# if myDatabase.exists():
# print("'{0}' successfully created.\n".format(databaseName))
query = Query(myDatabase, selector={"user_id": id})
doc = query()['docs'][0]
# for doc in query()['docs']:
# print(doc)
# client.disconnect()
return doc
def check_time(deviceid, check_value, username, device_name):
"""Check last watered time against value. Water if necessary.
Args:
readingdb (obj): Reading database object
deviceid (str): Device Id
check_value (int): Days to wait between watering
"""
print("Checking Time")
# Query DB
query = Query(readingdb,
selector={
'device_id': {
'$eq': deviceid
},
'type': {
'$eq': 'pump'
}
})
result = query(sort=[{'time_reading': 'desc'}], limit=1)['docs']
# Process reading
if len(result) > 0:
result = result[0]
days = check_value * 86400
if result['time_reading'] <= int(time()) - days:
water_plant(deviceid, username, device_name)
else:
# Pump has not been watered ever
# Water so there is a history
print("Watering for the first time")
water_plant(deviceid, username, device_name)
def test_retrieve_query_url(self):
"""
Test constructing the query test url
"""
query = Query(self.db)
self.assertEqual(query.url,
posixpath.join(self.db.database_url, '_find'))
def check_moisture(deviceid, check_value, username, device_name):
"""Check moisture level of device against watering value. Water if necessary.
Args:
deviceid (str): device id
check_value ([type]): [description]
"""
print("Checking Moisture")
# Query DB
query = Query(readingdb,
selector={
'device_id': {
'$eq': deviceid
},
'type': {
'$eq': 'moisture'
}
})
result = query(sort=[{'time_reading': 'desc'}], limit=1)['docs']
# Process reading
if len(result) > 0:
result = result[0]
print("Readging:", result)
if float(result['values']['moisture_level']) <= float(check_value):
water_plant(deviceid, username, device_name)
def retrieve_from_db():
# Retrieval will be based on geolocation of user: lat,long
# latitude: 0.001 degree = 111 m
lat = 5.989
lon = -50.14
selector = {
'_id': {
'$gt': 0,
}
}
qr= Query(db, selector=selector)
query_res = qr(limit=1000, skip=0)['docs']
# code to add into maps
green_marker=[]
red_marker=[]
orange_marker=[]
white_marker=[]
for each in query_res:
if (each['danger']=='Safe'):
green_marker.append((each['latitude'], each['longitude'], each['description']))
elif (each['danger']=='Medium'):
orange_marker = (each['latitude'], each['longitude'], each['description'])
elif (each['danger']=='High'):
red_marker = (each['latitude'], each['longitude'], each['description'])
else:
white_marker: (each['latitude'], each['longitude'], each['description'])
def test_constructor_with_kwargs(self):
"""
Test instantiating a Query by passing in query parameters
"""
query = Query(self.db, foo={'bar': 'baz'})
self.assertIsInstance(query, Query)
self.assertIsInstance(query.result, QueryResult)
self.assertEqual(query, {'foo': {'bar': 'baz'}})
def test_constructor_without_kwargs(self):
"""
Test instantiating a Query without parameters
"""
query = Query(self.db)
self.assertIsInstance(query, Query)
self.assertIsInstance(query.result, QueryResult)
self.assertEqual(query, {})
def test_custom_result_context_manager(self):
"""
Test that custom_result yields a context manager and returns expected
content
"""
self.populate_db_with_documents(100)
query = Query(
self.db,
selector={'_id': {'$lt': 'julia050'}},
fields=['_id'],
r=1
)
with query.custom_result(sort=[{'_id': 'desc'}]) as rslt:
self.assertEqual(
rslt[10:13],
[{'_id': 'julia039'}, {'_id': 'julia038'}, {'_id': 'julia037'}]
)
def getAllPlots(db):
query = Query(db)
resp = query(
selector={"type": {
"$eq": "plot"
}}
)
return resp
def test_custom_result_context_manager(self):
"""
Test that custom_result yields a context manager and returns expected
content
"""
self.populate_db_with_documents(100)
query = Query(
self.db,
selector={'_id': {'$lt': 'julia050'}},
fields=['_id'],
r=1
)
with query.custom_result(sort=[{'_id': 'desc'}]) as rslt:
self.assertEqual(
rslt[10:13],
[{'_id': 'julia039'}, {'_id': 'julia038'}, {'_id': 'julia037'}]
)
def create_summaries(conn):
"""
Get summary statistics from a view in the sentiments db and update the
summaries db.
"""
# query view
logging.info('Querying database for summary...')
db = conn['sentiments']
ddoc = DesignDocument(db, '_design/summary')
ddoc.fetch()
view = ddoc.get_view('summary-view')
view_results = view(group=True)['rows']
logging.info('Query completed.')
# switch to summaries db
db = conn['summaries']
for view_result in view_results:
symbol = view_result['key']
unix_date = int(time.mktime(date.today().timetuple()))
# query for today's summary record
query = Query(db,
fields=['_id', 'symbol', 'date', 'summary'],
selector={
'symbol': {
'$eq': symbol
},
'date': {
'$eq': unix_date
}
})
if query.result[0]:
# A record for today already exists so overwrite it. This should not normally happen.
record = db[query.result[0][0]['_id']]
summary = record['summary']
logging.info('Updating summary for %s', symbol)
record['summary'] = view_result['value']
summary.save()
else:
# Creating a new summary
logging.info('Creating summary for %s...', symbol)
data = {
'symbol': symbol,
'date': unix_date,
'summary': view_result['value']
}
new_summary = db.create_document(data)
if new_summary.exists():
logging.info('Finished creating summary.')
else:
logging.error('Failed to create summary.')
# don't exceed rate limit
time.sleep(1)
def find_by(cls, **kwargs):
""" Find records using selector """
query = Query(cls.database, selector=kwargs)
results = []
for doc in query.result:
customer = Customer()
customer.deserialize(doc)
results.append(customer)
return results
def find_by(cls, **kwargs):
""" Find records using selector """
query = Query(cls.database, selector=kwargs)
results = []
for doc in query.result:
pet = Pet()
pet.deserialize(doc)
results.append(pet)
return results
def get_product(id):
collection = client['products']
query = Query(collection, selector={'product_id': {'$eq': id}})
body = {}
for doc in query()['docs']:
body = doc
response_json = json.dumps(body, sort_keys=True, skipkeys=True, indent=4)
return response_json, 200
def create_result(self,
selector={'_id': {
'$gt': 0
}},
fields=['_id', 'name', 'age'],
**kwargs):
if kwargs.get('q_parms', None):
query = Query(self.db, **kwargs['q_parms'])
else:
query = Query(self.db)
if kwargs.get('qr_parms', None):
return QueryResult(query,
selector=selector,
fields=fields,
**kwargs['qr_parms'])
else:
return QueryResult(query, selector=selector, fields=fields)
def find_by_equal(cls, **kwargs):
""" Find records using selector """
query = Query(cls.database, selector=kwargs)
results = []
for doc in query.result:
supplier = Supplier()
supplier.deserialize(doc)
results.append(supplier)
return results
def find_by(cls, **kwargs):
""" Find records using selector """
query = Query(cls.database, selector=kwargs)
results = []
for doc in query.result:
promotion = Promotion()
promotion.deserialize(doc)
results.append(promotion)
return results
def test_make_result(self):
"""
Test that make_result wraps the query response as a QueryResult
"""
self.populate_db_with_documents(100)
query = Query(
self.db,
selector={'_id': {'$lt': 'julia050'}},
fields=['_id'],
sort=[{'_id': 'desc'}],
r=1
)
rslt = query.make_result()
self.assertIsInstance(rslt, QueryResult)
self.assertEqual(
rslt[10:13],
[{'_id': 'julia039'}, {'_id': 'julia038'}, {'_id': 'julia037'}]
)
def find_by(cls, **kwargs):
""" Find records using selector """
query = Query(cls.database, selector=kwargs)
results = []
for doc in query.result:
wishlist = Wishlist()
wishlist.deserialize(doc)
results.append(wishlist)
return results
def find_by_greater(cls, field: str, limit):
""" Find records using selector """
query = Query(cls.database, selector={field: {'$gt': limit}})
results = []
for doc in query.result:
supplier = Supplier()
supplier.deserialize(doc)
results.append(supplier)
return results
def getEmailDocID(my_database, doc_ID):
# Test if database exists
try:
my_database.exists()
except Exception:
return -1
# return all doc_id that match
query = Query(my_database, selector={'document_id': doc_ID})
return query(sort=[{'date': 'desc'}])['docs']
