class DB:
def __init__(self):
self.client = MongoClient(os.environ['MONGODB_URL'])
def getDBName(self):
for db_info in self.client.list_database_names():
print(db_info)
def saveSingleInfo(self, data, dataName):
db = self.client[dataName]
collection = db[dataName]
collection.insert_one(data)
def saveMultipleInfo(self, data: list, dataName):
db = self.client[dataName]
collection = db[dataName]
collection.insert_many(data)
def findSingleItem(self, ToFind: dict, WhichDB):
db = self.client[WhichDB]
results = db.find(ToFind)
for result in results:
print(result)
def deleteDB(self, dbToDelete):
self.client.drop_collection(dbToDelete)
print("Done deleting database")
class MyMongoClient:
"""[summary]
MongoDB CRUD Utility
"""
def __init__(self, url):
"""[summary]
Constructor
Args:
url ([string]): [url]
"""
self.db = MongoClient(url).get_default_database()
def drop_collection(self, collection_name):
self.db.drop_collection(collection_name)
def find_one(self, collection_name, filter, sort=[]):
collection = self.db[collection_name]
return collection.find_one(filter, sort=sort)
def insert_one(self, collection_name, value):
collection = self.db[collection_name]
return collection.insert_one(value)
def delete_one(self, collection_name, key):
collection = self.db[collection_name]
return collection.delete_one(key)
def setUp(self):
# Connect to test Database & drop collection faster than dropping whole DB
self.dal = dal
db = MongoClient('localhost', 27017).TestDB
db.drop_collection('urls')
self._id = db['urls'].insert_one({"url": 'https://www.python.org/'})
self.dal.db = db
def test_existing_map(self):
db = MongoClient().geo_example
db.drop_collection('disasters')
db.disasters.create_index([("loc", GEOSPHERE)])
result = db.disasters.insert_many([{"loc": { 'type': "Point", 'coordinates': [ -73, 40 ] }, "name": "blah1.44", "start_datetime": datetime.now()}])
print result.inserted_ids
m = Map(db)
m.get_map_id(datetime.now(), [0, 0])
class TestApp(unittest.TestCase):
def setUp(self):
self.db = MongoClient("52.17.26.163")['testtest']
app.set_db(self.db)
def test_create_new_id(self):
counter = self.db['counters'].find_one({"id": "testid"})
self.assertEquals(counter['seq'], 1)
def tearDown(self):
self.db.drop_collection('counters')
class TestApp(unittest.TestCase):
def setUp(self):
self.db = MongoClient("52.17.26.163")['testtest']
app.set_db(self.db)
def test_create_new_id(self):
counter = self.db['counters'].find_one({"id":"testid"})
self.assertEquals(counter['seq'], 1)
def tearDown(self):
self.db.drop_collection('counters')
def test_fuzzy_match(example_IRS_service_data,
example_IRS_search_object_with_spelled_out_saint,
mock_config_object):
client = MongoClient(
"mongodb+srv://" + os.environ.get('DBUSERNAME') + ":" +
os.environ.get('PW') +
"@shelter-rm3lc.azure.mongodb.net/shelter?retryWrites=true&w=majority"
)['shelter']
if 'pytest_fuzzy_test' in client.list_collection_names():
client.drop_collection('pytest_fuzzy_test')
client.create_collection('pytest_fuzzy_test')
insert_services(example_IRS_service_data, client, 'pytest_fuzzy_test')
refresh_ngrams(client, 'pytest_fuzzy_test')
name = example_IRS_search_object_with_spelled_out_saint['name']
zip_code = example_IRS_search_object_with_spelled_out_saint['zip']
dc = locate_potential_duplicate(name, zip_code, client,
'pytest_fuzzy_test')
client.drop_collection('pytest_fuzzy_test')
assert dc == 'ST FERIOLE ISLAND PARK'
def setUp(self):
self.app = app.test_client()
self.app.TESTING = True
# Inject test database into application
db = MongoClient('localhost', 27017).TestDB
# Drop collection (faster than dropping entire db)
if 'FirstCollection' in db.collection_names():
db.drop_collection('FirstCollection')
webapp.db = db
new_posters = [{
"name": "James",
"url": "my_url"
}, {
"name": "James",
"url": "my_url"
}]
webapp.db.FirstCollection.insert_many(new_posters)
def BuildDatabase(jmDictXml, connectionString=None):
jmDictDatabase = MongoClient(connectionString).jmdict
jmEntriesColl = jmDictDatabase.entries
def improveDb():
print("Making the DB better, really... Just wait")
totalEntries = jmEntriesColl.count()
for i, entry in zip(range(totalEntries), jmEntriesColl.find()):
if i % (totalEntries // 100) == 0:
print(".", end="", flush=True)
if i % (totalEntries // 10) == 0:
print("|", end="", flush=True)
if "k_ele" not in entry:
entry["k_ele"] = list()
if "r_ele" not in entry:
entry["r_ele"] = list()
kebs = (k_ele["keb"] for k_ele in entry["k_ele"])
rebs = (r_ele["reb"] for r_ele in entry["r_ele"])
glosses = (gloss.get("#text", None) for gloss in chain(*(sense.get("gloss", None) for sense in entry["sense"] if not None)) if not None)
entry["keys"] = list(chain(kebs, rebs, glosses))
jmEntriesColl.save(entry)
#http://stackoverflow.com/questions/2130016/splitting-a-list-of-arbitrary-size-into-only-roughly-n-equal-parts
def chunkify(lst,n):
return [ lst[i::n] for i in range(n) ]
j = open(jmDictXml, "rb")
def normalizeLists(path, key, value):
normalizeLists.needEnlisting = {
"entry" : ["sense", "k_ele", "r_ele"],
"sense" : ["stagk", "stagr", "pos", "xref", "ant", "field", "misc", "s_inf", "lsource", "dial", "gloss", "example"],
"r_ele": ["re_restr", "re_inf", "re_pri"],
"k_ele": ["k_inf", "ke_pri"]
}
if "entry" == key:
if int(value["ent_seq"]) % 1000 == 0:
print(value["ent_seq"])
if key in normalizeLists.needEnlisting:
for n in normalizeLists.needEnlisting[key]:
if n in value and not isinstance(value[n], list):
value[n] = [value[n]];
return key, value
parsed = xmltodict.parse(j, postprocessor=normalizeLists)
chunks = chunkify(parsed["JMdict"]["entry"], 5)
jmDictDatabase.drop_collection("entries")
jmDictDatabase.drop_collection("xrefs")
jmEntriesColl = jmDictDatabase.entries
for chunk in chunks:
jmEntriesColl.insert(chunk)
improveDb()
# jmEntriesColl.ensure_index("r_ele.reb")
# jmEntriesColl.ensure_index("k_ele.keb")
jmEntriesColl.ensure_index("keys")
jmEntriesColl.ensure_index("[keys, ent_seq]")
from bson.code import Code
mapper = Code(open("XrefMap.js").read())
reducer = Code(open("XrefReduce.js").read())
jmEntriesColl.map_reduce(mapper, reducer, "xrefs")
import json
import collections
from UTFToAscii import convert
from pymongo import MongoClient
from Constants import constants
restaurants_collection = MongoClient()[constants.DATABASE][constants.BUSINESS_COLLECTION]
reviews_collection = MongoClient()[constants.DATABASE][constants.REVIEWS_COLLECTION]
db = MongoClient()[constants.DATABASE]
db.drop_collection(restaurants_collection)
print restaurants_collection.find().count()
with open(constants.JSON_DATASET_FILE+'yelp_academic_dataset_business.json') as business:
for line in business:
business_info = json.loads(line)
if "Restaurants" in business_info["categories"] :
restaurants_collection.insert({
"_id" : business_info["business_id"],
"review_count" : business_info["review_count"],
"name" : business_info["name"],
"stars" : business_info["stars"],
"attributes" : business_info["attributes"],
"categories" : business_info["categories"]
})
print restaurants_collection.find().count()
#val = restaurants_collection.find_one( {"_id" : "SQ0j7bgSTazkVQlF5AnqyQ" })
class HRTDatabase:
def __init__(self):
self.database = MongoClient(os.environ['MONGODB_URI']).get_database(None)
def removeOldGTFS(self, date):
print "Removing Old GTFS Data"
collection_prefix = self.genCollectionName('', date)
for collection in self.database.collection_names():
if collection.find('_') != -1 and (not collection.endswith(collection_prefix)):
self.database.drop_collection(collection)
def insertGTFS(self, data, date):
self.insertData(self.genCollectionName('gtfs_', date), data)
def insertStops(self, data, date):
collection_name = self.genCollectionName('stops_', date)
self.insertData(collection_name, data)
self.database[collection_name].ensure_index([('location', GEO2D)])
def insertRoutes(self, data, date):
self.insertData(self.genCollectionName('routes_', date), data)
def insertDestinations(self, data, date):
self.insertData(self.genCollectionName('destinations_', date), data)
def getStopName(self, stop_id, date):
collection_name = self.genCollectionName('stops_', date)
stop = self.database[collection_name].find_one({"stopId": stop_id})
return stop['stopName']
def getFinalStops(self, date):
collection_name = self.genCollectionName('gtfs_', date)
final_stops = self.database[collection_name].aggregate([
{"$group": {
"_id": "$trip_id",
"stopId": {"$last": "$stop_id"},
"sequence": {"$last": "$stop_sequence"}
}}
])
return final_stops
def generateIndicesForGTFS(self, date):
collection_name = self.genCollectionName('gtfs_', date)
self.database[collection_name].create_index([
("block_id", ASCENDING)
], background=True)
self.database[collection_name].create_index([
("block_id", ASCENDING),
("arrival_time", ASCENDING)
], background=True)
self.database[collection_name].create_index([
("block_id", ASCENDING),
("actual_arrival_time", ASCENDING)
], background=True)
self.database[collection_name].create_index([
("stop_id", ASCENDING),
("arrival_time", ASCENDING),
("actual_arrival_time", ASCENDING)
], background=True)
self.database[collection_name].create_index([
("route_short_name", ASCENDING),
("stop_id", ASCENDING),
("direction_id", ASCENDING),
("arrival_time", ASCENDING)
], background=True)
self.database[collection_name].create_index([
("route_short_name", ASCENDING),
("stop_id", ASCENDING),
("direction_id", ASCENDING),
("departure_time", ASCENDING)
], background=True)
def genCollectionName(self, prefix, date):
return prefix + date.strftime('%Y%m%d')
def insertData(self, collection_name, data):
if len(data) > 0:
self.database[collection_name].remove()
self.database[collection_name].insert_many(data)
# get bus route mappings that are not more than 30 minutes old
def getBusRouteMappings(self):
mappings = {}
for mapping in self.database['busRouteMappings'].find():
if mapping['time'] > datetime.utcnow() + timedelta(minutes=-30):
mappings[mapping['busId']] = mapping
return mappings
def setBusRouteMappings(self, mappings):
self.database['busRouteMappings'].remove()
if len(mappings) > 0:
self.database['busRouteMappings'].insert(mappings)
# return the last time to the minute that a bus checked in and
# a list of all buses that checked in during that minute
def getLastCheckinSummary(self):
if self.database['checkins'].find().count() > 0:
last_time = self.database['checkins'].find().sort("$natural", -1)[0]["time"]
last_buses = self.database['checkins'].find({"time" : last_time}).distinct("busId")
return {"time": last_time.replace(tzinfo=pytz.UTC), "busIds": last_buses}
return None
def updateCheckins(self, checkins):
# purge checkins that are more than 2 hours
self.database['checkins'].remove({"time": {"$lt": datetime.utcnow() + timedelta(hours=-2)}})
if len(checkins) > 0:
self.database['checkins'].insert(checkins)
def getRealTimeArrivalUpdates(self, checkin):
checkin_local_time = checkin.time + timedelta(hours=-5)
collection_name = 'gtfs_' + checkin_local_time.strftime('%Y%m%d')
stop_times = self.database[collection_name].find({
'block_id': checkin.blockId,
'$or': [
{'arrival_time': {
'$gte': datetime.utcnow() + timedelta(minutes=-5-checkin.adherence),
'$lte': datetime.utcnow() + timedelta(minutes=30-checkin.adherence)
}},
{'actual_arrival_time': {
'$gte': datetime.utcnow() + timedelta(minutes=-5),
'$lte': datetime.utcnow() + timedelta(minutes=30)
}}
]
}, {'arrival_time': 1, 'actual_arrival_time': 1})
updates = []
for stoptime in stop_times:
new_arrival_time = stoptime['arrival_time'] - timedelta(minutes=checkin.adherence)
if 'actual_arrival_time' not in stoptime or new_arrival_time != stoptime['actual_arrival_time']:
updates.append(UpdateOne(
{'_id': stoptime['_id']},
{'$set': {'actual_arrival_time': new_arrival_time}}
))
return (collection_name, updates)
def updateRealTimeArrivals(self, updates):
for collection_name in updates:
if updates[collection_name]:
result = self.database[collection_name].bulk_write(updates[collection_name])
#print result.bulk_api_result
def getScheduledStop(self, checkin):
checkin_local_time = checkin.time + timedelta(hours=-5)
collection_name = 'gtfs_' + checkin_local_time.strftime('%Y%m%d')
scheduled_stop = self.database[collection_name].find_one({
"route_short_name" : checkin.routeShortName,
"stop_id": checkin.stopId,
"direction_id": {"$ne": checkin.direction},
"$or": [
{"arrival_time": {
"$gte": checkin.time + timedelta(minutes=checkin.adherence - 2),
"$lte": checkin.time + timedelta(minutes=checkin.adherence + 2)
}},
{"departure_time": {
"$gte": checkin.time + timedelta(minutes=checkin.adherence - 2),
"$lte": checkin.time + timedelta(minutes=checkin.adherence + 2)
}}
]
})
if scheduled_stop is None:
print "No scheduled stop found for the following checkin in {0}".format(collection_name)
print checkin.__dict__
return None
# get the stop sequence that OneBusAway uses
scheduled_stop['stop_sequence_OBA'] = self.database[collection_name].find({
"trip_id": scheduled_stop["trip_id"],
"stop_sequence": {"$lt": scheduled_stop["stop_sequence"]}
}).count()
return scheduled_stop
class MongoNamespaceManager(NamespaceManager):
"""
MongoDB backend for beaker
Configuration example:
beaker.session.type = mongo
beaker.session.uri = mongodb://localhost:27017/db_name.collection
"""
def __init__(self, namespace, uri=None, data_dir=None, lock_dir=None, **params):
NamespaceManager.__init__(self, namespace)
if not uri:
raise MissingCacheParameter("URI is required")
self.db_connection_params = pymongo.uri_parser.parse_uri(uri)
if not self.db_connection_params["collection"]:
raise MissingCacheParameter("invalid URI: missing collection")
elif not self.db_connection_params["database"]:
raise MissingCacheParameter("invalid URI: missing database")
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_mongodb_lock"
if hasattr(self, "lock_dir"):
verify_directory(self.lock_dir)
self.open_connection(uri)
def open_connection(self, uri):
self.db = MongoClient(uri)[self.db_connection_params["database"]]
self.db_collection = self.db_connection_params["collection"]
def get_creation_lock(self, key):
return file_synchronizer(identifier="mongodb_container/funclock/%s" % self.namespace,
lock_dir=self.lock_dir)
def _format_key(self, key):
return self.namespace + "_" + key
def __getitem__(self, key):
log.debug("Getting %s" % key)
return pickle.loads(str(self.db[self.db_collection].find_one({ "_id": self._format_key(key) },
fields=["data"])["data"]))
def __contains__(self, key):
log.debug("Contains %s" % key)
return self.db[self.db_collection].find({ "_id": self._format_key(key) }).count() > 0
def has_key(self, key):
return key in self
def set_value(self, key, value):
log.debug("Setting %s: %s" % (key, str(value)))
self.db[self.db_collection].update({ "_id": self._format_key(key) },
{ "$set": { "data": pickle.dumps(value), "timestamp": datetime.datetime.utcnow() } }, upsert=True)
def __setitem__(self, key, value):
self.set_value(key, value)
def __delitem__(self, key):
log.debug("Deleting %s" % key)
self.db[self.db_collection].remove({ "_id": self._format_key(key) })
def do_remove(self):
log.debug("Removing %s" % self.db_collection)
self.db.drop_collection(self.db_collection)
def keys(self):
log.debug("Retrieving keys from %s" % self.db_collection)
return [item["_id"].replace(self.namespace + "_", "") for item in self.db[self.db_collection].find()]
from pymongo import MongoClient
import time
db = MongoClient().performance_test
db.drop_collection("updates")
collection = db.updates
collection.insert({"x": 1})
collection.find_one()
start = time.time()
for i in range(10000):
collection.update({'x': 1}, {"$inc": {"x": 1}})
print "used time:", time.time() - start
def clean_db(self, addr):
db = MongoClient(addr).file_catalog
colls = db.list_collection_names()
for c in colls:
db.drop_collection(c)
from pymongo import MongoClient
import json
from bson import json_util
from pprint import pprint
from pprint import pformat
from pandas import DataFrame
import calendar
import seaborn as sns
from mpl_toolkits.basemap import Basemap
import matplotlib.pyplot as plt
# Connect to "singapore" collection in "map" database
mp = MongoClient('localhost:27017').map
sg = mp.singapore
# Clear data in singapore
mp.drop_collection("singapore")
# Import JSON file to "singapore" collection
with open("./input/singapore.json", "r") as f:
sg.insert_many(json.load(f, object_hook=json_util.object_hook))
# Show sample records
pprint(sg.find_one({"xml" : "osm"}))
pprint(sg.find_one({"xml" : "bounds"}))
pprint(list(sg.find({"xml" : "node"})[:3]))
pprint(list(sg.find({"xml" : "relation"})[:3]))
pprint(list(sg.find({"xml" : "way"})[:3]))
pprint(list(sg.aggregate([{"$match" : {"$and" : [{"xml" : "node"}, \
{"tag" : {"$exists" : True}}]}}, \
{"$sample" : {"size" : 3}}])))
class MongoOP:
def __init__(self, mongo_uri='mongodb://localhost:27017/test', \
collect_name='videos_update', old_collect_name='videos'):
self.db = MongoClient(mongo_uri).get_default_database()
self.collect_name = collect_name
self.old_collect_name = old_collect_name
def update_json_list(self, json_list, collect_name=None):
collect = self.get_collection(collect_name)
if collect_name == 'videos_new':
print("new Videos drop")
collect.drop()
for idx, json in enumerate(json_list):
if idx % 100 == 0 and idx > 0:
print("update into collect {} : {} / {}".format(
collect_name, idx, len(json_list)))
collect.update_one({'url': json['url']}, {'$set': json},
upsert=True)
def delete_url(self, url, collect_name=None):
collect = self.get_collection(collect_name)
collect.delete_one({'url': url})
def info_is_exists(self, url, collect_name=None):
collect = self.get_collection(collect_name)
return bool(collect.find_one({'url': url, 'title': {'$exists': True}}))
def get_unfinished_url_list(self, collect_name=None):
collect = self.get_collection(collect_name)
url_json_list = list(
collect.find({'update_date': {
'$exists': False
}}, {
'url': 1,
'_id': 0
}))
return url_json_list
def get_all_url_set(self, collect_name):
collect = self.get_collection(collect_name)
url_set = set(each['url'] for each \
in collect.find({'update_date': {'$exists':True}}, {'url':1, '_id':0}))
return url_set
def get_film_info_list(self, url_list, collect_name=None):
collect = self.get_collection(collect_name)
info_json_list = list(collect.find({'url': {'$in': url_list}}))
return info_json_list
def get_collection(self, collect_name):
if collect_name is None:
return self.db[self.collect_name]
else:
return self.db[collect_name]
def get_url_update_date(self, url, collect_name=None):
collect = self.get_collection(collect_name)
return collect.find_one({'url': url, 'update_date': {'$exists':True}}, \
{'update_date':1, '_id':0})
def get_logs(self, collect_name='logs'):
collect = self.db[collect_name]
for each in collect.find():
print(each)
def rename_collection(self, old_name, new_name, drop=False):
if new_name in self.db.collection_names() and drop:
self.drop_collection(new_name)
old_collect = self.db[old_name]
old_collect.rename(new_name)
def drop_collection(self, collect_name):
self.db.drop_collection(collect_name)
class CheapVol:
"""
Profits from buying shitcoins in accumulation phase.
Enters when price is still cheap and volume spikes
Config Requirements:
- periodsMA
- periodsVolLong
- periodsVolShort
- volCoef
- bolStd
"""
def __init__(self, stratName, assetList, isTest=False):
logging.debug("Initialising CheapVol()")
pd.options.mode.chained_assignment = None
self.assetList = assetList
self.isTest = isTest
with open("%s/Pipeline/resources/%s/config.yml" %
(Settings.BASE_PATH, stratName)) as configFile:
params = yaml.load(configFile)
self.enterParams = params["enter"]
self.exchangeList = params["assetSelection"]["exchangeList"]
self.db = MongoClient("localhost", 27017)[stratName]
self.col = self.db["PastPriceAction"]
self.initiateCollection() if not self.isTest else None
def _initSingle(self, asset, exchange, testData=[]):
logging.debug("Starting CheapVol._initSingle(asset=%s)" % asset)
logging.debug("1 second sleep to avoid rate limiters")
time.sleep(1.5 if not self.isTest else 0)
try:
pullData = (Pull(
emailOnFailure=False if self.isTest else True).candles(
asset="%sBTC" % asset,
exchange=exchange,
limit=max(
self.enterParams["periodsMA"],
self.enterParams["periodsVolLong"],
) + 1,
interval=self.enterParams["granularity"],
) if len(testData) == 0 else testData)
priceList = list(
pullData["close"])[-self.enterParams["periodsMA"]:]
volList = list(
pullData["volume"])[-self.enterParams["periodsVolLong"]:]
if (len(priceList) == self.enterParams["periodsMA"]
and len(volList) == self.enterParams["periodsVolLong"]):
self.col.insert_one({
"asset": asset,
"price": priceList,
"vol": volList,
"isLive": False,
})
return True
else:
logging.info("Not enough data for asset: %s" % asset)
except IndexError:
logging.warning("Failure on asset: %s" % asset)
return False
def initiateCollection(self):
"""
Creates mongo collection which contains the price action data required for CheapVol
"""
failList = []
logging.debug("Starting CheapVol.init()")
if "PastPriceAction" in self.db.collection_names():
self.db.drop_collection("PastPriceAction")
for asset, exchange in self.assetList:
if not self._initSingle(asset, exchange):
failList.append(asset)
logging.debug("No failed assets" if len(failList) ==
0 else "%s Failed assets: %s" %
(len(failList), failList))
logging.debug("Finished CheapVol.initiateCollection()")
return failList if self.isTest else None
def _getPADict(self, exchange):
logging.debug("Starting CheapVol._getPADict()")
startTS = int(time.time() - self.enterParams["granularity"])
dateStart = datetime.fromtimestamp(startTS).strftime(
"%Y-%m-%dT%H:%M:%S.000Z")
return Pull().getPriceAction(exchange=exchange,
startDate=dateStart,
baseAsset="BTC")
def before(self, testData=None):
"""
Runs before CheapVol on each asset and updates the mongo collection
"""
logging.debug("Starting CheapVol.before()")
newPA = {}
delistDict = {}
# using reversed to keep exchange priority
for exchange in reversed(self.exchangeList):
newPA.update(
self._getPADict(
exchange=exchange) if not self.isTest else testData)
delistDict.update(Pull().getDepositStatus(
exchange=exchange)) if not self.isTest else {}
for assetDict in list(self.col.find()):
assetDict["price"] = assetDict["price"][1:] + [
newPA[assetDict["asset"]]["price"]
]
assetDict["vol"] = assetDict["vol"][1:] + [
newPA[assetDict["asset"]]["vol"]
]
assetDict["isLive"] = (delistDict[assetDict["asset"]]
if not self.isTest else True)
assetDict.pop("_id", None)
self.col.find_one_and_replace({"asset": assetDict["asset"]},
assetDict)
logging.debug("Finished CheapVol.before()")
def run(self, asset):
logging.debug("Starting CheapVol.run(asset=%s)" % asset)
assetData = self.col.find_one({"asset": asset})
if assetData:
volL = np.round(
np.nanmean(np.array(assetData["vol"]).astype(np.float)), 5)
volS = np.round(
np.nanmean(
np.array(assetData["vol"]
[-self.enterParams["periodsVolShort"]:]).astype(
np.float)),
5,
)
priceData = np.array(assetData["price"]).astype(np.float)
bolDown = np.nanmean(
priceData) - self.enterParams["bolStd"] * np.nanstd(priceData)
logging.debug("volL: %s, volS: %s, price: %s, bolDown: %s" %
(volL, volS, priceData[-1], bolDown))
return (volS > self.enterParams["volCoef"] * volL
and priceData[-1] < bolDown and assetData["isLive"])
else:
return False
from faker import Faker
from pymongo import MongoClient
import uuid
import random
# constants for Coursera API
BASE_URL = "https://api.coursera.org/api/catalog.v1/courses"
FIELDS = 'shortName, name,language,largeIcon,photo,previewLink,shortDescription,smallIcon,subtitleLanguagesCsv,isTranslate,universityLogo,video,videoId,aboutTheCourse,targetAudience,faq,courseSyllabus,courseFormat,suggestedReadings,instructor,estimatedClassWorkload,aboutTheInstructor,sessions.fields(durationString,name,eligibleForCertificates,startDay,startMonth,startYear,active,status),instructors.fields(photo,bio,fullName,title,department,website,websiteTwitter),universities.fields(description,homeLink,location,website,websiteTwitter,logo),categories.fields(name,shortName,description)'
INCLUDES = 'universities,categories,instructors,sessions'
Q = 'search'
QUERY = 'computer science'
# MongoClient
db = MongoClient('mongodb://localhost:27017')['course_catalog']
db.drop_collection('course')
db.drop_collection('category')
db.drop_collection('session')
db.drop_collection('instructor')
db.drop_collection('university')
db.drop_collection('student')
db.drop_collection('course_taken')
fake = Faker()
def fetch_courses():
params = {
'fields': FIELDS,
'includes': INCLUDES,
#'q': Q,
from pprint import pprint
from faker import Faker
from pymongo import MongoClient
import uuid
import random
# constants for Coursera API
BASE_URL = "https://api.coursera.org/api/catalog.v1/courses"
FIELDS = 'shortName, name,language,largeIcon,photo,previewLink,shortDescription,smallIcon,subtitleLanguagesCsv,isTranslate,universityLogo,video,videoId,aboutTheCourse,targetAudience,faq,courseSyllabus,courseFormat,suggestedReadings,instructor,estimatedClassWorkload,aboutTheInstructor,sessions.fields(durationString,name,eligibleForCertificates,startDay,startMonth,startYear,active,status),instructors.fields(photo,bio,fullName,title,department,website,websiteTwitter),universities.fields(description,homeLink,location,website,websiteTwitter,logo),categories.fields(name,shortName,description)'
INCLUDES = 'universities,categories,instructors,sessions'
Q = 'search'
QUERY = 'computer science'
# MongoClient
db = MongoClient('mongodb://localhost:27017')['course_catalog']
db.drop_collection('course')
db.drop_collection('category')
db.drop_collection('session')
db.drop_collection('instructor')
db.drop_collection('university')
db.drop_collection('student')
db.drop_collection('course_taken')
fake = Faker()
def fetch_courses():
params = {
'fields': FIELDS,
'includes': INCLUDES,
class VideoViewTest(unittest.TestCase):
"""Test case docstring."""
def setUp(self):
self.database_uri = os.environ.get('DATABASE_URI_TEST')
self.config = testing.setUp()
db_url = urlparse(self.database_uri)
self.db = MongoClient(host=db_url.hostname, port=db_url.port)['videos']
self.v1 = {
'title': 'Radiohead Lucky Live',
'theme': 'Music',
'likes': 2,
'dislikes': 0
}
self.v2 = {
'title': 'Video1',
'theme': 'Music',
'likes': 6,
'dislikes': 10
}
self.v3 = {
'title': 'Video3',
'theme': 'Anime',
'likes': 0,
'dislikes': 0
}
self.v4 = {
'title': 'Video2',
'theme': 'Entretairment',
'likes': 5,
'dislikes': 5
}
def request(self, post=None):
r = testing.DummyRequest(post=post) if post else testing.DummyRequest()
r.db = self.db
return r
def tearDown(self):
self.db.drop_collection('videos')
testing.tearDown()
def test_index(self):
create_video(self.db, title=self.v1['title'], theme=self.v1['theme'])
create_video(self.db, title=self.v2['title'], theme=self.v2['theme'])
response = VideoViews(self.request()).index()
self.assertEqual(len(response['videos']), 2)
for v in zip(response['videos'], [self.v1, self.v2]):
self.assertEqual(v[0]['title'], v[1]['title'])
self.assertEqual(v[0]['theme'], v[1]['theme'])
def test_video_create(self):
from videos import main
app = main({}, mongo_uri=self.database_uri, testing=True)
from webtest import TestApp
self.testapp = TestApp(app)
self.v1['submit'] = True
r = self.testapp.post('/create', params=self.v1)
r = r.follow()
self.assertEqual(r.status_code, 200)
self.assertTrue(bool(get_video(self.db, title=self.v1['title'])))
def test_themes_list_score(self):
videos = self.db['videos']
expected = ['Music', 'Entretairment', 'Anime']
for v in [self.v1, self.v2, self.v3, self.v4]:
create_video(self.db, title=v['title'], theme=v['theme'])
v_doc = get_video(self.db, title=v['title'])
videos.update_one({'uid': v_doc['uid']}, {
'$set': {
k: v
for k, v in v.items() if k == 'likes' or k == 'dislikes'
}
})
response = VideoViews(self.request()).themes()
self.assertEqual(expected, [t for t, _ in response['themes']])
def drop_collection():
db = MongoClient("mongodb://localhost")['simple_example_for_pymongokeyset']
db.drop_collection('test_collect')
from pymongo import MongoClient
import time
db = MongoClient().performance_test
db.drop_collection("updates")
collection = db.updates
collection.insert({"x": 1})
collection.find_one()
start = time.time()
for i in range(10000):
collection.update({'x': 1}, {"$inc": {"x": 1}})
print "used time:", time.time()-start
class FlaskTestCase(unittest.TestCase):
def setUp(self):
app = Flask(__name__)
self.app = app
CSRFProtect(app)
Bootstrap(app)
nav = Nav(app)
app.secret_key = 'DUMMY'
@app.route('/')
def index():
if current_user.is_authenticated:
return 'OK'
else:
return redirect(url_for('commons.login'))
@nav.navigation()
def mynavbar():
"""Menu da aplicação."""
items = [View('Home', 'index')]
return Navbar('teste', *items)
app.testing = True
self.client = app.test_client()
self.db = MongoClient(host=MONGODB_URI).unit_test
login.configure(app)
login.DBUser.dbsession = self.db
login.DBUser.add('ajna', 'ajna')
def tearDown(self):
self.db.drop_collection('users')
rv = self.logout()
assert rv is not None
def get_token(self, url):
response = self.client.get(url, follow_redirects=True)
csrf_token = response.data.decode()
begin = csrf_token.find('csrf_token"') + 10
end = csrf_token.find('username"') - 10
csrf_token = csrf_token[begin:end]
begin = csrf_token.find('value="') + 7
end = csrf_token.find('/>')
self.csrf_token = csrf_token[begin:end]
return self.csrf_token
def login(self, username, senha):
self.get_token('/login')
return self.client.post('/login',
data=dict(username=username,
senha=senha,
csrf_token=self.csrf_token),
follow_redirects=True)
def logout(self):
return self.client.get('/logoout', follow_redirects=True)
def test_login_invalido(self):
rv = self.login('none', 'error')
print(rv.data)
assert rv is not None
assert b'username' in rv.data
def test_login(self):
rv = self.login('ajna', 'ajna')
assert rv is not None
assert b'OK' in rv.data
def test_DBuser(self):
auser = login.DBUser.get('ajna')
assert auser.name == 'ajna'
auser2 = login.DBUser.get('ajna', 'ajna')
assert auser2.name == 'ajna'
# Testa mundança de senha
login.DBUser.add('ajna', '1234')
auser3 = login.DBUser.get('ajna', 'ajna')
assert auser3 is None
auser4 = login.DBUser.get('ajna', '1234')
assert auser4.name == 'ajna'
def test_404(self):
rv = self.client.get('/non_ecsiste')
assert rv is not None
assert b'404' in rv.data
def tearDown(self):
from pymongo import MongoClient
db = MongoClient('localhost:27017')['hogehogehoge']
db.drop_collection('test')
