class database():
def __init__(self):
self.client = MongoClient()
def __del__(self):
self.client.disconnect()
def insert(self, fundamentals):
f = {
'Symbol': fundamentals['Symbol'],
'DownloadDate': fundamentals['Date'],
'Dividend': fundamentals['Dividend'],
'EPS': fundamentals['EPS'],
'Shares': fundamentals['Shares'],
'InstitutionalOwnership': fundamentals['InstitutionalOwnership'],
'Open': fundamentals['Open'],
'High': fundamentals['High'],
'Low': fundamentals['Low'],
'Close': fundamentals['Close'],
}
self.client.arbit.fundamentals.insert(f)
def fetch(self, currentDate, symbol):
d = currentDate
startDatetime = datetime.datetime(d.year, d.month, d.day, 0, 0, 0)
endDatetime = datetime.datetime(d.year, d.month, d.day, 23, 59, 59)
for result in self.client.arbit.fundamentals.find(
{'Symbol': symbol, 'DownloadDate': {'$gte': startDatetime, '$lt': endDatetime}}):
return (result)
return ([])
def getUserCount():
dbClient = MongoClient()
db = dbClient[conf.DB]
siteContent = db['content']
count = siteContent.count()
dbClient.disconnect()
return jsonify({"count": count})
class database():
def __init__(self):
self.client = MongoClient()
def __del__(self):
self.client.disconnect()
def insert(self, change):
# BSON (which mongodb uses) only supports datetime, so we need to convert the date.
# The hour and minute will be 0
d = change['RatingsChangeDate']
dt = datetime.datetime(d.year, d.month, d.day)
symbol = {
'RatingsChangeDate': dt,
'RatingsChangeType': change['RatingsChangeType'],
'Company': change['Company'],
'Ticker': change['Ticker'],
'BrokerageFirm': change['BrokerageFirm'],
'RatingsChange': change['RatingsChange'],
'PriceTarget': change['PriceTarget'],
}
self.client.arbit.ratingsChanges.insert(symbol)
def fetch(self, currentDate, ratingsChangeType):
d = currentDate
dt = datetime.datetime(d.year, d.month, d.day)
ratingsChanges = []
for change in self.client.arbit.ratingsChanges.find({'RatingsChangeDate': dt, 'RatingsChangeType': 'Upgrade'}):
ratingsChanges.append(change)
return ratingsChanges
def get_shard_nodes(host):
mc = MongoClient(host)
shard_coll = mc['config']['shards']
shard_cursor = shard_coll.find()
shards = [x for x in shard_cursor]
mc.disconnect()
return shards
class ReceiverProcess(object):
def __init__(self, config_file_name):
json_data = open(config_file_name)
config = json.load(json_data)
json_data.close()
self.urls = [self.__get_url_for_host(config, host) for host in config['hosts']]
self.mClient = MongoClient('localhost', 27017) #keep defaults for now
for host in config['hosts']:
self.mClient.test[host].remove() #remove all previous records
def __get_url_for_host(self, config, host):
return (config['base_url'] +
"?apikey={0}&limit={1}&host={2}"
).format(config['apikey'], config['limit'], host)
def __store_resp(self, response, **kwargs):
host = re.search(r'\host=([^&=]+)', response.url).group(1)
db = self.mClient.test
data = response.text.encode('utf-8')
db[host].insert(json.loads(data))
def make_requests(self):
rs = (grequests.get(url, hooks={'response':self.__store_resp}) for url in self.urls)
grequests.map(rs)
def cleanup(self):
self.mClient.disconnect()
def get_shard_nodes(host):
mc = MongoClient(host)
shard_coll = mc["config"]["shards"]
shard_cursor = shard_coll.find()
shards = [x for x in shard_cursor]
mc.disconnect()
return shards
def getUserCount():
dbClient = MongoClient()
db = dbClient[conf.DB]
siteContent = db['content']
count = siteContent.count()
dbClient.disconnect()
return jsonify({"count": count})
def test_copy_db(self):
authed_client = auth_context.client
if is_mongos(authed_client):
raise SkipTest("SERVER-6427")
c = MongoClient(host, port)
authed_client.admin.add_user("admin", "password")
c.admin.authenticate("admin", "password")
c.drop_database("pymongo_test")
c.drop_database("pymongo_test1")
c.pymongo_test.test.insert({"foo": "bar"})
try:
c.pymongo_test.add_user("mike", "password")
self.assertRaises(OperationFailure, c.copy_database,
"pymongo_test", "pymongo_test1",
username="******", password="******")
self.assertFalse("pymongo_test1" in c.database_names())
self.assertRaises(OperationFailure, c.copy_database,
"pymongo_test", "pymongo_test1",
username="******", password="******")
self.assertFalse("pymongo_test1" in c.database_names())
c.copy_database("pymongo_test", "pymongo_test1",
username="******", password="******")
self.assertTrue("pymongo_test1" in c.database_names())
self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
finally:
# Cleanup
remove_all_users(c.pymongo_test)
c.admin.remove_user("admin")
c.disconnect()
class database():
def __init__(self):
self.client = MongoClient()
def dropCollection(self):
self.client.arbit.symbols.drop()
def __del__(self):
self.client.disconnect()
def insert(self, symbolInformation):
symbol = {
'Symbol': symbolInformation['Symbol'],
'Exchange': symbolInformation['Exchange'],
'Name': symbolInformation['Name'],
'LastSale': symbolInformation['LastSale'],
'MarketCap': symbolInformation['MarketCap'],
'IPOYear': symbolInformation['IPOYear'],
'Sector': symbolInformation['Sector'],
'Industry': symbolInformation['Industry']
}
self.client.arbit.symbols.insert(symbol)
def getAllSymbols(self):
symbols = []
for symbolInformation in self.client.arbit.symbols.find():
symbols.append(symbolInformation['Symbol'])
return symbols
def cuckoo_clean():
"""Clean up cuckoo setup.
It deletes logs, all stored data from file system and configured databases (SQL
and MongoDB.
"""
# Init logging.
# This need to init a console logger handler, because the standard
# logger (init_logging()) logs to a file which will be deleted.
create_structure()
init_console_logging()
# Initialize the database connection.
db = Database()
# Drop all tables.
db.drop()
# Check if MongoDB reporting is enabled and drop that if it is.
cfg = Config("reporting")
if cfg.mongodb and cfg.mongodb.enabled:
from pymongo import MongoClient
host = cfg.mongodb.get("host", "127.0.0.1")
port = cfg.mongodb.get("port", 27017)
mdb = cfg.mongodb.get("db", "cuckoo")
try:
conn = MongoClient(host, port)
conn.drop_database(mdb)
conn.disconnect()
except:
log.warning("Unable to drop MongoDB database: %s", mdb)
# Paths to clean.
paths = [
os.path.join(CUCKOO_ROOT, "db"),
os.path.join(CUCKOO_ROOT, "log"),
os.path.join(CUCKOO_ROOT, "storage"),
]
# Delete various directories.
for path in paths:
if os.path.isdir(path):
try:
shutil.rmtree(path)
except (IOError, OSError) as e:
log.warning("Error removing directory %s: %s", path, e)
# Delete all compiled Python objects ("*.pyc").
for dirpath, dirnames, filenames in os.walk(CUCKOO_ROOT):
for fname in filenames:
if not fname.endswith(".pyc"):
continue
path = os.path.join(CUCKOO_ROOT, dirpath, fname)
try:
os.unlink(path)
except (IOError, OSError) as e:
log.warning("Error removing file %s: %s", path, e)
def cuckoo_clean():
"""Clean up cuckoo setup.
It deletes logs, all stored data from file system and configured databases (SQL
and MongoDB.
"""
# Init logging.
# This need to init a console logger handler, because the standard
# logger (init_logging()) logs to a file which will be deleted.
create_structure()
init_console_logging()
# Initialize the database connection.
db = Database()
# Drop all tables.
db.drop()
# Check if MongoDB reporting is enabled and drop that if it is.
cfg = Config("reporting")
if cfg.mongodb and cfg.mongodb.enabled:
from pymongo import MongoClient
host = cfg.mongodb.get("host", "127.0.0.1")
port = cfg.mongodb.get("port", 27017)
mdb = cfg.mongodb.get("db", "cuckoo")
try:
conn = MongoClient(host, port)
conn.drop_database(mdb)
conn.disconnect()
except:
log.warning("Unable to drop MongoDB database: %s", mdb)
# Paths to clean.
paths = [
os.path.join(CUCKOO_ROOT, "db"),
os.path.join(CUCKOO_ROOT, "log"),
os.path.join(CUCKOO_ROOT, "storage"),
]
# Delete various directories.
for path in paths:
if os.path.isdir(path):
try:
shutil.rmtree(path)
except (IOError, OSError) as e:
log.warning("Error removing directory %s: %s", path, e)
# Delete all compiled Python objects ("*.pyc").
for dirpath, dirnames, filenames in os.walk(CUCKOO_ROOT):
for fname in filenames:
if not fname.endswith(".pyc"):
continue
path = os.path.join(CUCKOO_ROOT, dirpath, fname)
try:
os.unlink(path)
except (IOError, OSError) as e:
log.warning("Error removing file %s: %s", path, e)
class TCMongoDB(object): """ Handling classes will inherit from this class. It gives useful tools for mongodb connections. """ def __init__(self): self.client = None self.db = None self.col = None self.grid_db = None self.fs = None def init_client(self): self.client = MongoClient(DB_ADDR, DB_PORT) """ try: self.client = MongoClient(DB_ADDR, DB_PORT) except pymongo.errors.ConnectionFailure, e: return False return True """ def close_client(self): if self.client: self.client.disconnect() def set_db(self, db_name=None): """Sets current db. I no client, try to connect to db""" if not self.client: self.init_client() self.db = self.client[DB_NAME if not db_name else db_name] if PROD: self.db.authenticate(DB_USER, DB_PASSWD, DB_NAME) def set_col(self, col): """Sets the collection. If db not selected, try to select default db""" if not self.db: self.set_db() self.col = self.db[col] # GridFS functions def set_grid_db(self, db_name=None): if not self.client: self.init_client() self.grid_db = self.client[DB_FILES if not db_name else db_name] if PROD: self.grid_db.authenticate(DB_USER, DB_PASSWD, DB_FILES) def set_grid(self, db_name=None): self.set_grid_db(db_name) def set_fs(self, db=None): self.fs = GridFS(self.grid_db if not db else db)
def delete_item(item_id):
c = MongoClient(mongodb_url)
try:
c.CodeConnect.items.remove({'_id': bson.ObjectId(item_id)})
return "OK"
except bson.errors.InvalidId:
abort(400)
finally:
c.disconnect()
def refine_snoclusters(db_name, db_host = "localhost", db_port = 27017):
client = MongoClient(db_host, db_port)
db = client[db_name]
#sort snoRNA clusters from the database per snoRNA class (2 different classes), per strand (2 different strands) and per genome (n different genomes).
genome_names = []
for genome in db['genomes'].find():
genome_names.append(genome['_id']+"@genomes")
sorted_clusters = []
for i in range(0, len(genome_names)*4): #if n genomes => n*2*2 sub-lists of clusters
sorted_clusters.append([])
for snocluster in db['ncRNAs'].find({'source': "tool:letitsno:NA"}):
index = 0 if snocluster['class'] == 'Gene, snRNA, snoRNA, CD-box' else 1
index += 0 if snocluster['genomicStrand'] == '+' else 2
j = 0
while j != len(genome_names):
genome_name = genome_names[j]
if snocluster['genome'] == genome_name:
index += range(0, len(genome_names)*4, 4)[j] #if n genomes => range_list=[0, 4, 8, 12, ..., n-1*2*2]
j += 1
snocluster['genomicStart'] = snocluster['genomicPositions'][0]
snocluster['genomicEnd'] = snocluster['genomicPositions'][1]
sorted_clusters[index].append(snocluster)
for cluster_annotations in sorted_clusters:
if len(cluster_annotations):
#clustering of snoRNA clusters whose genomic positions overlap
new_clusters = cluster_genomic_annotations(cluster_annotations, threshold = 2, fill_cluster_with_genomic_annotations = True)
#storage of enlarged snoRNA clusters in database and removing of all "old" clusters constituting these "new" clusters
if len(new_clusters):
for new_cluster in new_clusters:
new_cluster['_id'] = str(ObjectId())
new_cluster['name'] = new_cluster['genomic_annotations'][0]['name']
new_cluster['source'] = "tool:letitsno:NA"
new_cluster['class'] = new_cluster['genomic_annotations'][0]['class']
new_cluster['organism'] = new_cluster['genomic_annotations'][0]['organism']
new_cluster['genome'] = new_cluster['genomic_annotations'][0]['genome']
new_cluster['genomicStrand'] = new_cluster['genomic_annotations'][0]['genomicStrand']
new_cluster['genomicPositions'] = [new_cluster['genomicStart'], new_cluster['genomicEnd']]
new_cluster['ids'] = new_cluster['genomic_annotations'][0]['ids']
for cluster in new_cluster['genomic_annotations'][1:]:
new_cluster['name'] += cluster['name'][7:]
new_cluster['ids'].extend(cluster['ids']) #no double of ncRNAs _id
db["ncRNAs"].remove({'_id': cluster['_id']}) #db["ncRNAs"].remove(cluster) don't remove cluster
db["ncRNAs"].remove({'_id': new_cluster['genomic_annotations'][0]['_id']})
del new_cluster['genomicStart']
del new_cluster['genomicEnd']
del new_cluster['genomic_annotations']
del new_cluster['annotations_count']
db["ncRNAs"].insert(new_cluster)
client.disconnect()
def request_data():
c = MongoClient(URI)
collection = c.enavis.core
applicants = collection.find(fields=FIELDS)
json_projects = []
for applicant in applicants:
json_projects.append(applicant)
json_projects = json.dumps(json_projects, default=json_util.default)
c.disconnect()
return json_projects
def main():
log.startLogging(sys.stdout)
mongo_client = MongoClient(host='IP_OR_HOST_NAME', port=27017, document_class='DOC_CLASS')
log.msg('Mongo client {}.'.format(mongo_client))
app = MongoApp(mongo_client, '9f33566e-1f8f-11e2-8979-c42c030dd6a5', 'USER_NAME', 'PASSWORD')
transport = devicehive.client.ws.WebSocketFactory(app)
transport.connect('ws://IP_OR_HOST_NAME:8080/DeviceHiveJava/websocket/')
reactor.run()
mongo_client.disconnect()
def get_items():
c = MongoClient(mongodb_url)
try:
data = []
for item in list(c.CodeConnect.items.find()):
item['date'] = item['date'].isoformat()
item['_id'] = str(item['_id'])
data.append(item)
return jsonify({'items': data})
finally:
c.disconnect()
def check_unique_and_insert(self, items, host):
"""
Try to put data to DB.
"""
log.msg('class Db, method insert, items len ' + str(len(items)))
# Async mongo connection
async_mongo = yield txmongo.MongoConnection(host=self.db_host, port=self.db_port)
# Sync mongo connection
sync_mongo = MongoClient(self.db_host, self.db_port)
async_db = async_mongo[self.db_name]
async_records = async_db.records
sync_db = sync_mongo[self.db_name]
sync_records = sync_db.records
if host is None:
host = ''
# insert some data
for item in items:
log.msg('try to insert ' + item['name'] + ', ' + item['score'])
#
# This is a only one place with blocking request to DB, but I need this request to not insert
# data twice.
#
# TODO: find the way to remove this blocking request.
#
count = sync_records.find(
{
'record_id': item['record_id'],
'mode': item['mode'],
'score': int(item['score']),
'name': item['name'],
}).count()
if count == 0:
yield async_records.insert(
{
'name': item['name'],
'score': int(item['score']),
'timestamp': item['timestamp'],
'record_id': item['record_id'],
'mode': item['mode'],
'host': host,
}, safe=True)
log.msg('record inserted ' + item['record_id'])
else:
log.msg('record NOT inserted ' + item['record_id'])
sync_mongo.disconnect()
async_mongo.disconnect()
def get_facility_types():
facility_types = list()
client = MongoClient(MONGODB_HOST, MONGODB_PORT)
db = client[MONGODB_DB]
db.authenticate(MONGODB_USER, MONGODB_PW)
coll = db[MONGODB_COLL]
a_loc = coll.find_one()
for query_key in sorted(filter(lambda key: key not in NON_FACILITY_KEYS, a_loc.keys())):
facility_types.append({'short_name': query_key, 'long_name': _make_proper(query_key), 'selected': False})
client.disconnect()
return facility_types
def getAdvancedConfiguration():
connection = MongoClient('mongo')
db = connection['admin']
db.authenticate(
'drainware',
'JVxLZr9UmSMVobGGk0OKL172OsyHOQ6QophYpUu7B3DQhOvbcDZLYAHgNSnNXZTPXjd0OQ93kruR0aY2hLT26t14tnAedGhI9RiYCbCyc9JFw9srrRiLr3fRuiOFu0Jb'
)
db = connection['ddi']
collection = db.configuration
conf = collection.find_one({"id": "advanced"})
connection.disconnect()
return conf
def main():
context = zmq.Context()
subscriber = context.socket(zmq.SUB)
client = MongoClient()
db = client.mydb
coll = db.emd
#load in the system data
systems = []
fh = open(r'elitesystems.txt', 'r')
for line in fh.readlines():
out = line.rstrip('\n').split(',')
systems.append(out)
# Connect to the first publicly available relay.
subscriber.connect('tcp://firehose.elite-market-data.net:9500')
# Disable filtering.
subscriber.setsockopt(zmq.SUBSCRIBE, "")
i = 0
#while True:
while i < 1000000:
# Receive raw market JSON strings.
market_json = zlib.decompress(subscriber.recv())
# Un-serialize the JSON data to a Python dict.
market_data = simplejson.loads(market_json)
# Dump the market data to stdout. Or, you know, do more fun
# things here.
#print market_data
#fh.write(market_data)
#pid = coll.insert(market_data)
#Search for a matching system name
data = dict(market_data)
#print data['message']['stationName'][0:data['message']['stationName'].find(' (')]
#Loop through and find a matching system
for sys in systems:
if data['message']['stationName'][0:data['message']['stationName'].
find(' (')] == sys[0]:
data['system'] = {
'name': sys[0],
'coordX': float(sys[1]),
'coordY': float(sys[2]),
'coordZ': float(sys[3])
}
#print 'Matched'
break
#make a real date
s = data['message']['timestamp']
dt = datetime.strptime(s[0:19], "%Y-%m-%dT%H:%M:%S")
data['message']['realDate'] = dt
pid = coll.insert(data)
i += 1
#coll.update({'_id':pid},{"$set":{"systemName":sysName, "coordX":x, "coordY":y, "coordZ":z}})
print i
client.disconnect()
class MongoCacheBasePipeline(object):
def open_spider(self, spider):
mongohost = getattr(spider, "mongohost", "localhost")
try:
self.client = MongoClient(host=mongohost)
except:
log.msg("%s runs without caching because we can't find mongodb at '%s'" % (self.__class__.__name__, mongohost), level=log.WARNING)
def close_spider(self, spider):
if getattr(self, "client", None):
self.client.disconnect()
class WordlistMongoDatabasePipeline(object):
"""Saves the items to a mongodatabase"""
def process_item(self, item, spider):
if (self.connection.wordlist.words.find(dict(item)).count() == 0):
obj_id = self.connection.wordlist.words.insert(dict(item))
return item
def open_spider(self, spider):
self.connection = MongoClient('localhost', 1337)
def close_spider(self, spider):
self.connection.disconnect()
class WordlistMongoDatabasePipeline(object):
"""Saves the items to a mongodatabase"""
def process_item(self, item, spider):
if( self.connection.wordlist.words.find( dict(item) ).count()==0 ):
obj_id = self.connection.wordlist.words.insert( dict(item) )
return item
def open_spider(self, spider):
self.connection = MongoClient('localhost', 1337)
def close_spider(self, spider):
self.connection.disconnect()
class MongoDBStore():
class Transformer(SONManipulator):
def transform_incoming(self, son, collection):
pass
def transform_outgoing(self, son, collection):
pass
def __init__(self, host, port):
self.client = MongoClient(host, port)
self.db = self.client.flaskmap
self.db.add_son_manipulator(Transformer())
def all(self):
poiList = []
for poic in self.db['poi-containers']:
poiList.append(POIContainer(poic['name']))
for poi in poic.content:
poiList.content
return poiList
def create_container(self,
name="Nuevo mapa",
pois=[],
poi_bin_string=None):
uid = uuid4().hex
container = POIContainer(name)
container.id = uid
self.db['poi-containers'].insert(container)
return container
def save_container(self, uid, name, content):
c = self.db['poi-containers']
container = c.find_one({'uid': uid})
if container:
container.name = name
container.content = content
c.update({'uid': container.uid}, container)
return
raise Exception('Unknown UID')
def delete_container(self, uid):
raise Exception('Not implemented yet.')
def get_stream_ov2(self, uid):
raise Exception('Not implemented yet.')
def shutdown(self):
self.client.disconnect()
def __getc_poi_containers(self):
return self.db['poi-containers']
def getUsers():
dbClient = MongoClient()
db = dbClient[conf.DB]
siteContent = db['content']
skip = conf.NUMBER_OF_ITEMS * (int(request.args['page']) - 1)
result = []
for content in siteContent.find().skip(skip).limit(conf.NUMBER_OF_ITEMS):
del(content['_id'])
result.append(content)
response = make_response()
response.data = dumps(result)
dbClient.disconnect()
return response
def getUsers():
dbClient = MongoClient()
db = dbClient[conf.DB]
siteContent = db['content']
skip = conf.NUMBER_OF_ITEMS * (int(request.args['page']) - 1)
result = []
for content in siteContent.find().skip(skip).limit(conf.NUMBER_OF_ITEMS):
del (content['_id'])
result.append(content)
response = make_response()
response.data = dumps(result)
dbClient.disconnect()
return response
class CalculationProcess(object):
def __init__(self, config_file_name):
json_data = open(config_file_name)
config = json.load(json_data)
self.max_records = config['max_samples'] * config['limit']
self.max_samples = config['max_samples']
self.hosts = config['hosts']
self.mClient = MongoClient('localhost', 27017) #keep defaults for now
self.mClient['test']['increasing'].remove() #remove all prev records
def __get_change(self, records,path):
visitors = [record['visitors'] for record, i in zip(records, range(0, self.max_samples))]
avg = sum(visitors)/float(len(visitors))
variance = sum([(i-avg)**2 for i in visitors])/float(len(visitors))
std_dev_x2 = sqrt(variance) * 2.0
adj_visitors = filter(lambda x: abs(x - avg) <= std_dev_x2, visitors)
adj_avg = sum(adj_visitors)/len(adj_visitors)
return visitors[0] - adj_avg
def __get_tuple(self, host, collection, path, timestamp):
records = collection.find({'path':path}, sort=[('_id',-1)])
t = {
'host': host,
'change': self.__get_change(records,path),
'i': records.rewind()[0]['i'],
'path': path,
'created_at': timestamp
}
return t
def __store_calculation(self, tuples):
self.mClient['test']['increasing'].insert(tuples)
def __remove_previous_calculation(self, host, timestamp):
self.mClient['test']['increasing'].remove({
'host':host,
'created_at':{'$lt':timestamp}
})
def run(self):
for host in self.hosts:
collection = self.mClient['test'][host]
paths = collection.distinct('path')
timestamp = datetime.datetime.utcnow()
tuples = [self.__get_tuple(host, collection, path, timestamp) for path in paths]
if len(tuples)>0:
self.__store_calculation(tuples)
self.__remove_previous_calculation(host, timestamp)
def cleanup(self):
self.mClient.disconnect()
def TestMongo():
client = MongoClient("localhost", 27017)
db = client["alio2o"]
#posts = db["metaq_fail"]
posts = db["worklist_inconsist"]
#post = json.loads("{"method":"add","poiid":"0000000002"}")
#post_id = posts.insert(post)
#print post_id
#posts.remove({"poiid": "0000000002"})
print posts.count()
for item in posts.find():
print item
print item["poiid"]
client.disconnect()
def TestMongo():
client = MongoClient("localhost", 27017)
db = client["alio2o"]
#posts = db["metaq_fail"]
posts = db["worklist_inconsist"]
#post = json.loads("{"method":"add","poiid":"0000000002"}")
#post_id = posts.insert(post)
#print post_id
#posts.remove({"poiid": "0000000002"})
print posts.count()
for item in posts.find():
print item
print item["poiid"]
client.disconnect()
def home(request):
responseOut = {}
allOut = []
'''make some graphs from the scored profit data
'''
client = MongoClient()
db = client.mydb
coll = db.emd_profitdata
#data = coll.find_one()
#Get the most recent entry
d = coll.find().sort('procDTG',-1).limit(1)
#sort data
data = d.next()
runDate = data['procDTG']
sortedProfits = sorted(data['profitData'], key=itemgetter('avgProfit'), reverse=True)
x = []
labsX = []
labs = []
y = []
cnt = 0
td = []
#Build up list of bar graph numbers - commidities by score
#for i in data['profitData']:
for i in sortedProfits:
#print i
#print data['profitData']
allOut.append({'score':i['score'], 'commodity':i['commodity']})
td.append([i['commodity'], i['systemFrom'], i['systemTo'], i['dist'], i['avgProfit']])
#allOut.append({'score':i['score'], 'commodity':cnt})
y.append(i['score'])
#y.append(i['score'])
#x.append(cnt)
#labsX.append(cnt+0.3)
#labs.append(i['commodity'])
cnt+=1
client.disconnect()
print allOut
#Build Output
json_list = simplejson.dumps(allOut)
responseOut['procDTG']=runDate
responseOut['allOutput']= json_list
responseOut['tableData']=td
responseOut['yvals'] = y
responseOut['bins']=cnt
responseOut['page_title']='Trader Results'
responseOut['proc_pass_msg']= 'Results Below'
#return responseOut
#TODO: implement csrf checking
#responseOut[0](csrf(request))
return render_to_response('trader_results3.html',responseOut)
def add_item():
c = MongoClient(mongodb_url)
try:
item = request.json
item['date'] = datetime.now()
item['ip_addr'] = request.remote_addr
item_id = c.CodeConnect.items.insert(item)
# Make ID and Date JSON Serializable
item['_id'] = str(item_id)
item['date'] = item['date'].isoformat()
return jsonify({'items': item})
finally:
c.disconnect()
def main():
log.startLogging(sys.stdout)
mongo_client = MongoClient(host='IP_OR_HOST_NAME',
port=27017,
document_class='DOC_CLASS')
log.msg('Mongo client {}.'.format(mongo_client))
app = MongoApp(mongo_client, '9f33566e-1f8f-11e2-8979-c42c030dd6a5',
'USER_NAME', 'PASSWORD')
transport = devicehive.client.ws.WebSocketFactory(app)
transport.connect('ws://IP_OR_HOST_NAME:8080/DeviceHiveJava/websocket/')
reactor.run()
mongo_client.disconnect()
class database():
def __init__(self):
self.client = MongoClient()
def __del__(self):
self.client.disconnect()
def insert(self, form4Information):
year = int(form4Information['acceptanceDatetime'][0:4])
month = int(form4Information['acceptanceDatetime'][4:6])
day = int(form4Information['acceptanceDatetime'][6:8])
hour = int(form4Information['acceptanceDatetime'][8:10])
minute = int(form4Information['acceptanceDatetime'][10:12])
second = int(form4Information['acceptanceDatetime'][12:14])
acceptanceDatetime = datetime.datetime(year, month, day, hour, minute, second)
year = int(form4Information['transactionDate'][0:4])
month = int(form4Information['transactionDate'][5:7])
day = int(form4Information['transactionDate'][8:10])
transactionDate = datetime.datetime(year, month, day)
form4 = {
'SecDocument': form4Information['secDocument'],
'AcceptanceDatetime': acceptanceDatetime,
'IssuerTradingSymbol': form4Information['issuerTradingSymbol'],
'RptOwnerCik': form4Information['rptOwnerCik'],
'RptOwnerName': form4Information['rptOwnerName'],
'IsDirector': form4Information['isDirector'],
'IsOfficer': form4Information['isOfficer'],
'IsTenPercentOwner': form4Information['isTenPercentOwner'],
'IsOther': form4Information['isOther'],
'TransactionDate': transactionDate,
'TransactionShares': form4Information['transactionShares'],
'TransactionPricePerShare': form4Information['transactionPricePerShare'],
'TransactionAcquiredDisposed': form4Information['transactionAcquiredDisposedCode'],
'SharesOwned': form4Information['sharesOwned'],
}
self.client.arbit.form4.insert(form4)
def fetch(self, currentDate):
d = currentDate
startDatetime = datetime.datetime(d.year, d.month, d.day, 0, 0, 0)
endDatetime = datetime.datetime(d.year, d.month, d.day, 23, 59, 59)
forms = []
for form in self.client.arbit.form4.find({'TransactionAcquiredDisposed': 'A',
'AcceptanceDatetime': {'$gte': startDatetime, '$lt': endDatetime}}):
forms.append(form)
return forms
class Doinker:
def __init__(self, csv_path, database):
self.csv_path = csv_path
self.database = database
self.connection = MongoClient()
self.metadata = self.create_metadata()
def set_collection(self, collection):
self.metadata['collection'] = collection
def set_description(self, description):
self.metadata['collection'] = description
def generate_rows(self):
csv_file = open(self.csv_path, 'r')
dict_reader = DictReader(csv_file)
for row_dict in dict_reader:
yield row_dict
csv_file.close()
def get_schema(self):
for row in self.generate_rows():
return row.keys()
def create_metadata(self):
row = {
'metadata': True,
'schema': {},
'description': '',
'last_modified': None
}
metadata = {
'row': row,
'row_id': None,
'collection': ''
}
for field in self.get_schema():
metadata['row']['schema'][field] = ''
return metadata
def insert_data(self):
collection = self.connection[self.database][self.metadata['collection']]
self.metadata['row_id'] = collection.insert(self.metadata['row'])
for row in self.generate_rows():
collection.insert(row)
collection.update({'_id': self.metadata['row_id']}, {'$set': {'last_modified': datetime.utcnow()}})
def __del__(self):
self.connection.disconnect()
class ScavhqTestCase(unittest.TestCase):
@classmethod
def setUpClass(self):
scavhq.app.config['TESTING'] = True
self.app = scavhq.app.test_client()
self.client = MongoClient('localhost', 27017)
self.items = self.client['scavhq-dev'].items
def tearDown(self):
self.items.remove()
@classmethod
def tearDownClass(self):
self.client.disconnect()
class BaseBaseModel(object):
conn = None
database = None
collection = None
servers = "mongodb://localhost:27017"
def __init__(self, db=''):
self.connect()
self.selectDB(db)
def connect(self):
self.conn = MongoClient(self.servers)
def close(self):
return self.conn.disconnect()
def getConn(self):
return self.conn
def selectDB(self, name=''):
if name:
self.database = self.conn[name]
def selectAC(self, name=''):
if name:
self.collection = self.database[name]
def get_facility_types():
facility_types = list()
client = MongoClient(MONGODB_HOST, MONGODB_PORT)
db = client[MONGODB_DB]
db.authenticate(MONGODB_USER, MONGODB_PW)
coll = db[MONGODB_COLL]
a_loc = coll.find_one()
for query_key in sorted(
filter(lambda key: key not in NON_FACILITY_KEYS, a_loc.keys())):
facility_types.append({
'short_name': query_key,
'long_name': _make_proper(query_key),
'selected': False
})
client.disconnect()
return facility_types
class NGMongoConnect(object):
'''
mongodb连接
'''
conn = None
def __init__(self, *args, **kwargs):
self.args = args
self.kwargs = kwargs
self.connect()
def connect(self):
replica_set = self.kwargs.get("replica_set", None)
if replica_set:
self.conn = MongoClient(replica_set["connection_str"], replicaSet=replica_set['replica_set_name'])
else:
self.conn = MongoClient(self.args[0])
def disconnect(self):
return self.conn.disconnect()
def __getattr__(self, db_name):
return self.get_database(db_name)
def get_database(self, db_name):
'''
选择数据库
'''
return NGMongoDatabase(self.conn, db_name)
class BaseBaseModel(object):
conn = None
database = None
collection = None
servers = "mongodb://localhost:27017"
def __init__(self, db=''):
self.connect()
self.selectDB(db)
def connect(self):
self.conn = MongoClient(self.servers)
def close(self):
return self.conn.disconnect()
def getConn(self):
return self.conn
def selectDB(self, name=''):
if name:
self.database = self.conn[name]
def selectAC(self, name=''):
if name:
self.collection = self.database[name]
def render_GET(self, request):
"""
Function returns top100 for every game mode in JSON format.
"""
# Sync mongo connection
sync_mongo = MongoClient(self.db_host, self.db_port)
sync_db = sync_mongo[self.db_name]
sync_records = sync_db.records
result = {}
modes = [
'classic', 'relax', 'crazy',
'tetcolor_mega_classic', 'tetcolor_mega_relax', 'tetcolor_mega_crazy',
'tetcolor_columns_classic', 'tetcolor_columns_relax', 'tetcolor_columns_crazy',
'tetcolor_columns_mega_classic', 'tetcolor_columns_mega_relax', 'tetcolor_columns_mega_crazy',
'tetcolor_balls_classic', 'tetcolor_balls_relax', 'tetcolor_balls_crazy',
'tetcolor_balls_mega_classic', 'tetcolor_balls_mega_relax', 'tetcolor_balls_mega_crazy',
'tetcolor_tetris_classic', 'tetcolor_tetris_relax', 'tetcolor_tetris_crazy',
'tetcolor_tetris_mega_classic', 'tetcolor_tetris_mega_relax', 'tetcolor_tetris_mega_crazy',
]
for mode in modes:
r = []
records = sync_records.find({'mode': mode, })\
.sort([('score', -1)])\
.limit(100)
for item in records:
r.append({
'name': item['name'],
'score': item['score'],
'timestamp': item['timestamp'],
})
result[mode] = r
jsonp = json.dumps(result)
jsonp = 'jsonCallback(' + jsonp + ');'
request.setHeader('Content-Type', 'application/javascript')
sync_mongo.disconnect()
return jsonp
def test_copy_db(self):
authed_client = auth_context.client
if version.at_least(authed_client, (2, 7, 2)):
raise SkipTest("SERVER-17034")
if is_mongos(authed_client):
raise SkipTest("SERVER-6427")
c = MongoClient(host, port)
authed_client.admin.add_user("admin", "password")
c.admin.authenticate("admin", "password")
c.drop_database("pymongo_test")
c.drop_database("pymongo_test1")
c.pymongo_test.test.insert({"foo": "bar"})
try:
c.pymongo_test.add_user("mike", "password")
self.assertRaises(OperationFailure,
c.copy_database,
"pymongo_test",
"pymongo_test1",
username="******",
password="******")
self.assertFalse("pymongo_test1" in c.database_names())
self.assertRaises(OperationFailure,
c.copy_database,
"pymongo_test",
"pymongo_test1",
username="******",
password="******")
self.assertFalse("pymongo_test1" in c.database_names())
c.copy_database("pymongo_test",
"pymongo_test1",
username="******",
password="******")
self.assertTrue("pymongo_test1" in c.database_names())
self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
finally:
# Cleanup
remove_all_users(c.pymongo_test)
c.admin.remove_user("admin")
c.disconnect()
class MongoConnection():
client = None
def_db = None
def __init__(self):
try:
self.client = MongoClient(URI)
self.def_db = self.client.get_default_database()
except IOError:
print "Could not connect to mongo database !"
raise
def get_def_database(self):
return self.def_db
def close_connection(self):
self.client.disconnect()
def signPetition():
dbClient = MongoClient()
db = dbClient[conf.DB]
siteContent = db['content']
"""Check if the email already exists in the database. If an email exists
then do not add it again, send error to client and tell email already
exists.
"""
test = siteContent.find_one({"email": request.form['email']})
if test is None:
siteContent.insert({"name": request.form['name'],
"email": request.form['email']})
response = {"status": "Success",
"code": 1}
else:
response = {"status": "Duplicate email",
"code": 0}
dbClient.disconnect()
return jsonify(response)
def add_facility_type(csv_path, type_name):
print '%s: starting add facility type %s to all points in MongoDB' % \
(_get_current_time_str(), type_name)
# connect to MongoDB
client = MongoClient(MONGODB_HOST, MONGODB_PORT)
db = client[MONGODB_DB]
db.authenticate(MONGODB_USER, MONGODB_PW)
# parse csv to list of dicts
facilities = list()
for line in open(csv_path, 'r').readlines()[1:]:
line_split = line.split(FACILITY_FILE_DELIM)
name = line_split[0]
x = float(line_split[1])
y = float(line_split[2].split('\r\n')[0])
facilities.append({'name': name, 'x': x, 'y': y})
# insert facilities to MongoDB
fac_coll = db[type_name]
fac_coll.insert(facilities)
# update location to contain min distance information
loc_coll = db[MONGODB_LOC_COLL]
print '%s: Connected to DB (Host: %s, Port: %d, DB: %s)' % (_get_current_time_str(), MONGODB_HOST, MONGODB_PORT, MONGODB_DB)
index = 1
for loc in loc_coll.find().batch_size(LOC_BATCH_SIZE):
min_distance, min_distance_facility_name = _find_min_distance_facility_from_loc(loc, facilities)
loc_coll.update({'_id': ObjectId(loc['_id'])},
{'$set': {type_name:
{MONGODB_MIN_DISTANCE_FACILITY_FIELD_NAME: min_distance_facility_name,
MONGODB_MIN_DISTANCE_FIELD_NAME: min_distance}}
})
if index % NUM_POINTS_TO_PRINT == 0:
print '%s: %d locations are processed' % (_get_current_time_str(), index)
index += 1
print '%s: total %d locations now started having type %s' % (_get_current_time_str(), index - 1, type_name)
# disconnect from MongoDB
print '%s: disconnecting from MongoDB and exiting' % _get_current_time_str()
client.disconnect()
def signPetition():
dbClient = MongoClient()
db = dbClient[conf.DB]
siteContent = db['content']
"""Check if the email already exists in the database. If an email exists
then do not add it again, send error to client and tell email already
exists.
"""
test = siteContent.find_one({"email": request.form['email']})
if test is None:
siteContent.insert({
"name": request.form['name'],
"email": request.form['email']
})
response = {"status": "Success", "code": 1}
else:
response = {"status": "Duplicate email", "code": 0}
dbClient.disconnect()
return jsonify(response)
class charnDB:
"""
A class to connect to a charnDB instance (which is based on MongoDB)
"""
def __init__(self, host="localhost", port=27017):
self.client = MongoClient(host, port)
def list_databases(self):
"""
Return a list of String corresponding to the databases available.
"""
return self.client.database_names()
def get_database(self, database):
return self.client[database]
def disconnect(self):
"""
Close the connection to the database
"""
self.client.disconnect()
def get_locs(criteria, projections):
client = MongoClient(MONGODB_HOST, MONGODB_PORT)
db = client[MONGODB_DB]
db.authenticate(MONGODB_USER, MONGODB_PW)
coll = db[MONGODB_COLL]
# find locations that match the criteria
locs = coll.find(criteria, projections).batch_size(MAX_NUM_SEARCH_RESULTS)
# initialize return variables
results = list()
_num_results = 0
more_than_limit_flg = False
no_result_flg = False
for loc in locs:
# check if number of results exceeds the limit
_num_results += 1
if _num_results >= MAX_NUM_SEARCH_RESULTS:
more_than_limit_flg = True
break
# create a list of dicts containing minimum distance facility information
mdf = list()
for projection_key in projections.keys():
if projection_key not in NON_FACILITY_KEYS:
mdf.append({
'field_name': _make_proper(projection_key),
'md': loc[projection_key]['md'],
'mdfn': loc[projection_key]['mdfn']
})
results.append({X_KEY: loc[X_KEY], Y_KEY: loc[Y_KEY], 'mdf': mdf})
# check if number of results is zero
if _num_results == 0:
no_result_flg = True
client.disconnect()
return results, more_than_limit_flg, no_result_flg
class MongoDBConnection:
def __init__(self, host="localhost", port="27017"):
self.host = f"mongodb://{host}:{port}"
self.connection = None
def __enter__(self):
self.connection = MongoClient(self.host)
return self
def __exit__(self, a, b, c):
self.connection.close()
self.connection.disconnect()
with MongoDBConnection() as mongo:
conn = mongo.connection
db = conn.my_app
collection = db.people
collection.insert_many([{
"tests/lesson09/data/furniture/chair":
["metal_chair_back_isometric_400_clr_17527.png"],
"tests/lesson09/data/furniture/chair/couch":
["sofa_400_clr_10056.png"],
"tests/lesson09/data/furniture/table": [
"basic_desk_main_400_clr_17523.png",
"desk_isometric_back_400_clr_17524.png",
"table_with_cloth_400_clr_10664.png"
],
"tests/lesson09/data/new": [
"chairs_balancing_stacked_400_clr_11525.png",
"hotel_room_400_clr_12721.png"
],
"tests/lesson09/data/old": [
"couple_on_swing_bench_400_clr_12844.png",
"sitting_in_chair_relaxing_400_clr_6028.png"
]
}])
person = collection.find({"first_name": "Lisetta"})
pprint(list(person))
class DbConn(object):
def __init__(self, host="127.0.0.1", port=27017):
self.server = "mongodb://{host}:{port}".format(host=host, port=port)
self.conn = None
def connect(self):
self.conn = MongoClient(self.server)
def close(self):
return self.conn.disconnect()
def getConn(self):
return self.conn
def computeDistMatrix():
'''Computes distance between systems and stores in mongo
Really this should be a mongo job
'''
client = MongoClient()
db = client.mydb
coll = db.emd_dists
out = []
mat = {}
#out data: {fromsys:'',
# tosys:{draconis:222.567,
# ibootis:666.543,
# }
# }
#load in the system data
systems = []
fh = open(r'/Users/dusted-ipro/Documents/elitesystems.txt', 'r')
for line in fh.readlines():
out = line.rstrip('\n').split(',')
systems.append(out)
fh.close()
#print systems
for sys in systems:
#calc distance from this sys to every other one
mat['fromSys'] = sys[0]
mat['toSys'] = {}
for thisSys in systems:
d = distance(float(sys[1]), float(sys[2]), float(sys[3]),
float(thisSys[1]), float(thisSys[2]),
float(thisSys[3]))
mat['toSys'][thisSys[0]] = d
#dump to mongo
pid = coll.insert(mat.copy())
print pid
mat = {}
print 'Done'
client.disconnect()
return
class database():
def __init__(self):
self.client = MongoClient()
def __del__(self):
self.client.disconnect()
def insert(self, change):
# BSON (which mongodb uses) only supports datetime, so we need to convert the date.
# The hour and minute will be 0
d = change['RatingsChangeDate']
dt = datetime.datetime(d.year, d.month, d.day)
symbol = {
'RatingsChangeDate': dt,
'RatingsChangeType': change['RatingsChangeType'],
'Company': change['Company'],
'Ticker': change['Ticker'],
'BrokerageFirm': change['BrokerageFirm'],
'RatingsChange': change['RatingsChange'],
'PriceTarget': change['PriceTarget'],
}
self.client.arbit.ratingsChanges.insert(symbol)
def fetch(self, currentDate, ratingsChangeType):
d = currentDate
dt = datetime.datetime(d.year, d.month, d.day)
ratingsChanges = []
for change in self.client.arbit.ratingsChanges.find({
'RatingsChangeDate':
dt,
'RatingsChangeType':
'Upgrade'
}):
ratingsChanges.append(change)
return ratingsChanges
def graphing():
'''make some graphs from the scored profit data
'''
client = MongoClient()
db = client.mydb
coll = db.emd_profitdata
data = coll.find_one()
x = []
labsX = []
labs = []
y = []
cnt = 0
#Build up list of bar graph numbers
for i in data['profitData']:
#print i
#print data['profitData']
y.append(i['score'])
x.append(cnt)
labsX.append(cnt + 0.3)
labs.append(i['commodity'])
cnt += 1
print labs
client.disconnect()
bar = plt.bar(
x,
y,
0.3,
color='b',
label='Commodities by Profit Score (weighted by travel distance)')
plt.xticks(labsX, labs, rotation=90)
plt.legend()
plt.tight_layout()
plt.show()
return
class Stats(object):
def __init__(self):
self.stopping = False
mongo_uri = 'mongodb://{user}:{password}@{host}/{database}'.format(
user=urllib.quote(MONGO_USER),
password=urllib.quote(MONGO_PASSWORD),
host=MONGO_HOST,
database=MONGO_DATABASE_NAME)
connected = False
while not connected:
try:
self.client = MongoClient(MONGO_HOST,
replicaset=MONGO_REPLICA_SET)
connected = True
except pymongo.errors.ConnectionFailure as e:
logger.info(
'Failed to connect to mongodb [{}], retrying.'.format(e))
sleep(1)
self.db = self.client[MONGO_DATABASE_NAME]
self.logger = logging.getLogger('Stats')
def stop(self):
self.client.disconnect()
self.stopping = True
def update_minutes_and_hours_from_readings(self):
'''
This method updates the ten_minutes and hours collections with
documents that contain the average usage and temp_f for each
granularity for all of the data available from the envir_reading
collection.
Collections used:
1. envir_reading: Collection of readings from energy monitor
receiver every 6 seconds.
2. ten_minutes: Collection of usage and temp_f averages for each 10
minute window over all of the collected data.
2. hours: Collection of usage and temp_f averages for each hour
over all of the collected data.
'''
readings = self.db.envir_reading
hours = self.db.hours
ten_minutes = self.db.ten_minutes
bookmarks = self.db.bookmarks
logger = self.logger
logger.info('Updating minutes/hours from envir_reading.')
# Figure out where we left off by finding the timestamp of the last
# bookmark.
query = {}
reading_bookmark = bookmarks.find_one({'_id': 'envir_reading'})
if reading_bookmark is not None:
logger.info('Last bookmark was {}'.format(reading_bookmark))
query = {
'reading_timestamp': {
'$gt': reading_bookmark['timestamp']
}
}
logger.info('{} total readings.'.format(readings.count()))
index_ensured = False
while not index_ensured:
try:
readings.ensure_index('reading_timestamp', pymongo.ASCENDING)
index_ensured = True
except Exception as e:
logger.info('Failed to ensure index ({}), retrying.'.format(e))
sleep(1)
cursor = readings.find(query).sort('reading_timestamp',
pymongo.ASCENDING)
logger.info('{} new readings since last bookmark.'.format(
cursor.count()))
current_hour = None
reading_count = 0
for reading in cursor:
reading_count += 1
if (reading_count % 1000) == 0:
logger.info('Processed {} readings.'.format(reading_count))
if reading['total_watts'] == 0 or reading['temp_f'] == 0:
logger.info(
'Skipping reading at {}: total_watts({}) temp_f({})'.
format(reading['reading_timestamp'],
reading['total_watts'], reading['temp_f']))
continue
timestamp = reading['reading_timestamp']
if not current_hour or timestamp >= (current_hour['_id'] +
timedelta(hours=1)):
if current_hour:
saved = False
while not saved:
try:
new_id = hours.save(current_hour)
saved = True
except Exception as e:
logger.error(
'Failed to save current hour: {}'.format(e))
sleep(1)
logger.info("Moving to new hour {}".format(new_id))
current_hour_start = datetime(year=timestamp.year,
month=timestamp.month,
day=timestamp.day,
hour=timestamp.hour)
logger.debug(
'Looking for hour at {}'.format(current_hour_start))
current_hour = hours.find_one({'_id': current_hour_start})
if current_hour is None:
current_hour = {
'_id': current_hour_start,
'count': 0,
'average_usage': 0,
'average_tempf': 0,
'timestamps': []
}
logger.debug('Creating hour document: {}'.format(
current_hour_start))
if timestamp not in current_hour['timestamps']:
(current_hour['average_usage'], temp_count) = update_average(
old_average=current_hour['average_usage'],
old_count=current_hour['count'],
new_value=reading['total_watts'])
(current_hour['average_tempf'],
current_hour['count']) = update_average(
old_average=current_hour['average_tempf'],
old_count=current_hour['count'],
new_value=reading['temp_f'])
current_hour['timestamps'].append(timestamp)
if reading_bookmark is None:
reading_bookmark = {
'_id': 'envir_reading',
'timestamp': timestamp
}
else:
reading_bookmark['timestamp'] = timestamp
if current_hour:
saved = False
while not saved:
try:
new_id = hours.save(current_hour)
saved = True
except Exception as e:
logger.error('Failed to save current hour: {}'.format(e))
sleep(1)
logger.debug('Saved hour document {}'.format(new_id))
if reading_bookmark:
saved = False
while not saved:
try:
bookmarks.save(reading_bookmark)
saved = True
except Exception as e:
logger.error(
'Failed to save reading bookmark: {}'.format(e))
sleep(1)
logger.info('Saved bookmark at {}.'.format(reading_bookmark))
def update_hours_per_day_from_hours(self):
'''
This method updates two different collections:
1. Update averages for each hour in any day, so we should have 24
documents per collection that we updated.
2. Update averages for each hour for each day of the week, so we
should have 7 documents with 24 averages per day (both usage and
temp_f).
Collections used:
1. hours: Collection of usage and temp_f averages for each hour
over all of the collected data.
2. hours_per_dow: Collection of usage and temp_f averages for each
hour of each day of the week.
3. hours_in_day: Collection of usage and temp_f averages for each
hour in any day.
'''
logger = self.logger
hours = self.db.hours
hours_per_dow = self.db.hours_per_dow
hours_in_day = self.db.hours_in_day
bookmarks = self.db.bookmarks
logger.info('Updating hours per day/dow from hours collection.')
# We use a single bookmark for both hours per day of week and hours in
# day because we're building those off of the same collection of hours.
hour_bookmark = bookmarks.find_one({'_id': 'hours'})
if hour_bookmark is None:
hour_bookmark = {'_id': 'hours', 'timestamp': epoch}
current_dow = None
current_hour_of_day = None
hours_cache = {}
days_cache = {}
local_tz = pytz.timezone(LOCAL_TIMEZONE)
# We want $gte for finding our bookmark because we may have more values
# in the last hour we processed that weren't there when we processed
# them last.
cursor = hours.find({'_id': {'$gte': hour_bookmark['timestamp']}})
for hour in cursor:
# We use a localized datetime for building these collections
# because the day of the week values are really only relevant to
# the timezone where the data is being collected. The hours in a
# day don't really matter because they could be shifted by the
# timezone offset of the browser, but again, the hours make more
# sense when they're in the timezone of the collector. On top of
# that, because we're localizing, we can compensate for DST since
# we have the date.
local_timestamp = pytz.utc.localize(
hour['_id']).astimezone(local_tz)
current_hour_num = str(local_timestamp.hour)
logging.debug('Updating hour {}.'.format(current_hour_num))
# Find the current_hour_of_day document to update.
#
# First check the current hour of day to see if it's what we want.
if not current_hour_of_day or current_hour_of_day[
'_id'] != current_hour_num:
# Next check in the hours cache
if current_hour_num in hours_cache:
current_hour_of_day = hours_cache[current_hour_num]
else:
# Next we go to the DB
current_hour_of_day = hours_in_day.find_one(
{'_id': current_hour_num})
if not current_hour_of_day:
# Not in the DB either, so we need to create a new document.
current_hour_of_day = {
'_id': current_hour_num,
'timezone': local_tz.zone,
'average_usage': 0,
'average_tempf': 0,
'count': 0,
'timestamps': []
}
# Add the document (either what we pulled from the DB or
# just created) to the cache. We'll update the DB later
# when we're done processing the hours documents that we
# pulled from the DB.
hours_cache[current_hour_num] = current_hour_of_day
if not hour['_id'] in current_hour_of_day['timestamps']:
# Update the usage average.
(current_hour_of_day['average_usage'],
temp_count) = update_average(
old_average=current_hour_of_day['average_usage'],
old_count=current_hour_of_day['count'],
new_value=hour['average_usage'])
# Update the tempf average.
(current_hour_of_day['average_tempf'],
current_hour_of_day['count']) = update_average(
old_average=current_hour_of_day['average_tempf'],
old_count=current_hour_of_day['count'],
new_value=hour['average_tempf'])
current_hour_of_day['timestamps'].append(hour['_id'])
# Find the current day of the week document to update. These are
# indexed by the abbreviated day name (probably could have used day
# of week number, but the day name is easier to read and can be
# passed straight to whoever asks for it).
day_name = local_timestamp.strftime('%a')
logging.debug('Updating {}.'.format(day_name))
# Same dance as the hours, check if the current dow is the right one.
if not current_dow or current_dow['_id'] != day_name:
# Check the days cache next.
if day_name in days_cache:
current_dow = days_cache[day_name]
else:
# Check the DB
current_dow = hours_per_dow.find_one({'_id': day_name})
if not current_dow:
# Create a new document.
current_dow = {
'_id': day_name,
'timezone': local_tz.zone,
'hours': {},
}
# Add the new/pulled from DB document to the cache.
days_cache[day_name] = current_dow
if current_hour_num in current_dow['hours']:
current_hour_of_dow = current_dow['hours'][current_hour_num]
else:
current_hour_of_dow = {
'average_usage': 0,
'average_tempf': 0,
'count': 0,
'timestamps': []
}
current_dow['hours'][current_hour_num] = current_hour_of_dow
if not hour['_id'] in current_hour_of_dow['timestamps']:
# Update the usage average.
(current_hour_of_dow['average_usage'],
temp_count) = update_average(
old_average=current_hour_of_dow['average_usage'],
old_count=current_hour_of_dow['count'],
new_value=hour['average_usage'])
# Update the tempf average.
(current_hour_of_dow['average_tempf'],
current_hour_of_dow['count']) = update_average(
old_average=current_hour_of_dow['average_tempf'],
old_count=current_hour_of_dow['count'],
new_value=hour['average_tempf'])
current_hour_of_dow['timestamps'].append(hour['_id'])
# Update the naive (UTC) datetime bookmark
hour_bookmark['timestamp'] = hour['_id']
# Save our updates to the DB
for hour, doc in hours_cache.iteritems():
saved = False
while not saved:
try:
hours_in_day.save(doc)
saved = True
except Exception as e:
logger.error('Failed to save hours in day: {}'.format(e))
sleep(1)
logger.info('Saved hour {}.'.format(hour))
for day, doc in days_cache.iteritems():
saved = False
while not saved:
try:
hours_per_dow.save(doc)
saved = True
except Exception as e:
logger.error('Failed to save day of week: {}'.format(e))
sleep(1)
logger.info('Saved day {}.'.format(day))
# Save the last bookmark we processed.
saved = False
while not saved:
try:
bookmarks.save(hour_bookmark)
saved = True
except Exception as e:
logger.error('Failed to save hour bookmark: {}'.format(e))
sleep(1)
logger.info('Saved bookmark {}'.format(hour_bookmark))
def update_stats(self):
self.update_minutes_and_hours_from_readings()
self.update_hours_per_day_from_hours()
def run(self):
while not self.stopping:
self.update_stats()
sleep(60)
def Process(self):
# Initialize variables
usbRead = None
logfile = None
hidDevice = None
countrate = 0.0 # CPS
livetime = 0.0
realtime = 0.0
previousRealtime = 0.0
previousLivetime = 0.0
# Initialize counters
self.counts = {}
self.ratecounter = 0
self.totalcounter = 0 # keep track of total counts since start
channelsTotal = [0 for i in range(4096)]
# Cached data
try:
cachedData = jsonpickle.decode(
open("cached_%s.json" % self.deviceId, 'r').read())
os.remove("cached_%s.json" % self.deviceId)
except:
cachedData = []
if self.useDatabase:
try:
connection = MongoClient(
self.config.db_host,
self.config.db_port,
socketTimeoutMS=self.config.networkTimeout,
connectTimeoutMS=self.config.networkTimeout)
db = connection[self.config.db_name]
# MongoLab has user authentication
db.authenticate(self.config.db_user, self.config.db_passwd)
except:
print '-' * 60
traceback.print_exc(file=sys.stdout)
print '-' * 60
sys.exit(1)
try:
self.logPrint("Opening device id %s [%s]" %
(self.deviceId, self.devicePath))
hidDevice = hid.device()
try:
hidDevice.open_path(self.devicePath)
except:
hidDevice.open(USB_VENDOR_ID, USB_PRODUCT_ID)
self.logPrint("Manufacturer: %s" %
hidDevice.get_manufacturer_string())
self.logPrint("Product: %s" % hidDevice.get_product_string())
# Open log file
self.logPrint("Appending data to %s ..." % self.logFilename)
logfile = open(self.logFilename, "a", 1)
# Start timers
start_time = time.time()
countrate_start_time = start_time # countrate computation
passcount_start_time = start_time # realtime, livetime computation
# Start USB reading thread
self.logPrint("Start USB reading thread")
usbRead = RadAngel.USBReadThread(hidDevice, self)
usbRead.start()
# Main loop (Control-C to exit)
while True:
countrate_elapsed_time = time.time() - countrate_start_time
if (countrate_elapsed_time >= COUNTRATE_INTERVAL):
countrate_start_time = time.time()
countrate = float(
self.ratecounter) / countrate_elapsed_time
self.ratecounter = 0
passcount_elapsed_time = time.time() - passcount_start_time
if (passcount_elapsed_time >= PASSCOUNTS_INTERVAL):
passcount_start_time = time.time()
currentRealtime = realtime
elapsed_time = time.time() - start_time
realtime = realtime + passcount_elapsed_time
elapased = realtime - currentRealtime
livetime = livetime + elapased * (1.0 - countrate * 1E-05)
elapsed_time = time.time() - start_time
if (elapsed_time >= self.config.loggingInterval):
start_time = time.time()
# Copy the counter so USB read thread can continue
loggingCounts = dict(self.counts)
loggingCounter = sum(
[loggingCounts[i] for i in loggingCounts])
loggingRealtime = realtime - previousRealtime
loggingLivetime = livetime - previousLivetime
# Clear counter and channels
self.counts = {}
previousRealtime = realtime
previousLivetime = livetime
# Prepare for logging
now_utc = datetime.now(timezone('UTC'))
spectrum = [
"%d" % (loggingCounts[i] if i in loggingCounts else 0)
for i in range(4096)
]
cpm = float(loggingCounter) / loggingLivetime * 60.0
log = "%s,%s,%0.3f,%0.3f,%0.3f,%s,%s" % (
now_utc.strftime(zulu_fmt), self.deviceId,
loggingRealtime, loggingLivetime, cpm, loggingCounter,
",".join(spectrum))
logfile.write("%s\n" % log)
logfile.flush()
self.logPrint(log)
# Keep union
channelsTotal = [
x + y
for x, y in zip(channelsTotal, [(
loggingCounts[i] if i in loggingCounts else 0)
for i in range(4096)])
]
# Upload to database if needed
if self.useDatabase:
data = {
"deviceid":
self.deviceId,
"date":
now_utc,
"realtime":
loggingRealtime,
"livetime":
loggingLivetime,
"channels":
[(loggingCounts[i] if i in loggingCounts else 0)
for i in range(4096)],
"cpm":
cpm,
"counts":
loggingCounter
}
cachedData.append(data)
try:
if len(cachedData) > 1:
try:
# We failed previously so we need to reconnect
connection = MongoClient(
self.config.db_host,
self.config.db_port,
socketTimeoutMS=self.config.
networkTimeout,
connectTimeoutMS=self.config.
networkTimeout)
db = connection[self.config.db_name]
db.authenticate(self.config.db_user,
self.config.db_passwd)
except:
self.logPrint(
"Database connection failed [%d item(s)]"
% len(cachedData))
print '-' * 60
traceback.print_exc(file=sys.stdout)
print '-' * 60
pass
bulkDataInsert = copy.deepcopy(cachedData)
db.spectrum.insert(bulkDataInsert)
self.logPrint("Database updated [%d item(s)]" %
len(cachedData))
cachedData = []
except:
# Keep cached data and retry later
self.logPrint(
"Failed to update database [%d item(s)]" %
len(cachedData))
connection.disconnect()
print '-' * 60
traceback.print_exc(file=sys.stdout)
print '-' * 60
pass
if ((self.captureTime > 0) and
(realtime > self.captureTime)) or (
(self.captureCount > 0) and
(self.totalcounter > self.captureCount)):
# Union latest counts from unfinished period
channelsTotal = [
x + y
for x, y in zip(channelsTotal, [(
self.counts[i] if i in self.counts else 0)
for i in range(4096)])
]
self.logPrint("Total captured time %0.3f completed" %
realtime)
self.logPrint(
" realtime = %0.3f, livetime = %0.3f, total count = %d, countrate = %0.3f"
% (realtime, livetime, self.totalcounter, countrate))
break
time.sleep(0.0001) # force yield for other threads
except:
self.logPrint(
"You probably don't have the hard coded test hid. Update the hid.device line"
)
self.logPrint(
"in this script with one from the enumeration list output above and try again."
)
print '-' * 60
traceback.print_exc(file=sys.stdout)
print '-' * 60
finally:
self.logPrint("Cleanup resources")
if usbRead != None:
usbRead.stop()
usbRead.join()
if hidDevice != None: hidDevice.close()
if logfile != None: logfile.close()
if len(cachedData):
# Dump data that couldn't make it to database for later insert
jsonpickle.set_encoder_options('simplejson', sort_keys=True)
open("cached_%s.json" % self.deviceId,
"w").write(jsonpickle.encode(cachedData))
self.logPrint("Done")
return channelsTotal, realtime, livetime
def main(fileName, collectionName, dbName=DATABASE_NAME):
try:
#set up database connection
conn = MongoClient()
global collection
collection = conn[dbName][collectionName]
#make sure the spatial index is created for GeoJSON
collection.ensure_index([("geometry", "2dsphere")])
#open shapefile
shp = ogr.Open(fileName)
layer = shp.GetLayer(0)
#keep track of successes and failures
successCount = 0
featureCount = 0
failures = []
for fid in xrange(layer.GetFeatureCount()):
try:
feature = layer.GetFeature(fid)
geoJson = json.loads(feature.ExportToJson())
#convert all strings to their numerical values (if applicable)
for prop in geoJson['properties']:
geoJson['properties'][prop] = convertIfNumeric(
geoJson['properties'][prop])
#geoJson['properties']["center"] = [ float(geoJson['properties']['INTPTLON']), float(geoJson['properties']['INTPTLAT']) ]
if not feature.geometry().IsValid():
print 'yo'
geoJson['geometry'] = makePolygonValid(geoJson['geometry'])
print 'no'
#if this feature is a MultiPolygon, explode it into a group of Polygons
if geoJson['geometry']['type'].upper() == "MULTIPOLYGON":
multi = explodeMultiPolygon(geoJson)
for item in multi:
collection.insert(item)
featureCount += 1
else:
if 'id' in geoJson:
del geoJson['id']
collection.insert(geoJson)
featureCount += 1
successCount += 1
if successCount % 250 == 0:
print "Imported %d features!" % (successCount)
except Exception, e:
failures.append((fid, str(e)))
conn.close()
conn.disconnect()
print "Imported %d features." % successCount
print "Total (after multipolygons were split up): %d" % featureCount
print "%d features could not be imported" % (len(failures))
if len(failures) > 0:
val = raw_input("Would you like to see the error details (Y/N)?")
if val.upper() == "Y":
for fid, error in failures:
print "-" * 60
print "Feature ID: %d" % fid
print error[:400]
# }
# ])
# print("No. variants per chromosome:")
# for c in count_per_study['result']:
# print("{0} - {1} variants".format(c['_id']['chromosome'], c['num variants']))
indexes = collection.index_information()
print("No. indexes: {0}".format(len(indexes)))
print("Indexes: {0}".format(indexes.keys()))
#
# Program entry point
#
if __name__ == "__main__":
# Parse CLI arguments
cli_parser = build_cli_arguments()
args = cli_parser.parse_args()
# Create DB connection
mongo = MongoClient(args.host, args.port)
db = mongo[args.database]
db.authenticate(args.user, args.password)
print("Checking information in database '" + args.database + "'...\n")
check_files(db[args.collection_files])
print("")
check_variants(db[args.collection_variants])
mongo.disconnect()
