class database(): def __init__(self): self.client = MongoClient() def __del__(self): self.client.disconnect() def insert(self, fundamentals): f = { 'Symbol': fundamentals['Symbol'], 'DownloadDate': fundamentals['Date'], 'Dividend': fundamentals['Dividend'], 'EPS': fundamentals['EPS'], 'Shares': fundamentals['Shares'], 'InstitutionalOwnership': fundamentals['InstitutionalOwnership'], 'Open': fundamentals['Open'], 'High': fundamentals['High'], 'Low': fundamentals['Low'], 'Close': fundamentals['Close'], } self.client.arbit.fundamentals.insert(f) def fetch(self, currentDate, symbol): d = currentDate startDatetime = datetime.datetime(d.year, d.month, d.day, 0, 0, 0) endDatetime = datetime.datetime(d.year, d.month, d.day, 23, 59, 59) for result in self.client.arbit.fundamentals.find( {'Symbol': symbol, 'DownloadDate': {'$gte': startDatetime, '$lt': endDatetime}}): return (result) return ([])
def getUserCount(): dbClient = MongoClient() db = dbClient[conf.DB] siteContent = db['content'] count = siteContent.count() dbClient.disconnect() return jsonify({"count": count})
class database(): def __init__(self): self.client = MongoClient() def __del__(self): self.client.disconnect() def insert(self, change): # BSON (which mongodb uses) only supports datetime, so we need to convert the date. # The hour and minute will be 0 d = change['RatingsChangeDate'] dt = datetime.datetime(d.year, d.month, d.day) symbol = { 'RatingsChangeDate': dt, 'RatingsChangeType': change['RatingsChangeType'], 'Company': change['Company'], 'Ticker': change['Ticker'], 'BrokerageFirm': change['BrokerageFirm'], 'RatingsChange': change['RatingsChange'], 'PriceTarget': change['PriceTarget'], } self.client.arbit.ratingsChanges.insert(symbol) def fetch(self, currentDate, ratingsChangeType): d = currentDate dt = datetime.datetime(d.year, d.month, d.day) ratingsChanges = [] for change in self.client.arbit.ratingsChanges.find({'RatingsChangeDate': dt, 'RatingsChangeType': 'Upgrade'}): ratingsChanges.append(change) return ratingsChanges
def get_shard_nodes(host): mc = MongoClient(host) shard_coll = mc['config']['shards'] shard_cursor = shard_coll.find() shards = [x for x in shard_cursor] mc.disconnect() return shards
class ReceiverProcess(object): def __init__(self, config_file_name): json_data = open(config_file_name) config = json.load(json_data) json_data.close() self.urls = [self.__get_url_for_host(config, host) for host in config['hosts']] self.mClient = MongoClient('localhost', 27017) #keep defaults for now for host in config['hosts']: self.mClient.test[host].remove() #remove all previous records def __get_url_for_host(self, config, host): return (config['base_url'] + "?apikey={0}&limit={1}&host={2}" ).format(config['apikey'], config['limit'], host) def __store_resp(self, response, **kwargs): host = re.search(r'\host=([^&=]+)', response.url).group(1) db = self.mClient.test data = response.text.encode('utf-8') db[host].insert(json.loads(data)) def make_requests(self): rs = (grequests.get(url, hooks={'response':self.__store_resp}) for url in self.urls) grequests.map(rs) def cleanup(self): self.mClient.disconnect()
def get_shard_nodes(host): mc = MongoClient(host) shard_coll = mc["config"]["shards"] shard_cursor = shard_coll.find() shards = [x for x in shard_cursor] mc.disconnect() return shards
def test_copy_db(self): authed_client = auth_context.client if is_mongos(authed_client): raise SkipTest("SERVER-6427") c = MongoClient(host, port) authed_client.admin.add_user("admin", "password") c.admin.authenticate("admin", "password") c.drop_database("pymongo_test") c.drop_database("pymongo_test1") c.pymongo_test.test.insert({"foo": "bar"}) try: c.pymongo_test.add_user("mike", "password") self.assertRaises(OperationFailure, c.copy_database, "pymongo_test", "pymongo_test1", username="******", password="******") self.assertFalse("pymongo_test1" in c.database_names()) self.assertRaises(OperationFailure, c.copy_database, "pymongo_test", "pymongo_test1", username="******", password="******") self.assertFalse("pymongo_test1" in c.database_names()) c.copy_database("pymongo_test", "pymongo_test1", username="******", password="******") self.assertTrue("pymongo_test1" in c.database_names()) self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"]) finally: # Cleanup remove_all_users(c.pymongo_test) c.admin.remove_user("admin") c.disconnect()
class database(): def __init__(self): self.client = MongoClient() def dropCollection(self): self.client.arbit.symbols.drop() def __del__(self): self.client.disconnect() def insert(self, symbolInformation): symbol = { 'Symbol': symbolInformation['Symbol'], 'Exchange': symbolInformation['Exchange'], 'Name': symbolInformation['Name'], 'LastSale': symbolInformation['LastSale'], 'MarketCap': symbolInformation['MarketCap'], 'IPOYear': symbolInformation['IPOYear'], 'Sector': symbolInformation['Sector'], 'Industry': symbolInformation['Industry'] } self.client.arbit.symbols.insert(symbol) def getAllSymbols(self): symbols = [] for symbolInformation in self.client.arbit.symbols.find(): symbols.append(symbolInformation['Symbol']) return symbols
def cuckoo_clean(): """Clean up cuckoo setup. It deletes logs, all stored data from file system and configured databases (SQL and MongoDB. """ # Init logging. # This need to init a console logger handler, because the standard # logger (init_logging()) logs to a file which will be deleted. create_structure() init_console_logging() # Initialize the database connection. db = Database() # Drop all tables. db.drop() # Check if MongoDB reporting is enabled and drop that if it is. cfg = Config("reporting") if cfg.mongodb and cfg.mongodb.enabled: from pymongo import MongoClient host = cfg.mongodb.get("host", "127.0.0.1") port = cfg.mongodb.get("port", 27017) mdb = cfg.mongodb.get("db", "cuckoo") try: conn = MongoClient(host, port) conn.drop_database(mdb) conn.disconnect() except: log.warning("Unable to drop MongoDB database: %s", mdb) # Paths to clean. paths = [ os.path.join(CUCKOO_ROOT, "db"), os.path.join(CUCKOO_ROOT, "log"), os.path.join(CUCKOO_ROOT, "storage"), ] # Delete various directories. for path in paths: if os.path.isdir(path): try: shutil.rmtree(path) except (IOError, OSError) as e: log.warning("Error removing directory %s: %s", path, e) # Delete all compiled Python objects ("*.pyc"). for dirpath, dirnames, filenames in os.walk(CUCKOO_ROOT): for fname in filenames: if not fname.endswith(".pyc"): continue path = os.path.join(CUCKOO_ROOT, dirpath, fname) try: os.unlink(path) except (IOError, OSError) as e: log.warning("Error removing file %s: %s", path, e)
class TCMongoDB(object): """ Handling classes will inherit from this class. It gives useful tools for mongodb connections. """ def __init__(self): self.client = None self.db = None self.col = None self.grid_db = None self.fs = None def init_client(self): self.client = MongoClient(DB_ADDR, DB_PORT) """ try: self.client = MongoClient(DB_ADDR, DB_PORT) except pymongo.errors.ConnectionFailure, e: return False return True """ def close_client(self): if self.client: self.client.disconnect() def set_db(self, db_name=None): """Sets current db. I no client, try to connect to db""" if not self.client: self.init_client() self.db = self.client[DB_NAME if not db_name else db_name] if PROD: self.db.authenticate(DB_USER, DB_PASSWD, DB_NAME) def set_col(self, col): """Sets the collection. If db not selected, try to select default db""" if not self.db: self.set_db() self.col = self.db[col] # GridFS functions def set_grid_db(self, db_name=None): if not self.client: self.init_client() self.grid_db = self.client[DB_FILES if not db_name else db_name] if PROD: self.grid_db.authenticate(DB_USER, DB_PASSWD, DB_FILES) def set_grid(self, db_name=None): self.set_grid_db(db_name) def set_fs(self, db=None): self.fs = GridFS(self.grid_db if not db else db)
def delete_item(item_id): c = MongoClient(mongodb_url) try: c.CodeConnect.items.remove({'_id': bson.ObjectId(item_id)}) return "OK" except bson.errors.InvalidId: abort(400) finally: c.disconnect()
def refine_snoclusters(db_name, db_host = "localhost", db_port = 27017): client = MongoClient(db_host, db_port) db = client[db_name] #sort snoRNA clusters from the database per snoRNA class (2 different classes), per strand (2 different strands) and per genome (n different genomes). genome_names = [] for genome in db['genomes'].find(): genome_names.append(genome['_id']+"@genomes") sorted_clusters = [] for i in range(0, len(genome_names)*4): #if n genomes => n*2*2 sub-lists of clusters sorted_clusters.append([]) for snocluster in db['ncRNAs'].find({'source': "tool:letitsno:NA"}): index = 0 if snocluster['class'] == 'Gene, snRNA, snoRNA, CD-box' else 1 index += 0 if snocluster['genomicStrand'] == '+' else 2 j = 0 while j != len(genome_names): genome_name = genome_names[j] if snocluster['genome'] == genome_name: index += range(0, len(genome_names)*4, 4)[j] #if n genomes => range_list=[0, 4, 8, 12, ..., n-1*2*2] j += 1 snocluster['genomicStart'] = snocluster['genomicPositions'][0] snocluster['genomicEnd'] = snocluster['genomicPositions'][1] sorted_clusters[index].append(snocluster) for cluster_annotations in sorted_clusters: if len(cluster_annotations): #clustering of snoRNA clusters whose genomic positions overlap new_clusters = cluster_genomic_annotations(cluster_annotations, threshold = 2, fill_cluster_with_genomic_annotations = True) #storage of enlarged snoRNA clusters in database and removing of all "old" clusters constituting these "new" clusters if len(new_clusters): for new_cluster in new_clusters: new_cluster['_id'] = str(ObjectId()) new_cluster['name'] = new_cluster['genomic_annotations'][0]['name'] new_cluster['source'] = "tool:letitsno:NA" new_cluster['class'] = new_cluster['genomic_annotations'][0]['class'] new_cluster['organism'] = new_cluster['genomic_annotations'][0]['organism'] new_cluster['genome'] = new_cluster['genomic_annotations'][0]['genome'] new_cluster['genomicStrand'] = new_cluster['genomic_annotations'][0]['genomicStrand'] new_cluster['genomicPositions'] = [new_cluster['genomicStart'], new_cluster['genomicEnd']] new_cluster['ids'] = new_cluster['genomic_annotations'][0]['ids'] for cluster in new_cluster['genomic_annotations'][1:]: new_cluster['name'] += cluster['name'][7:] new_cluster['ids'].extend(cluster['ids']) #no double of ncRNAs _id db["ncRNAs"].remove({'_id': cluster['_id']}) #db["ncRNAs"].remove(cluster) don't remove cluster db["ncRNAs"].remove({'_id': new_cluster['genomic_annotations'][0]['_id']}) del new_cluster['genomicStart'] del new_cluster['genomicEnd'] del new_cluster['genomic_annotations'] del new_cluster['annotations_count'] db["ncRNAs"].insert(new_cluster) client.disconnect()
def request_data(): c = MongoClient(URI) collection = c.enavis.core applicants = collection.find(fields=FIELDS) json_projects = [] for applicant in applicants: json_projects.append(applicant) json_projects = json.dumps(json_projects, default=json_util.default) c.disconnect() return json_projects
def main(): log.startLogging(sys.stdout) mongo_client = MongoClient(host='IP_OR_HOST_NAME', port=27017, document_class='DOC_CLASS') log.msg('Mongo client {}.'.format(mongo_client)) app = MongoApp(mongo_client, '9f33566e-1f8f-11e2-8979-c42c030dd6a5', 'USER_NAME', 'PASSWORD') transport = devicehive.client.ws.WebSocketFactory(app) transport.connect('ws://IP_OR_HOST_NAME:8080/DeviceHiveJava/websocket/') reactor.run() mongo_client.disconnect()
def get_items(): c = MongoClient(mongodb_url) try: data = [] for item in list(c.CodeConnect.items.find()): item['date'] = item['date'].isoformat() item['_id'] = str(item['_id']) data.append(item) return jsonify({'items': data}) finally: c.disconnect()
def check_unique_and_insert(self, items, host): """ Try to put data to DB. """ log.msg('class Db, method insert, items len ' + str(len(items))) # Async mongo connection async_mongo = yield txmongo.MongoConnection(host=self.db_host, port=self.db_port) # Sync mongo connection sync_mongo = MongoClient(self.db_host, self.db_port) async_db = async_mongo[self.db_name] async_records = async_db.records sync_db = sync_mongo[self.db_name] sync_records = sync_db.records if host is None: host = '' # insert some data for item in items: log.msg('try to insert ' + item['name'] + ', ' + item['score']) # # This is a only one place with blocking request to DB, but I need this request to not insert # data twice. # # TODO: find the way to remove this blocking request. # count = sync_records.find( { 'record_id': item['record_id'], 'mode': item['mode'], 'score': int(item['score']), 'name': item['name'], }).count() if count == 0: yield async_records.insert( { 'name': item['name'], 'score': int(item['score']), 'timestamp': item['timestamp'], 'record_id': item['record_id'], 'mode': item['mode'], 'host': host, }, safe=True) log.msg('record inserted ' + item['record_id']) else: log.msg('record NOT inserted ' + item['record_id']) sync_mongo.disconnect() async_mongo.disconnect()
def get_facility_types(): facility_types = list() client = MongoClient(MONGODB_HOST, MONGODB_PORT) db = client[MONGODB_DB] db.authenticate(MONGODB_USER, MONGODB_PW) coll = db[MONGODB_COLL] a_loc = coll.find_one() for query_key in sorted(filter(lambda key: key not in NON_FACILITY_KEYS, a_loc.keys())): facility_types.append({'short_name': query_key, 'long_name': _make_proper(query_key), 'selected': False}) client.disconnect() return facility_types
def getAdvancedConfiguration(): connection = MongoClient('mongo') db = connection['admin'] db.authenticate( 'drainware', 'JVxLZr9UmSMVobGGk0OKL172OsyHOQ6QophYpUu7B3DQhOvbcDZLYAHgNSnNXZTPXjd0OQ93kruR0aY2hLT26t14tnAedGhI9RiYCbCyc9JFw9srrRiLr3fRuiOFu0Jb' ) db = connection['ddi'] collection = db.configuration conf = collection.find_one({"id": "advanced"}) connection.disconnect() return conf
def main(): context = zmq.Context() subscriber = context.socket(zmq.SUB) client = MongoClient() db = client.mydb coll = db.emd #load in the system data systems = [] fh = open(r'elitesystems.txt', 'r') for line in fh.readlines(): out = line.rstrip('\n').split(',') systems.append(out) # Connect to the first publicly available relay. subscriber.connect('tcp://firehose.elite-market-data.net:9500') # Disable filtering. subscriber.setsockopt(zmq.SUBSCRIBE, "") i = 0 #while True: while i < 1000000: # Receive raw market JSON strings. market_json = zlib.decompress(subscriber.recv()) # Un-serialize the JSON data to a Python dict. market_data = simplejson.loads(market_json) # Dump the market data to stdout. Or, you know, do more fun # things here. #print market_data #fh.write(market_data) #pid = coll.insert(market_data) #Search for a matching system name data = dict(market_data) #print data['message']['stationName'][0:data['message']['stationName'].find(' (')] #Loop through and find a matching system for sys in systems: if data['message']['stationName'][0:data['message']['stationName']. find(' (')] == sys[0]: data['system'] = { 'name': sys[0], 'coordX': float(sys[1]), 'coordY': float(sys[2]), 'coordZ': float(sys[3]) } #print 'Matched' break #make a real date s = data['message']['timestamp'] dt = datetime.strptime(s[0:19], "%Y-%m-%dT%H:%M:%S") data['message']['realDate'] = dt pid = coll.insert(data) i += 1 #coll.update({'_id':pid},{"$set":{"systemName":sysName, "coordX":x, "coordY":y, "coordZ":z}}) print i client.disconnect()
class MongoCacheBasePipeline(object): def open_spider(self, spider): mongohost = getattr(spider, "mongohost", "localhost") try: self.client = MongoClient(host=mongohost) except: log.msg("%s runs without caching because we can't find mongodb at '%s'" % (self.__class__.__name__, mongohost), level=log.WARNING) def close_spider(self, spider): if getattr(self, "client", None): self.client.disconnect()
class WordlistMongoDatabasePipeline(object): """Saves the items to a mongodatabase""" def process_item(self, item, spider): if (self.connection.wordlist.words.find(dict(item)).count() == 0): obj_id = self.connection.wordlist.words.insert(dict(item)) return item def open_spider(self, spider): self.connection = MongoClient('localhost', 1337) def close_spider(self, spider): self.connection.disconnect()
class WordlistMongoDatabasePipeline(object): """Saves the items to a mongodatabase""" def process_item(self, item, spider): if( self.connection.wordlist.words.find( dict(item) ).count()==0 ): obj_id = self.connection.wordlist.words.insert( dict(item) ) return item def open_spider(self, spider): self.connection = MongoClient('localhost', 1337) def close_spider(self, spider): self.connection.disconnect()
class MongoDBStore(): class Transformer(SONManipulator): def transform_incoming(self, son, collection): pass def transform_outgoing(self, son, collection): pass def __init__(self, host, port): self.client = MongoClient(host, port) self.db = self.client.flaskmap self.db.add_son_manipulator(Transformer()) def all(self): poiList = [] for poic in self.db['poi-containers']: poiList.append(POIContainer(poic['name'])) for poi in poic.content: poiList.content return poiList def create_container(self, name="Nuevo mapa", pois=[], poi_bin_string=None): uid = uuid4().hex container = POIContainer(name) container.id = uid self.db['poi-containers'].insert(container) return container def save_container(self, uid, name, content): c = self.db['poi-containers'] container = c.find_one({'uid': uid}) if container: container.name = name container.content = content c.update({'uid': container.uid}, container) return raise Exception('Unknown UID') def delete_container(self, uid): raise Exception('Not implemented yet.') def get_stream_ov2(self, uid): raise Exception('Not implemented yet.') def shutdown(self): self.client.disconnect() def __getc_poi_containers(self): return self.db['poi-containers']
def getUsers(): dbClient = MongoClient() db = dbClient[conf.DB] siteContent = db['content'] skip = conf.NUMBER_OF_ITEMS * (int(request.args['page']) - 1) result = [] for content in siteContent.find().skip(skip).limit(conf.NUMBER_OF_ITEMS): del(content['_id']) result.append(content) response = make_response() response.data = dumps(result) dbClient.disconnect() return response
def getUsers(): dbClient = MongoClient() db = dbClient[conf.DB] siteContent = db['content'] skip = conf.NUMBER_OF_ITEMS * (int(request.args['page']) - 1) result = [] for content in siteContent.find().skip(skip).limit(conf.NUMBER_OF_ITEMS): del (content['_id']) result.append(content) response = make_response() response.data = dumps(result) dbClient.disconnect() return response
class CalculationProcess(object): def __init__(self, config_file_name): json_data = open(config_file_name) config = json.load(json_data) self.max_records = config['max_samples'] * config['limit'] self.max_samples = config['max_samples'] self.hosts = config['hosts'] self.mClient = MongoClient('localhost', 27017) #keep defaults for now self.mClient['test']['increasing'].remove() #remove all prev records def __get_change(self, records,path): visitors = [record['visitors'] for record, i in zip(records, range(0, self.max_samples))] avg = sum(visitors)/float(len(visitors)) variance = sum([(i-avg)**2 for i in visitors])/float(len(visitors)) std_dev_x2 = sqrt(variance) * 2.0 adj_visitors = filter(lambda x: abs(x - avg) <= std_dev_x2, visitors) adj_avg = sum(adj_visitors)/len(adj_visitors) return visitors[0] - adj_avg def __get_tuple(self, host, collection, path, timestamp): records = collection.find({'path':path}, sort=[('_id',-1)]) t = { 'host': host, 'change': self.__get_change(records,path), 'i': records.rewind()[0]['i'], 'path': path, 'created_at': timestamp } return t def __store_calculation(self, tuples): self.mClient['test']['increasing'].insert(tuples) def __remove_previous_calculation(self, host, timestamp): self.mClient['test']['increasing'].remove({ 'host':host, 'created_at':{'$lt':timestamp} }) def run(self): for host in self.hosts: collection = self.mClient['test'][host] paths = collection.distinct('path') timestamp = datetime.datetime.utcnow() tuples = [self.__get_tuple(host, collection, path, timestamp) for path in paths] if len(tuples)>0: self.__store_calculation(tuples) self.__remove_previous_calculation(host, timestamp) def cleanup(self): self.mClient.disconnect()
def TestMongo(): client = MongoClient("localhost", 27017) db = client["alio2o"] #posts = db["metaq_fail"] posts = db["worklist_inconsist"] #post = json.loads("{"method":"add","poiid":"0000000002"}") #post_id = posts.insert(post) #print post_id #posts.remove({"poiid": "0000000002"}) print posts.count() for item in posts.find(): print item print item["poiid"] client.disconnect()
def home(request): responseOut = {} allOut = [] '''make some graphs from the scored profit data ''' client = MongoClient() db = client.mydb coll = db.emd_profitdata #data = coll.find_one() #Get the most recent entry d = coll.find().sort('procDTG',-1).limit(1) #sort data data = d.next() runDate = data['procDTG'] sortedProfits = sorted(data['profitData'], key=itemgetter('avgProfit'), reverse=True) x = [] labsX = [] labs = [] y = [] cnt = 0 td = [] #Build up list of bar graph numbers - commidities by score #for i in data['profitData']: for i in sortedProfits: #print i #print data['profitData'] allOut.append({'score':i['score'], 'commodity':i['commodity']}) td.append([i['commodity'], i['systemFrom'], i['systemTo'], i['dist'], i['avgProfit']]) #allOut.append({'score':i['score'], 'commodity':cnt}) y.append(i['score']) #y.append(i['score']) #x.append(cnt) #labsX.append(cnt+0.3) #labs.append(i['commodity']) cnt+=1 client.disconnect() print allOut #Build Output json_list = simplejson.dumps(allOut) responseOut['procDTG']=runDate responseOut['allOutput']= json_list responseOut['tableData']=td responseOut['yvals'] = y responseOut['bins']=cnt responseOut['page_title']='Trader Results' responseOut['proc_pass_msg']= 'Results Below' #return responseOut #TODO: implement csrf checking #responseOut[0](csrf(request)) return render_to_response('trader_results3.html',responseOut)
def add_item(): c = MongoClient(mongodb_url) try: item = request.json item['date'] = datetime.now() item['ip_addr'] = request.remote_addr item_id = c.CodeConnect.items.insert(item) # Make ID and Date JSON Serializable item['_id'] = str(item_id) item['date'] = item['date'].isoformat() return jsonify({'items': item}) finally: c.disconnect()
class database(): def __init__(self): self.client = MongoClient() def __del__(self): self.client.disconnect() def insert(self, form4Information): year = int(form4Information['acceptanceDatetime'][0:4]) month = int(form4Information['acceptanceDatetime'][4:6]) day = int(form4Information['acceptanceDatetime'][6:8]) hour = int(form4Information['acceptanceDatetime'][8:10]) minute = int(form4Information['acceptanceDatetime'][10:12]) second = int(form4Information['acceptanceDatetime'][12:14]) acceptanceDatetime = datetime.datetime(year, month, day, hour, minute, second) year = int(form4Information['transactionDate'][0:4]) month = int(form4Information['transactionDate'][5:7]) day = int(form4Information['transactionDate'][8:10]) transactionDate = datetime.datetime(year, month, day) form4 = { 'SecDocument': form4Information['secDocument'], 'AcceptanceDatetime': acceptanceDatetime, 'IssuerTradingSymbol': form4Information['issuerTradingSymbol'], 'RptOwnerCik': form4Information['rptOwnerCik'], 'RptOwnerName': form4Information['rptOwnerName'], 'IsDirector': form4Information['isDirector'], 'IsOfficer': form4Information['isOfficer'], 'IsTenPercentOwner': form4Information['isTenPercentOwner'], 'IsOther': form4Information['isOther'], 'TransactionDate': transactionDate, 'TransactionShares': form4Information['transactionShares'], 'TransactionPricePerShare': form4Information['transactionPricePerShare'], 'TransactionAcquiredDisposed': form4Information['transactionAcquiredDisposedCode'], 'SharesOwned': form4Information['sharesOwned'], } self.client.arbit.form4.insert(form4) def fetch(self, currentDate): d = currentDate startDatetime = datetime.datetime(d.year, d.month, d.day, 0, 0, 0) endDatetime = datetime.datetime(d.year, d.month, d.day, 23, 59, 59) forms = [] for form in self.client.arbit.form4.find({'TransactionAcquiredDisposed': 'A', 'AcceptanceDatetime': {'$gte': startDatetime, '$lt': endDatetime}}): forms.append(form) return forms
class Doinker: def __init__(self, csv_path, database): self.csv_path = csv_path self.database = database self.connection = MongoClient() self.metadata = self.create_metadata() def set_collection(self, collection): self.metadata['collection'] = collection def set_description(self, description): self.metadata['collection'] = description def generate_rows(self): csv_file = open(self.csv_path, 'r') dict_reader = DictReader(csv_file) for row_dict in dict_reader: yield row_dict csv_file.close() def get_schema(self): for row in self.generate_rows(): return row.keys() def create_metadata(self): row = { 'metadata': True, 'schema': {}, 'description': '', 'last_modified': None } metadata = { 'row': row, 'row_id': None, 'collection': '' } for field in self.get_schema(): metadata['row']['schema'][field] = '' return metadata def insert_data(self): collection = self.connection[self.database][self.metadata['collection']] self.metadata['row_id'] = collection.insert(self.metadata['row']) for row in self.generate_rows(): collection.insert(row) collection.update({'_id': self.metadata['row_id']}, {'$set': {'last_modified': datetime.utcnow()}}) def __del__(self): self.connection.disconnect()
class ScavhqTestCase(unittest.TestCase): @classmethod def setUpClass(self): scavhq.app.config['TESTING'] = True self.app = scavhq.app.test_client() self.client = MongoClient('localhost', 27017) self.items = self.client['scavhq-dev'].items def tearDown(self): self.items.remove() @classmethod def tearDownClass(self): self.client.disconnect()
class BaseBaseModel(object): conn = None database = None collection = None servers = "mongodb://localhost:27017" def __init__(self, db=''): self.connect() self.selectDB(db) def connect(self): self.conn = MongoClient(self.servers) def close(self): return self.conn.disconnect() def getConn(self): return self.conn def selectDB(self, name=''): if name: self.database = self.conn[name] def selectAC(self, name=''): if name: self.collection = self.database[name]
def get_facility_types(): facility_types = list() client = MongoClient(MONGODB_HOST, MONGODB_PORT) db = client[MONGODB_DB] db.authenticate(MONGODB_USER, MONGODB_PW) coll = db[MONGODB_COLL] a_loc = coll.find_one() for query_key in sorted( filter(lambda key: key not in NON_FACILITY_KEYS, a_loc.keys())): facility_types.append({ 'short_name': query_key, 'long_name': _make_proper(query_key), 'selected': False }) client.disconnect() return facility_types
class NGMongoConnect(object): ''' mongodb连接 ''' conn = None def __init__(self, *args, **kwargs): self.args = args self.kwargs = kwargs self.connect() def connect(self): replica_set = self.kwargs.get("replica_set", None) if replica_set: self.conn = MongoClient(replica_set["connection_str"], replicaSet=replica_set['replica_set_name']) else: self.conn = MongoClient(self.args[0]) def disconnect(self): return self.conn.disconnect() def __getattr__(self, db_name): return self.get_database(db_name) def get_database(self, db_name): ''' 选择数据库 ''' return NGMongoDatabase(self.conn, db_name)
def render_GET(self, request): """ Function returns top100 for every game mode in JSON format. """ # Sync mongo connection sync_mongo = MongoClient(self.db_host, self.db_port) sync_db = sync_mongo[self.db_name] sync_records = sync_db.records result = {} modes = [ 'classic', 'relax', 'crazy', 'tetcolor_mega_classic', 'tetcolor_mega_relax', 'tetcolor_mega_crazy', 'tetcolor_columns_classic', 'tetcolor_columns_relax', 'tetcolor_columns_crazy', 'tetcolor_columns_mega_classic', 'tetcolor_columns_mega_relax', 'tetcolor_columns_mega_crazy', 'tetcolor_balls_classic', 'tetcolor_balls_relax', 'tetcolor_balls_crazy', 'tetcolor_balls_mega_classic', 'tetcolor_balls_mega_relax', 'tetcolor_balls_mega_crazy', 'tetcolor_tetris_classic', 'tetcolor_tetris_relax', 'tetcolor_tetris_crazy', 'tetcolor_tetris_mega_classic', 'tetcolor_tetris_mega_relax', 'tetcolor_tetris_mega_crazy', ] for mode in modes: r = [] records = sync_records.find({'mode': mode, })\ .sort([('score', -1)])\ .limit(100) for item in records: r.append({ 'name': item['name'], 'score': item['score'], 'timestamp': item['timestamp'], }) result[mode] = r jsonp = json.dumps(result) jsonp = 'jsonCallback(' + jsonp + ');' request.setHeader('Content-Type', 'application/javascript') sync_mongo.disconnect() return jsonp
def test_copy_db(self): authed_client = auth_context.client if version.at_least(authed_client, (2, 7, 2)): raise SkipTest("SERVER-17034") if is_mongos(authed_client): raise SkipTest("SERVER-6427") c = MongoClient(host, port) authed_client.admin.add_user("admin", "password") c.admin.authenticate("admin", "password") c.drop_database("pymongo_test") c.drop_database("pymongo_test1") c.pymongo_test.test.insert({"foo": "bar"}) try: c.pymongo_test.add_user("mike", "password") self.assertRaises(OperationFailure, c.copy_database, "pymongo_test", "pymongo_test1", username="******", password="******") self.assertFalse("pymongo_test1" in c.database_names()) self.assertRaises(OperationFailure, c.copy_database, "pymongo_test", "pymongo_test1", username="******", password="******") self.assertFalse("pymongo_test1" in c.database_names()) c.copy_database("pymongo_test", "pymongo_test1", username="******", password="******") self.assertTrue("pymongo_test1" in c.database_names()) self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"]) finally: # Cleanup remove_all_users(c.pymongo_test) c.admin.remove_user("admin") c.disconnect()
class MongoConnection(): client = None def_db = None def __init__(self): try: self.client = MongoClient(URI) self.def_db = self.client.get_default_database() except IOError: print "Could not connect to mongo database !" raise def get_def_database(self): return self.def_db def close_connection(self): self.client.disconnect()
def signPetition(): dbClient = MongoClient() db = dbClient[conf.DB] siteContent = db['content'] """Check if the email already exists in the database. If an email exists then do not add it again, send error to client and tell email already exists. """ test = siteContent.find_one({"email": request.form['email']}) if test is None: siteContent.insert({"name": request.form['name'], "email": request.form['email']}) response = {"status": "Success", "code": 1} else: response = {"status": "Duplicate email", "code": 0} dbClient.disconnect() return jsonify(response)
def add_facility_type(csv_path, type_name): print '%s: starting add facility type %s to all points in MongoDB' % \ (_get_current_time_str(), type_name) # connect to MongoDB client = MongoClient(MONGODB_HOST, MONGODB_PORT) db = client[MONGODB_DB] db.authenticate(MONGODB_USER, MONGODB_PW) # parse csv to list of dicts facilities = list() for line in open(csv_path, 'r').readlines()[1:]: line_split = line.split(FACILITY_FILE_DELIM) name = line_split[0] x = float(line_split[1]) y = float(line_split[2].split('\r\n')[0]) facilities.append({'name': name, 'x': x, 'y': y}) # insert facilities to MongoDB fac_coll = db[type_name] fac_coll.insert(facilities) # update location to contain min distance information loc_coll = db[MONGODB_LOC_COLL] print '%s: Connected to DB (Host: %s, Port: %d, DB: %s)' % (_get_current_time_str(), MONGODB_HOST, MONGODB_PORT, MONGODB_DB) index = 1 for loc in loc_coll.find().batch_size(LOC_BATCH_SIZE): min_distance, min_distance_facility_name = _find_min_distance_facility_from_loc(loc, facilities) loc_coll.update({'_id': ObjectId(loc['_id'])}, {'$set': {type_name: {MONGODB_MIN_DISTANCE_FACILITY_FIELD_NAME: min_distance_facility_name, MONGODB_MIN_DISTANCE_FIELD_NAME: min_distance}} }) if index % NUM_POINTS_TO_PRINT == 0: print '%s: %d locations are processed' % (_get_current_time_str(), index) index += 1 print '%s: total %d locations now started having type %s' % (_get_current_time_str(), index - 1, type_name) # disconnect from MongoDB print '%s: disconnecting from MongoDB and exiting' % _get_current_time_str() client.disconnect()
def signPetition(): dbClient = MongoClient() db = dbClient[conf.DB] siteContent = db['content'] """Check if the email already exists in the database. If an email exists then do not add it again, send error to client and tell email already exists. """ test = siteContent.find_one({"email": request.form['email']}) if test is None: siteContent.insert({ "name": request.form['name'], "email": request.form['email'] }) response = {"status": "Success", "code": 1} else: response = {"status": "Duplicate email", "code": 0} dbClient.disconnect() return jsonify(response)
class charnDB: """ A class to connect to a charnDB instance (which is based on MongoDB) """ def __init__(self, host="localhost", port=27017): self.client = MongoClient(host, port) def list_databases(self): """ Return a list of String corresponding to the databases available. """ return self.client.database_names() def get_database(self, database): return self.client[database] def disconnect(self): """ Close the connection to the database """ self.client.disconnect()
def get_locs(criteria, projections): client = MongoClient(MONGODB_HOST, MONGODB_PORT) db = client[MONGODB_DB] db.authenticate(MONGODB_USER, MONGODB_PW) coll = db[MONGODB_COLL] # find locations that match the criteria locs = coll.find(criteria, projections).batch_size(MAX_NUM_SEARCH_RESULTS) # initialize return variables results = list() _num_results = 0 more_than_limit_flg = False no_result_flg = False for loc in locs: # check if number of results exceeds the limit _num_results += 1 if _num_results >= MAX_NUM_SEARCH_RESULTS: more_than_limit_flg = True break # create a list of dicts containing minimum distance facility information mdf = list() for projection_key in projections.keys(): if projection_key not in NON_FACILITY_KEYS: mdf.append({ 'field_name': _make_proper(projection_key), 'md': loc[projection_key]['md'], 'mdfn': loc[projection_key]['mdfn'] }) results.append({X_KEY: loc[X_KEY], Y_KEY: loc[Y_KEY], 'mdf': mdf}) # check if number of results is zero if _num_results == 0: no_result_flg = True client.disconnect() return results, more_than_limit_flg, no_result_flg
class MongoDBConnection: def __init__(self, host="localhost", port="27017"): self.host = f"mongodb://{host}:{port}" self.connection = None def __enter__(self): self.connection = MongoClient(self.host) return self def __exit__(self, a, b, c): self.connection.close() self.connection.disconnect() with MongoDBConnection() as mongo: conn = mongo.connection db = conn.my_app collection = db.people collection.insert_many([{ "tests/lesson09/data/furniture/chair": ["metal_chair_back_isometric_400_clr_17527.png"], "tests/lesson09/data/furniture/chair/couch": ["sofa_400_clr_10056.png"], "tests/lesson09/data/furniture/table": [ "basic_desk_main_400_clr_17523.png", "desk_isometric_back_400_clr_17524.png", "table_with_cloth_400_clr_10664.png" ], "tests/lesson09/data/new": [ "chairs_balancing_stacked_400_clr_11525.png", "hotel_room_400_clr_12721.png" ], "tests/lesson09/data/old": [ "couple_on_swing_bench_400_clr_12844.png", "sitting_in_chair_relaxing_400_clr_6028.png" ] }]) person = collection.find({"first_name": "Lisetta"}) pprint(list(person))
class DbConn(object): def __init__(self, host="127.0.0.1", port=27017): self.server = "mongodb://{host}:{port}".format(host=host, port=port) self.conn = None def connect(self): self.conn = MongoClient(self.server) def close(self): return self.conn.disconnect() def getConn(self): return self.conn
def computeDistMatrix(): '''Computes distance between systems and stores in mongo Really this should be a mongo job ''' client = MongoClient() db = client.mydb coll = db.emd_dists out = [] mat = {} #out data: {fromsys:'', # tosys:{draconis:222.567, # ibootis:666.543, # } # } #load in the system data systems = [] fh = open(r'/Users/dusted-ipro/Documents/elitesystems.txt', 'r') for line in fh.readlines(): out = line.rstrip('\n').split(',') systems.append(out) fh.close() #print systems for sys in systems: #calc distance from this sys to every other one mat['fromSys'] = sys[0] mat['toSys'] = {} for thisSys in systems: d = distance(float(sys[1]), float(sys[2]), float(sys[3]), float(thisSys[1]), float(thisSys[2]), float(thisSys[3])) mat['toSys'][thisSys[0]] = d #dump to mongo pid = coll.insert(mat.copy()) print pid mat = {} print 'Done' client.disconnect() return
class database(): def __init__(self): self.client = MongoClient() def __del__(self): self.client.disconnect() def insert(self, change): # BSON (which mongodb uses) only supports datetime, so we need to convert the date. # The hour and minute will be 0 d = change['RatingsChangeDate'] dt = datetime.datetime(d.year, d.month, d.day) symbol = { 'RatingsChangeDate': dt, 'RatingsChangeType': change['RatingsChangeType'], 'Company': change['Company'], 'Ticker': change['Ticker'], 'BrokerageFirm': change['BrokerageFirm'], 'RatingsChange': change['RatingsChange'], 'PriceTarget': change['PriceTarget'], } self.client.arbit.ratingsChanges.insert(symbol) def fetch(self, currentDate, ratingsChangeType): d = currentDate dt = datetime.datetime(d.year, d.month, d.day) ratingsChanges = [] for change in self.client.arbit.ratingsChanges.find({ 'RatingsChangeDate': dt, 'RatingsChangeType': 'Upgrade' }): ratingsChanges.append(change) return ratingsChanges
def graphing(): '''make some graphs from the scored profit data ''' client = MongoClient() db = client.mydb coll = db.emd_profitdata data = coll.find_one() x = [] labsX = [] labs = [] y = [] cnt = 0 #Build up list of bar graph numbers for i in data['profitData']: #print i #print data['profitData'] y.append(i['score']) x.append(cnt) labsX.append(cnt + 0.3) labs.append(i['commodity']) cnt += 1 print labs client.disconnect() bar = plt.bar( x, y, 0.3, color='b', label='Commodities by Profit Score (weighted by travel distance)') plt.xticks(labsX, labs, rotation=90) plt.legend() plt.tight_layout() plt.show() return
class Stats(object): def __init__(self): self.stopping = False mongo_uri = 'mongodb://{user}:{password}@{host}/{database}'.format( user=urllib.quote(MONGO_USER), password=urllib.quote(MONGO_PASSWORD), host=MONGO_HOST, database=MONGO_DATABASE_NAME) connected = False while not connected: try: self.client = MongoClient(MONGO_HOST, replicaset=MONGO_REPLICA_SET) connected = True except pymongo.errors.ConnectionFailure as e: logger.info( 'Failed to connect to mongodb [{}], retrying.'.format(e)) sleep(1) self.db = self.client[MONGO_DATABASE_NAME] self.logger = logging.getLogger('Stats') def stop(self): self.client.disconnect() self.stopping = True def update_minutes_and_hours_from_readings(self): ''' This method updates the ten_minutes and hours collections with documents that contain the average usage and temp_f for each granularity for all of the data available from the envir_reading collection. Collections used: 1. envir_reading: Collection of readings from energy monitor receiver every 6 seconds. 2. ten_minutes: Collection of usage and temp_f averages for each 10 minute window over all of the collected data. 2. hours: Collection of usage and temp_f averages for each hour over all of the collected data. ''' readings = self.db.envir_reading hours = self.db.hours ten_minutes = self.db.ten_minutes bookmarks = self.db.bookmarks logger = self.logger logger.info('Updating minutes/hours from envir_reading.') # Figure out where we left off by finding the timestamp of the last # bookmark. query = {} reading_bookmark = bookmarks.find_one({'_id': 'envir_reading'}) if reading_bookmark is not None: logger.info('Last bookmark was {}'.format(reading_bookmark)) query = { 'reading_timestamp': { '$gt': reading_bookmark['timestamp'] } } logger.info('{} total readings.'.format(readings.count())) index_ensured = False while not index_ensured: try: readings.ensure_index('reading_timestamp', pymongo.ASCENDING) index_ensured = True except Exception as e: logger.info('Failed to ensure index ({}), retrying.'.format(e)) sleep(1) cursor = readings.find(query).sort('reading_timestamp', pymongo.ASCENDING) logger.info('{} new readings since last bookmark.'.format( cursor.count())) current_hour = None reading_count = 0 for reading in cursor: reading_count += 1 if (reading_count % 1000) == 0: logger.info('Processed {} readings.'.format(reading_count)) if reading['total_watts'] == 0 or reading['temp_f'] == 0: logger.info( 'Skipping reading at {}: total_watts({}) temp_f({})'. format(reading['reading_timestamp'], reading['total_watts'], reading['temp_f'])) continue timestamp = reading['reading_timestamp'] if not current_hour or timestamp >= (current_hour['_id'] + timedelta(hours=1)): if current_hour: saved = False while not saved: try: new_id = hours.save(current_hour) saved = True except Exception as e: logger.error( 'Failed to save current hour: {}'.format(e)) sleep(1) logger.info("Moving to new hour {}".format(new_id)) current_hour_start = datetime(year=timestamp.year, month=timestamp.month, day=timestamp.day, hour=timestamp.hour) logger.debug( 'Looking for hour at {}'.format(current_hour_start)) current_hour = hours.find_one({'_id': current_hour_start}) if current_hour is None: current_hour = { '_id': current_hour_start, 'count': 0, 'average_usage': 0, 'average_tempf': 0, 'timestamps': [] } logger.debug('Creating hour document: {}'.format( current_hour_start)) if timestamp not in current_hour['timestamps']: (current_hour['average_usage'], temp_count) = update_average( old_average=current_hour['average_usage'], old_count=current_hour['count'], new_value=reading['total_watts']) (current_hour['average_tempf'], current_hour['count']) = update_average( old_average=current_hour['average_tempf'], old_count=current_hour['count'], new_value=reading['temp_f']) current_hour['timestamps'].append(timestamp) if reading_bookmark is None: reading_bookmark = { '_id': 'envir_reading', 'timestamp': timestamp } else: reading_bookmark['timestamp'] = timestamp if current_hour: saved = False while not saved: try: new_id = hours.save(current_hour) saved = True except Exception as e: logger.error('Failed to save current hour: {}'.format(e)) sleep(1) logger.debug('Saved hour document {}'.format(new_id)) if reading_bookmark: saved = False while not saved: try: bookmarks.save(reading_bookmark) saved = True except Exception as e: logger.error( 'Failed to save reading bookmark: {}'.format(e)) sleep(1) logger.info('Saved bookmark at {}.'.format(reading_bookmark)) def update_hours_per_day_from_hours(self): ''' This method updates two different collections: 1. Update averages for each hour in any day, so we should have 24 documents per collection that we updated. 2. Update averages for each hour for each day of the week, so we should have 7 documents with 24 averages per day (both usage and temp_f). Collections used: 1. hours: Collection of usage and temp_f averages for each hour over all of the collected data. 2. hours_per_dow: Collection of usage and temp_f averages for each hour of each day of the week. 3. hours_in_day: Collection of usage and temp_f averages for each hour in any day. ''' logger = self.logger hours = self.db.hours hours_per_dow = self.db.hours_per_dow hours_in_day = self.db.hours_in_day bookmarks = self.db.bookmarks logger.info('Updating hours per day/dow from hours collection.') # We use a single bookmark for both hours per day of week and hours in # day because we're building those off of the same collection of hours. hour_bookmark = bookmarks.find_one({'_id': 'hours'}) if hour_bookmark is None: hour_bookmark = {'_id': 'hours', 'timestamp': epoch} current_dow = None current_hour_of_day = None hours_cache = {} days_cache = {} local_tz = pytz.timezone(LOCAL_TIMEZONE) # We want $gte for finding our bookmark because we may have more values # in the last hour we processed that weren't there when we processed # them last. cursor = hours.find({'_id': {'$gte': hour_bookmark['timestamp']}}) for hour in cursor: # We use a localized datetime for building these collections # because the day of the week values are really only relevant to # the timezone where the data is being collected. The hours in a # day don't really matter because they could be shifted by the # timezone offset of the browser, but again, the hours make more # sense when they're in the timezone of the collector. On top of # that, because we're localizing, we can compensate for DST since # we have the date. local_timestamp = pytz.utc.localize( hour['_id']).astimezone(local_tz) current_hour_num = str(local_timestamp.hour) logging.debug('Updating hour {}.'.format(current_hour_num)) # Find the current_hour_of_day document to update. # # First check the current hour of day to see if it's what we want. if not current_hour_of_day or current_hour_of_day[ '_id'] != current_hour_num: # Next check in the hours cache if current_hour_num in hours_cache: current_hour_of_day = hours_cache[current_hour_num] else: # Next we go to the DB current_hour_of_day = hours_in_day.find_one( {'_id': current_hour_num}) if not current_hour_of_day: # Not in the DB either, so we need to create a new document. current_hour_of_day = { '_id': current_hour_num, 'timezone': local_tz.zone, 'average_usage': 0, 'average_tempf': 0, 'count': 0, 'timestamps': [] } # Add the document (either what we pulled from the DB or # just created) to the cache. We'll update the DB later # when we're done processing the hours documents that we # pulled from the DB. hours_cache[current_hour_num] = current_hour_of_day if not hour['_id'] in current_hour_of_day['timestamps']: # Update the usage average. (current_hour_of_day['average_usage'], temp_count) = update_average( old_average=current_hour_of_day['average_usage'], old_count=current_hour_of_day['count'], new_value=hour['average_usage']) # Update the tempf average. (current_hour_of_day['average_tempf'], current_hour_of_day['count']) = update_average( old_average=current_hour_of_day['average_tempf'], old_count=current_hour_of_day['count'], new_value=hour['average_tempf']) current_hour_of_day['timestamps'].append(hour['_id']) # Find the current day of the week document to update. These are # indexed by the abbreviated day name (probably could have used day # of week number, but the day name is easier to read and can be # passed straight to whoever asks for it). day_name = local_timestamp.strftime('%a') logging.debug('Updating {}.'.format(day_name)) # Same dance as the hours, check if the current dow is the right one. if not current_dow or current_dow['_id'] != day_name: # Check the days cache next. if day_name in days_cache: current_dow = days_cache[day_name] else: # Check the DB current_dow = hours_per_dow.find_one({'_id': day_name}) if not current_dow: # Create a new document. current_dow = { '_id': day_name, 'timezone': local_tz.zone, 'hours': {}, } # Add the new/pulled from DB document to the cache. days_cache[day_name] = current_dow if current_hour_num in current_dow['hours']: current_hour_of_dow = current_dow['hours'][current_hour_num] else: current_hour_of_dow = { 'average_usage': 0, 'average_tempf': 0, 'count': 0, 'timestamps': [] } current_dow['hours'][current_hour_num] = current_hour_of_dow if not hour['_id'] in current_hour_of_dow['timestamps']: # Update the usage average. (current_hour_of_dow['average_usage'], temp_count) = update_average( old_average=current_hour_of_dow['average_usage'], old_count=current_hour_of_dow['count'], new_value=hour['average_usage']) # Update the tempf average. (current_hour_of_dow['average_tempf'], current_hour_of_dow['count']) = update_average( old_average=current_hour_of_dow['average_tempf'], old_count=current_hour_of_dow['count'], new_value=hour['average_tempf']) current_hour_of_dow['timestamps'].append(hour['_id']) # Update the naive (UTC) datetime bookmark hour_bookmark['timestamp'] = hour['_id'] # Save our updates to the DB for hour, doc in hours_cache.iteritems(): saved = False while not saved: try: hours_in_day.save(doc) saved = True except Exception as e: logger.error('Failed to save hours in day: {}'.format(e)) sleep(1) logger.info('Saved hour {}.'.format(hour)) for day, doc in days_cache.iteritems(): saved = False while not saved: try: hours_per_dow.save(doc) saved = True except Exception as e: logger.error('Failed to save day of week: {}'.format(e)) sleep(1) logger.info('Saved day {}.'.format(day)) # Save the last bookmark we processed. saved = False while not saved: try: bookmarks.save(hour_bookmark) saved = True except Exception as e: logger.error('Failed to save hour bookmark: {}'.format(e)) sleep(1) logger.info('Saved bookmark {}'.format(hour_bookmark)) def update_stats(self): self.update_minutes_and_hours_from_readings() self.update_hours_per_day_from_hours() def run(self): while not self.stopping: self.update_stats() sleep(60)
def Process(self): # Initialize variables usbRead = None logfile = None hidDevice = None countrate = 0.0 # CPS livetime = 0.0 realtime = 0.0 previousRealtime = 0.0 previousLivetime = 0.0 # Initialize counters self.counts = {} self.ratecounter = 0 self.totalcounter = 0 # keep track of total counts since start channelsTotal = [0 for i in range(4096)] # Cached data try: cachedData = jsonpickle.decode( open("cached_%s.json" % self.deviceId, 'r').read()) os.remove("cached_%s.json" % self.deviceId) except: cachedData = [] if self.useDatabase: try: connection = MongoClient( self.config.db_host, self.config.db_port, socketTimeoutMS=self.config.networkTimeout, connectTimeoutMS=self.config.networkTimeout) db = connection[self.config.db_name] # MongoLab has user authentication db.authenticate(self.config.db_user, self.config.db_passwd) except: print '-' * 60 traceback.print_exc(file=sys.stdout) print '-' * 60 sys.exit(1) try: self.logPrint("Opening device id %s [%s]" % (self.deviceId, self.devicePath)) hidDevice = hid.device() try: hidDevice.open_path(self.devicePath) except: hidDevice.open(USB_VENDOR_ID, USB_PRODUCT_ID) self.logPrint("Manufacturer: %s" % hidDevice.get_manufacturer_string()) self.logPrint("Product: %s" % hidDevice.get_product_string()) # Open log file self.logPrint("Appending data to %s ..." % self.logFilename) logfile = open(self.logFilename, "a", 1) # Start timers start_time = time.time() countrate_start_time = start_time # countrate computation passcount_start_time = start_time # realtime, livetime computation # Start USB reading thread self.logPrint("Start USB reading thread") usbRead = RadAngel.USBReadThread(hidDevice, self) usbRead.start() # Main loop (Control-C to exit) while True: countrate_elapsed_time = time.time() - countrate_start_time if (countrate_elapsed_time >= COUNTRATE_INTERVAL): countrate_start_time = time.time() countrate = float( self.ratecounter) / countrate_elapsed_time self.ratecounter = 0 passcount_elapsed_time = time.time() - passcount_start_time if (passcount_elapsed_time >= PASSCOUNTS_INTERVAL): passcount_start_time = time.time() currentRealtime = realtime elapsed_time = time.time() - start_time realtime = realtime + passcount_elapsed_time elapased = realtime - currentRealtime livetime = livetime + elapased * (1.0 - countrate * 1E-05) elapsed_time = time.time() - start_time if (elapsed_time >= self.config.loggingInterval): start_time = time.time() # Copy the counter so USB read thread can continue loggingCounts = dict(self.counts) loggingCounter = sum( [loggingCounts[i] for i in loggingCounts]) loggingRealtime = realtime - previousRealtime loggingLivetime = livetime - previousLivetime # Clear counter and channels self.counts = {} previousRealtime = realtime previousLivetime = livetime # Prepare for logging now_utc = datetime.now(timezone('UTC')) spectrum = [ "%d" % (loggingCounts[i] if i in loggingCounts else 0) for i in range(4096) ] cpm = float(loggingCounter) / loggingLivetime * 60.0 log = "%s,%s,%0.3f,%0.3f,%0.3f,%s,%s" % ( now_utc.strftime(zulu_fmt), self.deviceId, loggingRealtime, loggingLivetime, cpm, loggingCounter, ",".join(spectrum)) logfile.write("%s\n" % log) logfile.flush() self.logPrint(log) # Keep union channelsTotal = [ x + y for x, y in zip(channelsTotal, [( loggingCounts[i] if i in loggingCounts else 0) for i in range(4096)]) ] # Upload to database if needed if self.useDatabase: data = { "deviceid": self.deviceId, "date": now_utc, "realtime": loggingRealtime, "livetime": loggingLivetime, "channels": [(loggingCounts[i] if i in loggingCounts else 0) for i in range(4096)], "cpm": cpm, "counts": loggingCounter } cachedData.append(data) try: if len(cachedData) > 1: try: # We failed previously so we need to reconnect connection = MongoClient( self.config.db_host, self.config.db_port, socketTimeoutMS=self.config. networkTimeout, connectTimeoutMS=self.config. networkTimeout) db = connection[self.config.db_name] db.authenticate(self.config.db_user, self.config.db_passwd) except: self.logPrint( "Database connection failed [%d item(s)]" % len(cachedData)) print '-' * 60 traceback.print_exc(file=sys.stdout) print '-' * 60 pass bulkDataInsert = copy.deepcopy(cachedData) db.spectrum.insert(bulkDataInsert) self.logPrint("Database updated [%d item(s)]" % len(cachedData)) cachedData = [] except: # Keep cached data and retry later self.logPrint( "Failed to update database [%d item(s)]" % len(cachedData)) connection.disconnect() print '-' * 60 traceback.print_exc(file=sys.stdout) print '-' * 60 pass if ((self.captureTime > 0) and (realtime > self.captureTime)) or ( (self.captureCount > 0) and (self.totalcounter > self.captureCount)): # Union latest counts from unfinished period channelsTotal = [ x + y for x, y in zip(channelsTotal, [( self.counts[i] if i in self.counts else 0) for i in range(4096)]) ] self.logPrint("Total captured time %0.3f completed" % realtime) self.logPrint( " realtime = %0.3f, livetime = %0.3f, total count = %d, countrate = %0.3f" % (realtime, livetime, self.totalcounter, countrate)) break time.sleep(0.0001) # force yield for other threads except: self.logPrint( "You probably don't have the hard coded test hid. Update the hid.device line" ) self.logPrint( "in this script with one from the enumeration list output above and try again." ) print '-' * 60 traceback.print_exc(file=sys.stdout) print '-' * 60 finally: self.logPrint("Cleanup resources") if usbRead != None: usbRead.stop() usbRead.join() if hidDevice != None: hidDevice.close() if logfile != None: logfile.close() if len(cachedData): # Dump data that couldn't make it to database for later insert jsonpickle.set_encoder_options('simplejson', sort_keys=True) open("cached_%s.json" % self.deviceId, "w").write(jsonpickle.encode(cachedData)) self.logPrint("Done") return channelsTotal, realtime, livetime
def main(fileName, collectionName, dbName=DATABASE_NAME): try: #set up database connection conn = MongoClient() global collection collection = conn[dbName][collectionName] #make sure the spatial index is created for GeoJSON collection.ensure_index([("geometry", "2dsphere")]) #open shapefile shp = ogr.Open(fileName) layer = shp.GetLayer(0) #keep track of successes and failures successCount = 0 featureCount = 0 failures = [] for fid in xrange(layer.GetFeatureCount()): try: feature = layer.GetFeature(fid) geoJson = json.loads(feature.ExportToJson()) #convert all strings to their numerical values (if applicable) for prop in geoJson['properties']: geoJson['properties'][prop] = convertIfNumeric( geoJson['properties'][prop]) #geoJson['properties']["center"] = [ float(geoJson['properties']['INTPTLON']), float(geoJson['properties']['INTPTLAT']) ] if not feature.geometry().IsValid(): print 'yo' geoJson['geometry'] = makePolygonValid(geoJson['geometry']) print 'no' #if this feature is a MultiPolygon, explode it into a group of Polygons if geoJson['geometry']['type'].upper() == "MULTIPOLYGON": multi = explodeMultiPolygon(geoJson) for item in multi: collection.insert(item) featureCount += 1 else: if 'id' in geoJson: del geoJson['id'] collection.insert(geoJson) featureCount += 1 successCount += 1 if successCount % 250 == 0: print "Imported %d features!" % (successCount) except Exception, e: failures.append((fid, str(e))) conn.close() conn.disconnect() print "Imported %d features." % successCount print "Total (after multipolygons were split up): %d" % featureCount print "%d features could not be imported" % (len(failures)) if len(failures) > 0: val = raw_input("Would you like to see the error details (Y/N)?") if val.upper() == "Y": for fid, error in failures: print "-" * 60 print "Feature ID: %d" % fid print error[:400]
# } # ]) # print("No. variants per chromosome:") # for c in count_per_study['result']: # print("{0} - {1} variants".format(c['_id']['chromosome'], c['num variants'])) indexes = collection.index_information() print("No. indexes: {0}".format(len(indexes))) print("Indexes: {0}".format(indexes.keys())) # # Program entry point # if __name__ == "__main__": # Parse CLI arguments cli_parser = build_cli_arguments() args = cli_parser.parse_args() # Create DB connection mongo = MongoClient(args.host, args.port) db = mongo[args.database] db.authenticate(args.user, args.password) print("Checking information in database '" + args.database + "'...\n") check_files(db[args.collection_files]) print("") check_variants(db[args.collection_variants]) mongo.disconnect()