def open(self):
try:
if self.replica_set:
self.conn = pymongo.ReplicaSetConnection(self.mongodb_uri, replicaSet=self.replica_set, fsync=self.mongodb_fsync)
else:
# Old versions of pymongo do not known about fsync
if ReplicaSetConnection:
self.conn = pymongo.Connection(self.mongodb_uri, fsync=self.mongodb_fsync)
else:
self.conn = pymongo.Connection(self.mongodb_uri)
self.db = self.conn[self.database]
self.db[self.collection].ensure_index([('host_name', pymongo.ASCENDING), ('time', pymongo.ASCENDING), ('lineno', pymongo.ASCENDING)], name='logs_idx')
self.db[self.collection].ensure_index([('time', pymongo.ASCENDING), ('lineno', pymongo.ASCENDING)], name='time_1_lineno_1')
if self.replica_set:
pass
# This might be a future option prefer_secondary
#self.db.read_preference = ReadPreference.SECONDARY
self.is_connected = CONNECTED
self.next_log_db_rotate = time.time()
except AutoReconnect as err:
# now what, ha?
logger.error("[LogStoreMongoDB] LiveStatusLogStoreMongoDB.AutoReconnect %s" % err)
# The mongodb is hopefully available until this module is restarted
raise LiveStatusLogStoreError(err)
except Exception as err:
# If there is a replica_set, but the host is a simple standalone one
# we get a "No suitable hosts found" here.
# But other reasons are possible too.
logger.error("[LogStoreMongoDB] Could not open the database: %s" % err)
raise LiveStatusLogStoreError(err)
def main(argv):
arg_parsing(argv)
global connection
global db
print "Welcome to the HW 6.x replica Checker. My job is to make sure you started a replica set with three nodes"
# connect to the db (mongostr was set in arg_parsing)
try:
connection = pymongo.ReplicaSetConnection(mongostr,
replicaSet=rs_name,
safe=True)
db = connection[db_name]
except:
print "can't connect to MongoDB replica", rs_name, " set using", mongostr, ". Is it running?"
exit(2) # no graceful exit if it is not connected
if (not repl_set_running(3)):
print "Sorry, the replica set does not seem to be running"
graceful_exit(1)
# if you are reading this in cleartext, you are violating the honor code.
# You can still redeem yourself. Get it working and don't submit the validation code until you do.
# All a man has at the end of the day is his word.
print "Tests Passed for HW 6.5. Your HW 6.5 validation code is kjvjkl3290mf0m20f2kjjv"
graceful_exit(0)
def open(self):
try:
if self.replica_set:
self.conn = pymongo.ReplicaSetConnection(
self.mongodb_uri,
replicaSet=self.replica_set,
fsync=self.mongodb_fsync)
else:
# Old versions of pymongo do not known about fsync
if ReplicaSetConnection:
self.conn = pymongo.Connection(self.mongodb_uri,
fsync=self.mongodb_fsync)
else:
self.conn = pymongo.Connection(self.mongodb_uri)
self.db = self.conn[self.database]
self.db[self.collection].ensure_index(
[('host_name', pymongo.ASCENDING), ('time', pymongo.ASCENDING),
('lineno', pymongo.ASCENDING)],
name='logs_idx')
if self.replica_set:
pass
# This might be a future option prefer_secondary
#self.db.read_preference = ReadPreference.SECONDARY
self.is_connected = CONNECTED
self.next_log_db_rotate = time.time()
except AutoReconnect, exp:
# now what, ha?
logger.error(
"[LogStoreMongoDB] LiveStatusLogStoreMongoDB.AutoReconnect %s"
% (exp))
# The mongodb is hopefully available until this module is restarted
raise LiveStatusLogStoreError
def __init__(self): # # TODO :: Add Error trapping / retries! # mongo = pymongo.ReplicaSetConnection( "data2:27017,data1:27017", replicaSet="linux",slaveOk=True, safe=True) self.db = mongo.fc_gluster
def database(
config,
read_preference=None,
):
conn = _connection(config)
host = conn['host']
replica_set = conn.get('replica-set')
db = conn['database']
colls = conn['collections']
if replica_set:
conn = pymongo.ReplicaSetConnection(
host,
replicaSet=replica_set,
)
# ReadPreference.PRIMARY is the default
if read_preference is not None:
conn.read_preference = read_preference
else:
conn = pymongo.Connection(host)
db = ProxyDatabase(conn, db, colls)
return db
'''
Extract the column number that needs updating
'''
for index, c in enumerate(header):
if columnName == c:
columnNumber = index
print "Column that needs updating is: " + str(columnNumber)
'''
Querying mongo db, for now development
'''
con = pymongo.ReplicaSetConnection(host='stsia0808.cluster.net', replicaSet = 'rs0')
#constr = "con.development.test"
constr ="con.preannotation."+str(CUR_CHROM)
a = eval(constr)
PreAnnotationDB = a
line = invar.readline()
while line:
# print line
"""
If this is the first line skip
"""
lin = line.strip().split("\t")
if (lin[1]=="Chromosome"):
line = invar.readline()
def getMongoInfos():
for uri in config.mongoMongoUri:
uristr = str(uri).strip()
if len(uristr) <= 1:
continue
try:
conn = pymongo.ReplicaSetConnection(uristr, replicaSet='SwallowCap')
#conn.read_preference = pymongo.read_preferences.ReadPreference.SECONDARY_PREFERRED
conn.read_preference= pymongo.ReadPreference.SECONDARY
except:
logger.error('[MongoInfo] Connect to ' + uristr + ' in replicaSet = SwallowCap failed. Try Normal connect.')
# print ('[MongoInfo] Connect to ' + uristr + ' in replicaSet = SwallowCap failed. Try Nomal connect.')
try:
conn = pymongo.Connection(uri)
except:
logger.error('[MongoInfo] Connect to ' + uristr + ' failed.')
# print ('[MongoInfo] Connect to ' + uristr + ' failed.')
continue
names = conn.database_names()
MPMap = dict()
for name in names:
if str(name).startswith('msg#'):
domains = str(name).split('#')
topic = domains[1]
info = next(conn[name]['c'].find().sort('_id', -1).limit(1), None)
# 新增代码，如果已经记录过该topic上次最新消息，则计算产生了多少新消息，否则将该topic数量作为产生了多少新消息
if(config.logMongoTopic.__contains__(topic.lower())):
lastPeroidCount = conn[name]['c'].find({'_id' : {'$gt' : config.logMongoTopic[topic.lower()]}}).count()
else:
lastPeroidCount = conn[name]['c'].count()
mongoStatInfo = MongoStatInfo()
mongoStatInfo.topicName = topic
mongoStatInfo.lastPeriodAmount = lastPeroidCount
mongoStatInfos.add(mongoStatInfo)
# 新增结束
if info == None:
MPMap[topic] = None
config.logMongoTopic[topic.lower()] = Timestamp(int(time.time()), 1)
else:
MPMap[topic] = info['_id']
config.logMongoTopic[topic.lower()] = info['_id']
for name in names:
if str(name).startswith('ack#'):
domains = str(name).split('#')
topic = domains[1]
cid = domains[2]
info = next(conn[name]['c'].find().sort('_id', -1).limit(1), None)
if info == None:
pass
else:
mi = MongoInfo()
mi.topic = topic
mi.consumerId = cid
mid = info['_id']
strmid = str(time.ctime(mid.time)) + ' NO.' + str(mid.inc)
mi.consumedMax = strmid
if MPMap.has_key(topic):
mid = MPMap[topic]
if mid != None:
strmid = str(time.ctime(mid.time)) + ' NO.' + str(mid.inc)
mi.producedMax = strmid
mi.delayTime = int(mid.time) - int(info['_id'].time)
else:
mi.producedMax = None
mi.delayTime = 0
else:
mi.producedMax = None
mi.delayTime = 0
mongoInfos.add(mi)
# 新增代码，用以统计mongo中consumer消费记录
lastPeroidAmount = 0
mongoConsumedStatInfo = MongoConsumedStatInfo()
if config.logMongoConsumeStatus.__contains__(topic.lower()):
if config.logMongoConsumeStatus[topic.lower()].__contains__(cid.lower()):
lastPeroidMsgId = config.logMongoConsumeStatus[topic.lower()][cid.lower()]
if names.__contains__('msg#' + topic):
lastPeroidAmount = conn['msg#' + topic]['c'].find({'_id' : {'$gt' : lastPeroidMsgId, '$lte' : info['_id']}}).count()
else:
if names.__contains__('msg#' + topic):
lastPeroidAmount = conn['msg#' + topic]['c'].find({'_id' : {'$gt' : Timestamp(int(info['_id'].time) - 360, 0), '$lte': Timestamp(int(info['_id'].time), info['_id'].inc)}}).count()
else:
if names.__contains__('msg#' + topic):
lastPeroidAmount = conn['msg#' + topic]['c'].find({'_id' : {'$gt' : Timestamp(int(info['_id'].time) - 360, 0), '$lte': Timestamp(int(info['_id'].time), info['_id'].inc)}}).count()
mongoConsumedStatInfo.consumerId = cid
mongoConsumedStatInfo.topicName = topic
mongoConsumedStatInfo.lastPeriodAmount = lastPeroidAmount
mongoConsumeStatInfos.add(mongoConsumedStatInfo)
if not dict(config.logMongoConsumeStatus).__contains__(topic.lower()):
config.logMongoConsumeStatus[topic.lower()] = dict()
config.logMongoConsumeStatus[topic.lower()][cid.lower()] = info['_id']
# 新增结束
for name in names:
if str(name).startswith('heartbeat#'):
ip = str(str(name)[10:]).strip().replace('_', '.')
if config.consumerServersIp.__contains__(ip):
latestMsgId = next(conn[name]['c'].find().sort('_id', -1).limit(1), None)['t']
lastHeartbeatTime = int(time.mktime(latestMsgId.timetuple()))
now = int(time.time())
if now - lastHeartbeatTime > 60 * 60 * 8 + 60:
consumerServerStat[ip] = False
else:
consumerServerStat[ip] = True
MPMap.clear()
conn.close()
__author__ = 'emilio.torrens'
import pymongo
connection = pymongo.ReplicaSetConnection('192.168.1.218:27017', replicaSet='devSet')
instances = connection.instances
print instances
except:
temp.append(line[4:])
all_vars.append(temp)
line = invar.readline()
invar.close()
var_it = 0
tracker = 0
content_it = 0
var_res = []
var_annot = []
content = []
#conn = pymongo.Connection(host='10.128.128.60')#host='10.128.143.255')
#conn = pymongo.Connection(host='stsia0808.cluster.net')
conn = pymongo.ReplicaSetConnection('hadoop00-adm.cluster.net',replicaSet='rs0')
a =eval("conn.development."+str(CUR_CHROM))
PreAnnotationDB = a
counter_found = 0
while var_it < len(all_vars):
hap = None
chrom = None #all_vars[var_it][0]
begpos = None #int(all_vars[var_it][1])
endpos = None #int(all_vars[var_it][2])
vartype = None #all_vars[var_it][4]
ref = None #all_vars[var_it][5]
obs = None #all_vars[var_it][6]
