def __init__(
self,
solrHost=None,
config=None,
addparams="",
):
self.config = AppConfig(config)
if solrHost is None:
self.solrURL = self.config.getConfig()["SOLR"]["host"]
else:
self.solrURL = solrHost
self.additionalparams = addparams
self.startTime = time.strftime('%H:%M:%S')
self.totalQueryTime = 0
self.totalNumberRequests = 0
self.timeCollection = {}
self.initializeCollecCurrentQueryTime()
cTimeUTC = datetime.utcnow()
nTList = [
str(cTimeUTC.date()), "T",
str(cTimeUTC.hour),
str(cTimeUTC.minute),
str(cTimeUTC.second), "Z"
]
self.currentTime = "".join(nTList)
def __init__(self, config: AppConfig):
self.config = config
self.parentPath = config.getConfig()["ParentPath"]
#self.solrURL = config.getConfig()["SolrURL"]
self.regex = re.compile(config.getConfig()["idregex"],
re.UNICODE | re.DOTALL | re.IGNORECASE)
self.solrWrapper = SolrWrapper(config)
self.outFile = open(config.getConfig()["outFile"], "a")
self.outline = Template("idnotavailable: $id / file: $file $linebreak")
def __init__(self,
appConfig: AppConfig = None,
userArg=None,
passwordArg=None):
self.config = appConfig.getConfig()
if not self.config["MONGO"]["HOST"]["user"] is None and not self.config[
"MONGO"]["HOST"]["password"] is None:
uri = 'mongodb://{USER}:{PASSWORD}@{SERVER}:{PORT}/{DB}'.format(
USER=self.config['HOST']['user'],
PASSWORD=self.config['HOST']['password'],
SERVER=self.config['HOST']['server'],
PORT=self.config['HOST']['port'],
DB=self.config['HOST']['authDB'])
elif not userArg is None and not passwordArg is None:
uri = 'mongodb://{USER}:{PASSWORD}@{SERVER}:{PORT}/{DB}'.format(
USER=userArg,
PASSWORD=passwordArg,
SERVER=self.config["MONGO"]['HOST']['server'],
PORT=self.config["MONGO"]['HOST']['port'],
DB=self.config["MONGO"]['HOST']['authDB'])
else:
uri = 'mongodb://{SERVER}:{PORT}'.format(
SERVER=self.config["MONGO"]['HOST']['server'],
PORT=self.config["MONGO"]['HOST']['port'],
)
self.client = MongoClient(uri)
self.database = self.client[self.config["MONGO"]['DB']['dbname']]
self.queryCollection = self.database[self.config["MONGO"]['DB']
['querycollection']]
self.responseCollection = self.database[self.config["MONGO"]['DB']
['responsecollection']]
def __init__(self, appConfig : AppConfig = None):
self.config = appConfig.getConfig()
if not self.config['HOST']['user'] is None:
uri = 'mongodb://{USER}:{PASSWORD}@{SERVER}:{PORT}/{DB}'.format(
USER=self.config['HOST']['user'],
PASSWORD=self.config['HOST']['password'],
SERVER=self.config['HOST']['server'],
PORT=self.config['HOST']['port'],
DB=self.config['HOST']['authDB']
)
else:
uri = 'mongodb://{SERVER}:{PORT}'.format(
SERVER=self.config['HOST']['server'],
PORT=self.config['HOST']['port'],
)
self.client = MongoClient( uri)
self.database = self.client[self.config['DB']['dbname']]
self.collection = self.database[self.config['DB']['collection']['name']]
def __init__(self, appConfig : AppConfig = None):
self.config = appConfig.getConfig()
if not self.config['HOST']['user'] is None:
uri = 'mongodb://{USER}:{PASSWORD}@{SERVER}:{PORT}/{DB}'.format(
USER=self.config['HOST']['user'],
PASSWORD=self.config['HOST']['password'],
SERVER=self.config['HOST']['server'],
PORT=self.config['HOST']['port'],
DB=self.config['HOST']['authDB']
)
else:
uri = 'mongodb://{SERVER}:{PORT}'.format(
SERVER=self.config['HOST']['server'],
PORT=self.config['HOST']['port'],
)
self.client = MongoClient( uri)
self.database = self.client[self.config['DB']['dbname']]
self.collection = self.database[self.config['DB']['collection']['name']]
id = pId.group(1)
#self.printId(id,subobject)
if not self.solrWrapper.checkId(id):
self.printId(id, subobject)
#print(strLine)
except Exception as ex:
print(ex)
def printId(self, currentId, filename):
self.outFile.write(
self.outline.substitute({
"id": currentId,
"file": filename,
"linebreak": linesep
}))
self.outFile.flush()
def closeResources(self):
if not self.outFile is None:
self.outFile.close()
if __name__ == '__main__':
config = AppConfig("config/files/idchecker/checker.yaml")
checker = IDChecker(config)
checker.checkIDs()
checker.closeResources()
print("work done")
def __init__(self, config: AppConfig):
self.sorlURL = config.getConfig()["SolrURL"]
self.template = Template(self.sorlURL)
from mq.topic import TopicConsumer
class TestExchange(object):
def __init__(self, exchangeName, config):
print exchangeName
print appConfig['rabbit']
self.topicConsumer = TopicConsumer(
config=config,
exchangeName=exchangeName,
bindingKey='admin.developmentParser.parser.acquilite',
consumeFunc=self.consumeMessages,
noArgs=True)
self.topicConsumer.startConsuming()
def consumeMessages(self, ch, method, properties, body):
print method.routing_key
print body
if __name__ == '__main__':
exchangeName = 'sis-admin-exchange'
from config.appConfig import AppConfig
config = AppConfig('developmentParser')
config.loadAll()
appControl = TestExchange(exchangeName, config)
class RunQueries():
def __init__(
self,
solrHost=None,
config=None,
addparams="",
):
self.config = AppConfig(config)
if solrHost is None:
self.solrURL = self.config.getConfig()["SOLR"]["host"]
else:
self.solrURL = solrHost
self.additionalparams = addparams
self.startTime = time.strftime('%H:%M:%S')
self.totalQueryTime = 0
self.totalNumberRequests = 0
self.timeCollection = {}
self.initializeCollecCurrentQueryTime()
cTimeUTC = datetime.utcnow()
nTList = [
str(cTimeUTC.date()), "T",
str(cTimeUTC.hour),
str(cTimeUTC.minute),
str(cTimeUTC.second), "Z"
]
self.currentTime = "".join(nTList)
def getURL(self):
return self.solrURL
def getAverTime(self):
return self.totalQueryTime / self.totalNumberRequests
def getStartTime(self):
return self.startTime
def collecCurrentQueryTime(self, queryTime):
if queryTime >= 0 and queryTime <= 100:
self.timeCollection['0_100'] += 1
elif queryTime >= 101 and queryTime <= 200:
self.timeCollection['100_200'] += 1
elif queryTime >= 201 and queryTime <= 300:
self.timeCollection['200_300'] += 1
elif queryTime >= 301 and queryTime <= 400:
self.timeCollection['300_400'] += 1
elif queryTime >= 401 and queryTime <= 500:
self.timeCollection['400_500'] += 1
elif queryTime >= 501 and queryTime <= 600:
self.timeCollection['500_600'] += 1
elif queryTime >= 601 and queryTime <= 700:
self.timeCollection['600_700'] += 1
elif queryTime >= 701 and queryTime <= 800:
self.timeCollection['700_800'] += 1
elif queryTime >= 801 and queryTime <= 900:
self.timeCollection['800_900'] += 1
elif queryTime >= 901 and queryTime <= 1000:
self.timeCollection['900_1000'] += 1
elif queryTime >= 1001 and queryTime <= 1200:
self.timeCollection['1000_1200'] += 1
elif queryTime >= 1201 and queryTime <= 1400:
self.timeCollection['1200_1400'] += 1
elif queryTime >= 1401 and queryTime <= 1600:
self.timeCollection['1400_1600'] += 1
elif queryTime >= 1601 and queryTime <= 1800:
self.timeCollection['1600_1800'] += 1
elif queryTime >= 1801 and queryTime <= 2000:
self.timeCollection['1800_2000'] += 1
elif queryTime >= 2001 and queryTime <= 2400:
self.timeCollection['2000_2400'] += 1
elif queryTime >= 2401 and queryTime <= 2800:
self.timeCollection['2400_2800'] += 1
elif queryTime >= 2801 and queryTime <= 3200:
self.timeCollection['2800_3200'] += 1
elif queryTime >= 3201 and queryTime <= 3800:
self.timeCollection['3200_3800'] += 1
elif queryTime >= 3801 and queryTime <= 4400:
self.timeCollection['3800_4400'] += 1
elif queryTime >= 4401 and queryTime <= 5000:
self.timeCollection['4400_5000'] += 1
elif queryTime >= 5001 and queryTime <= 6000:
self.timeCollection['5000_6000'] += 1
elif queryTime >= 6001 and queryTime <= 7000:
self.timeCollection['6000_7000'] += 1
elif queryTime >= 7001 and queryTime <= 8000:
self.timeCollection['7000_8000'] += 1
elif queryTime >= 8001 and queryTime <= 9000:
self.timeCollection['8000_9000'] += 1
elif queryTime >= 9001 and queryTime <= 10000:
self.timeCollection['9000_10000'] += 1
elif queryTime >= 10001 and queryTime <= 15000:
self.timeCollection['10000_15000'] += 1
elif queryTime >= 15001 and queryTime <= 20000:
self.timeCollection['15000_20000'] += 1
elif queryTime >= 20001:
self.timeCollection['gt20000'] += 1
def initializeCollecCurrentQueryTime(self):
self.timeCollection['0_100'] = 0
self.timeCollection['100_200'] = 0
self.timeCollection['200_300'] = 0
self.timeCollection['300_400'] = 0
self.timeCollection['400_500'] = 0
self.timeCollection['500_600'] = 0
self.timeCollection['600_700'] = 0
self.timeCollection['700_800'] = 0
self.timeCollection['800_900'] = 0
self.timeCollection['900_1000'] = 0
self.timeCollection['1000_1200'] = 0
self.timeCollection['1200_1400'] = 0
self.timeCollection['1400_1600'] = 0
self.timeCollection['1600_1800'] = 0
self.timeCollection['1800_2000'] = 0
self.timeCollection['2000_2400'] = 0
self.timeCollection['2400_2800'] = 0
self.timeCollection['2800_3200'] = 0
self.timeCollection['3200_3800'] = 0
self.timeCollection['3800_4400'] = 0
self.timeCollection['4400_5000'] = 0
self.timeCollection['5000_6000'] = 0
self.timeCollection['6000_7000'] = 0
self.timeCollection['7000_8000'] = 0
self.timeCollection['8000_9000'] = 0
self.timeCollection['9000_10000'] = 0
self.timeCollection['10000_15000'] = 0
self.timeCollection['15000_20000'] = 0
self.timeCollection['gt20000'] = 0
def getDistribution(self):
distribution = []
for k, v in self.timeCollection.items():
distribution.append(" (" + k + "=>" + str(v) + ") ")
return ''.join(distribution)
def executeRequest(self, query):
if not self.additionalparams == "":
query += "&" + self.additionalparams
result = requests.get(self.solrURL, params=query.encode("utf-8"))
io = StringIO(result.text)
result = json.load(io)
queryTime = result["responseHeader"]["QTime"]
self.totalQueryTime += queryTime
self.totalNumberRequests += 1
self.collecCurrentQueryTime(queryTime)
return result