def callback(self, ch, method, properties, body):
#进行压测
data = json.loads(body)
code = data.get('code')
host = data.get('host')
user = data.get('user')
qps = data.get('qps')
number = data.get('number')
with open("locustfile.py", "w") as file:
file.write(code)
cmd = "locust -f locustfile.py --host={0} --csv=testfan --no-web -c{1} -r{2} -t{3}".format(
host, user, qps, number)
print cmd
status = command(cmd)
print status
file = open("D:/workspace_py/testfan_623/testfan_requests.csv")
reader = csv.reader(file)
result = list(reader)
file.close()
data.setdefault('result', result)
mongo = Mongo()
mongo.insert('testfan', 'performance', data)
ch.basic_ack(delivery_tag=method.delivery_tag)
def toText(path='path to Offres Directorie'):
text = ''
#set up database
mydb = Mongo('TTProject', ['offres-info', 'offres-text'])
#move to directory that containes offres
os.chdir(path)
dirnames = open('names.txt').read().split(" ")
references = open('references.txt').read().split(" ")
i = 0
for directory in dirnames:
if len(directory):
x = len(directory)
dirname = str(directory)[1:x - 1]
#informations to store
reference = references[i]
name = dirname
text += '############################ ' + \
dirname+' ############################\n\n\n'
#loop trought Directory :
dirPath = path + '\\' + dirname
for filename in os.listdir(os.path.join(dirPath)):
text += '\n\n#################### ' + filename + ' ####################\n\n'
filePath = path + '\\' + dirname + '\\' + filename
# check extention
# PDFs
if filename.endswith('.pdf'):
pdfFile = wi(filename=os.path.join(filePath),
resolution=300)
images = pdfFile.convert('jpeg')
for page in images:
page.save('page.jpg', 'JPEG')
image = Image.open(io.BytesIO(page))
imgPage = wi(image=image)
Image = imgPage.make_blob('jpeg')
text += pytesseract.image_to_string(
Image.open('page.jpg'), lang='fra')
# DOCXs
if filename.endswith('.docx'):
document = Document(filePath)
tables = document.tables
for p in document.paragraphs:
text += p.text
text += "\n ###### Tables's Content ###### \n"
for table in tables:
for row in table.rows:
for cell in row.cells:
for paragraph in cell.paragraphs:
text += (paragraph.text)
#inserting to mongo
offre = {'_id': reference, "name": name, 'text': text}
mydb.insert(offre, 'offres-text')
text = ''
i += 1
def record(self, data):
"""
:param data: 记录到数据库里机器的性能数据。
:param host: 数据库地址
:param port: 数据库端口
:return:
"""
print data
mongo = Mongo()
mongo.insert("testfan", "monitor", data)
mongo.close()
def index():
m = Mongo()
if request.method == 'POST':
if request.form['xliuyan'] != '':
xinliuyan = {
"content": request.form['xliuyan'],
"time": time.strftime('%Y-%m-%d %H:%M:%S')
}
m.insert(xinliuyan)
return render_template('index.html', newmessage=m.show())
return render_template('index.html', newmessage=m.show())
return render_template('index.html', newmessage=m.show())
def create_objects(collection_name, objects, limit=None, offset=None, *args, **kwargs):
try:
response = createResponse200()
mongo = Mongo()
mongo.connect()
database = mongo.use_db(database_name)
if type(objects) is not list:
objects = [objects]
# Checking if any of the object already exist by _id
objects_id = [ object.get('_id', None) for object in objects ]
query = {'_id' : {'$in' : objects_id }}
existing = mongo.find(collection_name, query)
existing_ids = [ e.get('_id') for e in existing ]
if existing:
raise Exception("Some objects already exist in '%s' (same _id). Trying using UPDATE method. No objects created." % collection_name)
# no limits or offset so far
created = mongo.insert(collection_name, objects)
response['result']['objects'] = created
response['result']['count'] = len(created)
response['result']['limit'] = limit
response['result']['offset'] = offset
except Exception, e:
response = createResponse401(str(e))
def insert():
data = request.get_json()
client = Mongo()
collection = data.pop("collection", "user")
result = client.insert("testfan", collection, data)
if result is None:
return jsonify({'status': 400, 'message': '插入失败', 'data': data})
else:
return jsonify({'status': 200, 'message': '插入成功', 'data': str(result)})
def FindInsert(self):
"""Find and Insert record in MongoDB and check"""
db = Mongo(0)
db_insert = db.insert({
'session': 'FindInsertTest',
'data': 'FindInsertTest Test data'
})
db_findinsert = db.find_insert({
'session': 'FindInsertTest',
'data': 'FindInsertTest Test data'
})
return self.assertEqual('FindInsertTest', db_findinsert['data'])
def MongoUpdate(self):
"""Update record in MongoDB and check"""
db = Mongo(0)
db_insert = db.insert({
'session': 'UpdateTest',
'data': 'Update Test data'
})
db_update = db.update({
'session': 'UpdateTest',
'data': 'Test data Updated'
})
db_fetch = db.fetch({'session': 'UpdateTest', 'data': ''})
return self.assertEqual('Test data Updated', db_fetch['data'])
def create_objects(collection_name,
objects,
limit=None,
offset=None,
*args,
**kwargs):
try:
response = createResponse200()
mongo = Mongo()
mongo.connect()
database = mongo.use_db(database_name)
if type(objects) is not list:
objects = [objects]
# Checking if any of the object already exist by _id
objects_id = [object.get('_id', None) for object in objects]
query = {'_id': {'$in': objects_id}}
existing = mongo.find(collection_name, query)
existing_ids = [e.get('_id') for e in existing]
if existing:
raise Exception(
"Some objects already exist in '%s' (same _id). Trying using UPDATE method. No objects created."
% collection_name)
# no limits or offset so far
created = mongo.insert(collection_name, objects)
response['result']['objects'] = created
response['result']['count'] = len(created)
response['result']['limit'] = limit
response['result']['offset'] = offset
except Exception, e:
response = createResponse401(str(e))
def MongoInsert(self):
"""Insert record in MongoDB and check"""
db = Mongo(0)
db_insert = db.insert({'session': 'MongoTest', 'data': 'Test data'})
db_fetch = db.fetch({'session': 'MongoTest', 'data': 'Test data'})
return self.assertEqual('MongoTest', db_fetch['session'])
def testMongo(self):
mongo = Mongo('config.ini')
data = [{'name':'1'},{'name':'2'}]
mongo.insert(data, 'test')
'//*[@id="ctl0_CONTENU_PAGE_idEntrepriseConsultationSummary_reference"]'
)
reference = Navigateur.find_element_by_xpath(
'//*[@id="ctl0_CONTENU_PAGE_idEntrepriseConsultationSummary_reference"]'
).text
result = mydb.find('offres-info', '_id', reference)
if result.count() == 0: #if not exist insert to db & continue
#collect elements to insert
lien = Navigateur.current_url
offre = {'_id': reference, 'lien': lien}
#add reference to .txt file
f.write(str(reference + " "))
#inserting
mydb.insert(offre, 'offres-info')
#continue action
wait("//*[@id=\"ctl0_CONTENU_PAGE_linkDownloadDce\"]")
Navigateur.find_element_by_xpath(
"//*[@id=\"ctl0_CONTENU_PAGE_linkDownloadDce\"]").click()
wait(
"//*[@id=\"ctl0_CONTENU_PAGE_EntrepriseFormulaireDemande_accepterConditions\"]"
)
Navigateur.find_element_by_xpath(
"//*[@id=\"ctl0_CONTENU_PAGE_EntrepriseFormulaireDemande_accepterConditions\"]"
).click()
wait(
'//*[@id="ctl0_CONTENU_PAGE_EntrepriseFormulaireDemande_nom"]')
class Bird():
def __init__(self, name=None, family=None, continent=None, visible=False):
self.name = name
self.family = family
self.continent = continent
self.visible = visible
self.M = Mongo()
def list(self, id=None):
""" Lists all birds if id is None.
Lists one bird if id is set.
"""
if id is None:
# get data from Mongo
data = self.M.get()
if not data:
return False
# Build a list of our object-id's
response = []
for b in data:
if b['visible'] == True:
response.append(str(b['_id']))
# if we have any id's return to user
if len(response) > 0:
return response
return False
else:
# get data from Mongo
data = self.M.get(id)
if not data:
return False
# here we convert _id to id for valid JSON
data["id"] = str(data["_id"])
# and pops the _id
data.pop("_id")
return data
return False
def insert(self):
""" Creates our bird dict and ensures insertion.
"""
# create utc-date for when bird is added
self.added = datetime.utcnow().strftime("%Y-%m-%d")
# build our bird-dict
bird = {
"name": self.name,
"family": self.family,
"continents": self.continent,
"visible": self.visible,
"added": self.added
}
# insert bird
id = self.M.insert(bird)
return id
def delete(self, id):
""" Deletes the bird with supplied id from collection
"""
status = self.M.delete(id)
# if deleted documents > 0 we've deleted the bird
if status['n'] > 0:
return True
else:
return False
class TextProcess:
def __init__(self, config_path):
self.uni = universities.API()
self.reg = r".*(\s+\w+\s+University).*"
self.config = Config(config_path)
self.mongo = Mongo(config_path)
self.collection = 'data'
self.minimum_length = int(
self.config.getValueDefault('Data', 'MINLENGTH', 400))
self.max_items = int(self.config.getValue('Data', 'ITEMS'))
self.start = int(self.config.getValue('Data', 'START'))
if not self.max_items:
raise Exception('Please set max_items inside config.ini')
def readblock(self, f):
ret = []
line = f.readline()
#if encounterring empty line, skip it
while not line.strip():
line = f.readline()
while line.strip():
ret.append(line)
line = f.readline()
if not ret:
return None
return ''.join(ret)
def findcountry(self, aff):
places = GeoText(aff)
countries = []
#match country firstly
for item in places.country_mentions:
countries.append(item)
#match customized country_map
for key, val in country_map.items():
if key in aff and val not in countries:
countries.append(val)
#match universities
m = re.match(self.reg, aff)
if m is not None:
for group in m.groups():
group = group.strip()
for university in self.uni.search(name=group):
if university.country_code not in countries:
countries.append(university.country_code)
return countries
def readfile(self, path):
with open(path) as f:
n = self.start
rake = Rake()
while True:
line = self.readblock(f)
#check new record starts
if line.startswith('%d. ' % n):
data = {}
data['seqid'] = n
#get time
data['time'] = line.split('.')[2].split(';')[0].strip()
#get title
data['title'] = self.readblock(f).replace('\n',
' ').strip()
#get author
data['author'] = self.readblock(f).replace('\n',
' ').strip()
#detect affiliation
line = self.readblock(f)
if line.startswith('Author information:'):
data['affiliation'] = line.replace('\n', ' ').strip()
countries = self.findcountry(data['affiliation'])
if countries:
data['country'] = countries
else:
#could not find country info, skip it
LOGGER.info(
'could not find country info: of paper id: [%d], [%s]'
% (int(n), data['affiliation']))
n += 1
continue
line = self.readblock(f)
#abstract
while line.startswith('Comment'):
line = self.readblock(f)
#no abstract
if line.find('DOI:') >= 0 or line.find(
'PMID:') >= 0 or line.find('PMCID:') >= 0:
data['DOI'] = line.replace(
'\n', ' ').rstrip('[Indexed for MEDLINE]').strip()
#skip this record as it does not have abstract
n += 1
continue
else:
data['abstract'] = line.replace('\n', ' ').strip()
#abstract length is too short
if len(data['abstract']) < self.minimum_length:
n += 1
continue
line = self.readblock(f)
#skip all other elements until encountering DOI
while line.find('DOI:') == -1 and line.find(
'PMID:') == -1 and line.find('PMCID:') == -1:
line = self.readblock(f)
data['DOI'] = line.replace(
'\n', ' ').rstrip('[Indexed for MEDLINE]').strip()
n += 1
#here data is ready for processing
rake.extract_keywords_from_text(data['abstract'])
words = rake.get_ranked_phrases()
data['keywords'] = words[:2]
LOGGER.debug(data)
LOGGER.info('starts saving id: %d data into mongodb' %
int(n - 1))
self.mongo.insert([data], self.collection)
#record ends
if n > self.max_items:
break
class Bird():
def __init__(self, name=None, family=None, continent=None, visible=False):
self.name = name
self.family = family
self.continent = continent
self.visible = visible
self.M = Mongo()
def list(self, id=None):
""" Lists all birds if id is None.
Lists one bird if id is set.
"""
if id is None:
# get data from Mongo
data = self.M.get()
if not data:
return False
# Build a list of our object-id's
response = []
for b in data:
if b['visible'] == True:
response.append(str(b['_id']))
# if we have any id's return to user
if len(response) > 0:
return response
return False
else:
# get data from Mongo
data = self.M.get(id)
if not data:
return False
# here we convert _id to id for valid JSON
data["id"] = str(data["_id"])
# and pops the _id
data.pop("_id")
return data
return False
def insert(self):
""" Creates our bird dict and ensures insertion.
"""
# create utc-date for when bird is added
self.added = datetime.utcnow().strftime("%Y-%m-%d")
# build our bird-dict
bird = {
"name": self.name,
"family": self.family,
"continents": self.continent,
"visible": self.visible,
"added": self.added
}
# insert bird
id = self.M.insert(bird)
return id
def delete(self, id):
""" Deletes the bird with supplied id from collection
"""
status = self.M.delete(id)
# if deleted documents > 0 we've deleted the bird
if status['n'] > 0:
return True
else:
return False
