def callback(self, ch, method, properties, body): #进行压测 data = json.loads(body) code = data.get('code') host = data.get('host') user = data.get('user') qps = data.get('qps') number = data.get('number') with open("locustfile.py", "w") as file: file.write(code) cmd = "locust -f locustfile.py --host={0} --csv=testfan --no-web -c{1} -r{2} -t{3}".format( host, user, qps, number) print cmd status = command(cmd) print status file = open("D:/workspace_py/testfan_623/testfan_requests.csv") reader = csv.reader(file) result = list(reader) file.close() data.setdefault('result', result) mongo = Mongo() mongo.insert('testfan', 'performance', data) ch.basic_ack(delivery_tag=method.delivery_tag)
def toText(path='path to Offres Directorie'): text = '' #set up database mydb = Mongo('TTProject', ['offres-info', 'offres-text']) #move to directory that containes offres os.chdir(path) dirnames = open('names.txt').read().split(" ") references = open('references.txt').read().split(" ") i = 0 for directory in dirnames: if len(directory): x = len(directory) dirname = str(directory)[1:x - 1] #informations to store reference = references[i] name = dirname text += '############################ ' + \ dirname+' ############################\n\n\n' #loop trought Directory : dirPath = path + '\\' + dirname for filename in os.listdir(os.path.join(dirPath)): text += '\n\n#################### ' + filename + ' ####################\n\n' filePath = path + '\\' + dirname + '\\' + filename # check extention # PDFs if filename.endswith('.pdf'): pdfFile = wi(filename=os.path.join(filePath), resolution=300) images = pdfFile.convert('jpeg') for page in images: page.save('page.jpg', 'JPEG') image = Image.open(io.BytesIO(page)) imgPage = wi(image=image) Image = imgPage.make_blob('jpeg') text += pytesseract.image_to_string( Image.open('page.jpg'), lang='fra') # DOCXs if filename.endswith('.docx'): document = Document(filePath) tables = document.tables for p in document.paragraphs: text += p.text text += "\n ###### Tables's Content ###### \n" for table in tables: for row in table.rows: for cell in row.cells: for paragraph in cell.paragraphs: text += (paragraph.text) #inserting to mongo offre = {'_id': reference, "name": name, 'text': text} mydb.insert(offre, 'offres-text') text = '' i += 1
def record(self, data): """ :param data: 记录到数据库里机器的性能数据。 :param host: 数据库地址 :param port: 数据库端口 :return: """ print data mongo = Mongo() mongo.insert("testfan", "monitor", data) mongo.close()
def index(): m = Mongo() if request.method == 'POST': if request.form['xliuyan'] != '': xinliuyan = { "content": request.form['xliuyan'], "time": time.strftime('%Y-%m-%d %H:%M:%S') } m.insert(xinliuyan) return render_template('index.html', newmessage=m.show()) return render_template('index.html', newmessage=m.show()) return render_template('index.html', newmessage=m.show())
def create_objects(collection_name, objects, limit=None, offset=None, *args, **kwargs): try: response = createResponse200() mongo = Mongo() mongo.connect() database = mongo.use_db(database_name) if type(objects) is not list: objects = [objects] # Checking if any of the object already exist by _id objects_id = [ object.get('_id', None) for object in objects ] query = {'_id' : {'$in' : objects_id }} existing = mongo.find(collection_name, query) existing_ids = [ e.get('_id') for e in existing ] if existing: raise Exception("Some objects already exist in '%s' (same _id). Trying using UPDATE method. No objects created." % collection_name) # no limits or offset so far created = mongo.insert(collection_name, objects) response['result']['objects'] = created response['result']['count'] = len(created) response['result']['limit'] = limit response['result']['offset'] = offset except Exception, e: response = createResponse401(str(e))
def insert(): data = request.get_json() client = Mongo() collection = data.pop("collection", "user") result = client.insert("testfan", collection, data) if result is None: return jsonify({'status': 400, 'message': '插入失败', 'data': data}) else: return jsonify({'status': 200, 'message': '插入成功', 'data': str(result)})
def FindInsert(self): """Find and Insert record in MongoDB and check""" db = Mongo(0) db_insert = db.insert({ 'session': 'FindInsertTest', 'data': 'FindInsertTest Test data' }) db_findinsert = db.find_insert({ 'session': 'FindInsertTest', 'data': 'FindInsertTest Test data' }) return self.assertEqual('FindInsertTest', db_findinsert['data'])
def MongoUpdate(self): """Update record in MongoDB and check""" db = Mongo(0) db_insert = db.insert({ 'session': 'UpdateTest', 'data': 'Update Test data' }) db_update = db.update({ 'session': 'UpdateTest', 'data': 'Test data Updated' }) db_fetch = db.fetch({'session': 'UpdateTest', 'data': ''}) return self.assertEqual('Test data Updated', db_fetch['data'])
def create_objects(collection_name, objects, limit=None, offset=None, *args, **kwargs): try: response = createResponse200() mongo = Mongo() mongo.connect() database = mongo.use_db(database_name) if type(objects) is not list: objects = [objects] # Checking if any of the object already exist by _id objects_id = [object.get('_id', None) for object in objects] query = {'_id': {'$in': objects_id}} existing = mongo.find(collection_name, query) existing_ids = [e.get('_id') for e in existing] if existing: raise Exception( "Some objects already exist in '%s' (same _id). Trying using UPDATE method. No objects created." % collection_name) # no limits or offset so far created = mongo.insert(collection_name, objects) response['result']['objects'] = created response['result']['count'] = len(created) response['result']['limit'] = limit response['result']['offset'] = offset except Exception, e: response = createResponse401(str(e))
def MongoInsert(self): """Insert record in MongoDB and check""" db = Mongo(0) db_insert = db.insert({'session': 'MongoTest', 'data': 'Test data'}) db_fetch = db.fetch({'session': 'MongoTest', 'data': 'Test data'}) return self.assertEqual('MongoTest', db_fetch['session'])
def testMongo(self): mongo = Mongo('config.ini') data = [{'name':'1'},{'name':'2'}] mongo.insert(data, 'test')
'//*[@id="ctl0_CONTENU_PAGE_idEntrepriseConsultationSummary_reference"]' ) reference = Navigateur.find_element_by_xpath( '//*[@id="ctl0_CONTENU_PAGE_idEntrepriseConsultationSummary_reference"]' ).text result = mydb.find('offres-info', '_id', reference) if result.count() == 0: #if not exist insert to db & continue #collect elements to insert lien = Navigateur.current_url offre = {'_id': reference, 'lien': lien} #add reference to .txt file f.write(str(reference + " ")) #inserting mydb.insert(offre, 'offres-info') #continue action wait("//*[@id=\"ctl0_CONTENU_PAGE_linkDownloadDce\"]") Navigateur.find_element_by_xpath( "//*[@id=\"ctl0_CONTENU_PAGE_linkDownloadDce\"]").click() wait( "//*[@id=\"ctl0_CONTENU_PAGE_EntrepriseFormulaireDemande_accepterConditions\"]" ) Navigateur.find_element_by_xpath( "//*[@id=\"ctl0_CONTENU_PAGE_EntrepriseFormulaireDemande_accepterConditions\"]" ).click() wait( '//*[@id="ctl0_CONTENU_PAGE_EntrepriseFormulaireDemande_nom"]')
class Bird(): def __init__(self, name=None, family=None, continent=None, visible=False): self.name = name self.family = family self.continent = continent self.visible = visible self.M = Mongo() def list(self, id=None): """ Lists all birds if id is None. Lists one bird if id is set. """ if id is None: # get data from Mongo data = self.M.get() if not data: return False # Build a list of our object-id's response = [] for b in data: if b['visible'] == True: response.append(str(b['_id'])) # if we have any id's return to user if len(response) > 0: return response return False else: # get data from Mongo data = self.M.get(id) if not data: return False # here we convert _id to id for valid JSON data["id"] = str(data["_id"]) # and pops the _id data.pop("_id") return data return False def insert(self): """ Creates our bird dict and ensures insertion. """ # create utc-date for when bird is added self.added = datetime.utcnow().strftime("%Y-%m-%d") # build our bird-dict bird = { "name": self.name, "family": self.family, "continents": self.continent, "visible": self.visible, "added": self.added } # insert bird id = self.M.insert(bird) return id def delete(self, id): """ Deletes the bird with supplied id from collection """ status = self.M.delete(id) # if deleted documents > 0 we've deleted the bird if status['n'] > 0: return True else: return False
class TextProcess: def __init__(self, config_path): self.uni = universities.API() self.reg = r".*(\s+\w+\s+University).*" self.config = Config(config_path) self.mongo = Mongo(config_path) self.collection = 'data' self.minimum_length = int( self.config.getValueDefault('Data', 'MINLENGTH', 400)) self.max_items = int(self.config.getValue('Data', 'ITEMS')) self.start = int(self.config.getValue('Data', 'START')) if not self.max_items: raise Exception('Please set max_items inside config.ini') def readblock(self, f): ret = [] line = f.readline() #if encounterring empty line, skip it while not line.strip(): line = f.readline() while line.strip(): ret.append(line) line = f.readline() if not ret: return None return ''.join(ret) def findcountry(self, aff): places = GeoText(aff) countries = [] #match country firstly for item in places.country_mentions: countries.append(item) #match customized country_map for key, val in country_map.items(): if key in aff and val not in countries: countries.append(val) #match universities m = re.match(self.reg, aff) if m is not None: for group in m.groups(): group = group.strip() for university in self.uni.search(name=group): if university.country_code not in countries: countries.append(university.country_code) return countries def readfile(self, path): with open(path) as f: n = self.start rake = Rake() while True: line = self.readblock(f) #check new record starts if line.startswith('%d. ' % n): data = {} data['seqid'] = n #get time data['time'] = line.split('.')[2].split(';')[0].strip() #get title data['title'] = self.readblock(f).replace('\n', ' ').strip() #get author data['author'] = self.readblock(f).replace('\n', ' ').strip() #detect affiliation line = self.readblock(f) if line.startswith('Author information:'): data['affiliation'] = line.replace('\n', ' ').strip() countries = self.findcountry(data['affiliation']) if countries: data['country'] = countries else: #could not find country info, skip it LOGGER.info( 'could not find country info: of paper id: [%d], [%s]' % (int(n), data['affiliation'])) n += 1 continue line = self.readblock(f) #abstract while line.startswith('Comment'): line = self.readblock(f) #no abstract if line.find('DOI:') >= 0 or line.find( 'PMID:') >= 0 or line.find('PMCID:') >= 0: data['DOI'] = line.replace( '\n', ' ').rstrip('[Indexed for MEDLINE]').strip() #skip this record as it does not have abstract n += 1 continue else: data['abstract'] = line.replace('\n', ' ').strip() #abstract length is too short if len(data['abstract']) < self.minimum_length: n += 1 continue line = self.readblock(f) #skip all other elements until encountering DOI while line.find('DOI:') == -1 and line.find( 'PMID:') == -1 and line.find('PMCID:') == -1: line = self.readblock(f) data['DOI'] = line.replace( '\n', ' ').rstrip('[Indexed for MEDLINE]').strip() n += 1 #here data is ready for processing rake.extract_keywords_from_text(data['abstract']) words = rake.get_ranked_phrases() data['keywords'] = words[:2] LOGGER.debug(data) LOGGER.info('starts saving id: %d data into mongodb' % int(n - 1)) self.mongo.insert([data], self.collection) #record ends if n > self.max_items: break
class Bird(): def __init__(self, name=None, family=None, continent=None, visible=False): self.name = name self.family = family self.continent = continent self.visible = visible self.M = Mongo() def list(self, id=None): """ Lists all birds if id is None. Lists one bird if id is set. """ if id is None: # get data from Mongo data = self.M.get() if not data: return False # Build a list of our object-id's response = [] for b in data: if b['visible'] == True: response.append(str(b['_id'])) # if we have any id's return to user if len(response) > 0: return response return False else: # get data from Mongo data = self.M.get(id) if not data: return False # here we convert _id to id for valid JSON data["id"] = str(data["_id"]) # and pops the _id data.pop("_id") return data return False def insert(self): """ Creates our bird dict and ensures insertion. """ # create utc-date for when bird is added self.added = datetime.utcnow().strftime("%Y-%m-%d") # build our bird-dict bird = { "name": self.name, "family": self.family, "continents": self.continent, "visible": self.visible, "added": self.added } # insert bird id = self.M.insert(bird) return id def delete(self, id): """ Deletes the bird with supplied id from collection """ status = self.M.delete(id) # if deleted documents > 0 we've deleted the bird if status['n'] > 0: return True else: return False