def read(self, fn): """ Read a tabbed format line Each line consists of: sent, prob, pred, arg1, arg2, ... """ d = {} ex_index = 0 with open(fn) as fin: for line in fin: if not line.strip(): continue data = line.strip().split('\t') try: text, confidence, rel = data[:3] except: continue curExtraction = Extraction( pred=rel, head_pred_index=None, sent=text, confidence=float(confidence), question_dist= "./question_distributions/dist_wh_sbj_obj1.json", index=ex_index) ex_index += 1 for arg in data[3:]: curExtraction.addArg(arg) d[text] = d.get(text, []) + [curExtraction] self.oie = d
def __init__(self, nodeName="slam_ekf"): super(SLAM_Localization, self).__init__(nodeName) self.icp = SubICP() self.extraction = Extraction() self.isFirstScan = True self.laser_count = 0 # interval self.laser_interval = 5 # State Vector [x y yaw].T, column vector. # self.xOdom = np.zeros((STATE_SIZE,1)) self.xEst = np.zeros((STATE_SIZE, 1)) # Covariance. Initial is very certain. self.PEst = np.zeros((STATE_SIZE, STATE_SIZE)) # landMark Estimation. Like former self.tar_pc self.lEst = np.zeros((LM_SIZE, 0)) # lEst should be of 2*N size # ros topic self.laser_sub = rospy.Subscriber('/course_agv/laser/scan', LaserScan, self.laserCallback) # self.location_pub = rospy.Publisher('ekf_location',Odometry,queue_size=3) ## localization parameters # minimum landmark matches to update. self.min_match = int(rospy.get_param('/slam/min_match', 2)) # minimum number of points for a landmark cluster self.extraction.landMark_min_pt = int( rospy.get_param('/slam/landMark_min_pt', 1)) # maximum radius to be identified as landmark self.extraction.radius_max_th = float( rospy.get_param('/slam/radius_max_th', 0.4)) OBSTACLE_RADIUS = 0.35
def __init__(self): self.icp = ICP() self.ekf = EKF() self.extraction = Extraction() # odom robot init states self.robot_x = rospy.get_param('/icp/robot_x',0) self.robot_y = rospy.get_param('/icp/robot_y',0) self.robot_theta = rospy.get_param('/icp/robot_theta',0) self.sensor_sta = [self.robot_x,self.robot_y,self.robot_theta] self.isFirstScan = True self.src_pc = [] self.tar_pc = [] # State Vector [x y yaw] self.xOdom = np.zeros((3,1)) self.xEst = np.zeros((3,1)) self.PEst = np.eye(3) # map observation self.obstacle = [] # radius self.obstacle_r = 10 # init map self.updateMap() # ros topic self.laser_sub = rospy.Subscriber('/course_agv/laser/scan',LaserScan,self.laserCallback) self.location_pub = rospy.Publisher('ekf_location',Odometry,queue_size=3) self.odom_pub = rospy.Publisher('icp_odom',Odometry,queue_size=3) self.odom_broadcaster = tf.TransformBroadcaster() self.landMark_pub = rospy.Publisher('/landMarks',MarkerArray,queue_size=1)
def __init__(self, cam_type='zed', realtime=False): if cam_type is 'zed': odom_file = '/export/patraval/robo_car_new_loop_all/zed_front/gps/fix.txt' self.initial_offset = 6070 self.offset_length = 6634 elif cam_type is 'pg': # odom_file = '/export/patraval/robo_car_loop2/pg_cam/gps/fix.txt' odom_file = '/export/patraval/robo_loop_pg_only/pg_cam/gps/fix.txt' self.initial_offset = 5953 self.offset_length = 6522 self.initial_offset = 1600 # pg #1417 #-- zed # self.initial_offset = 1948 self.offset_length = 6000 # self.initial_offset = 3510 # self.offset_length = 6634 print(self.initial_offset, self.offset_length) self.transformer = State_Transition(odom_file, cam_type, self.initial_offset, realtime) self.extractor = Extraction(cam_type)
def address(): question = request.args.get('question', 0, type=str) filtered_sentence = Parssing.get_main_words(datas_management, question) try: url_googleaddress = Extraction.get_urladdress(datas_extraction, filtered_sentence) googleapi_data = Extraction.get_googleapi_data(datas_extraction, url_googleaddress) address_location = Extraction.get_address(datas_extraction, googleapi_data) random_sentence = random.choice(sentences_list.sentences_address) address_sentence = random_sentence + address_location return jsonify(result=address_sentence) except (ValueError, TypeError): return jsonify(result="Mince, je ne trouve pas :)")
def map(): question = request.args.get('question', 0, type=str) filtered_sentence = Parssing.get_main_words(datas_management, question) url_googleaddress = Extraction.get_urladdress(datas_extraction, filtered_sentence) googleapi_data = Extraction.get_googleapi_data(datas_extraction, url_googleaddress) lat = Extraction.get_lat(datas_extraction, googleapi_data) lng = Extraction.get_lng(datas_extraction, googleapi_data) if lat is None or lng is None: return jsonify(result="") else: url_googlemap = Extraction.get_urlmap(datas_extraction, filtered_sentence, lat, lng) return jsonify(result=url_googlemap)
def requires(self): """ Depends on list of Extraction tasks. """ for url in self.urls: yield Extraction(url)
def read(self, fn): d = defaultdict(lambda: []) with open(fn) as fin: for line_ind, line in enumerate(fin): data = line.strip().split('\t') text, rel = data[:2] args = data[2:] confidence = 1 curExtraction = Extraction(pred=rel, head_pred_index=None, sent=text, confidence=float(confidence), index=line_ind) for arg in args: curExtraction.addArg(arg) d[text].append(curExtraction) self.oie = d
def wikilink(): question = request.args.get('question', 0, type=str) filtered_sentence = Parssing.get_main_words(datas_management, question) first_word = Parssing.wiki_firstword(datas_management, filtered_sentence) try: link_wiki = Extraction.wiki_link(datas_extraction, first_word) return jsonify(result=link_wiki) except wikipedia.exceptions.DisambiguationError: return jsonify(result="Ou faire une nouvelle recherche...") except wikipedia.exceptions.PageError: return jsonify(result="Essaie autre chose...")
def setUpClass(cls): cls.extraction_task_1 = Extraction(url=TEST_URL_1) cls.extraction_task_2 = Extraction(url=TEST_URL_2) cls.saving_task = Saving([TEST_URL_1, TEST_URL_2]) cls.saving_task.input = lambda: [ cls.extraction_task_1.output(), cls.extraction_task_2.output() ] cls.cmd("hadoop fs -rm -r /{app_name}".format( app_name=APPLICATION_NAME).split()) cls.cmd("hadoop fs -mkdir -p /{app_name}/{ex_out}".format( app_name=APPLICATION_NAME, ex_out=EXTRACTION_OUTPUT).split()) cls.cmd( "hadoop fs -put test-resources/https--en.wikipedia.org-wiki-Battle_of_Austerlitz.html /{app_name}/{ex_out}" .format(app_name=APPLICATION_NAME, ex_out=EXTRACTION_OUTPUT).split()) cls.cmd( "hadoop fs -put test-resources/https--en.wikipedia.org-wiki-Napoleon.html /{app_name}/{ex_out}" .format(app_name=APPLICATION_NAME, ex_out=EXTRACTION_OUTPUT).split())
def __init__(self): # ros param self.robot_x = rospy.get_param('/slam/robot_x',0) self.robot_y = rospy.get_param('/slam/robot_y',0) self.robot_theta = rospy.get_param('/slam/robot_theta',0) ## ros param of mapping self.map_x_width = rospy.get_param('/slam/map_width') self.map_y_width = rospy.get_param('/slam/map_height') self.map_reso = rospy.get_param('/slam/map_resolution') self.map_cellx_width = int(round(self.map_x_width/self.map_reso)) self.map_celly_width = int(round(self.map_y_width/self.map_reso)) self.icp = ICP() self.ekf = EKF() self.extraction = Extraction() self.mapping = Mapping(self.map_cellx_width,self.map_celly_width,self.map_reso) # odom robot init states self.sensor_sta = [self.robot_x,self.robot_y,self.robot_theta] self.isFirstScan = True self.src_pc = [] self.tar_pc = [] # State Vector [x y yaw] self.xOdom = np.zeros((STATE_SIZE, 1)) self.xEst = np.zeros((STATE_SIZE, 1)) self.PEst = np.eye(STATE_SIZE) # map observation self.obstacle = [] # radius self.obstacle_r = 10 # ros topic self.laser_sub = rospy.Subscriber('/course_agv/laser/scan',LaserScan,self.laserCallback) self.location_pub = rospy.Publisher('ekf_location',Odometry,queue_size=3) self.odom_pub = rospy.Publisher('icp_odom',Odometry,queue_size=3) self.odom_broadcaster = tf.TransformBroadcaster() self.landMark_pub = rospy.Publisher('/landMarks',MarkerArray,queue_size=1) self.map_pub = rospy.Publisher('/slam_map',OccupancyGrid,queue_size=1)
def non_zero_row(self, test_row, col_names, q_alias): non_zero = 0 check = False check3 = False nil = 0 flag = 0 q_col = 0 number = '' for i in col_names: try: if Extraction.capitalised(self, i[0]) in q_alias: q_col = i[1] break else: for temp_val in q_alias: if temp_val in Extraction.capitalised(self, i[0]): q_col = i[1] break except Exception as e: pass for i in range(len(test_row)): if test_row[i] == '': nil += 1 if test_row[q_col] != '': if type(test_row[q_col]) == int or type(test_row[q_col]) == float: check3 = True if check3: if len(test_row) - 2 * nil >= 0: return True else: return True else: return False
def __init__(self, nodeName="ekf_icp"): super(EKF_Landmark_Localization, self).__init__(nodeName) self.icp = SubICP() self.extraction = Extraction() self.src_pc = None self.isFirstScan = True self.laser_count = 0 # interval self.laser_interval = 5 # State Vector [x y yaw].T, column vector. self.xEst = np.zeros((STATE_SIZE, 1)) # Covariance. Initial state is certain. # self.PEst=np.eye(STATE_SIZE) self.PEst = np.zeros((STATE_SIZE, STATE_SIZE)) # init map # map observation self.tar_pc = None self.updateMap() # ros topic self.laser_sub = rospy.Subscriber('/course_agv/laser/scan', LaserScan, self.laserCallback) # self.location_pub = rospy.Publisher('ekf_location',Odometry,queue_size=3) # parameters from launch file. # minimum landmark matches to update. Actually even 1 point is acceptable. self.min_match = int(rospy.get_param('/localization/min_match', 1)) # minimum number of points for a landmark cluster self.extraction.landMark_min_pt = int( rospy.get_param('/localization/landMark_min_pt', 2)) # maximum radius to be identified as landmark self.extraction.radius_max_th = float( rospy.get_param('/localization/radius_max_th', 0.4))
def formatting_data(self, convert_nb_2_i, size_type, source_wb, dest_wb, data_col, alias_data, material_check, a105, f22, f91, f92, wcb, wcc, wc9, c12a, c12ab): sheet = source_wb.sheet_by_name('Standard Sheet') sheet_f = dest_wb.get_sheet_by_name('Standard Sheet') redFill = PatternFill(start_color='00CED1FF', end_color='00CED1FF', fill_type='solid') for data in alias_data: print "FORMATTING FOR -" + str(data) for row_s in range(sheet.nrows): try: cap = sheet.cell(row_s, data_col[data]).value except Exception as e: break if '-' not in str(cap)[:1]: cap = Extraction.capitalised(self, str(cap)) for key in alias_data[data]: if cap in alias_data[data][key]: sheet_f.cell(row=row_s + 1, column=data_col[data] + 1).value = key if data == 'VALVE MATERIAL' and material_check == 'YES' and key not in [ 'F316', 'CF3', 'CF8' ]: try: check, material = self.verify_material( convert_nb_2_i, size_type, key, sheet.cell(row_s, data_col['TEMPERATURE']).value, sheet.cell(row_s, data_col['VALVE SIZE']).value, a105, f22, f91, f92, wcb, wcc, wc9, c12a, c12ab) if check: sheet_f.cell(row=row_s + 1, column=data_col[data] + 1).value = material else: sheet_f.cell(row=row_s + 1, column=data_col[data] + 1).value = material sheet_f.cell(row=row_s + 1, column=data_col[data] + 1).fill = redFill except Exception as e: print e pass break dest_wb.save(os.getcwd() + '/' + 'Output.xls')
def wiki(): question = request.args.get('question', 0, type=str) filtered_sentence = Parssing.get_main_words(datas_management, question) first_word = Parssing.wiki_firstword(datas_management, filtered_sentence) try: sentence_wiki = Extraction.wiki_datas(datas_extraction, first_word) random_sentence = random.choice(sentences_list.sentences_wiki) result_wiki = random_sentence + sentence_wiki return jsonify(result=result_wiki) except wikipedia.exceptions.DisambiguationError: return jsonify( result= "Je peux trouver des infos mais il y a beaucoup de choses avec le même nom, peux-tu préciser s'il te plait ?" ) except wikipedia.exceptions.PageError: return jsonify(result="Hum, ça ne me dit vraiment rien...")
def format_n_filter(self, length, alias_data, sheet_row, data_column, size, i2n): temp_l = [] temp = '' for data in data_column: try: cap = sheet_row[data[1]] except Exception as e: print e continue if ('-' not in str(cap) or '/' not in str(cap)) and data[0] != 'SIZE': cap = Extraction.capitalised(self, str(cap)) for key in alias_data: for alias in alias_data[key]: if cap in alias_data[key][alias]: temp_l.append(alias) if data[0] == 'SIZE': if size == 'IN': cap += '\"' temp_l.append(int(self.i2n[cap])) else: if cap != '': try: temp_l.append(int(cap)) except Exception as e: print e pass rat = '' if data[0] == 'RATING': for val in str(cap): try: if type(int(val)) == int or type(float(val)) == float: rat += str(val) except: pass temp_l.append(int(rat)) for i in range(length): try: temp += '-' + str(temp_l[i]) except: temp += '-' temp = temp[1:] + '-' return temp
def GET(self, datafile, method): params = web.input(output="xml") """Returns some extracted information from a file""" extractor = Extraction() data = '' txtfile = TMP_FOLDER + datafile + '.txt' """Check if the file exists, if not return a 404""" if not os.path.exists(txtfile): return web.notfound() try: if method == 'text': txtfile = TMP_FOLDER + datafile + '.txt' web.header('Content-Type', 'text/text') # Set the Header return open(txtfile,"rb").read() elif method == 'file': pdffile = TMP_FOLDER + datafile typeFilterStatus = utilities.typeFilter(pdffile) web.header('Content-Type', typeFilterStatus) # Set the Header return open(pdffile,"rb").read() else: if method == 'header': data = data + extractor.extractHeaders(txtfile) elif method == 'citations': data = data + extractor.extractCitations(txtfile) elif method == 'body': data = data + extractor.extractBody(txtfile) elif method == 'keyphrases': data = data + extractor.extractKeyphrases(txtfile) #Print XML or JSON if params.output == 'xml' or params.output == '': web.header('Content-Type','text/xml; charset=utf-8') return utilities.printXML(data) elif params.output == 'json': jsondata = xmltodict.parse(data) web.header('Content-Type','text/json; charset=utf-8') return json.dumps(jsondata) else: web.ctx.status = '400' return 'Unsupported output format. Options are: "xml" (default) and "json"' except (IOError, OSError) as er: #Internal error, i.e. during extraction web.debug(er) return web.internalerror()
def read(self, fn): d = {} with open(fn) as fin: for line in fin: data = line.strip().split('\t') if len(data) >= 4: arg1 = data[3] rel = data[2] arg_else = data[4:] confidence = data[1] text = data[0] curExtraction = Extraction(pred=rel, head_pred_index=-1, sent=text, confidence=float(confidence)) curExtraction.addArg(arg1) for arg in arg_else: curExtraction.addArg(arg) d[text] = d.get(text, []) + [curExtraction] self.oie = d
# -*- coding: utf-8 -*- # Import library import numpy as np from utils import Data from extraction import Extraction from sklearn.neighbors import KNeighborsClassifier from sklearn.neural_network import MLPClassifier from sklearn.model_selection import train_test_split, cross_val_score from sklearn.metrics import accuracy_score, confusion_matrix # Import dataset #X, Y, lb = Data.numbers() X, Y, lb = Extraction.huMoments() #X, Y, lb = Extraction.lbp() #X, Y, lb = Extraction.glcm() n_iter = 10 scores_list = [] for i in range(n_iter): # k-Fold Cross-Validation #model = KNeighborsClassifier(n_neighbors=10) #cv_scores = cross_val_score(model, X, Y, cv=5) #scores = np.mean(cv_scores) #scores_list.append(scores) # Holdout x_train, x_test, y_train, y_test = train_test_split(X, Y, test_size=0.2) #model = KNeighborsClassifier(n_neighbors=10) model = MLPClassifier(hidden_layer_sizes=(64, 128, 64), max_iter=1000,
set_logger() if EXTRACTION: """ The extraction phase involves the process of obtaining a set of documents from a repository, such as Scopus or the Web of Science, or it can involve the steps of scraping a publisher’s website to retrieve full-text articles (typically in PDF format). Scopus generally provides publication abstracts, including all the meta-data (journal, authors, affiliations, publication date) through various APIs. The upside of using an API is that publication content is easily obtained for a large number of documents simultaneously, however, these APIs often do not provide full-text for all the publications or journals of interest. In these cases, scraping a publisher’s websites can be an alternative solution. This process involves building many handcrafted crawlers, as each publisher lists their publications in a different manner on their website. Download limits should always be respected when building such scripts. Another option would be to manually download articles, although such approaches might not be feasible if the document collection of interest contains thousands or tens of thousands of articles. To enable a comparison of topics by time, or a comparison of topics by journals, it is important to store this information alongside the content of the document. """ # instantiate Extraction class extraction = Extraction() # extract publications from NIPS website extraction.extract_publications() if PREPROCESSING: """ The pre-processing phase can be seen as the process of going from a document source to an interpretable representation for the topic model algorithm. This phase is typically different for full-text and abstract data. One of the main differences is that abstract data is often provided in a clean format, whereas full-text is commonly obtained by converting a PDF document into its plain text representation. Within this phase, an important part is to filter out the content that is not important from a topic model's point-of-view, rather than from a human’s point-of-view. Abstract data usually comes in a clean format of around 300--400 words, and little additional text is added to it; typically the copyright statement is the only text that should be removed. In contrast, full-text articles can contain a lot of additional text that has been added by the publisher. This is article meta-data and boilerplate. It is important that such additional text is removed, and various methods to do so exist. Examples include: deleting the first cover page; deleting the first n-bits of the content; using regular expressions or other pattern matching techniques to find and remove additional text, or more advanced methods. For full-text articles, a choice can be made to also exclude the reference
class SLAM_EKF(): def __init__(self): # ros param self.robot_x = rospy.get_param('/slam/robot_x',0) self.robot_y = rospy.get_param('/slam/robot_y',0) self.robot_theta = rospy.get_param('/slam/robot_theta',0) ## ros param of mapping self.map_x_width = rospy.get_param('/slam/map_width') self.map_y_width = rospy.get_param('/slam/map_height') self.map_reso = rospy.get_param('/slam/map_resolution') self.map_cellx_width = int(round(self.map_x_width/self.map_reso)) self.map_celly_width = int(round(self.map_y_width/self.map_reso)) self.icp = ICP() self.ekf = EKF() self.extraction = Extraction() self.mapping = Mapping(self.map_cellx_width,self.map_celly_width,self.map_reso) # odom robot init states self.sensor_sta = [self.robot_x,self.robot_y,self.robot_theta] self.isFirstScan = True self.src_pc = [] self.tar_pc = [] # State Vector [x y yaw] self.xOdom = np.zeros((STATE_SIZE, 1)) self.xEst = np.zeros((STATE_SIZE, 1)) self.PEst = np.eye(STATE_SIZE) # map observation self.obstacle = [] # radius self.obstacle_r = 10 # ros topic self.laser_sub = rospy.Subscriber('/course_agv/laser/scan',LaserScan,self.laserCallback) self.location_pub = rospy.Publisher('ekf_location',Odometry,queue_size=3) self.odom_pub = rospy.Publisher('icp_odom',Odometry,queue_size=3) self.odom_broadcaster = tf.TransformBroadcaster() self.landMark_pub = rospy.Publisher('/landMarks',MarkerArray,queue_size=1) self.map_pub = rospy.Publisher('/slam_map',OccupancyGrid,queue_size=1) def laserCallback(self,msg): np_msg = self.laserToNumpy(msg) lm = self.extraction.process(np_msg) # u = self.calc_odometry(self.lm2pc(lm)) u = self.calc_odometry(np_msg) z = self.observation(lm) self.xEst,self.PEst = self.ekf.estimate(self.xEst,self.PEst,z,u) # TODO # obs = self.u2T(self.xEst[0:3]).dot(np_msg) # pmap = self.mapping.update(obs[0], obs[1], self.xEst[0], self.xEst[1]) # self.publishMap(pmap) self.publishLandMark(lm) self.publishResult() pass def observation(self,lm): landmarks = lm z = np.zeros((0, 3)) for i in range(len(landmarks.id)): dx = landmarks.position_x[i] dy = landmarks.position_y[i] d = math.hypot(dx, dy) angle = self.ekf.pi_2_pi(math.atan2(dy, dx)) zi = np.array([d, angle, i]) z = np.vstack((z, zi)) return z def calc_odometry(self,np_msg): if self.isFirstScan: self.tar_pc = np_msg self.isFirstScan = False return np.array([[0.0,0.0,0.0]]).T self.src_pc = np_msg transform_acc = self.icp.process(self.tar_pc,self.src_pc) self.tar_pc = np_msg return self.T2u(transform_acc) def laserToNumpy(self,msg): total_num = len(msg.ranges) pc = np.ones([3,total_num]) range_l = np.array(msg.ranges) range_l[range_l == np.inf] = MAX_LASER_RANGE angle_l = np.linspace(msg.angle_min,msg.angle_max,total_num) pc[0:2,:] = np.vstack((np.multiply(np.cos(angle_l),range_l),np.multiply(np.sin(angle_l),range_l))) return pc def T2u(self,t): dw = math.atan2(t[1,0],t[0,0]) u = np.array([[t[0,2],t[1,2],dw]]) return u.T def u2T(self,u): w = u[2] dx = u[0] dy = u[1] return np.array([ [ math.cos(w),-math.sin(w), dx], [ math.sin(w), math.cos(w), dy], [0,0,1] ]) def lm2pc(self,lm): total_num = len(lm.id) dy = lm.position_y dx = lm.position_x range_l = np.hypot(dy,dx) angle_l = np.arctan2(dy,dx) pc = np.ones((3,total_num)) pc[0:2,:] = np.vstack((np.multiply(np.cos(angle_l),range_l),np.multiply(np.sin(angle_l),range_l))) print("mdbg ",total_num) return pc def publishResult(self): # tf s = self.xEst.reshape(-1) q = tf.transformations.quaternion_from_euler(0,0,s[2]) self.odom_broadcaster.sendTransform((s[0],s[1],0.001),(q[0],q[1],q[2],q[3]), rospy.Time.now(),"ekf_location","world_base") # odom odom = Odometry() odom.header.stamp = rospy.Time.now() odom.header.frame_id = "world_base" odom.pose.pose.position.x = s[0] odom.pose.pose.position.y = s[1] odom.pose.pose.position.z = 0.001 odom.pose.pose.orientation.x = q[0] odom.pose.pose.orientation.y = q[1] odom.pose.pose.orientation.z = q[2] odom.pose.pose.orientation.w = q[3] self.location_pub.publish(odom) s = self.xOdom q = tf.transformations.quaternion_from_euler(0,0,s[2]) # odom odom = Odometry() odom.header.stamp = rospy.Time.now() odom.header.frame_id = "world_base" odom.pose.pose.position.x = s[0] odom.pose.pose.position.y = s[1] odom.pose.pose.position.z = 0.001 odom.pose.pose.orientation.x = q[0] odom.pose.pose.orientation.y = q[1] odom.pose.pose.orientation.z = q[2] odom.pose.pose.orientation.w = q[3] self.odom_pub.publish(odom) print("mdbg ",self.xEst.shape) # self.laser_pub.publish(self.target_laser) pass def publishLandMark(self,msg): # msg = LandMarkSet() if len(msg.id) <= 0: return landMark_array_msg = MarkerArray() for i in range(len(msg.id)): marker = Marker() marker.header.frame_id = "course_agv__hokuyo__link" marker.header.stamp = rospy.Time(0) marker.ns = "lm" marker.id = i marker.type = Marker.SPHERE marker.action = Marker.ADD marker.pose.position.x = msg.position_x[i] marker.pose.position.y = msg.position_y[i] marker.pose.position.z = 0 # 2D marker.pose.orientation.x = 0.0 marker.pose.orientation.y = 0.0 marker.pose.orientation.z = 0.0 marker.pose.orientation.w = 1.0 marker.scale.x = 0.2 marker.scale.y = 0.2 marker.scale.z = 0.2 marker.color.a = 0.5 # Don't forget to set the alpha marker.color.r = 0.0 marker.color.g = 0.0 marker.color.b = 1.0 landMark_array_msg.markers.append(marker) for i in range(((self.xEst.shape[0]-STATE_SIZE)/2)): marker = Marker() marker.header.frame_id = "world_base" marker.header.stamp = rospy.Time(0) marker.ns = "lm" marker.id = i+len(msg.id) marker.type = Marker.SPHERE marker.action = Marker.ADD marker.pose.position.x = self.xEst[STATE_SIZE+i*2,0] marker.pose.position.y = self.xEst[STATE_SIZE+i*2+1,0] marker.pose.position.z = 0 # 2D marker.pose.orientation.x = 0.0 marker.pose.orientation.y = 0.0 marker.pose.orientation.z = 0.0 marker.pose.orientation.w = 1.0 marker.scale.x = 0.2 marker.scale.y = 0.2 marker.scale.z = 0.2 marker.color.a = 0.5 # Don't forget to set the alpha marker.color.r = 0.0 marker.color.g = 1.0 marker.color.b = 0.0 landMark_array_msg.markers.append(marker) self.landMark_pub.publish(landMark_array_msg) def publishMap(self,pmap): map_msg = OccupancyGrid() map_msg.header.seq = 1 map_msg.header.stamp = rospy.Time().now() map_msg.header.frame_id = "map" map_msg.info.map_load_time = rospy.Time().now() map_msg.info.resolution = self.map_reso map_msg.info.width = self.map_cellx_width map_msg.info.height = self.map_celly_width map_msg.info.origin.position.x = -self.map_cellx_width*self.map_reso/2.0 map_msg.info.origin.position.y = -self.map_celly_width*self.map_reso/2.0 map_msg.info.origin.position.z = 0 map_msg.info.origin.orientation.x = 0 map_msg.info.origin.orientation.y = 0 map_msg.info.origin.orientation.z = 0 map_msg.info.origin.orientation.w = 1.0 map_msg.data = list(pmap.T.reshape(-1)*100) self.map_pub.publish(map_msg)
#!/usr/bin/env python # -*- coding: utf-8 -*- from conversion import Conversion from extraction import Extraction from distance import Distance from sys import argv if __name__ == "__main__": y = argv[1] if "-c" in y: c = Conversion(y) elif "-e" in y: e = Extraction(y) elif y == "-d": d = Distance() else: print "There is no %r option. Try -cmp3 -cm4a -e or -d." % (y)
def run(self): resume_extract = Extraction(self.file_path) resumes = resume_extract.get_resumes() degree_map = self.__build_degree_map(resumes, "computer") self.__plot_degree_data(degree_map)
song_features = db.song_features_collection # #####REPLACE##### distance_features = db.distance_features_collection # #####REPLACE##### #__________________________________________________________________________ options, rem = getopt(argv[1:], 'c:e:d:n:g:h', [ 'conversion=', 'extraction=', 'distance', 'neighbour', 'graphdist', 'help' ]) for opt, arg in options: if opt in ('-c', '--conversion'): from conversion import Conversion c = Conversion(arg, rem[0]) elif opt in ('-e', '--extraction'): from extraction import Extraction e = Extraction(arg, song_features) elif opt in ('-d', '--distance'): from distance import Distance d = Distance(song_features, distance_features) elif opt in ('-n', '--neighbour'): from neighbour import Neighbour n = Neighbour(song_features, distance_features) elif opt in ('-g', '--graphdist'): from graphdist import Graphdist g = Graphdist(song_features, distance_features) elif opt in ('-h', '--help'): print """The following options are available: -c, --conversion mp3/m4a => conversion of mp4 and mp3 files to wave files -e, --extraction
class Localization(): def __init__(self): self.icp = ICP() self.ekf = EKF() self.extraction = Extraction() # odom robot init states self.robot_x = rospy.get_param('/icp/robot_x',0) self.robot_y = rospy.get_param('/icp/robot_y',0) self.robot_theta = rospy.get_param('/icp/robot_theta',0) self.sensor_sta = [self.robot_x,self.robot_y,self.robot_theta] self.isFirstScan = True self.src_pc = [] self.tar_pc = [] # State Vector [x y yaw] self.xOdom = np.zeros((3,1)) self.xEst = np.zeros((3,1)) self.PEst = np.eye(3) # map observation self.obstacle = [] # radius self.obstacle_r = 10 # init map self.updateMap() # ros topic self.laser_sub = rospy.Subscriber('/course_agv/laser/scan',LaserScan,self.laserCallback) self.location_pub = rospy.Publisher('ekf_location',Odometry,queue_size=3) self.odom_pub = rospy.Publisher('icp_odom',Odometry,queue_size=3) self.odom_broadcaster = tf.TransformBroadcaster() self.landMark_pub = rospy.Publisher('/landMarks',MarkerArray,queue_size=1) def updateMap(self): print("debug: try update map obstacle") rospy.wait_for_service('/static_map') try: getMap = rospy.ServiceProxy('/static_map',GetMap) self.map = getMap().map # print(self.map) except: e = sys.exc_info()[0] print('Service call failed: %s'%e) # Update for planning algorithm map_data = np.array(self.map.data).reshape((-1,self.map.info.height)).transpose() tx,ty = np.nonzero((map_data > 20)|(map_data < -0.5)) ox = (tx*self.map.info.resolution+self.map.info.origin.position.x)*1.0 oy = (ty*self.map.info.resolution+self.map.info.origin.position.y)*1.0 self.obstacle = np.vstack((ox,oy)).transpose() self.obstacle_r = self.map.info.resolution print("debug: update map obstacle success! ") def laserCallback(self,msg): u = self.calc_odometry(msg) z = self.ekf.odom_model(self.xEst,self.calc_map_observation(self.laserToNumpy(msg))) self.xEst,self.PEst = self.ekf.estimate(self.xEst,self.PEst,z,u) self.publishResult() pass def laserEstimation(self): # TODO return pc def calc_map_observation(self,msg): landmarks_src = self.extraction.process(msg,True) landmarks_tar = self.extraction.process(self.laserEstimation(),True) print("lm : ",len(landmarks_src.id),len(landmarks_tar.id)) self.publishLandMark(landmarks_src,landmarks_tar) src_pc = self.lm2pc(landmarks_src) tar_pc = self.lm2pc(landmarks_tar) transform_acc = self.icp.process(tar_pc,src_pc) return self.T2u(transform_acc) def calc_odometry(self,msg): if self.isFirstScan: self.tar_pc = self.laserToNumpy(msg) self.isFirstScan = False return np.array([[0.0,0.0,0.0]]).T self.src_pc = self.laserToNumpy(msg) transform_acc = self.icp.process(self.tar_pc,self.src_pc) self.tar_pc = self.laserToNumpy(msg) return self.T2u(transform_acc) def lm2pc(self,lm): total_num = len(lm.id) dy = lm.position_y dx = lm.position_x range_l = np.hypot(dy,dx) angle_l = np.arctan2(dy,dx) pc = np.ones((3,total_num)) pc[0:2,:] = np.vstack((np.multiply(np.cos(angle_l),range_l),np.multiply(np.sin(angle_l),range_l))) print("mdbg ",total_num) return pc def laserToNumpy(self,msg): total_num = len(msg.ranges) pc = np.ones([3,total_num]) range_l = np.array(msg.ranges) angle_l = np.linspace(msg.angle_min,msg.angle_max,total_num) pc[0:2,:] = np.vstack((np.multiply(np.cos(angle_l),range_l),np.multiply(np.sin(angle_l),range_l))) return pc def T2u(self,t): dw = math.atan2(t[1,0],t[0,0]) u = np.array([[t[0,2],t[1,2],dw]]) return u.T def publishResult(self): # tf s = self.xEst.reshape(-1) q = tf.transformations.quaternion_from_euler(0,0,s[2]) self.odom_broadcaster.sendTransform((s[0],s[1],0.001),(q[0],q[1],q[2],q[3]), rospy.Time.now(),"ekf_location","world_base") # odom odom = Odometry() odom.header.stamp = rospy.Time.now() odom.header.frame_id = "world_base" odom.pose.pose.position.x = s[0] odom.pose.pose.position.y = s[1] odom.pose.pose.position.z = 0.001 odom.pose.pose.orientation.x = q[0] odom.pose.pose.orientation.y = q[1] odom.pose.pose.orientation.z = q[2] odom.pose.pose.orientation.w = q[3] self.location_pub.publish(odom) s = self.xOdom q = tf.transformations.quaternion_from_euler(0,0,s[2]) # odom odom = Odometry() odom.header.stamp = rospy.Time.now() odom.header.frame_id = "world_base" odom.pose.pose.position.x = s[0] odom.pose.pose.position.y = s[1] odom.pose.pose.position.z = 0.001 odom.pose.pose.orientation.x = q[0] odom.pose.pose.orientation.y = q[1] odom.pose.pose.orientation.z = q[2] odom.pose.pose.orientation.w = q[3] self.odom_pub.publish(odom) # self.laser_pub.publish(self.target_laser) pass def publishLandMark(self,msg,msg2): # msg = LandMarkSet() if len(msg.id) <= 0: return landMark_array_msg = MarkerArray() for i in range(len(msg.id)): marker = Marker() marker.header.frame_id = "course_agv__hokuyo__link" marker.header.stamp = rospy.Time(0) marker.ns = "lm" marker.id = i marker.type = Marker.SPHERE marker.action = Marker.ADD marker.pose.position.x = msg.position_x[i] marker.pose.position.y = msg.position_y[i] marker.pose.position.z = 0 # 2D marker.pose.orientation.x = 0.0 marker.pose.orientation.y = 0.0 marker.pose.orientation.z = 0.0 marker.pose.orientation.w = 1.0 marker.scale.x = 0.2 marker.scale.y = 0.2 marker.scale.z = 0.2 marker.color.a = 0.5 # Don't forget to set the alpha marker.color.r = 0.0 marker.color.g = 0.0 marker.color.b = 1.0 landMark_array_msg.markers.append(marker) for i in range(len(msg2.id)): marker = Marker() marker.header.frame_id = "course_agv__hokuyo__link" marker.header.stamp = rospy.Time(0) marker.ns = "lm" marker.id = i+len(msg.id) marker.type = Marker.SPHERE marker.action = Marker.ADD marker.pose.position.x = msg2.position_x[i] marker.pose.position.y = msg2.position_y[i] marker.pose.position.z = 0 # 2D marker.pose.orientation.x = 0.0 marker.pose.orientation.y = 0.0 marker.pose.orientation.z = 0.0 marker.pose.orientation.w = 1.0 marker.scale.x = 0.2 marker.scale.y = 0.2 marker.scale.z = 0.2 marker.color.a = 0.5 # Don't forget to set the alpha marker.color.r = 0.0 marker.color.g = 1.0 marker.color.b = 0.0 landMark_array_msg.markers.append(marker) self.landMark_pub.publish(landMark_array_msg)
class Segmentation_Pipeline(): def __init__(self, cam_type='zed', realtime=False): if cam_type is 'zed': odom_file = '/export/patraval/robo_car_new_loop_all/zed_front/gps/fix.txt' self.initial_offset = 6070 self.offset_length = 6634 elif cam_type is 'pg': # odom_file = '/export/patraval/robo_car_loop2/pg_cam/gps/fix.txt' odom_file = '/export/patraval/robo_loop_pg_only/pg_cam/gps/fix.txt' self.initial_offset = 5953 self.offset_length = 6522 self.initial_offset = 1600 # pg #1417 #-- zed # self.initial_offset = 1948 self.offset_length = 6000 # self.initial_offset = 3510 # self.offset_length = 6634 print(self.initial_offset, self.offset_length) self.transformer = State_Transition(odom_file, cam_type, self.initial_offset, realtime) self.extractor = Extraction(cam_type) def start_process_disk(self): for i in range(self.initial_offset, self.offset_length): points = self.extractor.execute_extraction(i) self.transformer.point_transformation(points, i) print("Processed: ", i) # self.extractor.display_final_image(self.transformer.vehicle_coords_base, self.transformer.pop_index, points) # raw_input("Press Enter to continue...") def start_process_realtime2(self, index=None): for index in range(self.initial_offset, self.offset_length): points = self.extractor.execute_extraction(index) self.transformer.point_transformation(points, index) print("Processed: ", index) # self.extractor.display_final_image(self.transformer.vehicle_coords_base, points, self.transformer.pop_index) def start_process_realtime(self, index): points = self.extractor.execute_extraction(index + self.initial_offset) self.transformer.point_transformation(points, index + self.initial_offset) # print("Processed: ", index+70+6000) self.extractor.display_final_image( self.transformer.vehicle_coords_base, points, self.transformer.pop_index) def start_process_cluster_realtime2(self, index=None): for index in range(self.initial_offset, self.offset_length): print("Processing frame: ", index) points, points_disp = self.extractor.execute_extraction_cluster( index) self.transformer.cluster_transformation(points, points_disp, index) self.transformer.init_origin_state(self.initial_offset) for index in range(self.initial_offset, self.offset_length): print("logging frame: ", index) self.transformer.log_transformations( self.transformer.logged_cluster, index)
class SLAM_Localization(LandmarkLocalization, EKF_SLAM): alpha = 3.0 # factor in estimating covariance. def __init__(self, nodeName="slam_ekf"): super(SLAM_Localization, self).__init__(nodeName) self.icp = SubICP() self.extraction = Extraction() self.isFirstScan = True self.laser_count = 0 # interval self.laser_interval = 5 # State Vector [x y yaw].T, column vector. # self.xOdom = np.zeros((STATE_SIZE,1)) self.xEst = np.zeros((STATE_SIZE, 1)) # Covariance. Initial is very certain. self.PEst = np.zeros((STATE_SIZE, STATE_SIZE)) # landMark Estimation. Like former self.tar_pc self.lEst = np.zeros((LM_SIZE, 0)) # lEst should be of 2*N size # ros topic self.laser_sub = rospy.Subscriber('/course_agv/laser/scan', LaserScan, self.laserCallback) # self.location_pub = rospy.Publisher('ekf_location',Odometry,queue_size=3) ## localization parameters # minimum landmark matches to update. self.min_match = int(rospy.get_param('/slam/min_match', 2)) # minimum number of points for a landmark cluster self.extraction.landMark_min_pt = int( rospy.get_param('/slam/landMark_min_pt', 1)) # maximum radius to be identified as landmark self.extraction.radius_max_th = float( rospy.get_param('/slam/radius_max_th', 0.4)) OBSTACLE_RADIUS = 0.35 # feed icp landmark instead of laser. def laserCallback(self, msg): print('------seq: ', msg.header.seq) if self.isFirstScan: # feed in landmarks. # self.icp.tar_pc=self.extraction.process(msg) self.icp.tar_pc = self.icp.laserToNumpy(msg) self.isFirstScan = False return # Update once every 5 laser scan because the we cannot distinguish rotation if interval is too small. self.laser_count += 1 if self.laser_count < self.laser_interval: return # Updating process self.laser_count = 0 # z is the landmarks in self frame as a 2*n array. z = self.extraction.process(msg, True) self.publishLandMark(z, "b") # relative displacement u = self.calc_odometry(msg) # xEst,lEst,PEst is both predicted and updated in the ekf. self.xEst, self.lEst, self.PEst = self.estimate( self.xEst, self.lEst, self.PEst, z, u) self.publishResult() return def calc_odometry(self, msg): ''' Let icp handle the odometry, returns a relative odometry u. ''' # laser callback is manually fed by its owner class. return self.icp.laserCallback(msg) # EKF virtual function. def observation_model(self, xEst, lEst): ''' returns an 2*n column vector list. [z1,z2,....zn] ''' rotation = tf.transformations.euler_matrix(0, 0, xEst[2, 0])[0:2, 0:2] z = np.dot(rotation.T, lEst - xEst[0:2]) return z def estimate(self, xEst, lEst, PEst, z, u): G, Fx = self.jacob_motion(xEst, lEst, u) # FIXME: the G and Fx transpose is confused. Thanks god they are all symmetric ones... covariance = np.dot(G, np.dot(PEst, G.T)) + np.dot( Fx, np.dot(Cx, Fx.T)) # Predict xPredict = self.odom_model(xEst, u) zEst = self.observation_model(xPredict, lEst) self.publishLandMark(zEst, color="r", namespace="estimate", frame=self.nodeName) neighbour = self.icp.findNearest(z, zEst) # Predicted zPredict = zEst[:, neighbour.tar_indices] self.publishLandMark(zPredict, color="g", namespace="paired", frame=self.nodeName) lSize = np.size(lEst, 1) zSize = np.size(z, 1) # how many observed landmarks are missed, how many new ones will be added. missedIndices = sorted( list(set(range(zSize)) - set(neighbour.src_indices))) paired = len(neighbour.src_indices) missed = len(missedIndices) neighbour.src_indices += missedIndices # the newly-added landmarks neighbour.tar_indices += range(lSize, missed + lSize) zObserved = z[:, neighbour.src_indices] # add new landmarks to newL (new lEst) # newL=np.repeat(xPredict[0:2],missed,axis=1) # delicately select the spawning position... newL = np.dot( tf.transformations.euler_matrix(0, 0, xPredict[2, 0])[0:2, 0:2], z[:, missedIndices]) + xPredict[0:2] newL = np.hstack((lEst, newL)) zEst = self.observation_model(xPredict, newL) zPredict = zEst[:, neighbour.tar_indices] # the newly-added landmarks' uncertainty is very large # block is so powerful! covariance = scilinalg.block_diag(covariance, np.diag([INF] * LM_SIZE * missed)) yPredict = self.XLtoY(xPredict, newL) variance = self.alpha / (zSize + self.alpha) * OBSTACLE_RADIUS print("\n\npaired: {} variance: {} missed: {} landmarks: {}".format( paired, variance, missed, lSize + missed)) if zSize < self.min_match: print("Observed landmarks are too little to execute update.") # only update according to the prediction stage. return xPredict, newL, covariance m = self.jacob_h(newL, neighbour, xPredict) # z (2*n)array->(2n*1) array zPredict = np.vstack(np.hsplit(zPredict, np.size(zPredict, 1))) zObserved = np.vstack(np.hsplit(zObserved, np.size(zObserved, 1))) # print("delta z: \n{}\n".format(zObserved-zPredict)) # print("covariance:\n{}\n".format(covariance)) # Karman factor. Universal formula. K = np.dot( np.dot(covariance, m.T), np.linalg.inv( np.dot(m, np.dot(covariance, m.T)) + np.diag([variance**2] * 2 * zSize))) # Update fix = np.dot(K, zObserved - zPredict) # print("fix: \n{}\n\n".format(fix)) yEst = yPredict + fix PEst = covariance - np.dot(K, np.dot(m, covariance)) xEst, lEst = self.YtoXL(yEst) return xEst, lEst, PEst def XLtoY(self, x, l): # split y (2*N) to 2N*1 y = np.vstack(np.hsplit(l, np.size(l, 1))) y = np.vstack((x, y)) return y def YtoXL(self, y): x, l = np.vsplit(y, [STATE_SIZE]) l = np.hstack(np.vsplit(l, np.size(l) / LM_SIZE)) return x, l
def main_code(self, fname): wb = open_workbook(fname, formatting_info=True) standard_list = open_workbook(os.getcwd() + '/Standard Values.xlsx') standard_table = Extraction.extract_data(self, standard_list) ptm_data = Extraction.extract_data(self, standard_list, 'alias_col') font = wb.font_list sheet_name = [] cordinates = Extraction.search_titles(self, wb, standard_table) cordinates_row = Extraction.extract_rows(self, cordinates) #below code finds the max column max_col = 1 max_col_sheet = '' row_x = 0 for sheetname in cordinates.keys(): for value in cordinates[sheetname]: x, y = value if y > max_col and y != '': max_col = y max_col_sheet = sheetname row_x = x sheet_m = wb.sheet_by_name(max_col_sheet) header = {} cordinates_dict = Extraction.get_cordinates(self, cordinates_row, wb) work = Workbook() w_sheet = work.active w_sheet.title = 'Standard Sheet' counter = 0 heading_count = 0 x = row_x for col in range(sheet_m.ncols): if col < sheet_m.ncols - 1 and sheet_m.cell( x, col).value != '' and sheet_m.cell( x, col + 1).value == '' and sheet_m.cell( x + 1, col).value != '': w_sheet.cell(row=counter + 1, column=col + 1).value = sheet_m.cell(x + 1, col).value header[sheet_m.cell(x + 1, col).value] = col elif col < sheet_m.ncols - 1 and sheet_m.cell( x, col).value == '' and x < sheet_m.nrows - 1: if sheet_m.cell(x, col + 1).value != '' and sheet_m.cell( x + 1, col).value != '': w_sheet.cell(row=counter + 1, column=col + 1).value = sheet_m.cell( x + 1, col).value header[sheet_m.cell(x + 1, col).value] = col elif sheet_m.cell(x, col + 1).value == '' and sheet_m.cell( x + 1, col).value != '': w_sheet.cell(row=counter + 1, column=col + 1).value = sheet_m.cell( x + 1, col).value header[sheet_m.cell(x + 1, col).value] = col elif sheet_m.cell(x, col + 1).value != '' and sheet_m.cell( x - 1, col).value != '': w_sheet.cell(row=counter + 1, column=col + 1).value = sheet_m.cell( x - 1, col).value header[sheet_m.cell(x - 1, col).value] = col elif sheet_m.cell(x, col + 1).value == '' and sheet_m.cell( x - 1, col).value != '': w_sheet.cell(row=counter + 1, column=col + 1).value = sheet_m.cell( x - 1, col).value header[sheet_m.cell(x - 1, col).value] = col else: if sheet_m.cell(x, col).value != '': w_sheet.cell(row=counter + 1, column=col + 1).value = sheet_m.cell(x, col).value header[sheet_m.cell(row_x, col).value] = col counter += 1 for sheet in wb.sheets(): if sheet.visibility == 0: col_n = None second_max = 0 print sheet.name counting = 1 order_row = (cordinates_dict[sheet.name].keys()) order_row.sort() for tempp in range(len(order_row) - 1, -1, -1): if order_row[tempp] < sheet.nrows: second_max = order_row[tempp] break for row_n in order_row: #row_n is row in increasing form row_count = row_n if row_count < second_max: #counting is the number of rows in a sheet while row_count < order_row[counting]: for column_name in cordinates_dict[ sheet.name][row_n]: cell_formatting = wb.xf_list[ sheet.cell_xf_index( row_count, column_name[1])] # tests for the presence of striked out cells and skips them check2 = font[ cell_formatting.font_index].struck_out != 1 if not check2: if sheet.cell( row_count, column_name[1] ).ctype == XL_CELL_BLANK or sheet.cell( row_count, column_name[1] ).ctype == XL_CELL_EMPTY: check2 = True if check2: check = Filter.non_zero_row( self, sheet.row_values(row_count), cordinates_dict[sheet.name][row_n], ptm_data['QUANTITY']) if check: try: w_sheet.cell( row=counter + 1, column=header[column_name[0]] + 1).value = sheet.cell( row_count, column_name[1]).value except Exception as e: cap1 = Extraction.capitalised( self, column_name[0]) for key in header.keys(): cap2 = Extraction.capitalised( self, key) for tempi in ptm_data: for j in ptm_data[tempi]: if (cap1 in j or j in cap1 ) and (j in cap2 or tempi in cap2): col_n = key break if col_n == key: break if col_n == key: break w_sheet.cell( row=counter + 1, column=header[key] + 1).value = sheet.cell( row_count, column_name[1]).value else: break else: break row_count += 1 if check2 and check: counter += 1 elif row_count == second_max: while row_count < sheet.nrows: for column_name in cordinates_dict[ sheet.name][row_n]: cell_formatting = wb.xf_list[ sheet.cell_xf_index( row_count, column_name[1])] check2 = font[ cell_formatting.font_index].struck_out != 1 if not check2: if sheet.cell( row_count, column_name[1] ).ctype == XL_CELL_BLANK or sheet.cell( row_count, column_name[1] ).ctype == XL_CELL_EMPTY: check2 = True if check2: check = Filter.non_zero_row( self, sheet.row_values(row_count), cordinates_dict[sheet.name][row_n], ptm_data['QUANTITY']) if check: try: w_sheet.cell( row=counter + 1, column=header[column_name[0]] + 1).value = sheet.cell( row_count, column_name[1]).value except Exception as e: cap1 = Extraction.capitalised( self, column_name[0]) for key in header.keys(): cap2 = Extraction.capitalised( self, key) if cap1 == cap2: break else: for i in ptm_data: for j in ptm_data[i]: if (cap1 in j or j in cap1 ) and ( j in cap2 or i in cap2): col_n = key break if col_n == key: break if col_n == key: break w_sheet.cell( row=counter + 1, column=header[key] + 1).value = sheet.cell( row_count, column_name[1]).value else: break else: break if check2 and check: counter += 1 row_count += 1 counting += 1 else: continue counting += 1 print 'writing done' print sheet.name work.save(os.getcwd() + '/' + 'Output.xls') wb2 = open_workbook(os.getcwd() + '/' + 'Output.xls') sheet = wb2.sheet_by_index(0) data_column = {} for vtemp in header: stamp = 0 for key in ptm_data: for value in ptm_data[key]: cap = str(Extraction.capitalised(self, vtemp)) cap2 = str(Extraction.capitalised(self, value)) if cap != '' and cap == cap2 or cap2 in cap or cap in cap2: data_column[key] = header[vtemp] stamp = 1 break if stamp == 1: break '''below code is used to construct a nested dictionary of aliases for material, operation, valve type, etc''' alias_s = standard_list.sheet_by_name('alias') alias_db = standard_list.sheet_by_name('db alias') size_con = standard_list.sheet_by_name('conversion') convert_nb_2_i = {} convert_i_2_nb = {} for row in range(size_con.nrows): word = '' for val in str(size_con.cell(row, 1).value): word += val convert_nb_2_i[size_con.cell(row, 0).value] = word convert_i_2_nb[word] = size_con.cell(row, 0).value if self.size_value == 'IN': conversion_data = convert_nb_2_i for row in range(1, sheet.nrows): d_value = '' value_string = str( sheet.cell(row, data_column['VALVE SIZE']).value).lstrip() for i in str(value_string): if i in '0123456789': d_value += i else: break try: w_sheet.cell( row=row + 1, column=data_column['VALVE SIZE'] + 1).value = conversion_data[int(d_value)].strip("\"") except: pass elif self.size_value == 'NB': conversion_data = convert_i_2_nb for row in range(1, sheet.nrows): d_value = '' value_string = str( sheet.cell( row, data_column['VALVE SIZE']).value).strip().strip("\"") for i in str(value_string): if i in '0123456789': d_value += i else: break try: w_sheet.cell(row=row + 1, column=data_column['VALVE SIZE'] + 1).value = conversion_data[int(d_value)] except: pass alias_data = {} for col in range(alias_s.ncols): alias_d = {} temp = [] for row in range(1, alias_s.nrows): if alias_s.cell(row, col).value != '': if alias_s.cell(row, col).value != '-' and alias_s.cell( row, col).value != '--': temp.append( str( Extraction.capitalised( self, alias_s.cell(row, col).value))) else: temp.append(str(alias_s.cell(row, col).value)) elif alias_s.cell(row, col).value == '' and len(temp) >= 1: alias_d[temp[0]] = temp temp = [] alias_data[alias_s.cell(0, col).value] = alias_d alias_data2 = {} for col in range(alias_db.ncols): alias_d = {} temp = [] for row in range(1, alias_db.nrows): if alias_db.cell(row, col).value != '': if alias_db.cell(row, col).value != '-' and alias_db.cell( row, col).value != '--': temp.append( str( Extraction.capitalised( self, alias_db.cell(row, col).value))) else: temp.append(str(alias_db.cell(row, col).value)) elif alias_db.cell(row, col).value == '' and len(temp) >= 1: alias_d[temp[0]] = temp temp = [] alias_data2[alias_db.cell(0, col).value] = alias_d C = 0 '''Below code is used to standardise the material name given in the original excel sheetusing alias excel sheet''' work.save(os.getcwd() + '/' + 'Output.xls') wb2 = open_workbook(os.getcwd() + '/' + 'Output.xls') Filter.formatting_data(self, convert_nb_2_i, self.size_value, wb2, work, data_column, alias_data, self.material_value, self.a105, self.f22, self.f91, self.f92, self.wcb, self.wcc, self.wc9, self.c12a, self.c12ab) work.save(os.getcwd() + '/' + 'Output.xls') wb2 = open_workbook(os.getcwd() + '/' + 'Output.xls') sheet = wb2.sheet_by_index(0) if self.rating_value == 'YES': cliet = Client('enter ip address for the soap service') r = [] w_sheet.cell(row=1, column=sheet.ncols + 1).value = "RATING(S)" data_column['VALVE RATING'] = sheet.ncols print("FETCHING RATING FROM SERVER") for rows in range(1, sheet.nrows): w_sheet.cell( row=rows + 1, column=sheet.ncols + 1).value = cliet.service.GetMinClassRating( sheet.cell(rows, data_column['TEMPERATURE']).value, sheet.cell(rows, data_column['PRESSURE']).value, sheet.cell(rows, data_column['VALVE MATERIAL']).value) work.save(os.getcwd() + '/' + 'Output.xls') order = [] ssd = standard_list.sheet_by_name('standard') for col in range(ssd.ncols): order.append(ssd.cell(0, col).value) final_s = work.create_sheet('final') regret_s = work.create_sheet('regret') wb2 = open_workbook(os.getcwd() + '/' + 'Output.xls') sheet = wb2.sheet_by_index(0) counter = 1 counter2 = 1 head = 1 redFill = PatternFill(start_color='00CED1FF', end_color='00CED1FF', fill_type='solid') for row in range(sheet.nrows): count = 0 while count < len(order): if head == 1: for col in range(len(order)): if order[col] in data_column: final_s.cell(row=counter, column=col + 3).value = order[col] final_s.cell(row=counter, column=2).value = 'VALVE PROJECT' final_s.cell(row=counter, column=1).value = 'VALVE SCHEDULE' count += 1 counter += 1 head = 0 elif sheet.cell(row, data_column[order[count]] ).value in alias_data['VALVE TYPE'].keys(): for col in range(len(order)): if order[col] in data_column: if w_sheet.cell(row=row + 1, column=data_column[order[col]] + 1).fill == redFill: final_s.cell(row=counter, column=col + 3).fill = redFill final_s.cell(row=counter, column=col + 3).value = sheet.cell( row, data_column[order[col]]).value final_s.cell(row=counter, column=2).value = self.vp final_s.cell(row=counter, column=1).value = self.vs else: final_s.cell(row=counter, column=col + 3).value = sheet.cell( row, data_column[order[col]]).value final_s.cell(row=counter, column=2).value = self.vp final_s.cell(row=counter, column=1).value = self.vs counter += 1 count += 1 break else: for col in range(len(order)): if order[col] in data_column: if w_sheet.cell(row=row + 1, column=data_column[order[col]] + 1).fill == redFill: regret_s.cell(row=counter2, column=col + 3).fill = redFill regret_s.cell( row=counter2, column=col + 3).value = sheet.cell( row, data_column[order[col]]).value regret_s.cell(row=counter2, column=2).value = self.vp regret_s.cell(row=counter2, column=1).value = self.vs else: regret_s.cell( row=counter2, column=col + 3).value = sheet.cell( row, data_column[order[col]]).value regret_s.cell(row=counter2, column=2).value = self.vp regret_s.cell(row=counter2, column=1).value = self.vs counter2 += 1 count += 1 break work.save(os.getcwd() + '/' + 'Output.xls') wb2 = open_workbook(os.getcwd() + '/' + 'Output.xls') Part_number_extraction(alias_data2, standard_list, wb2, work, self.size_value, convert_i_2_nb, self.part_value, self.part_value2, convert_nb_2_i) work.save(os.getcwd() + '/' + 'Output.xls')
def read(self, fn): d = defaultdict(lambda: []) with open(fn) as fin: data = json.load(fin) for sentence in data: tuples = data[sentence] for t in tuples: if t["pred"].strip() == "<be>": rel = "[is]" else: rel = t["pred"].replace("<be> ", "") confidence = 1 curExtraction = Extraction(pred=rel, head_pred_index=None, sent=sentence, confidence=float(confidence), index=None) if t["arg0"] != "": curExtraction.addArg(t["arg0"]) if t["arg1"] != "": curExtraction.addArg(t["arg1"]) if t["arg2"] != "": curExtraction.addArg(t["arg2"]) if t["arg3"] != "": curExtraction.addArg(t["arg3"]) if t["temp"] != "": curExtraction.addArg(t["temp"]) if t["loc"] != "": curExtraction.addArg(t["loc"]) d[sentence].append(curExtraction) self.oie = d
from flask import Flask, render_template, request, jsonify import sentences_list import random from extraction import Extraction from parssing import Parssing import wikipedia app = Flask(__name__) app.config['SECRET_KEY'] = '7TKUKe09wW1PlrtSL066lsN18uWA7iuO' # Instances creation datas_extraction = Extraction() datas_management = Parssing() @app.route('/') def home(): return render_template('index.html', title="Bienvenue chez GrandPy Bot") @app.route('/_answer') def answer(): question = request.args.get('question', 0, type=str) return jsonify(result=question) @app.route('/_address') def address(): question = request.args.get('question', 0, type=str) filtered_sentence = Parssing.get_main_words(datas_management, question)
from extraction import Extraction import time startTime = time.time() print time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(startTime)) extract1 = Extraction() extract1.loadData() extract1.run() endTime = time.time() print time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(endTime)) seconds = endTime - startTime print seconds m, s = divmod(seconds, 60) h, m = divmod(m, 60) print "%d:%02d:%02d" % (h, m, s)