def import_all(cls): cls.objects.all().delete() prm = create_prm(cls.RDF_TYPE) res = request_metro(prm) data = parse_response(res) instances = [] for d in data: try: date = dp.parse(d['dc:date']) conn_railways = ','.join(d['odpt:connectingRailway']) exits = ','.join(d['odpt:exit']) if d['odpt:exit'] else '' surveys = ','.join(d['odpt:passengerSurvey']) instances.append( cls(ld_context=d['@context'], ld_id=d['@id'], ld_type=d['@type'], same_as=d['owl:sameAs'], date=date, title=d['dc:title'], connecting_railways=conn_railways, exits=exits, facility=d['odpt:facility'], operator=d['odpt:operator'], passenger_surveys=surveys, railway_id=d['odpt:railway'], station_code=d['odpt:stationCode'], region=d['ug:region'])) except: print d raise for ins in instances: ins.save()
def import_all(cls): cls.objects.all().delete() prm = create_prm(cls.RDF_TYPE) res = request_metro(prm) data = parse_response(res) for d in data: try: date = dp.parse(d['dc:date']) rw = cls(ld_context=d['@context'], ld_id=d['@id'], ld_type=d['@type'], same_as=d['owl:sameAs'], date=date, title=d['dc:title'], operator=d['odpt:operator'], line_code=d['odpt:lineCode'], region=d['ug:region']) rw.save() # save StationOrder so_list = d['odpt:stationOrder'] for so in so_list: st = Station.get_or_create(so['odpt:station']) StationOrder.get_or_create(rw, st, so['odpt:index']) # save TravelTime tt_list = d['odpt:travelTime'] for tt in tt_list: fs = Station.get_or_create(tt['odpt:fromStation']) ts = Station.get_or_create(tt['odpt:toStation']) TravelTime.get_or_create(rw, fs, ts, tt['odpt:necessaryTime'], tt['odpt:trainType']) # save WomenOnlyCar if 'odpt:womenOnlyCar' in d: woc_list = d['odpt:womenOnlyCar'] for woc in woc_list: fs = Station.get_or_create(woc['odpt:fromStation']) ts = Station.get_or_create(woc['odpt:toStation']) time_from = dp.parse( woc['odpt:availableTimeFrom']).time() time_until = dp.parse( woc['odpt:availableTimeUntil']).time() WomenOnlyCar.get_or_create(rw, fs, ts, woc['odpt:operationDay'], time_from, time_until, woc['odpt:carComposition'], woc['odpt:carNumber']) except: print d raise
def handle_index(page=1): args = request.query.decode("utf-8") page = int(args.page) searchType = args.searchType query = args.query # search solr = Solr(SOLR_URL + '/' + solr_core[searchType] + '/') start = (page - 1) * PER_PAGE solr_query_params = { 'start':start, 'rows':10, 'hl': 'true', 'hl.fl': 'body', 'hl.fragsize': 100, 'hl.snippets': 3, 'df': 'text' } responseObj = solr.search(query, **solr_query_params) ResultObj = parse_response(responseObj) resultNum = ResultObj['hits'] print resultNum if resultNum == 0: return "No match Docs find! What's a pity" resultNum = resultNum if resultNum < TOTAL else TOTAL paginator = Pagination(page, PER_PAGE, resultNum) # result results = results_for_page(ResultObj, page, PER_PAGE, resultNum) # ad if resultNum < 2: ads = ads_for_page(ResultObj, 1) else: ads = ads_for_page(ResultObj, AD_NUM) #spellcheck spell_flag = True query_right = spellcheck.spellCorret(query, wordSet, bagOfword, parameter) if len(query_right) == 0: spell_flag = False template_params = { 'query': query, 'results': results, 'ads': ads, 'paginator': paginator, 'resultNum': resultNum, 'searchType': searchType, 'spellFlag': spell_flag, 'query_right': query_right } return template('result.ftl', **template_params)
def import_all(cls): cls.objects.all().delete() prm = create_prm(cls.RDF_TYPE) res = request_metro(prm) data = parse_response(res) for d in data: try: date = dp.parse(d['dc:date']) rw = cls(ld_context = d['@context'], ld_id = d['@id'], ld_type = d['@type'], same_as = d['owl:sameAs'], date = date, title = d['dc:title'], operator = d['odpt:operator'], line_code = d['odpt:lineCode'], region = d['ug:region'] ) rw.save() # save StationOrder so_list = d['odpt:stationOrder'] for so in so_list: st = Station.get_or_create(so['odpt:station']) StationOrder.get_or_create(rw, st, so['odpt:index']) # save TravelTime tt_list = d['odpt:travelTime'] for tt in tt_list: fs = Station.get_or_create(tt['odpt:fromStation']) ts = Station.get_or_create(tt['odpt:toStation']) TravelTime.get_or_create(rw, fs, ts, tt['odpt:necessaryTime'], tt['odpt:trainType']) # save WomenOnlyCar if 'odpt:womenOnlyCar' in d: woc_list = d['odpt:womenOnlyCar'] for woc in woc_list: fs = Station.get_or_create(woc['odpt:fromStation']) ts = Station.get_or_create(woc['odpt:toStation']) time_from = dp.parse(woc['odpt:availableTimeFrom']).time() time_until = dp.parse(woc['odpt:availableTimeUntil']).time() WomenOnlyCar.get_or_create(rw, fs, ts, woc['odpt:operationDay'], time_from, time_until, woc['odpt:carComposition'], woc['odpt:carNumber']) except: print d raise
def import_all(cls): prm = create_prm(cls.RDF_TYPE) res = request_metro(prm) data = parse_response(res) instances = [] for d in data: try: date = dp.parse(d['dc:date']) valid = dp.parse(d['dct:valid']) from_station = Station.get_or_create(d['odpt:fromStation']) starting_station = Station.get_or_create(d['odpt:startingStation']) terminal_station = Station.get_or_create(d['odpt:terminalStation']) to_station = Station.get_or_create(d['odpt:toStation']) cls(ld_context = d['@context'], ld_id = d['@id'], ld_type = d['@type'], date = date, valid = valid, delay = d['odpt:delay'], frequency = d['odpt:frequency'], from_station = from_station, rail_direction = d['odpt:railDirection'], railway = d['odpt:railway'], starting_station = starting_station, terminal_station = terminal_station, to_station = to_station, train_number = d['odpt:trainNumber'], train_owner = d['odpt:trainOwner'], train_type = d['odpt:trainType'], same_as = d['owl:sameAs'], from_station_raw = d['odpt:fromStation'], starting_station_raw = d['odpt:startingStation'], terminal_station_raw = d['odpt:terminalStation'], to_station_raw = d['odpt:toStation'], ).save() except: print sys.exc_info() print d
def import_all(cls): cls.objects.all().delete() prm = create_prm(cls.RDF_TYPE) res = request_metro(prm) data = parse_response(res) instances = [] for d in data: try: date = dp.parse(d["dc:date"]) conn_railways = ",".join(d["odpt:connectingRailway"]) exits = ",".join(d["odpt:exit"]) if d["odpt:exit"] else "" surveys = ",".join(d["odpt:passengerSurvey"]) instances.append( cls( ld_context=d["@context"], ld_id=d["@id"], ld_type=d["@type"], same_as=d["owl:sameAs"], date=date, title=d["dc:title"], connecting_railways=conn_railways, exits=exits, facility=d["odpt:facility"], operator=d["odpt:operator"], passenger_surveys=surveys, railway_id=d["odpt:railway"], station_code=d["odpt:stationCode"], region=d["ug:region"], ) ) except: print d raise for ins in instances: ins.save()
def system_eval(): global trec_eval_results,baseline baseline = OrderedDict(baseline) if len(trec_eval_results) != 0: template_params = { 'trec_eval_results': trec_eval_results, 'baseline': baseline } return template('syseval_result.ftl', **template_params) trec_list = ['Trec14', 'Trec09'] R = lambda x:int(filter(str.isdigit,x)) for trec in trec_list: qrels = trec_dict[trec]['qrels.adhoc'] query_dict = evaluate.get_query_info(trec_dict[trec]['queries']) print 'query len %d' % len(query_dict) core = 'Trec09' if R(trec) < 12 else 'Trec12' solr = Solr(SOLR_URL + '/' + solr_core[core] + '/', timeout=100) results = [] for query_id, query in query_dict.items(): responseObj = solr.search(query, **solr_query_params) ResultObj = parse_response(responseObj) results.append([query_id, ResultObj]) # gen submit and save evaluate.gen_save_submmit(results, RUN_FILE_PATH) # eval by eval_program cmd = EVAL_EXE + ' -m all_trec -M 1000 ' + qrels + ' ' + RUN_FILE_PATH + ' > ' + RESULT_PATH exe_cmd(cmd) # read eval result and display eval_results = evaluate.read_eval(RESULT_PATH) trec_eval_results[trec] = eval_results print trec + ' is Done!' template_params = { 'trec_eval_results': trec_eval_results, 'baseline': baseline } return template('syseval_result.ftl', **template_params)