def parsefile(filename): filename, extension = _preamble(filename) fileobject = open(filename + '.' + extension, 'r') currentline = 0 while True: line = fileobject.readline() if not line: break line = line.strip() currentline += 1 if not line: continue parse = line.split() if parse[0] == '--': continue elif parse[0] == 'RELATION': relname, attributes, types, currentline = \ _handle_declaration(fileobject, parse, currentline) relations[relname] = relation.Relation(relname, types, attributes) elif parse[0] == 'INSERT': values = _handle_insertion(line, currentline) if values == None: break relations[relname].insert(values) else: _callerror(ERROR07 % ('', currentline)) return return relations
def _create(query): ''' Create a tuple, and add it to the enviroment, given the name, the type of it and the attributes or schema (create relname (t1 ... tn) (a1 ... am)) where relname is a valid string representing the name of the new relation ([A-Z][a-z])*. n, m are integers, and ti, represents the type of the ith attribute in the schema (ai). For succesfull application, n has to be equal to m. Each type could be one of: STRING, INTEGER, CHAR, BOOL, REAL A valid attribute name matches ([a-z]*) Adds the relation to the enviroment, and returns an expression, that evaluates to a relation ''' global env if not _check_query_length(lambda a, b: a != b, query, 'create', 4): return # The arguments, the name of the new relation, its types and attributes relname, types, attributes = query[1], query[2], set(query[3]) if not _handle_name(relname, 'relation'): # Wrong identifier for relation's name return if len(types) == 0 or len(attributes) == 0: # Empty schema or empty types _println(errors.ERROR_CREATE_ARITY_ZERO \ % relname) return if len(types) != len(attributes): # Arity mismatch _println(str(errors.ERROR_CREATE_ARITY_MISMATCH \ % relname)) return attributes = query[3] for i, _ in enumerate(types): # Check for each type t, a = types[i], attributes[i] if t not in dmlparser.TYPES: _println(errors.ERROR_CREATE_TYPE % (relname, t)) return if not _handle_name(a, 'attribute'): # Check for each attribute return # Finally, insert the relation in the enviroment, and return its name env[relname] = relation.Relation(relname, types, query[3]) return relname
def import_schemas(f_folder): """returns a list of relations built from each file in parameterized folder""" new_relations = {} #get filenames from parameterized folder files = glob.glob(str(".\\" + f_folder + "\\*.csv")) #build new relation from each file for each_file in files: filename = str(each_file.replace(str(".\\" + str(f_folder) + "\\"), "").replace(".csv", "")) print("Converting " + str(filename) + ".csv to relation...") this_relation = relation.Relation(each_file, f_folder) new_relations[filename] = this_relation # print() #return list of filenames as strings return new_relations
def init_param(): param = {} train_list = lg.train('./data/freebase15k/train.txt') # param['graph'] = G param['train'] = train_list # param['validation'] = lg.test('./data/freebase15k/valid.txt') param['test'] = lg.test('./data/freebase15k/test.txt') param['relation'] = lg.read_relation('./data/freebase15k/train.txt') param['vocabulary'] = lg.read_vocab('./data/freebase15k/train.txt') param['evaluation_metric'] = 'mean_rank' param['iteration'] = 0 param['model'] = 1 param['window'] = 1 param['kns'] = 20 param['alpha'] = 0.07 param['beta'] = 0.03 param['dim'] = 200 param['iter_count'] = 500 param['ns_size'] = 1000 param['delta'] = 20 param['k'] = 0 param['vocab'] = v.Vocabulary(param['vocabulary']) param['rel'] = r.Relation(param['relation']) param['relation'] = [] param['vocabulary'] = [] param['nn0'] = np.random.uniform(low=-0.5 / param['dim'], high=0.5 / param['dim'], size=(len(param['vocab']), param['dim'])) param['nn1'] = np.zeros(shape=(len(param['vocab']), param['dim'])) param['nn2'] = np.random.uniform(low=-0.5 / param['dim'], high=0.5 / param['dim'], size=(len(param['rel']), param['dim'])) param['table'] = t.TableForNegativeSamples(param['vocab']) param['test_corrupted_all'] = Co._all(param, 'test') # param['test_corrupted_filtered'] = Co._filtered(param, 'test') # param['validation_corrupted'] = Co._all(param, 'validation') # param['validation_corrupted_filtered'] = Co._filtered(param, 'validation') return param
def add_relations_from_json(self, json_data): """ Add the states defined in json_data to the object. :param json_data: python representation of a json_object, containing at least: { 'relations': [['a', 1, 'b'], ['c', 2, 'd']], } :return: void """ for [source_name, agent, destination_name] in json_data['relations']: try: self.add_relation( relation.Relation( str(agent), self.get_state_by_name(source_name), self.get_state_by_name(destination_name))) except errors.ModelError: raise errors.ModelError( "The relationship {source}R_{agent}{destination} is not between existing states" .format(source=source_name, destination=destination_name, agent=agent))
def main(): print time.strftime("%H:%M:%S", time.localtime()), " - script started" print " Searching RelationIDs and Lines in route..." # Create connection to DB server. connection = connect( "dbname='{}' user='******' password='******' port='{}' host='{}'".format( os.environ['POSTGRES_DB'], os.environ['POSTGRES_USER'], os.environ['POSTGRES_PASSWORD'], os.environ['POSTGRES_PORT'], os.environ['POSTGRES_HOST'])) relationCursor = connection.cursor() auxiliaryCursor = connection.cursor() wayCursor = connection.cursor() # Find relation IDs to be parsed, ie. those with osmc:symbol or some mtb values # Treat lines with useful attributes as relations (osm_id >= 0) relationIDs = [] relations = [] relationCursor.execute(''' SELECT osm_id, "mtb:scale", "mtb:scale:uphill", "sac_scale", network, "osmc:symbol", "color", "colour", "kct_blue", "kct_green", "kct_yellow", "kct_red", route, ref FROM osm_route WHERE route IN ('mtb','horse','ski','bicycle','hiking','foot','mountain hiking') AND ( "access" IS NULL OR "access" NOT IN ('private','no') OR COALESCE(bicycle,foot,horse) = 'yes' ); ''') while True: # Fetch some of the result. rows = relationCursor.fetchmany(100) # Empty result means end of query. if not rows: break #relations have negative osm_id in route table #lines have positive osm_id in route table for row in rows: if (row[0] < 0): #osm_id is not a primary key if not (row[0] in relationIDs): relationIDs.append(-row[0]) else: # 0: osm_id; 1: mtb:scale; 2: mtb:scale:uphill; 3: network; 4: "osmc:symbol; 5: route; 6: ref" lineInfo = "LINE;" + str(row[0]) + ";" + str( row[1]) + ";" + str(row[2]) + ";" + str( row[3]) + ";" + str(row[4]) + ";" + str( row[5]) + ";" + str(row[6]) + ";" + str( row[7]) + ";" + str(row[8]) + ";" + str( row[9]) + ";" + str(row[10]) + ";" + str( row[11]) + ";" + str( row[12]) + ";" + str(row[13]) relations.append(relation.Relation(lineInfo)) print time.strftime("%H:%M:%S", time.localtime()), " - RelationIDs and Lines found." print " Getting Relation details from route_rels..." # Select important columns just for our IDs for id in relationIDs: relationCursor.execute(''' SELECT id, members, tags FROM route_rels WHERE id=%s ''' % id) row = relationCursor.fetchone() # Make Relation object with parsed data relations.append(relation.Relation(row)) print time.strftime("%H:%M:%S", time.localtime()), " - relations details found." print " Making single routes from relations with all osmc:symbols..." # Find final routes and append all corresponding osmcSymbols routes = routesFromRels(relations) listOfRoutes = routes.values() listOfRoutes.sort() print time.strftime("%H:%M:%S", time.localtime()), " - routes now have osmc:symbols." print " Finding firstNode and lastNode for each route in route_ways..." # Clean previous routes. auxiliaryCursor.execute("DROP TABLE IF EXISTS routes") auxiliaryCursor.execute( "DELETE FROM geometry_columns WHERE f_table_name = 'routes'") auxiliaryCursor.execute( "CREATE TABLE routes AS SELECT osm_id, way, highway, tracktype,oneway FROM route WHERE osm_id = 0" ) auxiliaryCursor.execute( "DELETE FROM geometry_columns WHERE f_table_name = 'routes'") auxiliaryCursor.execute( "INSERT INTO geometry_columns VALUES ('', 'public', 'routes', 'way', 2, 900913, 'LINESTRING')" ) # Add important information to each route i = 0 for r in listOfRoutes: i += 1 #print "%d/%d" % (i,len(listOfRoutes)) auxiliaryCursor.execute(''' SELECT way, highway, tracktype,sac_scale,oneway FROM osm_route WHERE osm_id=%s AND (("access"<>'private' AND "access"<>'no') OR "access" IS NULL OR ("access" IN ('private', 'no') AND (bicycle='yes' or foot='yes') or horse='yes')) ''' % r.id) row = auxiliaryCursor.fetchone() # Some route IDs from relations may not be present in line table, ie. out of bounding box, those are ignored if row is not None: routes[r.id].geometry = row[0] routes[r.id].highway = row[1] routes[r.id].tracktype = row[2] routes[r.id].sacScale = row[3] routes[r.id].oneway = row[4] wayCursor.execute(''' SELECT nodes[1], nodes[array_upper(nodes, 1)] FROM route_ways WHERE id=%s ''' % r.id) firstEndNodes = wayCursor.fetchone() routes[r.id].firstNode = firstEndNodes[0] routes[r.id].lastNode = firstEndNodes[1] # print r.id, ": ", routes[r.id].firstNode, ", ", routes[r.id].lastNode else: routes.pop(r.id) print time.strftime( "%H:%M:%S", time.localtime()), " - firstNodes and lastNodes are found." print " Finding route neighbours based on first and last nodes..." # Find end nodes and their routes nodes = findNodes(routes) # Find previous and next route neighbours for r in routes: nextRouteIDs = deepcopy(nodes[routes[r].lastNode]) nextRouteIDs.remove(routes[r].id) previousRouteIDs = deepcopy(nodes[routes[r].firstNode]) previousRouteIDs.remove(routes[r].id) for rid in nextRouteIDs: routes[routes[r].id].nextRoutes.append(rid) for rid in previousRouteIDs: routes[routes[r].id].previousRoutes.append(rid) #remove unconnected tracks with highway=track and tracktype=grade1 and mtb:scale is null print time.strftime("%H:%M:%S", time.localtime()), " Removing disconnected tracks." routes = removeUnconnected(routes, nodes) print " Tracks removed." print time.strftime( "%H:%M:%S", time.localtime()), " Finding dangerous nodes (column warning)." # Find nodeIDs, where track's attribute mtb:scale changes rapidly (difference >= 2), # create new column warning in routes with the difference dangerNodes = findDangerousNodes(nodes, routes) pointCursor = connection.cursor() insertDangerNodes(dangerNodes, pointCursor) pointCursor.close() print time.strftime("%H:%M:%S", time.localtime()), " - neighbours are found." print " Determining offset for each route..." # Find offset polarity # listOfRoutes = routes.values() listOfRoutes = sorted(routes.values(), key=lambda route: route.osmcSigns[0], reverse=True) if len(listOfRoutes) > 1000: setrecursionlimit(len(listOfRoutes)) for r in listOfRoutes: # print "For cycle: ", r.id, r.osmcSigns[0] setOffset(routes, r.id, "next") setOffset(routes, r.id, "previous") print time.strftime("%H:%M:%S", time.localtime()), " - offset is found." print " Inserting of routes into new empty table routes..." # Determine maximum number of different osmcSymbols at one route maxSigns = 0 for r in routes.values(): if (maxSigns < r.numOfSigns): maxSigns = r.numOfSigns if maxSigns < 8: maxSigns = 8 # Prepare database table for data insertion auxiliaryCursor.execute(''' ALTER TABLE routes ADD "mtb:scale" text; ''') auxiliaryCursor.execute(''' ALTER TABLE routes ADD "mtb:scale:uphill" text; ''') auxiliaryCursor.execute(''' ALTER TABLE routes ADD "sac_scale" text; ''') auxiliaryCursor.execute(''' ALTER TABLE routes ADD offsetSide integer; ''') # Add columns for maximum number of osmcSymbols for column in range(maxSigns): auxiliaryCursor.execute(''' ALTER TABLE routes ADD osmcSymbol%s text; ''' % (str(column))) auxiliaryCursor.execute(''' ALTER TABLE routes ADD color%s text; ''' % (str(column))) auxiliaryCursor.execute(''' ALTER TABLE routes ADD colour%s text; ''' % (str(column))) auxiliaryCursor.execute(''' ALTER TABLE routes ADD kct_blue%s text; ''' % (str(column))) auxiliaryCursor.execute(''' ALTER TABLE routes ADD kct_green%s text; ''' % (str(column))) auxiliaryCursor.execute(''' ALTER TABLE routes ADD kct_yellow%s text; ''' % (str(column))) auxiliaryCursor.execute(''' ALTER TABLE routes ADD kct_red%s text; ''' % (str(column))) auxiliaryCursor.execute(''' ALTER TABLE routes ADD network%s text; ''' % (str(column))) auxiliaryCursor.execute(''' ALTER TABLE routes ADD route%s text; ''' % (str(column))) auxiliaryCursor.execute(''' ALTER TABLE routes ADD ref%s text; ''' % (str(column))) # Insert route values into the table for r in listOfRoutes: if r.geometry is not None: row = r.getValuesRow() auxiliaryCursor.execute(''' INSERT INTO routes VALUES (%s) ''' % (row)) print " Finished inserting routes into new table." auxiliaryCursor.execute(''' DROP TABLE IF EXISTS route_centroids; ''') auxiliaryCursor.execute(''' CREATE TABLE route_centroids AS SELECT H1.osm_id,St_Centroid(St_Envelope(H1.way)) AS centroid FROM routes H1 ''') auxiliaryCursor.execute(''' CREATE INDEX i__route_centroids__controid ON route_centroids USING GIST (centroid) ''') auxiliaryCursor.execute(''' DROP TABLE IF EXISTS route_density; ''') auxiliaryCursor.execute(''' CREATE TABLE route_density AS SELECT H1.osm_id AS osm_id,Count(H2.osm_id) AS density FROM route_centroids H1 LEFT JOIN route_centroids H2 ON ST_DWithin(H1.centroid,H2.centroid,1000) AND H1.osm_id <> H2.osm_id GROUP BY H1.osm_id ''') auxiliaryCursor.execute(''' CREATE INDEX i__route_density__osm_id ON route_density (osm_id) ''') auxiliaryCursor.execute(''' DROP TABLE IF EXISTS routes2 ''') auxiliaryCursor.execute(''' CREATE TABLE routes2 AS ( SELECT osm_id,route,way,offsetside,highway,tracktype,"mtb:scale","mtb:scale:uphill",sac_scale,color,network,oneway,rank() OVER (PARTITION BY osm_id ORDER BY (CASE WHEN route IS NULL THEN 100 WHEN color = 'violet' THEN 1 WHEN color = 'turquoise' THEN 2 WHEN color = 'red' THEN 3 WHEN color = 'green' THEN 4 WHEN color = 'blue' THEN 5 WHEN color = 'yellow' THEN 6 WHEN color = 'black' THEN 7 WHEN color = 'white' THEN 8 WHEN color = 'brown' THEN 9 WHEN color = 'orange' THEN 10 WHEN color = 'purple' THEN 11 END) ) AS offset,density,osmcsymbol FROM ( SELECT osm_id,way, offsetside,network,oneway, (CASE WHEN route IN ('bicycle','mtb') THEN 'bicycle' WHEN route IS NULL AND (osmcsymbol IS NOT NULL OR network IS NOT NULL OR ref IS NOT NULL) THEN 'hiking' WHEN route IN ('foot','hiking','mountain hiking') THEN 'hiking' ELSE route END) AS route, highway,tracktype,"mtb:scale","mtb:scale:uphill",sac_scale, (CASE WHEN route IN ('bicycle','mtb') THEN 'violet' WHEN route IN ('ski') THEN 'turquoise' WHEN COALESCE(kct_blue,'') <> '' THEN 'blue' WHEN COALESCE(kct_green,'') <> '' THEN 'green' WHEN COALESCE(kct_yellow,'') <> '' THEN 'yellow' WHEN COALESCE(kct_red,'') <> '' THEN 'red' WHEN osmcsymbol LIKE 'red:%' THEN 'red' WHEN osmcsymbol LIKE 'blue:%' THEN 'blue' WHEN osmcsymbol LIKE 'green:%' THEN 'green' WHEN osmcsymbol LIKE 'yellow:%' THEN 'yellow' WHEN osmcsymbol LIKE 'black:%' THEN 'black' WHEN osmcsymbol LIKE 'white:%' THEN 'white' WHEN osmcsymbol LIKE 'brown:%' THEN 'brown' WHEN osmcsymbol LIKE 'orange:%' THEN 'orange' WHEN osmcsymbol LIKE 'purple:%' THEN 'purple' WHEN COALESCE(colour,color,'') <> '' THEN COALESCE(colour,color) ELSE 'red' END) AS color, density, osmcsymbol FROM ( SELECT Y.osm_id,way, offsetside,highway,tracktype,"mtb:scale","mtb:scale:uphill",sac_scale,oneway, (CASE X.Which WHEN '0' THEN route0 WHEN '1' THEN route1 WHEN '2' THEN route2 WHEN '3' THEN route3 WHEN '4' THEN route4 WHEN '5' THEN route5 WHEN '6' THEN route6 WHEN '7' THEN route7 END) AS route, (CASE X.Which WHEN '0' THEN osmcsymbol0 WHEN '1' THEN osmcsymbol1 WHEN '2' THEN osmcsymbol2 WHEN '3' THEN osmcsymbol3 WHEN '4' THEN osmcsymbol4 WHEN '5' THEN osmcsymbol5 WHEN '6' THEN osmcsymbol6 WHEN '7' THEN osmcsymbol7 END) AS osmcsymbol, (CASE X.Which WHEN '0' THEN network0 WHEN '1' THEN network1 WHEN '2' THEN network2 WHEN '3' THEN network3 WHEN '4' THEN network4 WHEN '5' THEN network5 WHEN '6' THEN network6 WHEN '7' THEN network7 END) AS network, (CASE X.Which WHEN '0' THEN ref0 WHEN '1' THEN ref1 WHEN '2' THEN ref2 WHEN '3' THEN ref3 WHEN '4' THEN ref4 WHEN '5' THEN ref5 WHEN '6' THEN ref6 WHEN '7' THEN ref7 END) AS "ref", (CASE X.Which WHEN '0' THEN color0 WHEN '1' THEN color1 WHEN '2' THEN color2 WHEN '3' THEN color3 WHEN '4' THEN color4 WHEN '5' THEN color5 WHEN '6' THEN color6 WHEN '7' THEN color7 END) AS color, (CASE X.Which WHEN '0' THEN colour0 WHEN '1' THEN colour1 WHEN '2' THEN colour2 WHEN '3' THEN colour3 WHEN '4' THEN colour4 WHEN '5' THEN colour5 WHEN '6' THEN colour6 WHEN '7' THEN colour7 END) AS colour, (CASE X.Which WHEN '0' THEN kct_blue0 WHEN '1' THEN kct_blue1 WHEN '2' THEN kct_blue2 WHEN '3' THEN kct_blue3 WHEN '4' THEN kct_blue4 WHEN '5' THEN kct_blue5 WHEN '6' THEN kct_blue6 WHEN '7' THEN kct_blue7 END) AS kct_blue, (CASE X.Which WHEN '0' THEN kct_green0 WHEN '1' THEN kct_green1 WHEN '2' THEN kct_green2 WHEN '3' THEN kct_green3 WHEN '4' THEN kct_green4 WHEN '5' THEN kct_green5 WHEN '6' THEN kct_green6 WHEN '7' THEN kct_green7 END) AS kct_green, (CASE X.Which WHEN '0' THEN kct_yellow0 WHEN '1' THEN kct_yellow1 WHEN '2' THEN kct_yellow2 WHEN '3' THEN kct_yellow3 WHEN '4' THEN kct_yellow4 WHEN '5' THEN kct_yellow5 WHEN '6' THEN kct_yellow6 WHEN '7' THEN kct_yellow7 END) AS kct_yellow, (CASE X.Which WHEN '0' THEN kct_red0 WHEN '1' THEN kct_red1 WHEN '2' THEN kct_red2 WHEN '3' THEN kct_red3 WHEN '4' THEN kct_red4 WHEN '5' THEN kct_red5 WHEN '6' THEN kct_red6 WHEN '7' THEN kct_red7 END) AS kct_red, (SELECT Max(RD.density) FROM route_density RD WHERE RD.osm_id = Y.osm_id) AS density FROM routes Y CROSS JOIN (SELECT '0' UNION ALL SELECT '1' UNION ALL SELECT '2' UNION ALL SELECT '3' UNION ALL SELECT '4' UNION ALL SELECT '5' UNION ALL SELECT '6' UNION ALL SELECT '7' UNION ALL SELECT '8') X (Which) ) AS R ) R2 WHERE COALESCE(route,'') <> '' GROUP BY R2.osm_id,R2.way,R2.route,R2.offsetside,R2.highway,R2.tracktype,R2."mtb:scale",R2."mtb:scale:uphill",R2.sac_scale,R2.color,R2.density,R2.osmcsymbol,R2.network,R2.oneway ) ''') print "Relations: ", len(relations) print "max Signs: ", maxSigns print "Routes: ", len(routes) print "Nodes: ", len(nodes) print "Danger nodes:", len(dangerNodes) # print routes[39952857].nextRoutes, routes[44013159].previousRoutes # print nodes[559611826] # commit the result into the database auxiliaryCursor.close() connection.commit() print time.strftime( "%H:%M:%S", time.localtime()), " - Relations2lines finished successfully."
if state.state_data[blk].get_neighbors( ) != goal.state_data[blk].get_neighbors(): neighbor_diff += 1 sblk_x, sblk_y, sblk_z = state.state_data[blk].get_location() gblk_x, gblk_y, gblk_z = goal.state_data[blk].get_location() dist_diff += abs(gblk_x - sblk_x) + abs(gblk_y - sblk_y) + abs(gblk_z - sblk_z) return dist_diff + neighbor_diff if __name__ == "__main__": # Local variables to store state data initial_state = r.Relation('initial_state') goal_state = r.Relation('goal_state') # Populate local variables setup(initial_state, goal_state) # Conditionally print debug information if print_debug_flag: print("-------- INITIAL STATE --------") print(initial_state) print("\n\n-------- GOAL STATE --------") print(goal_state) # Verify initial and goal states if flag indicates to do so if validate_flag: # Verify initial state
def parser(text, D, N): url_get_base = "http://api.ltp-cloud.com/analysis/" args = { 'api_key': 'o9g0E8g50ZtwuZsmfx4A7juNyw0E0M7fic4dgHSK', 'text': text, 'pattern': 'dp', 'format': 'xml' } result = urllib.urlopen(url_get_base, urllib.urlencode(args)) # POST method content = result.read().strip(" ") # print content # 结果解析 relation_list = [] root = Etree.fromstring(content) word = root.findall('doc/para/sent/word') # 抽取6种依存关系:ADV, ATT, COO, SBV, VOB, CMP relates = ["ADV", "ATT", "COO", "SBV", "VOB", "CMP"] for w in word: relate = w.get("relate") if relates.__contains__(relate): cword = word[int(w.get("parent"))] mw = w.get("cont") # 修饰词 cw = cword.get("cont") # 支配词 pos_mw = w.get("pos") pos_cw = cword.get("pos") id_mw = w.get("id") id_cw = cword.get("id") dd_mw = False dd_cw = False nd_mw = False nd_cw = False # 判断是否程度副词 if pos_mw == "d" and D.has_key(mw): dd_mw = True if pos_cw == "d" and D.has_key(cw): dd_cw = True # 判断是否否定词 if pos_mw == "d" and N.__contains__(mw): nd_mw = True if pos_cw == "d" and N.__contains__(cw): nd_cw = True print mw, cw, relate, pos_mw, pos_cw, id_mw, id_cw, dd_mw, dd_cw, nd_mw, nd_cw, "\n" add_relation = relation.Relation(mw, cw, relate, pos_mw, pos_cw, id_mw, id_cw, dd_mw, dd_cw, nd_mw, nd_cw) relation_list.append(add_relation) # try: # except Exception: # print '异常',Exception return relation_list