def parse(database, statement): parser = Parser(statement) parser.expect("CREATE") parser.expect_optional("OR", "REPLACE") parser.expect("VIEW") viewName = parser.parse_identifier() columnsExist = parser.expect_optional("(") columnNames = list() if (columnsExist): while not parser.expect_optional(")"): columnNames.append( ParserUtils.get_object_name(parser.parse_identifier())) parser.expect_optional(",") parser.expect("AS") query = parser.get_rest() view = PgView(ParserUtils.get_object_name(viewName)) view.columnNames = columnNames view.query = query schemaName = ParserUtils.get_schema_name(viewName, database) schema = database.getSchema(schemaName) if schema is None: raise Exception("CannotFindSchema" % (schemaName, statement)) schema.addView(view)
def collect_rank(cid, start_date, end_date): params = Parser.get_params(cid, end_date=end_date, start_date=start_date) page = 1 params['page'] = page keyword_rank_list = Parser().get_keyword_rank(params, use_random_proxy=True) if len(keyword_rank_list) < 2: return for keyword_rank in keyword_rank_list[1]: DbConnector().print_rank(keyword_rank=keyword_rank, cid=cid, end_date=end_date, start_date=start_date)
def parse(database, statement): parser = Parser(statement) parser.expect("CREATE", "SEQUENCE") sequenceName = parser.parse_identifier() sequence = PgSequence(ParserUtils.get_object_name(sequenceName)) schemaName = ParserUtils.get_schema_name(sequenceName, database) schema = database.getSchema(schemaName) if schema is None: raise Exception( "Cannot find schema '%s' for statement '%s'. Missing CREATE SCHEMA statement?" % (schemaName, statement)) schema.addSequence(sequence) while not parser.expect_optional(";"): if parser.expect_optional("INCREMENT"): parser.expect_optional("BY") sequence.increment = parser.parse_string() elif parser.expect_optional("MINVALUE"): sequence.minValue = parser.parse_string() elif parser.expect_optional("MAXVALUE"): sequence.maxValue = parser.parse_string() elif parser.expect_optional("START"): parser.expect_optional("WITH") sequence.startWith = parser.parse_string() elif parser.expect_optional("CACHE"): sequence.cache = parser.parse_string() elif parser.expect_optional("CYCLE"): sequence.cycle = True elif parser.expect_optional("OWNED", "BY"): if parser.expect_optional("NONE"): sequence.ownedBy = None else: sequence.ownedBy = ParserUtils.get_object_name( parser.parse_identifier()) elif parser.expect_optional("NO"): if parser.expect_optional("MINVALUE"): sequence.minValue = None elif parser.expect_optional("MAXVALUE"): sequence.maxValue = None elif parser.expect_optional("CYCLE"): sequence.cycle = False else: parser.throw_unsupported_command() else: parser.throw_unsupported_command()
def parse(database, statement): parser = Parser(statement) parser.expect('CREATE', 'TABLE') # Optional IF NOT EXISTS, irrelevant for our purposes parser.expect_optional("IF", "NOT", "EXISTS") tableName = ParserUtils.get_object_name(parser.parse_identifier()) table = PgTable(tableName) # Figure it out why do we need this schemaName = ParserUtils.get_schema_name(tableName, database) schema = database.getSchema(schemaName) if schema is None: raise Exception( "Cannot find schema \'%s\' for statement \'%s\'. Missing CREATE SCHEMA statement?" % (schemaName, statement)) schema.addTable(table) parser.expect("(") while not parser.expect_optional(")"): if parser.expect_optional("CONSTRAINT"): CreateTableParser.parseConstraint(parser, table) else: CreateTableParser.parseColumn(parser, table) if parser.expect_optional(")"): break else: parser.expect(",") while not parser.expect_optional(";"): if parser.expect_optional("INHERITS"): CreateTableParser.parseInherits(parser, table) elif parser.expect_optional("WITHOUT"): table.oids = "OIDS=false" elif parser.expect_optional("WITH"): if (parser.expect_optional("OIDS") or parser.expect_optional("OIDS=true")): table.oids = "OIDS=true" elif parser.expect_optional("OIDS=false"): table.oids = "OIDS=false" else: print 'table.setWith(parser.getExpression())' elif parser.expect_optional("TABLESPACE"): print 'table.setTablespace(parser.parseString()' else: parser.throw_unsupported_command()
def parse(database, statement): parser = Parser(statement) parser.expect("ALTER", "VIEW") viewName = parser.parse_identifier() schemaName = ParserUtils.get_schema_name(viewName, database) schema = database.getSchema(schemaName) if schema is None: raise Exception( "Cannot find schema '%s' for statement '%s'. Missing CREATE SCHEMA statement?" % (schemaName, statement)) objectName = ParserUtils.get_object_name(viewName) view = schema.views[objectName] if view is None: raise Exception( "Cannot find view '%s' for statement '%s'. Missing CREATE VIEW statement?" % (viewName, statement)) while not parser.expect_optional(";"): if parser.expect_optional("ALTER"): parser.expect_optional("COLUMN") columnName = ParserUtils.get_object_name( parser.parse_identifier()) if parser.expect_optional("SET", "DEFAULT"): expression = parser.get_expression() view.addColumnDefaultValue(columnName, expression) elif parser.expect_optional("DROP", "DEFAULT"): view.removeColumnDefaultValue(columnName) else: parser.throw_unsupported_command() elif parser.expect_optional("OWNER", "TO"): # we do not parse this one so we just consume the identifier # if (outputIgnoredStatements) { # database.addIgnoredStatement("ALTER TABLE " + viewName # + " OWNER TO " + parser.parseIdentifier() + ';'); # } else { parser.parse_identifier() # } else: parser.throw_unsupported_command()
def main(): # Get file name file_name = parseArguments(sys.argv) # Read file lines = readFile(file_name) # Create parser object parser = Parser(lines) # Compute errors errors = [ line.parsingError for line in parser.parse() if line.parsingError ] # Print results if not errors: print("File is valid.") else: for error in errors: print(error, end='') print("File is not valid.")
def parse(database, statement): parser = Parser(statement) parser.expect("CREATE") unique = parser.expect_optional("UNIQUE") parser.expect("INDEX") parser.expect_optional("CONCURRENTLY") indexName = ParserUtils.get_object_name(parser.parse_identifier()) parser.expect("ON") tableName = parser.parse_identifier() definition = parser.get_rest() schemaName = ParserUtils.get_schema_name(tableName, database) schema = database.getSchema(schemaName) if (schema is None): print 'ERROR: CreateIndexParser[Line 21]' # throw new RuntimeException(MessageFormat.format( # Resources.getString("CannotFindSchema"), schemaName, # statement)); objectName = ParserUtils.get_object_name(tableName) table = schema.getTable(objectName) if (table is None): print 'ERROR: CreateIndexParser[Line 32]' # throw new RuntimeException(MessageFormat.format( # Resources.getString("CannotFindTable"), tableName, # statement)); index = PgIndex(indexName) table.addIndex(index) schema.addIndex(index) index.definition = definition.strip() index.tableName = table.name index.unique = unique
def collect_keyword_trend(cid, keyword, start_date="2017-08-01", end_date="2019-01-31"): params = Parser.get_params(cid, end_date=end_date, start_date=start_date) params['keyword'] = keyword shopping_param, click_trend_info_list = Parser().shopping_request( Parser.CATEGORY_CLICK_TREND, params, use_random_proxy=True) if len(click_trend_info_list) < 1: return click_trend_info_id = DbConnector().insert_info( table_name="click_trend_info", shopping_param=shopping_param, shopping_info_list=click_trend_info_list) for age_rate_info in click_trend_info_list: DbConnector().insert_trend(click_trend_info_id, age_rate_info.data_list)
def parse(urls, proxies, visibility=False): """ Запускаем программу """ queue = Queue() # Даем очереди нужные нам ссылки для скачивания for url in urls: queue.put(url) with open("cookies.pickle", "rb") as f: cookies = pickle.load(f) # Запускаем потом и очередь threads = [] for i in range(len(proxies)): t = Parser(queue, i, cookies, proxies[i], visibility=visibility) t.start() threads.append(t) # Ждем завершения работы очереди for t in threads: t.join()
def parse(database, statement): parser = Parser(statement) parser.expect("CREATE", "SCHEMA") if parser.expect_optional("AUTHORIZATION"): schema = PgSchema( ParserUtils.get_object_name(parser.parse_identifier())) database.addSchema(schema) schema.authorization = schema.name else: schemaName = ParserUtils.get_object_name(parser.parse_identifier()) schema = PgSchema(schemaName) database.schemas[schemaName] = schema if parser.expect_optional("AUTHORIZATION"): schema.authorization = ParserUtils.get_object_name( parser.parse_identifier()) definition = parser.get_rest() if definition: schema.definition = definition
def parse(database, statement): parser = Parser(statement) parser.expect("COMMENT", "ON") if parser.expect_optional("TABLE"): CommentParser.parseTable(parser, database) elif parser.expect_optional("COLUMN"): CommentParser.parseColumn(parser, database) elif parser.expect_optional("CONSTRAINT"): CommentParser.parseConstraint(parser, database) elif parser.expect_optional("DATABASE"): CommentParser.parseDatabase(parser, database) elif parser.expect_optional("FUNCTION"): CommentParser.parseFunction(parser, database) elif parser.expect_optional("INDEX"): CommentParser.parseIndex(parser, database) elif parser.expect_optional("SCHEMA"): CommentParser.parseSchema(parser, database) elif parser.expect_optional("SEQUENCE"): CommentParser.parseSequence(parser, database) elif parser.expect_optional("TRIGGER"): CommentParser.parseTrigger(parser, database) elif parser.expect_optional("VIEW"): CommentParser.parseView(parser, database)
def main(): logging.getLogger().setLevel(logging.INFO) etl_date = (datetime.now(pytz.timezone('Asia/Seoul')) - timedelta(days=1)).strftime('%Y-%m-%d') params = Parser().get_params("50001768,50001769", etl_date) age_rate_info_list = Parser().shopping_request(Parser.CATEGORY_AGE_RATE, params) logging.info(age_rate_info_list) click_trend_rate_info_list = Parser().shopping_request(Parser.CATEGORY_CLICK_TREND, params) logging.info(click_trend_rate_info_list) device_rate_info_list = Parser().shopping_request(Parser.CATEGORY_DEVICE_RATE, params) logging.info(device_rate_info_list) gender_rate_info_list = Parser().shopping_request(Parser.CATEGORY_GENDER_RATE, params) logging.info(gender_rate_info_list) range, keyword_rank = Parser().get_keyword_rank(params) logging.info(keyword_rank)
def parse(database, statement, ignoreSlonyTriggers): parser = Parser(statement) parser.expect("CREATE", "TRIGGER") triggerName = parser.parse_identifier() objectName = ParserUtils.get_object_name(triggerName) trigger = PgTrigger() trigger.name = objectName if parser.expect_optional("BEFORE"): trigger.event = PgTrigger.EVENT_BEFORE elif parser.expect_optional("AFTER"): trigger.event = PgTrigger.EVENT_AFTER elif parser.expect_optional("INSTEAD OF"): trigger.event = PgTrigger.EVENT_INSTEAD_OF first = True while True: if not first and not parser.expect_optional("OR"): break elif parser.expect_optional("INSERT"): trigger.onInsert = True elif parser.expect_optional("UPDATE"): trigger.onUpdate = True if parser.expect_optional("OF"): while True: trigger.updateColumns.append(parser.parse_identifier()) if not parser.expect_optional(","): break elif parser.expect_optional("DELETE"): trigger.onDelete = True elif parser.expect_optional("TRUNCATE"): trigger.onTruncate = True elif (first): break else: parser.throw_unsupported_command() first = False parser.expect("ON") trigger.tableName = ParserUtils.get_object_name(parser.parse_identifier()) if parser.expect_optional("FOR"): parser.expect_optional("EACH") if parser.expect_optional("ROW"): trigger.forEachRow = True elif parser.expect_optional("STATEMENT"): trigger.forEachRow = False else: parser.throw_unsupported_command() if parser.expect_optional("WHEN"): parser.expect("(") trigger.when = parser.get_expression() parser.expect(")") parser.expect("EXECUTE", "PROCEDURE") trigger.function = parser.get_rest() ignoreSlonyTrigger = ignoreSlonyTriggers and ("_slony_logtrigger" == trigger.name or "_slony_denyaccess" == trigger.name) if (not ignoreSlonyTrigger): schema = database.getSchema(ParserUtils.get_schema_name(trigger.tableName, database)) container = schema.tables.get(trigger.tableName) if not container: container = schema.views.get(trigger.tableName) if container: container.triggers[trigger.name] = trigger else: raise Exception()
from tokenizer.Readers import Reader from tokenizer.Readers import PosReader from tokenizer.Tokenizer import Tokenizer from tokenizer.TokenList import * from parser.Parser import Parser # r = Reader('{"1" : True, "3" : [1,2,3,{"4" : "Alter"}]}') # t = Tokenizer(r).getTokens() # for i in t.tokenList : # print(i.getType()) result = Parser('{"1" : True, "3" : [1,2,3,{"4" : "Alter"}]}').parse() print(result) print(result['3']) print(result['3'][3]) for i in result['3']: print(i) # # Example - 1 import urllib.request # request = urllib.request.Request('https://news-at.zhihu.com/api/4/news/latest') # Zhihu # resp = urllib.request.urlopen(request) # raw_data = resp.read() # data = raw_data.decode() # json_data = Parser(data).parse() # print(json_data) # # Example - 2 # request = urllib.request.Request('http://local.shiyiquan.net/message/global/?time_update=0&Ajax=true') # Shiyiquan # resp = urllib.request.urlopen(request) # raw_data = resp.read().decode('unicode-escape')
def parse(database, statement): parser = Parser(statement) parser.expect("CREATE") parser.expect_optional("OR", "REPLACE") parser.expect("FUNCTION") functionName = parser.parse_identifier() schemaName = ParserUtils.get_schema_name(functionName, database) schema = database.getSchema(schemaName) if schema is None: raise Exception( "Cannot find schema '%s' for statement '%s'. Missing CREATE SCHEMA statement?" % (schemaName, statement)) function = PgFunction() function.name = ParserUtils.get_object_name(functionName) parser.expect("(") while not parser.expect_optional(")"): mode = None if parser.expect_optional("IN"): mode = "IN" elif parser.expect_optional("OUT"): mode = "OUT" elif parser.expect_optional("INOUT"): mode = "INOUT" elif parser.expect_optional("VARIADIC"): mode = "VARIADIC" position = parser.position argumentName = None dataType = parser.parse_data_type() position2 = parser.position if (not parser.expect_optional(")") and not parser.expect_optional(",") and not parser.expect_optional("=") and not parser.expect_optional("DEFAULT")): parser.position = position argumentName = ParserUtils.get_object_name( parser.parse_identifier()) dataType = parser.parse_data_type() else: parser.position = position2 defaultExpression = '' if (parser.expect_optional("=") or parser.expect_optional("DEFAULT")): defaultExpression = parser.get_expression() else: defaultExpression = None argument = Argument() argument.dataType = dataType argument.defaultExpression = defaultExpression argument.mode = mode argument.name = argumentName function.addArgument(argument) if (parser.expect_optional(")")): break else: parser.expect(",") function.body = parser.get_rest() schema.addFunction(function)
def setUp(self): self.parser = Parser("STATEMENT FOR TEST")
def parse(database, statement): parser = Parser(statement) parser.expect("ALTER", "TABLE") parser.expect_optional("ONLY") tableName = parser.parse_identifier() schemaName = ParserUtils.get_schema_name(tableName, database) schema = database.getSchema(schemaName) if schema is None: raise Exception("CannotFindSchema") objectName = ParserUtils.get_object_name(tableName) table = schema.tables.get(objectName) if table is None: logging.debug('Table %s was not fount in schema %s' % (tableName, schema)) view = schema.views.get(objectName) if view is not None: AlterTableParser.parseView(parser, view, tableName, database) return sequence = schema.sequences.get(objectName) if sequence is not None: AlterTableParser.parseSequence(parser, sequence, tableName, database) return raise Exception("Cannot find object '%s' for statement '%s'." % (tableName, statement)) while not parser.expect_optional(";"): if parser.expect_optional("ALTER"): AlterTableParser.parseAlterColumn(parser, table) elif (parser.expect_optional("CLUSTER", "ON")): table.clusterIndexName = ParserUtils.get_object_name( parser.parse_identifier()) elif (parser.expect_optional("OWNER", "TO")): # we do not parse this one so we just consume the identifier # if (outputIgnoredStatements): # print 'database.addIgnoredStatement("ALTER TABLE " + tableName + " OWNER TO " + parser.parseIdentifier() + ';')' # else: parser.parse_identifier() elif (parser.expect_optional("ADD")): if (parser.expect_optional("FOREIGN", "KEY")): print 'parseAddForeignKey(parser, table);' elif (parser.expect_optional("CONSTRAINT")): AlterTableParser.parseAddConstraint(parser, table, schema) else: parser.throw_unsupported_command() elif (parser.expect_optional("ENABLE")): print 'parseEnable(parser, outputIgnoredStatements, tableName, database);' elif (parser.expect_optional("DISABLE")): print 'parseDisable(parser, outputIgnoredStatements, tableName, database);' else: parser.throw_unsupported_command() if (parser.expect_optional(";")): break else: parser.expect(",")
import sys #sys.path.insert(0,'/home/youssefsb/VS_workspace/Comp442/Comp442/parser/Grammar') import graphviz from subprocess import call from parser.Parser import Parser from parser.Grammar.Grammar import Grammar from semanticAnalyzer.SemAnalyzer import SemAnalyzer grammar = Grammar('parser/Grammar/outfile.grm', 'parser/Grammar/outfile.grm.first', 'parser/Grammar/outfile.grm.follow') parser = Parser(grammar, 'testCases/mainOnly.src') semAnalyzer = SemAnalyzer() parser.parse() globalSymTable = semAnalyzer.analyzeTree(parser.astRoot) print(str(globalSymTable.getTypeSizesHashMap())) globalSymTable.symTableAsHtml() call(["moon/moon", "moon/outFile.m", "moon/lib.m"])