def parse(database, statement):
parser = Parser(statement)
parser.expect("CREATE")
parser.expect_optional("OR", "REPLACE")
parser.expect("VIEW")
viewName = parser.parse_identifier()
columnsExist = parser.expect_optional("(")
columnNames = list()
if (columnsExist):
while not parser.expect_optional(")"):
columnNames.append(
ParserUtils.get_object_name(parser.parse_identifier()))
parser.expect_optional(",")
parser.expect("AS")
query = parser.get_rest()
view = PgView(ParserUtils.get_object_name(viewName))
view.columnNames = columnNames
view.query = query
schemaName = ParserUtils.get_schema_name(viewName, database)
schema = database.getSchema(schemaName)
if schema is None:
raise Exception("CannotFindSchema" % (schemaName, statement))
schema.addView(view)
def collect_rank(cid, start_date, end_date):
params = Parser.get_params(cid, end_date=end_date, start_date=start_date)
page = 1
params['page'] = page
keyword_rank_list = Parser().get_keyword_rank(params,
use_random_proxy=True)
if len(keyword_rank_list) < 2:
return
for keyword_rank in keyword_rank_list[1]:
DbConnector().print_rank(keyword_rank=keyword_rank,
cid=cid,
end_date=end_date,
start_date=start_date)
def parse(database, statement):
parser = Parser(statement)
parser.expect("CREATE", "SEQUENCE")
sequenceName = parser.parse_identifier()
sequence = PgSequence(ParserUtils.get_object_name(sequenceName))
schemaName = ParserUtils.get_schema_name(sequenceName, database)
schema = database.getSchema(schemaName)
if schema is None:
raise Exception(
"Cannot find schema '%s' for statement '%s'. Missing CREATE SCHEMA statement?"
% (schemaName, statement))
schema.addSequence(sequence)
while not parser.expect_optional(";"):
if parser.expect_optional("INCREMENT"):
parser.expect_optional("BY")
sequence.increment = parser.parse_string()
elif parser.expect_optional("MINVALUE"):
sequence.minValue = parser.parse_string()
elif parser.expect_optional("MAXVALUE"):
sequence.maxValue = parser.parse_string()
elif parser.expect_optional("START"):
parser.expect_optional("WITH")
sequence.startWith = parser.parse_string()
elif parser.expect_optional("CACHE"):
sequence.cache = parser.parse_string()
elif parser.expect_optional("CYCLE"):
sequence.cycle = True
elif parser.expect_optional("OWNED", "BY"):
if parser.expect_optional("NONE"):
sequence.ownedBy = None
else:
sequence.ownedBy = ParserUtils.get_object_name(
parser.parse_identifier())
elif parser.expect_optional("NO"):
if parser.expect_optional("MINVALUE"):
sequence.minValue = None
elif parser.expect_optional("MAXVALUE"):
sequence.maxValue = None
elif parser.expect_optional("CYCLE"):
sequence.cycle = False
else:
parser.throw_unsupported_command()
else:
parser.throw_unsupported_command()
def parse(database, statement):
parser = Parser(statement)
parser.expect('CREATE', 'TABLE')
# Optional IF NOT EXISTS, irrelevant for our purposes
parser.expect_optional("IF", "NOT", "EXISTS")
tableName = ParserUtils.get_object_name(parser.parse_identifier())
table = PgTable(tableName)
# Figure it out why do we need this
schemaName = ParserUtils.get_schema_name(tableName, database)
schema = database.getSchema(schemaName)
if schema is None:
raise Exception(
"Cannot find schema \'%s\' for statement \'%s\'. Missing CREATE SCHEMA statement?"
% (schemaName, statement))
schema.addTable(table)
parser.expect("(")
while not parser.expect_optional(")"):
if parser.expect_optional("CONSTRAINT"):
CreateTableParser.parseConstraint(parser, table)
else:
CreateTableParser.parseColumn(parser, table)
if parser.expect_optional(")"):
break
else:
parser.expect(",")
while not parser.expect_optional(";"):
if parser.expect_optional("INHERITS"):
CreateTableParser.parseInherits(parser, table)
elif parser.expect_optional("WITHOUT"):
table.oids = "OIDS=false"
elif parser.expect_optional("WITH"):
if (parser.expect_optional("OIDS")
or parser.expect_optional("OIDS=true")):
table.oids = "OIDS=true"
elif parser.expect_optional("OIDS=false"):
table.oids = "OIDS=false"
else:
print 'table.setWith(parser.getExpression())'
elif parser.expect_optional("TABLESPACE"):
print 'table.setTablespace(parser.parseString()'
else:
parser.throw_unsupported_command()
def parse(database, statement):
parser = Parser(statement)
parser.expect("ALTER", "VIEW")
viewName = parser.parse_identifier()
schemaName = ParserUtils.get_schema_name(viewName, database)
schema = database.getSchema(schemaName)
if schema is None:
raise Exception(
"Cannot find schema '%s' for statement '%s'. Missing CREATE SCHEMA statement?"
% (schemaName, statement))
objectName = ParserUtils.get_object_name(viewName)
view = schema.views[objectName]
if view is None:
raise Exception(
"Cannot find view '%s' for statement '%s'. Missing CREATE VIEW statement?"
% (viewName, statement))
while not parser.expect_optional(";"):
if parser.expect_optional("ALTER"):
parser.expect_optional("COLUMN")
columnName = ParserUtils.get_object_name(
parser.parse_identifier())
if parser.expect_optional("SET", "DEFAULT"):
expression = parser.get_expression()
view.addColumnDefaultValue(columnName, expression)
elif parser.expect_optional("DROP", "DEFAULT"):
view.removeColumnDefaultValue(columnName)
else:
parser.throw_unsupported_command()
elif parser.expect_optional("OWNER", "TO"):
# we do not parse this one so we just consume the identifier
# if (outputIgnoredStatements) {
# database.addIgnoredStatement("ALTER TABLE " + viewName
# + " OWNER TO " + parser.parseIdentifier() + ';');
# } else {
parser.parse_identifier()
# }
else:
parser.throw_unsupported_command()
def main():
# Get file name
file_name = parseArguments(sys.argv)
# Read file
lines = readFile(file_name)
# Create parser object
parser = Parser(lines)
# Compute errors
errors = [
line.parsingError for line in parser.parse() if line.parsingError
]
# Print results
if not errors:
print("File is valid.")
else:
for error in errors:
print(error, end='')
print("File is not valid.")
def parse(database, statement):
parser = Parser(statement)
parser.expect("CREATE")
unique = parser.expect_optional("UNIQUE")
parser.expect("INDEX")
parser.expect_optional("CONCURRENTLY")
indexName = ParserUtils.get_object_name(parser.parse_identifier())
parser.expect("ON")
tableName = parser.parse_identifier()
definition = parser.get_rest()
schemaName = ParserUtils.get_schema_name(tableName, database)
schema = database.getSchema(schemaName)
if (schema is None):
print 'ERROR: CreateIndexParser[Line 21]'
# throw new RuntimeException(MessageFormat.format(
# Resources.getString("CannotFindSchema"), schemaName,
# statement));
objectName = ParserUtils.get_object_name(tableName)
table = schema.getTable(objectName)
if (table is None):
print 'ERROR: CreateIndexParser[Line 32]'
# throw new RuntimeException(MessageFormat.format(
# Resources.getString("CannotFindTable"), tableName,
# statement));
index = PgIndex(indexName)
table.addIndex(index)
schema.addIndex(index)
index.definition = definition.strip()
index.tableName = table.name
index.unique = unique
def collect_keyword_trend(cid,
keyword,
start_date="2017-08-01",
end_date="2019-01-31"):
params = Parser.get_params(cid, end_date=end_date, start_date=start_date)
params['keyword'] = keyword
shopping_param, click_trend_info_list = Parser().shopping_request(
Parser.CATEGORY_CLICK_TREND, params, use_random_proxy=True)
if len(click_trend_info_list) < 1:
return
click_trend_info_id = DbConnector().insert_info(
table_name="click_trend_info",
shopping_param=shopping_param,
shopping_info_list=click_trend_info_list)
for age_rate_info in click_trend_info_list:
DbConnector().insert_trend(click_trend_info_id,
age_rate_info.data_list)
def parse(urls, proxies, visibility=False):
"""
Запускаем программу
"""
queue = Queue()
# Даем очереди нужные нам ссылки для скачивания
for url in urls:
queue.put(url)
with open("cookies.pickle", "rb") as f:
cookies = pickle.load(f)
# Запускаем потом и очередь
threads = []
for i in range(len(proxies)):
t = Parser(queue, i, cookies, proxies[i], visibility=visibility)
t.start()
threads.append(t)
# Ждем завершения работы очереди
for t in threads:
t.join()
def parse(database, statement):
parser = Parser(statement)
parser.expect("CREATE", "SCHEMA")
if parser.expect_optional("AUTHORIZATION"):
schema = PgSchema(
ParserUtils.get_object_name(parser.parse_identifier()))
database.addSchema(schema)
schema.authorization = schema.name
else:
schemaName = ParserUtils.get_object_name(parser.parse_identifier())
schema = PgSchema(schemaName)
database.schemas[schemaName] = schema
if parser.expect_optional("AUTHORIZATION"):
schema.authorization = ParserUtils.get_object_name(
parser.parse_identifier())
definition = parser.get_rest()
if definition:
schema.definition = definition
def parse(database, statement):
parser = Parser(statement)
parser.expect("COMMENT", "ON")
if parser.expect_optional("TABLE"):
CommentParser.parseTable(parser, database)
elif parser.expect_optional("COLUMN"):
CommentParser.parseColumn(parser, database)
elif parser.expect_optional("CONSTRAINT"):
CommentParser.parseConstraint(parser, database)
elif parser.expect_optional("DATABASE"):
CommentParser.parseDatabase(parser, database)
elif parser.expect_optional("FUNCTION"):
CommentParser.parseFunction(parser, database)
elif parser.expect_optional("INDEX"):
CommentParser.parseIndex(parser, database)
elif parser.expect_optional("SCHEMA"):
CommentParser.parseSchema(parser, database)
elif parser.expect_optional("SEQUENCE"):
CommentParser.parseSequence(parser, database)
elif parser.expect_optional("TRIGGER"):
CommentParser.parseTrigger(parser, database)
elif parser.expect_optional("VIEW"):
CommentParser.parseView(parser, database)
def main():
logging.getLogger().setLevel(logging.INFO)
etl_date = (datetime.now(pytz.timezone('Asia/Seoul')) - timedelta(days=1)).strftime('%Y-%m-%d')
params = Parser().get_params("50001768,50001769", etl_date)
age_rate_info_list = Parser().shopping_request(Parser.CATEGORY_AGE_RATE, params)
logging.info(age_rate_info_list)
click_trend_rate_info_list = Parser().shopping_request(Parser.CATEGORY_CLICK_TREND, params)
logging.info(click_trend_rate_info_list)
device_rate_info_list = Parser().shopping_request(Parser.CATEGORY_DEVICE_RATE, params)
logging.info(device_rate_info_list)
gender_rate_info_list = Parser().shopping_request(Parser.CATEGORY_GENDER_RATE, params)
logging.info(gender_rate_info_list)
range, keyword_rank = Parser().get_keyword_rank(params)
logging.info(keyword_rank)
def parse(database, statement, ignoreSlonyTriggers):
parser = Parser(statement)
parser.expect("CREATE", "TRIGGER")
triggerName = parser.parse_identifier()
objectName = ParserUtils.get_object_name(triggerName)
trigger = PgTrigger()
trigger.name = objectName
if parser.expect_optional("BEFORE"):
trigger.event = PgTrigger.EVENT_BEFORE
elif parser.expect_optional("AFTER"):
trigger.event = PgTrigger.EVENT_AFTER
elif parser.expect_optional("INSTEAD OF"):
trigger.event = PgTrigger.EVENT_INSTEAD_OF
first = True
while True:
if not first and not parser.expect_optional("OR"):
break
elif parser.expect_optional("INSERT"):
trigger.onInsert = True
elif parser.expect_optional("UPDATE"):
trigger.onUpdate = True
if parser.expect_optional("OF"):
while True:
trigger.updateColumns.append(parser.parse_identifier())
if not parser.expect_optional(","):
break
elif parser.expect_optional("DELETE"):
trigger.onDelete = True
elif parser.expect_optional("TRUNCATE"):
trigger.onTruncate = True
elif (first):
break
else:
parser.throw_unsupported_command()
first = False
parser.expect("ON")
trigger.tableName = ParserUtils.get_object_name(parser.parse_identifier())
if parser.expect_optional("FOR"):
parser.expect_optional("EACH")
if parser.expect_optional("ROW"):
trigger.forEachRow = True
elif parser.expect_optional("STATEMENT"):
trigger.forEachRow = False
else:
parser.throw_unsupported_command()
if parser.expect_optional("WHEN"):
parser.expect("(")
trigger.when = parser.get_expression()
parser.expect(")")
parser.expect("EXECUTE", "PROCEDURE")
trigger.function = parser.get_rest()
ignoreSlonyTrigger = ignoreSlonyTriggers and ("_slony_logtrigger" == trigger.name or "_slony_denyaccess" == trigger.name)
if (not ignoreSlonyTrigger):
schema = database.getSchema(ParserUtils.get_schema_name(trigger.tableName, database))
container = schema.tables.get(trigger.tableName)
if not container:
container = schema.views.get(trigger.tableName)
if container:
container.triggers[trigger.name] = trigger
else:
raise Exception()
from tokenizer.Readers import Reader
from tokenizer.Readers import PosReader
from tokenizer.Tokenizer import Tokenizer
from tokenizer.TokenList import *
from parser.Parser import Parser
# r = Reader('{"1" : True, "3" : [1,2,3,{"4" : "Alter"}]}')
# t = Tokenizer(r).getTokens()
# for i in t.tokenList :
# print(i.getType())
result = Parser('{"1" : True, "3" : [1,2,3,{"4" : "Alter"}]}').parse()
print(result)
print(result['3'])
print(result['3'][3])
for i in result['3']:
print(i)
# # Example - 1
import urllib.request
# request = urllib.request.Request('https://news-at.zhihu.com/api/4/news/latest') # Zhihu
# resp = urllib.request.urlopen(request)
# raw_data = resp.read()
# data = raw_data.decode()
# json_data = Parser(data).parse()
# print(json_data)
# # Example - 2
# request = urllib.request.Request('http://local.shiyiquan.net/message/global/?time_update=0&Ajax=true') # Shiyiquan
# resp = urllib.request.urlopen(request)
# raw_data = resp.read().decode('unicode-escape')
def parse(database, statement):
parser = Parser(statement)
parser.expect("CREATE")
parser.expect_optional("OR", "REPLACE")
parser.expect("FUNCTION")
functionName = parser.parse_identifier()
schemaName = ParserUtils.get_schema_name(functionName, database)
schema = database.getSchema(schemaName)
if schema is None:
raise Exception(
"Cannot find schema '%s' for statement '%s'. Missing CREATE SCHEMA statement?"
% (schemaName, statement))
function = PgFunction()
function.name = ParserUtils.get_object_name(functionName)
parser.expect("(")
while not parser.expect_optional(")"):
mode = None
if parser.expect_optional("IN"):
mode = "IN"
elif parser.expect_optional("OUT"):
mode = "OUT"
elif parser.expect_optional("INOUT"):
mode = "INOUT"
elif parser.expect_optional("VARIADIC"):
mode = "VARIADIC"
position = parser.position
argumentName = None
dataType = parser.parse_data_type()
position2 = parser.position
if (not parser.expect_optional(")")
and not parser.expect_optional(",")
and not parser.expect_optional("=")
and not parser.expect_optional("DEFAULT")):
parser.position = position
argumentName = ParserUtils.get_object_name(
parser.parse_identifier())
dataType = parser.parse_data_type()
else:
parser.position = position2
defaultExpression = ''
if (parser.expect_optional("=")
or parser.expect_optional("DEFAULT")):
defaultExpression = parser.get_expression()
else:
defaultExpression = None
argument = Argument()
argument.dataType = dataType
argument.defaultExpression = defaultExpression
argument.mode = mode
argument.name = argumentName
function.addArgument(argument)
if (parser.expect_optional(")")):
break
else:
parser.expect(",")
function.body = parser.get_rest()
schema.addFunction(function)
def setUp(self):
self.parser = Parser("STATEMENT FOR TEST")
def parse(database, statement):
parser = Parser(statement)
parser.expect("ALTER", "TABLE")
parser.expect_optional("ONLY")
tableName = parser.parse_identifier()
schemaName = ParserUtils.get_schema_name(tableName, database)
schema = database.getSchema(schemaName)
if schema is None:
raise Exception("CannotFindSchema")
objectName = ParserUtils.get_object_name(tableName)
table = schema.tables.get(objectName)
if table is None:
logging.debug('Table %s was not fount in schema %s' %
(tableName, schema))
view = schema.views.get(objectName)
if view is not None:
AlterTableParser.parseView(parser, view, tableName, database)
return
sequence = schema.sequences.get(objectName)
if sequence is not None:
AlterTableParser.parseSequence(parser, sequence, tableName,
database)
return
raise Exception("Cannot find object '%s' for statement '%s'." %
(tableName, statement))
while not parser.expect_optional(";"):
if parser.expect_optional("ALTER"):
AlterTableParser.parseAlterColumn(parser, table)
elif (parser.expect_optional("CLUSTER", "ON")):
table.clusterIndexName = ParserUtils.get_object_name(
parser.parse_identifier())
elif (parser.expect_optional("OWNER", "TO")):
# we do not parse this one so we just consume the identifier
# if (outputIgnoredStatements):
# print 'database.addIgnoredStatement("ALTER TABLE " + tableName + " OWNER TO " + parser.parseIdentifier() + ';')'
# else:
parser.parse_identifier()
elif (parser.expect_optional("ADD")):
if (parser.expect_optional("FOREIGN", "KEY")):
print 'parseAddForeignKey(parser, table);'
elif (parser.expect_optional("CONSTRAINT")):
AlterTableParser.parseAddConstraint(parser, table, schema)
else:
parser.throw_unsupported_command()
elif (parser.expect_optional("ENABLE")):
print 'parseEnable(parser, outputIgnoredStatements, tableName, database);'
elif (parser.expect_optional("DISABLE")):
print 'parseDisable(parser, outputIgnoredStatements, tableName, database);'
else:
parser.throw_unsupported_command()
if (parser.expect_optional(";")):
break
else:
parser.expect(",")
import sys
#sys.path.insert(0,'/home/youssefsb/VS_workspace/Comp442/Comp442/parser/Grammar')
import graphviz
from subprocess import call
from parser.Parser import Parser
from parser.Grammar.Grammar import Grammar
from semanticAnalyzer.SemAnalyzer import SemAnalyzer
grammar = Grammar('parser/Grammar/outfile.grm',
'parser/Grammar/outfile.grm.first',
'parser/Grammar/outfile.grm.follow')
parser = Parser(grammar, 'testCases/mainOnly.src')
semAnalyzer = SemAnalyzer()
parser.parse()
globalSymTable = semAnalyzer.analyzeTree(parser.astRoot)
print(str(globalSymTable.getTypeSizesHashMap()))
globalSymTable.symTableAsHtml()
call(["moon/moon", "moon/outFile.m", "moon/lib.m"])