def test_age_rate_parsing(mocker):
with open("%s/../resources/age_rate_sample.txt" %
os.path.dirname(os.path.abspath(__file__))) as f:
mocker.patch.object(Parser, "datalab_api_call")
Parser.datalab_api_call.return_value = json.loads(f.readlines()[0])
params = Parser.get_params("50001768", "2019-02-01")
shopping_param, age_rate_info_list = Parser.shopping_request(
Parser.CATEGORY_AGE_RATE, params)
assert len(age_rate_info_list) == 1
age_rate_info: ShoppingInfo = age_rate_info_list[0]
age_rate_list: list = age_rate_info.data_list
assert len(age_rate_list) == 6
assert age_rate_list[0].code == "10"
assert age_rate_list[0].label == "10대"
assert age_rate_list[0].ratio == 1.83823
assert age_rate_list[len(age_rate_list) - 1].code == "60"
assert age_rate_list[len(age_rate_list) - 1].label == "60대"
assert age_rate_list[len(age_rate_list) - 1].ratio == 8.61796
assert age_rate_info.code is None
assert age_rate_info.full_title is None
assert age_rate_info.title == "50001768"
def test_device_rate_parsing(mocker):
with open("%s/../resources/device_rate_sample.txt" %
os.path.dirname(os.path.abspath(__file__))) as f:
mocker.patch.object(Parser, "datalab_api_call")
Parser.datalab_api_call.return_value = json.loads(f.readlines()[0])
params = Parser.get_params("50001768", "2019-02-01")
shopping_param, device_rate_info_list = Parser.shopping_request(
Parser.CATEGORY_DEVICE_RATE, params)
assert len(device_rate_info_list) == 1
device_rate_info: ShoppingInfo = device_rate_info_list[0]
device_rate_info_list: list = device_rate_info.data_list
assert len(device_rate_info_list) == 2
assert device_rate_info_list[0].code == "mo"
assert device_rate_info_list[0].label == "모바일"
assert device_rate_info_list[0].ratio == 100.0
assert device_rate_info_list[len(device_rate_info_list) -
1].code == "pc"
assert device_rate_info_list[len(device_rate_info_list) -
1].label == "PC"
assert device_rate_info_list[len(device_rate_info_list) -
1].ratio == 14.39998
assert device_rate_info.code is None
assert device_rate_info.full_title is None
assert device_rate_info.title == "50001768"
def parse(database, statement):
parser = Parser(statement)
parser.expect("ALTER", "SEQUENCE")
sequenceName = parser.parseIdentifier()
schemaName = ParserUtils.getSchemaName(sequenceName, database)
schema = database.getSchema(schemaName)
if schema is None:
raise Exception("CannotFindSchema")
objectName = ParserUtils.getObjectName(sequenceName);
sequence = schema.sequences[objectName]
if sequence is None:
raise Exception("Cannot find sequence '%s' for statement '%s'. Missing CREATE SEQUENCE?" % (sequenceName, statement))
while not parser.expectOptional(";"):
if (parser.expectOptional("OWNED", "BY")):
if parser.expectOptional("NONE"):
sequence.ownedBy = None
else:
sequence.ownedBy = parser.getExpression()
else:
parser.throwUnsupportedCommand()
def create(self,configfile):
Config.load_config(configfile)
sortedEvents = SortedRawAwareEvents()
sortedEvents.get_events_from_SQLite()
#sortedEvents.get_events_from_MySQL()
print "sorted"
parser = Parser()
result = list()
temp_learning_element = LearningSetElement()
i =0
for event in sortedEvents:
learning_element = temp_learning_element.clone2()
parsed_events = parser.parse_event(event)
if self.filter_out(parsed_events):
continue
UpdateElement.update(learning_element,parsed_events)
result.append(learning_element)
temp_learning_element = learning_element
i += 1
percent = float(i)/len(sortedEvents)*100
sys.stdout.write('%s percent \r' %( percent ))
sys.stdout.flush()
return result
def test_gender_rate_parsing(mocker):
with open("%s/../resources/gender_rate_sample.txt" %
os.path.dirname(os.path.abspath(__file__))) as f:
mocker.patch.object(Parser, "datalab_api_call")
Parser.datalab_api_call.return_value = json.loads(f.readlines()[0])
params = Parser.get_params("50001768", "2019-02-01")
shopping_param, gender_rate_info_list = Parser.shopping_request(
Parser.CATEGORY_GENDER_RATE, params)
assert len(gender_rate_info_list) == 1
gender_rate_info: ShoppingInfo = gender_rate_info_list[0]
gender_rate_list: list = gender_rate_info.data_list
assert len(gender_rate_list) == 2
assert gender_rate_list[0].code == "f"
assert gender_rate_list[0].label == "여성"
assert gender_rate_list[0].ratio == 59.05537
assert gender_rate_list[len(gender_rate_list) - 1].code == "m"
assert gender_rate_list[len(gender_rate_list) - 1].label == "남성"
assert gender_rate_list[len(gender_rate_list) - 1].ratio == 100.0
assert gender_rate_info.code is None
assert gender_rate_info.full_title is None
assert gender_rate_info.title == "50001768"
assert True
def test_keyword_rank_parsing(mocker):
with open("%s/../resources/keyword_rank_sample.txt" %
os.path.dirname(os.path.abspath(__file__))) as f:
mocker.patch.object(Parser, "datalab_api_call")
Parser.datalab_api_call.return_value = json.loads(f.readlines()[0])
params = Parser.get_params("50001768", "2019-02-01")
keyword_rank_list = Parser.get_keyword_rank(params)
assert len(keyword_rank_list) == 20
keyword_rank: KeywordRank = keyword_rank_list[0]
assert keyword_rank.keyword == "등산화"
assert keyword_rank.link_id == "등산화"
def collect_rank(cid, start_date, end_date):
params = Parser.get_params(cid, end_date=end_date, start_date=start_date)
page = 1
params['page'] = page
keyword_rank_list = Parser().get_keyword_rank(params,
use_random_proxy=True)
if len(keyword_rank_list) < 2:
return
for keyword_rank in keyword_rank_list[1]:
DbConnector().print_rank(keyword_rank=keyword_rank,
cid=cid,
end_date=end_date,
start_date=start_date)
def main():
# Get file name
file_name = parseArguments(sys.argv)
# Read file
lines = readFile(file_name)
# Create parser object
parser = Parser(lines)
# Compute errors
errors = [
line.parsingError for line in parser.parse() if line.parsingError
]
# Print results
if not errors:
print("File is valid.")
else:
for error in errors:
print(error, end='')
print("File is not valid.")
def collect_keyword_trend(cid,
keyword,
start_date="2017-08-01",
end_date="2019-01-31"):
params = Parser.get_params(cid, end_date=end_date, start_date=start_date)
params['keyword'] = keyword
shopping_param, click_trend_info_list = Parser().shopping_request(
Parser.CATEGORY_CLICK_TREND, params, use_random_proxy=True)
if len(click_trend_info_list) < 1:
return
click_trend_info_id = DbConnector().insert_info(
table_name="click_trend_info",
shopping_param=shopping_param,
shopping_info_list=click_trend_info_list)
for age_rate_info in click_trend_info_list:
DbConnector().insert_trend(click_trend_info_id,
age_rate_info.data_list)
def test_click_trend_parsing(mocker):
with open("%s/../resources/click_trend_sample.txt" %
os.path.dirname(os.path.abspath(__file__))) as f:
mocker.patch.object(Parser, "datalab_api_call")
Parser.datalab_api_call.return_value = json.loads(f.readlines()[0])
params = Parser.get_params("50001768", "2019-02-01")
shopping_param, click_trend_info_list = Parser.shopping_request(
Parser.CATEGORY_CLICK_TREND, params)
assert len(click_trend_info_list) == 1
click_trend_info: ShoppingInfo = click_trend_info_list[0]
click_trend_list: list = click_trend_info.data_list
assert len(click_trend_list) == 549
assert click_trend_list[0].period == "20170801"
assert click_trend_list[0].value == 22.82894
assert click_trend_list[len(click_trend_list) - 1].period == "20190131"
assert click_trend_list[len(click_trend_list) - 1].value == 21.27836
assert click_trend_info.code == "50001768"
assert click_trend_info.full_title == "스포츠/레저 > 등산 > 등산화"
assert click_trend_info.title == "등산화"
def parse(database, statement):
parser = Parser(statement)
parser.expect("CREATE", "SCHEMA")
if parser.expect_optional("AUTHORIZATION"):
schema = PgSchema(
ParserUtils.get_object_name(parser.parse_identifier()))
database.addSchema(schema)
schema.authorization = schema.name
else:
schemaName = ParserUtils.get_object_name(parser.parse_identifier())
schema = PgSchema(schemaName)
database.schemas[schemaName] = schema
if parser.expect_optional("AUTHORIZATION"):
schema.authorization = ParserUtils.get_object_name(
parser.parse_identifier())
definition = parser.get_rest()
if definition:
schema.definition = definition
def parse(database, statement):
parser = Parser(statement)
parser.expect("CREATE", "SCHEMA")
if parser.expectOptional("AUTHORIZATION"):
schema = PgSchema(ParserUtils.getObjectName(parser.parseIdentifier()))
database.addSchema(schema)
schema.authorization = schema.name
else:
schemaName = ParserUtils.getObjectName(parser.parseIdentifier())
schema = PgSchema(schemaName)
database.schemas[schemaName] = schema
if parser.expectOptional("AUTHORIZATION"):
schema.authorization = ParserUtils.getObjectName(parser.parseIdentifier())
definition = parser.getRest()
if definition:
schema.definition = definition
def parse(self, result: str, url: str) -> Data:
"""
Get title and links from html
Save parsed data to DB
:param result: html
:param url: source url
:return: parsed data
"""
data: Data = None
if result:
data = Parser.parse(result, url)
if data:
self.__data_dao.save_data(
ParseData(url=data.uri,
title=data.title,
html=data.html,
created=datetime.now(),
task_id=self.__task_id))
return data
def main():
logging.getLogger().setLevel(logging.INFO)
etl_date = (datetime.now(pytz.timezone('Asia/Seoul')) - timedelta(days=1)).strftime('%Y-%m-%d')
params = Parser().get_params("50001768,50001769", etl_date)
age_rate_info_list = Parser().shopping_request(Parser.CATEGORY_AGE_RATE, params)
logging.info(age_rate_info_list)
click_trend_rate_info_list = Parser().shopping_request(Parser.CATEGORY_CLICK_TREND, params)
logging.info(click_trend_rate_info_list)
device_rate_info_list = Parser().shopping_request(Parser.CATEGORY_DEVICE_RATE, params)
logging.info(device_rate_info_list)
gender_rate_info_list = Parser().shopping_request(Parser.CATEGORY_GENDER_RATE, params)
logging.info(gender_rate_info_list)
range, keyword_rank = Parser().get_keyword_rank(params)
logging.info(keyword_rank)
class TestParser(unittest.TestCase):
def setUp(self):
self.parser = Parser("STATEMENT FOR TEST")
# Expect tests
def test_expect_word(self):
self.assertEqual(self.parser.expect("STATEMENT"), None)
def test_expect_words(self):
self.assertEqual(self.parser.expect("STATEMENT","FOR","TEST"), None)
def test_not_expected_word(self):
self.assertRaises(Exception, self.parser.expect, "WRONG_STATEMENT")
def test_not_expected_words(self):
self.assertRaises(Exception, self.parser.expect, ("NOT","EXPECTED","WORDS"))
def test_one_not_expected_word(self):
self.assertRaises(Exception, self.parser.expect, ("STATEMENT", "WRONG","FOR","TEST"))
def test_one_wrong_expected_word(self):
self.assertRaises(Exception, self.parser.expect, ("STATEMENT", "WRONG","TEST"))
# Expect optional tests
def test_expectOptional_word(self):
self.assertTrue(self.parser.expect_optional("STATEMENT"))
def test_expectOptional_words(self):
self.assertTrue(self.parser.expect_optional("STATEMENT","FOR","TEST"))
def test_not_expectOptional_word(self):
self.assertFalse(self.parser.expect_optional("WRONG_STATEMENT"))
def test_not_expectOptional_words(self):
self.assertFalse(self.parser.expect_optional("NOT","EXPECTED","WORDS"))
def test_one_not_expectOptional_word(self):
self.assertRaises(Exception, self.parser.expect_optional, ("WRONG", "STATEMENT","FOR","TEST"))
def test_first_fine_other_not_expectOptional_word(self):
self.assertRaises(Exception, self.parser.expect_optional, ("STATEMENT","WRONG","FOR","TEST"))
def parse(urls, proxies, visibility=False):
"""
Запускаем программу
"""
queue = Queue()
# Даем очереди нужные нам ссылки для скачивания
for url in urls:
queue.put(url)
with open("cookies.pickle", "rb") as f:
cookies = pickle.load(f)
# Запускаем потом и очередь
threads = []
for i in range(len(proxies)):
t = Parser(queue, i, cookies, proxies[i], visibility=visibility)
t.start()
threads.append(t)
# Ждем завершения работы очереди
for t in threads:
t.join()
from tokenizer.Readers import Reader
from tokenizer.Readers import PosReader
from tokenizer.Tokenizer import Tokenizer
from tokenizer.TokenList import *
from parser.Parser import Parser
# r = Reader('{"1" : True, "3" : [1,2,3,{"4" : "Alter"}]}')
# t = Tokenizer(r).getTokens()
# for i in t.tokenList :
# print(i.getType())
result = Parser('{"1" : True, "3" : [1,2,3,{"4" : "Alter"}]}').parse()
print(result)
print(result['3'])
print(result['3'][3])
for i in result['3']:
print(i)
# # Example - 1
import urllib.request
# request = urllib.request.Request('https://news-at.zhihu.com/api/4/news/latest') # Zhihu
# resp = urllib.request.urlopen(request)
# raw_data = resp.read()
# data = raw_data.decode()
# json_data = Parser(data).parse()
# print(json_data)
# # Example - 2
# request = urllib.request.Request('http://local.shiyiquan.net/message/global/?time_update=0&Ajax=true') # Shiyiquan
# resp = urllib.request.urlopen(request)
# raw_data = resp.read().decode('unicode-escape')
def parse(database, statement):
parser = Parser(statement)
parser.expect("CREATE", "SEQUENCE")
sequenceName = parser.parse_identifier()
sequence = PgSequence(ParserUtils.get_object_name(sequenceName))
schemaName = ParserUtils.get_schema_name(sequenceName, database)
schema = database.getSchema(schemaName)
if schema is None:
raise Exception(
"Cannot find schema '%s' for statement '%s'. Missing CREATE SCHEMA statement?"
% (schemaName, statement))
schema.addSequence(sequence)
while not parser.expect_optional(";"):
if parser.expect_optional("INCREMENT"):
parser.expect_optional("BY")
sequence.increment = parser.parse_string()
elif parser.expect_optional("MINVALUE"):
sequence.minValue = parser.parse_string()
elif parser.expect_optional("MAXVALUE"):
sequence.maxValue = parser.parse_string()
elif parser.expect_optional("START"):
parser.expect_optional("WITH")
sequence.startWith = parser.parse_string()
elif parser.expect_optional("CACHE"):
sequence.cache = parser.parse_string()
elif parser.expect_optional("CYCLE"):
sequence.cycle = True
elif parser.expect_optional("OWNED", "BY"):
if parser.expect_optional("NONE"):
sequence.ownedBy = None
else:
sequence.ownedBy = ParserUtils.get_object_name(
parser.parse_identifier())
elif parser.expect_optional("NO"):
if parser.expect_optional("MINVALUE"):
sequence.minValue = None
elif parser.expect_optional("MAXVALUE"):
sequence.maxValue = None
elif parser.expect_optional("CYCLE"):
sequence.cycle = False
else:
parser.throw_unsupported_command()
else:
parser.throw_unsupported_command()
def parse(database, statement):
parser = Parser(statement)
parser.expect("CREATE")
parser.expect_optional("OR", "REPLACE")
parser.expect("FUNCTION")
functionName = parser.parse_identifier()
schemaName = ParserUtils.get_schema_name(functionName, database)
schema = database.getSchema(schemaName)
if schema is None:
raise Exception(
"Cannot find schema '%s' for statement '%s'. Missing CREATE SCHEMA statement?"
% (schemaName, statement))
function = PgFunction()
function.name = ParserUtils.get_object_name(functionName)
parser.expect("(")
while not parser.expect_optional(")"):
mode = None
if parser.expect_optional("IN"):
mode = "IN"
elif parser.expect_optional("OUT"):
mode = "OUT"
elif parser.expect_optional("INOUT"):
mode = "INOUT"
elif parser.expect_optional("VARIADIC"):
mode = "VARIADIC"
position = parser.position
argumentName = None
dataType = parser.parse_data_type()
position2 = parser.position
if (not parser.expect_optional(")")
and not parser.expect_optional(",")
and not parser.expect_optional("=")
and not parser.expect_optional("DEFAULT")):
parser.position = position
argumentName = ParserUtils.get_object_name(
parser.parse_identifier())
dataType = parser.parse_data_type()
else:
parser.position = position2
defaultExpression = ''
if (parser.expect_optional("=")
or parser.expect_optional("DEFAULT")):
defaultExpression = parser.get_expression()
else:
defaultExpression = None
argument = Argument()
argument.dataType = dataType
argument.defaultExpression = defaultExpression
argument.mode = mode
argument.name = argumentName
function.addArgument(argument)
if (parser.expect_optional(")")):
break
else:
parser.expect(",")
function.body = parser.get_rest()
schema.addFunction(function)
def setUp(self):
self.parser = Parser("STATEMENT FOR TEST")
def parse(database, statement, ignoreSlonyTriggers):
parser = Parser(statement)
parser.expect("CREATE", "TRIGGER")
triggerName = parser.parse_identifier()
objectName = ParserUtils.get_object_name(triggerName)
trigger = PgTrigger()
trigger.name = objectName
if parser.expect_optional("BEFORE"):
trigger.event = PgTrigger.EVENT_BEFORE
elif parser.expect_optional("AFTER"):
trigger.event = PgTrigger.EVENT_AFTER
elif parser.expect_optional("INSTEAD OF"):
trigger.event = PgTrigger.EVENT_INSTEAD_OF
first = True
while True:
if not first and not parser.expect_optional("OR"):
break
elif parser.expect_optional("INSERT"):
trigger.onInsert = True
elif parser.expect_optional("UPDATE"):
trigger.onUpdate = True
if parser.expect_optional("OF"):
while True:
trigger.updateColumns.append(parser.parse_identifier())
if not parser.expect_optional(","):
break
elif parser.expect_optional("DELETE"):
trigger.onDelete = True
elif parser.expect_optional("TRUNCATE"):
trigger.onTruncate = True
elif (first):
break
else:
parser.throw_unsupported_command()
first = False
parser.expect("ON")
trigger.tableName = ParserUtils.get_object_name(parser.parse_identifier())
if parser.expect_optional("FOR"):
parser.expect_optional("EACH")
if parser.expect_optional("ROW"):
trigger.forEachRow = True
elif parser.expect_optional("STATEMENT"):
trigger.forEachRow = False
else:
parser.throw_unsupported_command()
if parser.expect_optional("WHEN"):
parser.expect("(")
trigger.when = parser.get_expression()
parser.expect(")")
parser.expect("EXECUTE", "PROCEDURE")
trigger.function = parser.get_rest()
ignoreSlonyTrigger = ignoreSlonyTriggers and ("_slony_logtrigger" == trigger.name or "_slony_denyaccess" == trigger.name)
if (not ignoreSlonyTrigger):
schema = database.getSchema(ParserUtils.get_schema_name(trigger.tableName, database))
container = schema.tables.get(trigger.tableName)
if not container:
container = schema.views.get(trigger.tableName)
if container:
container.triggers[trigger.name] = trigger
else:
raise Exception()
def parse(database, statement):
parser = Parser(statement)
parser.expect("COMMENT", "ON")
if parser.expect_optional("TABLE"):
CommentParser.parseTable(parser, database)
elif parser.expect_optional("COLUMN"):
CommentParser.parseColumn(parser, database)
elif parser.expect_optional("CONSTRAINT"):
CommentParser.parseConstraint(parser, database)
elif parser.expect_optional("DATABASE"):
CommentParser.parseDatabase(parser, database)
elif parser.expect_optional("FUNCTION"):
CommentParser.parseFunction(parser, database)
elif parser.expect_optional("INDEX"):
CommentParser.parseIndex(parser, database)
elif parser.expect_optional("SCHEMA"):
CommentParser.parseSchema(parser, database)
elif parser.expect_optional("SEQUENCE"):
CommentParser.parseSequence(parser, database)
elif parser.expect_optional("TRIGGER"):
CommentParser.parseTrigger(parser, database)
elif parser.expect_optional("VIEW"):
CommentParser.parseView(parser, database)
def parse(database, statement):
parser = Parser(statement)
parser.expect("COMMENT", "ON")
if parser.expect_optional("TABLE"):
CommentParser.parseTable(parser, database)
elif parser.expect_optional("COLUMN"):
CommentParser.parseColumn(parser, database)
elif parser.expect_optional("CONSTRAINT"):
CommentParser.parseConstraint(parser, database)
elif parser.expect_optional("DATABASE"):
CommentParser.parseDatabase(parser, database)
elif parser.expect_optional("FUNCTION"):
CommentParser.parseFunction(parser, database)
elif parser.expect_optional("INDEX"):
CommentParser.parseIndex(parser, database)
elif parser.expect_optional("SCHEMA"):
CommentParser.parseSchema(parser, database)
elif parser.expect_optional("SEQUENCE"):
CommentParser.parseSequence(parser, database)
elif parser.expect_optional("TRIGGER"):
CommentParser.parseTrigger(parser, database)
elif parser.expect_optional("VIEW"):
CommentParser.parseView(parser, database)
import sys
#sys.path.insert(0,'/home/youssefsb/VS_workspace/Comp442/Comp442/parser/Grammar')
import graphviz
from subprocess import call
from parser.Parser import Parser
from parser.Grammar.Grammar import Grammar
from semanticAnalyzer.SemAnalyzer import SemAnalyzer
grammar = Grammar('parser/Grammar/outfile.grm',
'parser/Grammar/outfile.grm.first',
'parser/Grammar/outfile.grm.follow')
parser = Parser(grammar, 'testCases/mainOnly.src')
semAnalyzer = SemAnalyzer()
parser.parse()
globalSymTable = semAnalyzer.analyzeTree(parser.astRoot)
print(str(globalSymTable.getTypeSizesHashMap()))
globalSymTable.symTableAsHtml()
call(["moon/moon", "moon/outFile.m", "moon/lib.m"])
def parse(database, statement):
parser = Parser(statement)
parser.expect('CREATE', 'TABLE')
# Optional IF NOT EXISTS, irrelevant for our purposes
parser.expect_optional("IF", "NOT", "EXISTS")
tableName = ParserUtils.get_object_name(parser.parse_identifier())
table = PgTable(tableName)
# Figure it out why do we need this
schemaName = ParserUtils.get_schema_name(tableName, database)
schema = database.getSchema(schemaName)
if schema is None:
raise Exception(
"Cannot find schema \'%s\' for statement \'%s\'. Missing CREATE SCHEMA statement?"
% (schemaName, statement))
schema.addTable(table)
parser.expect("(")
while not parser.expect_optional(")"):
if parser.expect_optional("CONSTRAINT"):
CreateTableParser.parseConstraint(parser, table)
else:
CreateTableParser.parseColumn(parser, table)
if parser.expect_optional(")"):
break
else:
parser.expect(",")
while not parser.expect_optional(";"):
if parser.expect_optional("INHERITS"):
CreateTableParser.parseInherits(parser, table)
elif parser.expect_optional("WITHOUT"):
table.oids = "OIDS=false"
elif parser.expect_optional("WITH"):
if (parser.expect_optional("OIDS")
or parser.expect_optional("OIDS=true")):
table.oids = "OIDS=true"
elif parser.expect_optional("OIDS=false"):
table.oids = "OIDS=false"
else:
print 'table.setWith(parser.getExpression())'
elif parser.expect_optional("TABLESPACE"):
print 'table.setTablespace(parser.parseString()'
else:
parser.throw_unsupported_command()
def parse(database, statement):
parser = Parser(statement)
parser.expect("CREATE")
unique = parser.expectOptional("UNIQUE")
parser.expect("INDEX")
parser.expectOptional("CONCURRENTLY")
indexName = ParserUtils.getObjectName(parser.parseIdentifier())
parser.expect("ON")
tableName = parser.parseIdentifier()
definition = parser.getRest()
schemaName =ParserUtils.getSchemaName(tableName, database)
schema = database.getSchema(schemaName)
if (schema is None):
print 'ERROR: CreateIndexParser[Line 21]'
# throw new RuntimeException(MessageFormat.format(
# Resources.getString("CannotFindSchema"), schemaName,
# statement));
objectName = ParserUtils.getObjectName(tableName)
table = schema.getTable(objectName)
if (table is None):
print 'ERROR: CreateIndexParser[Line 32]'
# throw new RuntimeException(MessageFormat.format(
# Resources.getString("CannotFindTable"), tableName,
# statement));
index = PgIndex(indexName)
table.addIndex(index)
schema.addIndex(index)
index.definition = definition.strip()
index.tableName = table.name
index.unique = unique
def parse(database, statement):
parser = Parser(statement)
parser.expect("ALTER", "TABLE")
parser.expect_optional("ONLY")
tableName = parser.parse_identifier()
schemaName = ParserUtils.get_schema_name(tableName, database)
schema = database.getSchema(schemaName)
if schema is None:
raise Exception("CannotFindSchema")
objectName = ParserUtils.get_object_name(tableName)
table = schema.tables.get(objectName)
if table is None:
logging.debug('Table %s was not fount in schema %s' %
(tableName, schema))
view = schema.views.get(objectName)
if view is not None:
AlterTableParser.parseView(parser, view, tableName, database)
return
sequence = schema.sequences.get(objectName)
if sequence is not None:
AlterTableParser.parseSequence(parser, sequence, tableName,
database)
return
raise Exception("Cannot find object '%s' for statement '%s'." %
(tableName, statement))
while not parser.expect_optional(";"):
if parser.expect_optional("ALTER"):
AlterTableParser.parseAlterColumn(parser, table)
elif (parser.expect_optional("CLUSTER", "ON")):
table.clusterIndexName = ParserUtils.get_object_name(
parser.parse_identifier())
elif (parser.expect_optional("OWNER", "TO")):
# we do not parse this one so we just consume the identifier
# if (outputIgnoredStatements):
# print 'database.addIgnoredStatement("ALTER TABLE " + tableName + " OWNER TO " + parser.parseIdentifier() + ';')'
# else:
parser.parse_identifier()
elif (parser.expect_optional("ADD")):
if (parser.expect_optional("FOREIGN", "KEY")):
print 'parseAddForeignKey(parser, table);'
elif (parser.expect_optional("CONSTRAINT")):
AlterTableParser.parseAddConstraint(parser, table, schema)
else:
parser.throw_unsupported_command()
elif (parser.expect_optional("ENABLE")):
print 'parseEnable(parser, outputIgnoredStatements, tableName, database);'
elif (parser.expect_optional("DISABLE")):
print 'parseDisable(parser, outputIgnoredStatements, tableName, database);'
else:
parser.throw_unsupported_command()
if (parser.expect_optional(";")):
break
else:
parser.expect(",")
def parse(database, statement):
parser = Parser(statement)
parser.expect("ALTER", "TABLE")
parser.expectOptional("ONLY")
tableName = parser.parseIdentifier()
schemaName = ParserUtils.getSchemaName(tableName, database)
schema = database.getSchema(schemaName)
if schema is None:
raise Exception("CannotFindSchema")
objectName = ParserUtils.getObjectName(tableName)
table = schema.tables.get(objectName)
if table is None:
view = schema.views.get(objectName)
if view is not None:
AlterTableParser.parseView(parser, view, tableName, database)
return
sequence = schema.sequences.get(objectName)
if sequence is not None:
AlterTableParser.parseSequence(parser, sequence, tableName, database);
return
raise Exception("Cannot find object '%s' for statement '%s'." % (tableName, statement))
while not parser.expectOptional(";"):
if parser.expectOptional("ALTER"):
AlterTableParser.parseAlterColumn(parser, table)
elif (parser.expectOptional("CLUSTER", "ON")):
table.clusterIndexName = ParserUtils.getObjectName(parser.parseIdentifier())
elif (parser.expectOptional("OWNER", "TO")):
# we do not parse this one so we just consume the identifier
# if (outputIgnoredStatements):
# print 'database.addIgnoredStatement("ALTER TABLE " + tableName + " OWNER TO " + parser.parseIdentifier() + ';')'
# else:
parser.parseIdentifier()
elif (parser.expectOptional("ADD")):
if (parser.expectOptional("FOREIGN", "KEY")):
print 'parseAddForeignKey(parser, table);'
elif (parser.expectOptional("CONSTRAINT")):
AlterTableParser.parseAddConstraint(parser, table, schema)
else:
parser.throwUnsupportedCommand()
elif (parser.expectOptional("ENABLE")):
print 'parseEnable(parser, outputIgnoredStatements, tableName, database);'
elif (parser.expectOptional("DISABLE")):
print 'parseDisable(parser, outputIgnoredStatements, tableName, database);'
else:
parser.throwUnsupportedCommand()
if (parser.expectOptional(";")):
break
else:
parser.expect(",")
def parse(database, statement):
parser = Parser(statement)
parser.expect("ALTER", "VIEW")
viewName = parser.parseIdentifier()
schemaName = ParserUtils.getSchemaName(viewName, database)
schema = database.getSchema(schemaName)
if schema is None:
raise Exception("Cannot find schema '%s' for statement '%s'. Missing CREATE SCHEMA statement?" % (schemaName, statement))
objectName = ParserUtils.getObjectName(viewName)
view = schema.views[objectName]
if view is None:
raise Exception("Cannot find view '%s' for statement '%s'. Missing CREATE VIEW statement?" % (viewName, statement))
while not parser.expectOptional(";"):
if parser.expectOptional("ALTER"):
parser.expectOptional("COLUMN")
columnName = ParserUtils.getObjectName(parser.parseIdentifier())
if parser.expectOptional("SET", "DEFAULT"):
expression = parser.getExpression()
view.addColumnDefaultValue(columnName, expression)
elif parser.expectOptional("DROP", "DEFAULT"):
view.removeColumnDefaultValue(columnName)
else:
parser.throwUnsupportedCommand()
elif parser.expectOptional("OWNER", "TO"):
# we do not parse this one so we just consume the identifier
# if (outputIgnoredStatements) {
# database.addIgnoredStatement("ALTER TABLE " + viewName
# + " OWNER TO " + parser.parseIdentifier() + ';');
# } else {
parser.parseIdentifier()
# }
else:
parser.throwUnsupportedCommand()
return
self._get_events_from_MySQL(username=Config.username, password=Config.password, databasename=Config.databasename, tables=Config.tables)
def get_events_from_SQLite(self):
con = Config
if (Config.dbfile or Config.tables) == None:
print "Load config file!"
print Config.dbfile, Config.tables
return
self._get_events_from_SQLite(filename=Config.dbfile, tables=Config.tables)
def filter_by_occurance_time(self):
#todo there is much more logs from network screen and network_traffic then application, I mean in some point of time application logs do not occur any more
pass
if __name__ == '__main__':
Config.load_config(ConfigVals)
sortedEvents = SortedRawAwareEvents()
sortedEvents.get_events_from_MySQL()
for x in sortedEvents:
time = datetime.datetime.fromtimestamp(int(x['values'].timestamp/1000)).strftime('%Y-%m-%d %H:%M:%S')
print x['table'].name+" "+str(time)
parser = Parser()
for event in sortedEvents:
parser.parse_event(event)
def parse(database, statement):
parser = Parser(statement)
parser.expect('CREATE', 'TABLE')
# Optional IF NOT EXISTS, irrelevant for our purposes
parser.expect_optional("IF", "NOT", "EXISTS")
tableName = ParserUtils.get_object_name(parser.parse_identifier())
table = PgTable(tableName)
# Figure it out why do we need this
schemaName = ParserUtils.get_schema_name(tableName, database)
schema = database.getSchema(schemaName)
if schema is None:
raise Exception("Cannot find schema \'%s\' for statement \'%s\'. Missing CREATE SCHEMA statement?" % (schemaName, statement))
schema.addTable(table)
parser.expect("(")
while not parser.expect_optional(")"):
if parser.expect_optional("CONSTRAINT"):
CreateTableParser.parseConstraint(parser, table)
else:
CreateTableParser.parseColumn(parser, table)
if parser.expect_optional(")"):
break
else:
parser.expect(",")
while not parser.expect_optional(";"):
if parser.expect_optional("INHERITS"):
CreateTableParser.parseInherits(parser, table)
elif parser.expect_optional("WITHOUT"):
table.oids = "OIDS=false"
elif parser.expect_optional("WITH"):
if (parser.expect_optional("OIDS") or parser.expect_optional("OIDS=true")):
table.oids = "OIDS=true"
elif parser.expect_optional("OIDS=false"):
table.oids = "OIDS=false"
else:
print 'table.setWith(parser.getExpression())'
elif parser.expect_optional("TABLESPACE"):
print 'table.setTablespace(parser.parseString()'
else:
parser.throw_unsupported_command()
def parse(database, statement):
parser = Parser(statement)
parser.expect("ALTER", "VIEW")
viewName = parser.parse_identifier()
schemaName = ParserUtils.get_schema_name(viewName, database)
schema = database.getSchema(schemaName)
if schema is None:
raise Exception(
"Cannot find schema '%s' for statement '%s'. Missing CREATE SCHEMA statement?"
% (schemaName, statement))
objectName = ParserUtils.get_object_name(viewName)
view = schema.views[objectName]
if view is None:
raise Exception(
"Cannot find view '%s' for statement '%s'. Missing CREATE VIEW statement?"
% (viewName, statement))
while not parser.expect_optional(";"):
if parser.expect_optional("ALTER"):
parser.expect_optional("COLUMN")
columnName = ParserUtils.get_object_name(
parser.parse_identifier())
if parser.expect_optional("SET", "DEFAULT"):
expression = parser.get_expression()
view.addColumnDefaultValue(columnName, expression)
elif parser.expect_optional("DROP", "DEFAULT"):
view.removeColumnDefaultValue(columnName)
else:
parser.throw_unsupported_command()
elif parser.expect_optional("OWNER", "TO"):
# we do not parse this one so we just consume the identifier
# if (outputIgnoredStatements) {
# database.addIgnoredStatement("ALTER TABLE " + viewName
# + " OWNER TO " + parser.parseIdentifier() + ';');
# } else {
parser.parse_identifier()
# }
else:
parser.throw_unsupported_command()
def parse(database, statement):
parser = Parser(statement)
parser.expect("CREATE")
parser.expect_optional("OR", "REPLACE")
parser.expect("VIEW")
viewName = parser.parse_identifier()
columnsExist = parser.expect_optional("(")
columnNames = list()
if (columnsExist):
while not parser.expect_optional(")"):
columnNames.append(
ParserUtils.get_object_name(parser.parse_identifier()))
parser.expect_optional(",")
parser.expect("AS")
query = parser.get_rest()
view = PgView(ParserUtils.get_object_name(viewName))
view.columnNames = columnNames
view.query = query
schemaName = ParserUtils.get_schema_name(viewName, database)
schema = database.getSchema(schemaName)
if schema is None:
raise Exception("CannotFindSchema" % (schemaName, statement))
schema.addView(view)
def parse(database, statement):
parser = Parser(statement)
parser.expect("CREATE")
parser.expectOptional("OR", "REPLACE")
parser.expect("FUNCTION")
functionName = parser.parseIdentifier();
schemaName = ParserUtils.getSchemaName(functionName, database)
schema = database.getSchema(schemaName)
if schema is None:
raise Exception("Cannot find schema '%s' for statement '%s'. Missing CREATE SCHEMA statement?" % (schemaName, statement))
function = PgFunction()
function.name = ParserUtils.getObjectName(functionName)
parser.expect("(")
while not parser.expectOptional(")"):
mode = ''
if parser.expectOptional("IN"):
mode = "IN"
elif parser.expectOptional("OUT"):
mode = "OUT"
elif parser.expectOptional("INOUT"):
mode = "INOUT"
elif parser.expectOptional("VARIADIC"):
mode = "VARIADIC"
else:
mode = None
position = parser.position
argumentName = None
dataType = parser.parseDataType()
position2 = parser.position
if (not parser.expectOptional(")") and not parser.expectOptional(",")
and not parser.expectOptional("=")
and not parser.expectOptional("DEFAULT")):
parser.position = position
argumentName = ParserUtils.getObjectName(parser.parseIdentifier())
dataType = parser.parseDataType()
else:
parser.position = position2
defaultExpression = ''
if (parser.expectOptional("=")
or parser.expectOptional("DEFAULT")):
defaultExpression = parser.getExpression()
else:
defaultExpression = None
argument = Argument()
argument.dataType = dataType
argument.defaultExpression = defaultExpression
argument.mode = mode
argument.name = argumentName
function.addArgument(argument)
if (parser.expectOptional(")")):
break
else:
parser.expect(",")
function.body = parser.getRest()
schema.addFunction(function)
def parse(database, statement):
parser = Parser(statement)
parser.expect("CREATE")
parser.expect_optional("OR", "REPLACE")
parser.expect("VIEW")
viewName = parser.parse_identifier()
columnsExist = parser.expect_optional("(")
columnNames = list()
if (columnsExist):
while not parser.expect_optional(")"):
columnNames.append(ParserUtils.get_object_name(parser.parse_identifier()))
parser.expect_optional(",")
parser.expect("AS")
query = parser.get_rest()
view = PgView(ParserUtils.get_object_name(viewName))
view.columnNames = columnNames
view.query = query
schemaName = ParserUtils.get_schema_name(viewName, database)
schema = database.getSchema(schemaName)
if schema is None:
raise Exception("CannotFindSchema" % (schemaName, statement))
schema.addView(view)
def parse(database, statement):
parser = Parser(statement)
parser.expect("CREATE", "SEQUENCE")
sequenceName = parser.parseIdentifier();
sequence = PgSequence(ParserUtils.getObjectName(sequenceName))
schemaName = ParserUtils.getSchemaName(sequenceName, database)
schema = database.getSchema(schemaName)
if schema is None:
raise Exception("Cannot find schema '%s' for statement '%s'. Missing CREATE SCHEMA statement?" % (schemaName, statement))
schema.addSequence(sequence)
while not parser.expectOptional(";"):
if parser.expectOptional("INCREMENT"):
parser.expectOptional("BY")
sequence.increment = parser.parseString()
elif parser.expectOptional("MINVALUE"):
sequence.minValue = parser.parseString()
elif parser.expectOptional("MAXVALUE"):
sequence.maxValue = parser.parseString()
elif parser.expectOptional("START"):
parser.expectOptional("WITH")
sequence.startWith = parser.parseString()
elif parser.expectOptional("CACHE"):
sequence.cache = parser.parseString()
elif parser.expectOptional("CYCLE"):
sequence.cycle = True
elif parser.expectOptional("OWNED", "BY"):
if parser.expectOptional("NONE"):
sequence.ownedBy = None
else:
sequence.ownedBy = ParserUtils.getObjectName(parser.parseIdentifier())
elif parser.expectOptional("NO"):
if parser.expectOptional("MINVALUE"):
sequence.minValue = None
elif parser.expectOptional("MAXVALUE"):
sequence.maxValue = None
elif parser.expectOptional("CYCLE"):
sequence.cycle = False
else:
parser.throwUnsupportedCommand()
else:
parser.throwUnsupportedCommand()
def parse(database, statement):
parser = Parser(statement)
parser.expect("CREATE")
unique = parser.expect_optional("UNIQUE")
parser.expect("INDEX")
parser.expect_optional("CONCURRENTLY")
indexName = ParserUtils.get_object_name(parser.parse_identifier())
parser.expect("ON")
tableName = parser.parse_identifier()
definition = parser.get_rest()
schemaName = ParserUtils.get_schema_name(tableName, database)
schema = database.getSchema(schemaName)
if (schema is None):
print 'ERROR: CreateIndexParser[Line 21]'
# throw new RuntimeException(MessageFormat.format(
# Resources.getString("CannotFindSchema"), schemaName,
# statement));
objectName = ParserUtils.get_object_name(tableName)
table = schema.getTable(objectName)
if (table is None):
print 'ERROR: CreateIndexParser[Line 32]'
# throw new RuntimeException(MessageFormat.format(
# Resources.getString("CannotFindTable"), tableName,
# statement));
index = PgIndex(indexName)
table.addIndex(index)
schema.addIndex(index)
index.definition = definition.strip()
index.tableName = table.name
index.unique = unique
