def test_tokenize_blank():
tokenize_blank = token_rules.tokenize_blank
scanner = Scanner("\n")
token = tokenize_blank(scanner.next(), scanner)
assert_equals(token.symbol, 'BlankLine')
scanner = Scanner(["foo\n"])
token = tokenize_blank(scanner.next(), scanner)
ok_(token is None, "Got token %r, expected None" % token)
def test_tokenize_trigger():
text = textwrap.dedent("""
DELIMITER ;;
/*!50003 CREATE*/ /*!50017 DEFINER=`root`@`localhost`*/ /*!50003 TRIGGER `customer_create_date` BEFORE INSERT ON `customer` FOR EACH ROW SET NEW.create_date = NOW() */;;
DELIMITER ;
""").lstrip()
scanner = Scanner(StringIO(text))
token = token_rules.tokenize_delimiter(scanner.next(), scanner)
assert_equals(token.symbol, 'CreateTrigger')
assert_equals(token.text, text)
def test_make_token():
stream = textwrap.dedent("""
-- Host: localhost Database: sakila
-- Dumping data for table `actor`
""").splitlines()
scanner = Scanner(stream)
line = scanner.next()
token = token_rules.make_token('SqlComment', line, scanner)
assert_equals(token.text, line)
assert_equals(token.symbol, 'SqlComment')
assert_equals(token.offset, 0)
def test_tokenizer_prefix():
"""Test that we tokenize a line that matches a prefix"""
stream = textwrap.dedent("""\
-- Dumping data for table `actor`
""").splitlines()
scanner = Scanner(stream)
tokenize_prefix = token_rules.tokenize_prefix
make_token = token_rules.make_token
tokenizer = tokenize_prefix('-- Dumping data',
make_token,
'TableData')
token = tokenizer(scanner.next(), scanner)
assert_equals(token.symbol, 'TableData')
def test_tokenize_multiline():
text = textwrap.dedent("""
CREATE TABLE `actor` (
`actor_id` smallint(5) unsigned NOT NULL AUTO_INCREMENT,
`first_name` varchar(45) NOT NULL,
`last_name` varchar(45) NOT NULL,
`last_update` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`actor_id`),
KEY `idx_actor_last_name` (`last_name`)
) ENGINE=InnoDB AUTO_INCREMENT=201 DEFAULT CHARSET=utf8;
""").lstrip()
tokenize_multi_line = token_rules.tokenize_multi_line
scanner = Scanner(StringIO(text))
token = tokenize_multi_line('CreateTable', ';', scanner.next(), scanner)
assert_equals(token.symbol, 'CreateTable')
assert_equals(token.line_range, (1, text.count("\n")))
ok_(token.text.startswith('CREATE TABLE `actor`'))
ok_(token.text.rstrip().endswith(';'))
def test_tokenize_stored_procedure():
text = textwrap.dedent("""
DELIMITER ;;
/*!50003 CREATE*/ /*!50020 DEFINER=`root`@`localhost`*/ /*!50003 PROCEDURE `film_in_stock`(IN p_film_id INT, IN p_store_id INT, OUT p_film_count INT)
READS SQL DATA
BEGIN
SELECT inventory_id
FROM inventory
WHERE film_id = p_film_id
AND store_id = p_store_id
AND inventory_in_stock(inventory_id);
SELECT FOUND_ROWS() INTO p_film_count;
END */;;
DELIMITER ;
""").lstrip()
scanner = Scanner(StringIO(text))
tokenize_delimiter = token_rules.tokenize_delimiter
token = tokenize_delimiter(scanner.next(),
scanner)
assert_equals(token.symbol, 'CreateRoutine')
assert_equals(token.text, text)
