def elem_init(self, sessiondata=None):
"""
Override this method to perform validation and initialization
after a DeleteQuery is parsed
"""
metadata = self.elements[1].string.strip(" '\"")
# Verify if a metadata is provided
if not metadata:
raise TINCMMException("Invalid syntax for a delete query. Metadata not specified: %s" %self.string)
self.metadata = metadata
self.value = None
# Verify if a value is provided.
# Value is optional because it might be required only for certain kind of metadata
# (like tags , product_version).
# Note: delete metadata=vale will be supported only for tags and product_version
# Hardcoding the check here as we do not have a metadata language at this point
# to make this metadata agnostic. In an ideal world every metadata should have a type
# and this should be allowed for sequence metadata types
if len(self.elements) > 2 and self.elements[2]:
# Verify if a value is provided , value will be the second element in the optional grammar
if not len(self.elements[2].elements) == 2:
raise TINCMMException("Invalid syntax for a delete query: %s" %self.string)
if not self.elements[2].elements[1].string.strip(" '\""):
raise TINCMMException("Invalid syntax for a delete query. Value not specified: %s" %self.string)
if not (self.metadata == 'tags' or self.metadata == 'product_version'):
raise TINCMMException("Invalid syntax for a delete query. Value should be specified only for tags and product_version: %s" %self.string)
self.value = self.elements[2].elements[1].string.strip(" '\"")
def elem_init(self, sessiondata=None):
"""
Override this method to perform validation and initialization
after an UpdateQuery is parsed
"""
value = self.elements[3].string.strip(" '\"")
metadata = self.elements[1].string.strip(" '\"")
# Verify if a value is provided
if not value:
raise TINCMMException("Invalid syntax for an update query. Value not specified: %s" %self.string)
# Verify if a metadata is provided
if not value:
raise TINCMMException("Invalid syntax for an update query. Metadata not specified: %s" %self.string)
self.metadata = metadata
self.value = value
def update_file(self, file_to_write=None):
"""
This method will update the original python file with the new docstring that is stored in docstring_tuples.
If file_to_write is specified, the original python file will be left as is, and file_to_write will be used
with new docstring.
@type file_to_write: string
@param file_to_write: Absolute path to the file to use to write back the original file with new docstring.
Defaults to None. If None, the original file will be updated.
"""
if not file_to_write:
file_to_write = self.filename
with open(self.filename, 'r') as file_object:
token_list = []
for (token_type, token_string, token_start_position,
token_end_position,
token_full_line) in tokenize.generate_tokens(
file_object.next):
token_list.append((token_type, token_string))
update_needed = False
tokens_inserted = 0
for docstring_tuple in self.docstring_tuples:
# If there is a new docstring and an original docstring, replace the corresponding token string
if docstring_tuple.new_docstring and docstring_tuple.original_docstring:
original_token_type = token_list[docstring_tuple.token_idx][0]
token_list[docstring_tuple.token_idx] = (
original_token_type, docstring_tuple.new_docstring)
update_needed = True
# If there is a new docstring, but no original docstring, a new token has to be inserted at the right token_idx
if docstring_tuple.new_docstring and not docstring_tuple.original_docstring:
token = (tokenize.STRING, docstring_tuple.new_docstring)
# Assuming we will be inserting in the right order here
# Account for previous tokens added in the original list
token_list.insert(
int(docstring_tuple.token_idx) + tokens_inserted, token)
tokens_inserted += 1
token = (tokenize.NEWLINE, '')
token_list.insert(
int(docstring_tuple.token_idx) + tokens_inserted, token)
tokens_inserted += 1
update_needed = True
if not update_needed:
tinctest.logger.debug("No update required for file %s" %
self.filename)
if file_to_write == self.filename:
return
else:
try:
shutil.copyfile(self.filename, file_to_write)
except Exception, e:
tinctest.logger.exception(e)
raise TINCMMException(
"Encountered error while copying file %s to %s" %
(self.filename, file_to_write))
def _parse_query(self, query):
"""
Parse the given query string using TINCMM query grammar and return the result
"""
result = None
query_parser = TINCMMQueryGrammar.parser()
try:
result = query_parser.parse_string(query, reset=True, eof=True, matchtype='longest')
except ParseError,e:
tinctest.logger.exception("Error while parsing query %s" %query)
raise TINCMMException("Invalid query %s. Check the syntax of the query." %query)
def elem_init(self, sessiondata=None):
"""
Override this method to perform validation and initialization
after a SelectQuery is parsed
"""
metadata = self.elements[1].string.strip(" '\"")
if not metadata:
raise TINCMMException("Invalid syntax for select query. Metadata not specified: %s" %self.string)
self.metadata = metadata
self.value = None
def __init__(self, argv=None):
self.test_suite = None
self.queries = []
self.logfiles = []
self.discover_options = None
if not argv:
argv = sys.argv
try:
self.parse_args(argv)
self.run()
except Exception, e:
tinctest.logger.exception(e)
raise TINCMMException("tincmm encountered an error: %s" % e)
def parse_file(self):
"""
This method will parse through the file and find the docstring.
"""
try:
file_object = open(self.filename, 'r')
file_object.close()
except:
raise TINCMMException("Cannot open file %s" % self.filename)
with open(self.filename, 'r') as file_object:
for line in file_object:
stripped_line = line.strip()
if stripped_line.find('--') != 0:
break
self.original_docstring += line
def __init__(self, queries):
"""
@param queries: An iterable of query strings
@type queries: list or any iterable of query strings or a single query string
"""
self.query_list = []
self.query_result_list = []
if isinstance(queries, basestring):
self.query_list.append(queries.strip())
else:
try:
for query in queries:
self.query_list.append(query.strip())
except TypeError, e:
tinctest.logger.exception("Argument 'queries' should be a string or an iterable")
raise TINCMMException("Argument 'queries' should be a string or an iterable")
def elem_init(self, sessiondata=None):
self.query_kind = ''
if isinstance(self.elements[0], SelectQuery):
self.query_kind = 'select'
elif isinstance(self.elements[0], InsertQuery):
self.query_kind = 'insert'
elif isinstance(self.elements[0], UpdateQuery):
self.query_kind = 'update'
elif isinstance(self.elements[0], DeleteQuery):
self.query_kind = 'delete'
else:
raise TINCMMException("Invalid query specified: %s" %self.string)
self.metadata = self.elements[0].metadata
self.value = self.elements[0].value
def update_file(self, file_to_write=None):
"""
This method will update the original sql file with the new docstring.
If file_to_write is specified, the original sql file will be left as is, and file_to_write will be used
with new docstring.
@type file_to_write: string
@param file_to_write: Absolute path to the file to use to write back the original file with new docstring.
Defaults to None. If None, the original file will be updated.
"""
if not file_to_write:
file_to_write = self.filename
if self.original_docstring == self.new_docstring:
if file_to_write == self.filename:
return
else:
try:
shutil.copyfile(self.filename, file_to_write)
except Exception, e:
tinctest.logger.exception(e)
raise TINCMMException(
"Encountered error while copying file %s to %s" %
(self.filename, file_to_write))
class SQLFileHandler(object):
"""
This class handles parsing and update a SQL file. It provides similar services to PythonFileHandler for SQL files.
"""
def __init__(self, filename):
"""
This constructor requires a sql filename to parse and store all its information. The information is stored in original_docstring
and new_docstring.
@type filename: string
@param filename: Absolute path to the sql file that needs docstring handling.
"""
self.filename = filename
# Docstring in SQL File
self.original_docstring = ""
self.new_docstring = ""
self.parse_file()
def parse_file(self):
"""
This method will parse through the file and find the docstring.
"""
try:
file_object = open(self.filename, 'r')
file_object.close()
except:
raise TINCMMException("Cannot open file %s" % self.filename)
with open(self.filename, 'r') as file_object:
for line in file_object:
stripped_line = line.strip()
if stripped_line.find('--') != 0:
break
self.original_docstring += line
def update_file(self, file_to_write=None):
"""
This method will update the original sql file with the new docstring.
If file_to_write is specified, the original sql file will be left as is, and file_to_write will be used
with new docstring.
@type file_to_write: string
@param file_to_write: Absolute path to the file to use to write back the original file with new docstring.
Defaults to None. If None, the original file will be updated.
"""
if not file_to_write:
file_to_write = self.filename
if self.original_docstring == self.new_docstring:
if file_to_write == self.filename:
return
else:
try:
shutil.copyfile(self.filename, file_to_write)
except Exception, e:
tinctest.logger.exception(e)
raise TINCMMException(
"Encountered error while copying file %s to %s" %
(self.filename, file_to_write))
try:
file_object = open(self.filename, 'r')
file_object.close()
except:
raise TINCMMException("Cannot open file %s" % self.filename)
code_string = self.new_docstring
rest_of_code = ""
with open(self.filename, 'r') as file_object:
for line in file_object:
stripped_line = line.strip()
if stripped_line.find('--') == 0:
continue
rest_of_code += line
code_string += rest_of_code
try:
with open(file_to_write, 'w') as file_object:
file_object.write(code_string)
except Exception, e:
tinctest.logger.exception(e)
raise TINCMMException(
"Encountered error while modifying file: %s" % file_to_write)
def parse_file(self):
"""
This method will parse through the python file and populate docstring_tuples. Even if a class/method doesn't have docstring, it would be stored in tuples list.
"""
try:
file_object = open(self.filename, 'r')
file_object.close()
except:
raise TINCMMException("Cannot open file %s" % self.filename)
tinctest.logger.debug("Parsing file %s" % self.filename)
with open(self.filename, 'r') as file_object:
current_class_name = None
current_method_name = None
current_docstring_tuple = None
check_indent = False
check_dedent = False
check_docstring = False
token_idx = -1
for (token_type, token_string, token_start_position,
token_end_position,
token_full_line) in tokenize.generate_tokens(
file_object.next):
token_idx += 1
# Look for current class name
if token_type == tokenize.NAME and token_string == "class":
class_name = _ParserHelp.find_test_class(token_full_line)
if class_name:
tinctest.logger.debug("Found class definition: %s" %
class_name)
# Add any previously constructued tuple
if current_docstring_tuple:
self.docstring_tuples.append(
current_docstring_tuple)
current_class_name = class_name
current_method_name = None
check_indent = True
row, column = token_start_position
current_docstring_tuple = PythonFileHandler.docstring_tuple(
class_name=current_class_name,
method_name=current_method_name,
offset=column,
original_docstring=None,
new_docstring=None,
token_idx=None)
# Check if class doc string is not found
if check_docstring and (
current_class_name and not current_method_name
) and (
(token_type != tokenize.STRING) or
(token_type == tokenize.STRING and
not _ParserHelp.find_docstring_quotes(token_full_line))):
# This is the point where the class docstring should be inserted if any
tinctest.logger.debug(
"FileHandler %s: Did not find class docstring for: %s | %s"
% (self.filename, current_docstring_tuple.class_name,
current_docstring_tuple.method_name))
current_docstring_tuple = current_docstring_tuple._replace(
token_idx=token_idx)
check_docstring = False
check_indent = False
# Check for method definition
if ((not check_docstring) and current_class_name
and token_type == tokenize.NAME
and token_string == "def"):
method_name = _ParserHelp.find_test_method(token_full_line)
tinctest.logger.debug("Found method definition: %s.%s" %
(current_class_name, method_name))
if method_name and class_name:
if current_docstring_tuple:
self.docstring_tuples.append(
current_docstring_tuple)
current_method_name = method_name
check_indent = True
row, column = token_start_position
current_docstring_tuple = PythonFileHandler.docstring_tuple(
class_name=current_class_name,
method_name=current_method_name,
offset=column,
original_docstring=None,
new_docstring=None,
token_idx=None)
# Look for indent
if check_indent and token_type == tokenize.INDENT:
tinctest.logger.debug(
"Found indent after definition: %s | %s" %
(current_class_name, current_method_name))
check_docstring = True
check_dedent = True
continue
# Terminate docstring check if the token following a definition is not a docstring token
if check_docstring and current_class_name and current_method_name and (
(token_type != tokenize.STRING) or
(token_type == tokenize.STRING and
not _ParserHelp.find_docstring_quotes(token_full_line))):
# Token following a defintion is not a docstring token
tinctest.logger.debug(
"FileHandler %s: Did not find method docstring for: %s | %s"
% (self.filename, current_docstring_tuple.class_name,
current_docstring_tuple.method_name))
check_docstring = False
current_docstring_tuple = current_docstring_tuple._replace(
token_idx=token_idx)
# Look for docstring after a definition of class or a method
if check_docstring and token_type == tokenize.STRING and _ParserHelp.find_docstring_quotes(
token_full_line):
tinctest.logger.debug(
"FileHandler %s: Found a docstring for: %s | %s" %
(self.filename, current_docstring_tuple.class_name,
current_docstring_tuple.method_name))
tinctest.logger.debug("Docstring found: %s" % token_string)
# This is docstring!
current_docstring_tuple = current_docstring_tuple._replace(
original_docstring=token_string)
current_docstring_tuple = current_docstring_tuple._replace(
token_idx=token_idx)
check_docstring = False
# Look for dedent
if check_dedent and token_type == tokenize.DEDENT:
start_position_row, start_position_column = token_start_position
if start_position_column == 0:
# New class...
current_class_name = None
current_method_name = None
else:
# New method...
current_method_name = None
# Reset check_docstring to start over
check_indent = False
check_dedent = False
check_docstring = False
# Look for stray methods
elif token_type == tokenize.NAME and token_string == "def":
method_name = _ParserHelp.find_test_method(token_full_line)
# Add last def/class's tuple
if current_docstring_tuple:
self.docstring_tuples.append(current_docstring_tuple)
class PythonFileHandler(object):
"""
This class handles tokenizing a python file, and provides services with storing all the docstring information, updating the docstring,
updating the original python file, or getting a metadata dictionary from a given docstring.
"""
# Tuple to keep track of all docstring properties:
# class_name: class that docstring belongs to.
# method_name: method that docstring belongs to. If it is class docstring, method_name is None
# offset: offset (column number) of the definition (NOT docstring)
# original_docstring: the original docstring as found in the file.
# new_docstring: the new docstring that has the latest information.
# token_idx: The index at which the docstring token is found. If no docstring found, then this will be the index at which the new docstring should be inserted
docstring_tuple = namedtuple(
'docstring_tuple',
'class_name method_name offset original_docstring new_docstring token_idx'
)
def __init__(self, filename):
"""
This constructor requires a filename to parse and store all its information. The information is stored in a list of named tuples.
@type filename: string
@param filename: Absolute path to the python file that needs docstring handling.
"""
self.filename = filename
# List of docstring namedtuple
self.docstring_tuples = []
self.parse_file()
def parse_file(self):
"""
This method will parse through the python file and populate docstring_tuples. Even if a class/method doesn't have docstring, it would be stored in tuples list.
"""
try:
file_object = open(self.filename, 'r')
file_object.close()
except:
raise TINCMMException("Cannot open file %s" % self.filename)
tinctest.logger.debug("Parsing file %s" % self.filename)
with open(self.filename, 'r') as file_object:
current_class_name = None
current_method_name = None
current_docstring_tuple = None
check_indent = False
check_dedent = False
check_docstring = False
token_idx = -1
for (token_type, token_string, token_start_position,
token_end_position,
token_full_line) in tokenize.generate_tokens(
file_object.next):
token_idx += 1
# Look for current class name
if token_type == tokenize.NAME and token_string == "class":
class_name = _ParserHelp.find_test_class(token_full_line)
if class_name:
tinctest.logger.debug("Found class definition: %s" %
class_name)
# Add any previously constructued tuple
if current_docstring_tuple:
self.docstring_tuples.append(
current_docstring_tuple)
current_class_name = class_name
current_method_name = None
check_indent = True
row, column = token_start_position
current_docstring_tuple = PythonFileHandler.docstring_tuple(
class_name=current_class_name,
method_name=current_method_name,
offset=column,
original_docstring=None,
new_docstring=None,
token_idx=None)
# Check if class doc string is not found
if check_docstring and (
current_class_name and not current_method_name
) and (
(token_type != tokenize.STRING) or
(token_type == tokenize.STRING and
not _ParserHelp.find_docstring_quotes(token_full_line))):
# This is the point where the class docstring should be inserted if any
tinctest.logger.debug(
"FileHandler %s: Did not find class docstring for: %s | %s"
% (self.filename, current_docstring_tuple.class_name,
current_docstring_tuple.method_name))
current_docstring_tuple = current_docstring_tuple._replace(
token_idx=token_idx)
check_docstring = False
check_indent = False
# Check for method definition
if ((not check_docstring) and current_class_name
and token_type == tokenize.NAME
and token_string == "def"):
method_name = _ParserHelp.find_test_method(token_full_line)
tinctest.logger.debug("Found method definition: %s.%s" %
(current_class_name, method_name))
if method_name and class_name:
if current_docstring_tuple:
self.docstring_tuples.append(
current_docstring_tuple)
current_method_name = method_name
check_indent = True
row, column = token_start_position
current_docstring_tuple = PythonFileHandler.docstring_tuple(
class_name=current_class_name,
method_name=current_method_name,
offset=column,
original_docstring=None,
new_docstring=None,
token_idx=None)
# Look for indent
if check_indent and token_type == tokenize.INDENT:
tinctest.logger.debug(
"Found indent after definition: %s | %s" %
(current_class_name, current_method_name))
check_docstring = True
check_dedent = True
continue
# Terminate docstring check if the token following a definition is not a docstring token
if check_docstring and current_class_name and current_method_name and (
(token_type != tokenize.STRING) or
(token_type == tokenize.STRING and
not _ParserHelp.find_docstring_quotes(token_full_line))):
# Token following a defintion is not a docstring token
tinctest.logger.debug(
"FileHandler %s: Did not find method docstring for: %s | %s"
% (self.filename, current_docstring_tuple.class_name,
current_docstring_tuple.method_name))
check_docstring = False
current_docstring_tuple = current_docstring_tuple._replace(
token_idx=token_idx)
# Look for docstring after a definition of class or a method
if check_docstring and token_type == tokenize.STRING and _ParserHelp.find_docstring_quotes(
token_full_line):
tinctest.logger.debug(
"FileHandler %s: Found a docstring for: %s | %s" %
(self.filename, current_docstring_tuple.class_name,
current_docstring_tuple.method_name))
tinctest.logger.debug("Docstring found: %s" % token_string)
# This is docstring!
current_docstring_tuple = current_docstring_tuple._replace(
original_docstring=token_string)
current_docstring_tuple = current_docstring_tuple._replace(
token_idx=token_idx)
check_docstring = False
# Look for dedent
if check_dedent and token_type == tokenize.DEDENT:
start_position_row, start_position_column = token_start_position
if start_position_column == 0:
# New class...
current_class_name = None
current_method_name = None
else:
# New method...
current_method_name = None
# Reset check_docstring to start over
check_indent = False
check_dedent = False
check_docstring = False
# Look for stray methods
elif token_type == tokenize.NAME and token_string == "def":
method_name = _ParserHelp.find_test_method(token_full_line)
# Add last def/class's tuple
if current_docstring_tuple:
self.docstring_tuples.append(current_docstring_tuple)
def update_file(self, file_to_write=None):
"""
This method will update the original python file with the new docstring that is stored in docstring_tuples.
If file_to_write is specified, the original python file will be left as is, and file_to_write will be used
with new docstring.
@type file_to_write: string
@param file_to_write: Absolute path to the file to use to write back the original file with new docstring.
Defaults to None. If None, the original file will be updated.
"""
if not file_to_write:
file_to_write = self.filename
with open(self.filename, 'r') as file_object:
token_list = []
for (token_type, token_string, token_start_position,
token_end_position,
token_full_line) in tokenize.generate_tokens(
file_object.next):
token_list.append((token_type, token_string))
update_needed = False
tokens_inserted = 0
for docstring_tuple in self.docstring_tuples:
# If there is a new docstring and an original docstring, replace the corresponding token string
if docstring_tuple.new_docstring and docstring_tuple.original_docstring:
original_token_type = token_list[docstring_tuple.token_idx][0]
token_list[docstring_tuple.token_idx] = (
original_token_type, docstring_tuple.new_docstring)
update_needed = True
# If there is a new docstring, but no original docstring, a new token has to be inserted at the right token_idx
if docstring_tuple.new_docstring and not docstring_tuple.original_docstring:
token = (tokenize.STRING, docstring_tuple.new_docstring)
# Assuming we will be inserting in the right order here
# Account for previous tokens added in the original list
token_list.insert(
int(docstring_tuple.token_idx) + tokens_inserted, token)
tokens_inserted += 1
token = (tokenize.NEWLINE, '')
token_list.insert(
int(docstring_tuple.token_idx) + tokens_inserted, token)
tokens_inserted += 1
update_needed = True
if not update_needed:
tinctest.logger.debug("No update required for file %s" %
self.filename)
if file_to_write == self.filename:
return
else:
try:
shutil.copyfile(self.filename, file_to_write)
except Exception, e:
tinctest.logger.exception(e)
raise TINCMMException(
"Encountered error while copying file %s to %s" %
(self.filename, file_to_write))
try:
with open(file_to_write, 'w') as file_object:
code_string = tokenize.untokenize(token_list)
file_object.write(code_string)
except Exception, e:
tinctest.logger.exception(e)
raise TINCMMException(
"Encountered error while modifying file: %s" % file_to_write)
for query in self.query_list:
self.query_result_list.append(self._parse_query(query))
def _parse_query(self, query):
"""
Parse the given query string using TINCMM query grammar and return the result
"""
result = None
query_parser = TINCMMQueryGrammar.parser()
try:
result = query_parser.parse_string(query, reset=True, eof=True, matchtype='longest')
except ParseError,e:
tinctest.logger.exception("Error while parsing query %s" %query)
raise TINCMMException("Invalid query %s. Check the syntax of the query." %query)
if query_parser.remainder():
tinctest.logger.error("No complete match found while parsing query %s: Matched text: %s Remainder: %s" %(query,
result,
query_parser.remainder()))
raise TINCMMException("Error while parsing query %s" %query)
if not result:
tinctest.logger.error("No result obtained from parsing the query: %s" %query)
raise TINCMMException("Error while parsing query %s" %query)
return result
if __name__ == '__main__':
parser = SelectQuery.parser()
parser.parse_string("select metadata1,metadata2;")