def test_nested_select_statement(self):
parser = Parser()
sub_query = """SELECT CLASS FROM TAIPAI WHERE CLASS = 'VAN'"""
nested_query = """SELECT ID FROM ({});""".format(sub_query)
parsed_sub_query = parser.parse(sub_query)[0]
actual_stmt = parser.parse(nested_query)[0]
self.assertEqual(actual_stmt.stmt_type, StatementType.SELECT)
self.assertEqual(actual_stmt.target_list[0].col_name, 'ID')
self.assertEqual(actual_stmt.from_table, parsed_sub_query)
sub_query = """SELECT Yolo(frame).bbox FROM autonomous_vehicle_1
WHERE Yolo(frame).label = 'vehicle'"""
nested_query = """SELECT Licence_plate(bbox) FROM
({})
WHERE Is_suspicious(bbox) = 1 AND
Licence_plate(bbox) = '12345';
""".format(sub_query)
query = """SELECT Licence_plate(bbox) FROM TAIPAI
WHERE Is_suspicious(bbox) = 1 AND
Licence_plate(bbox) = '12345';
"""
query_stmt = parser.parse(query)[0]
actual_stmt = parser.parse(nested_query)[0]
sub_query_stmt = parser.parse(sub_query)[0]
self.assertEqual(actual_stmt.from_table, sub_query_stmt)
self.assertEqual(actual_stmt.where_clause, query_stmt.where_clause)
self.assertEqual(actual_stmt.target_list, query_stmt.target_list)
def test_parse(self):
parser = Parser(Environment({'a': 'b'}))
self.assertEqual([['a', 'b', 'c'], ['d', 'e', 'f']],
parser.parse("a b c | d e f"))
self.assertEqual([['a', 'b | b', 'c'], ['d', 'e', 'f']],
parser.parse("a \"b | b\" c | \"d\" e f"))
self.assertEqual([['a', 'b | b', 'c'], ['d', 'e', 'b', 'b']],
parser.parse("a \"b | b\" c | \"d\" e $a \"$a\""))
class ParserTests(unittest.TestCase):
TEST_FILE = '../data/test_data'
def setUp(self):
self.parser = Parser()
def tearDown(self):
parser = None
def test_iterative(self):
i = 0
with open(self.TEST_FILE, 'r') as fp:
while True:
try:
value = next(fp) + next(fp) + next(fp)
value = value[:-1]
next(fp)
expected = next(fp)[3:-1]
if i <= 13:
actual = self.parser.parse(value, validate_checksum_iterative=True, guess_possible_number=False)
else:
actual = self.parser.parse(value, validate_checksum_iterative=True, guess_possible_number=True)
if expected == actual:
print(expected + ' == ' + actual)
else:
print(expected + ' != ' + actual)
i += 1
# self.assertEqual(expected, actual)
except StopIteration:
break
def test_recursive(self):
i = 0
with open(self.TEST_FILE, 'r') as fp:
while True:
try:
value = next(fp) + next(fp) + next(fp)
value = value[:-1]
next(fp)
expected = next(fp)[3:-1]
if i <= 13:
actual = self.parser.parse(value, validate_checksum_iterative=False, guess_possible_number=False)
else:
actual = self.parser.parse(value, validate_checksum_iterative=False, guess_possible_number=True)
if expected == actual:
print(expected + ' == ' + actual)
else:
print(expected + ' != ' + actual)
i += 1
# self.assertEqual(expected, actual)
except StopIteration:
break
def test_should_insert_row_in_table(self):
dummy_info = TableInfo('MyVideo')
dummy_table = TableRef(dummy_info)
columns = [
DataFrameColumn('Frame_ID', ColumnType.INTEGER),
DataFrameColumn('Frame_Path', ColumnType.TEXT, array_dimensions=50)
]
plan_node = CreatePlan(dummy_table, columns, False)
createExec = CreateExecutor(plan_node)
url = createExec.exec()
parser = Parser()
insert_query = """INSERT INTO MyVideo (Frame_ID, Frame_Path)
VALUES (1, '/mnt/frames/1.png');
"""
eva_statement_list = parser.parse(insert_query)
insert_stmt = eva_statement_list[0]
convertor = StatementToPlanConvertor()
convertor.visit(insert_stmt)
logical_plan_node = convertor.plan
print("logical", logical_plan_node)
phy_plan_node = InsertPlan(logical_plan_node.video_catalog_id,
logical_plan_node.column_list,
logical_plan_node.value_list)
insertExec = InsertExecutor(phy_plan_node)
insertExec.exec()
# test if we have a added the in our storage
df = load_dataframe(url)
self.assertEqual(df.collect()[0][0], 1)
self.assertEqual(df.collect()[0][1], "'/mnt/frames/1.png'")
def test_insert_statement(self):
parser = Parser()
insert_query = """INSERT INTO MyVideo (Frame_ID, Frame_Path)
VALUES (1, '/mnt/frames/1.png');
"""
eva_statement_list = parser.parse(insert_query)
self.assertIsInstance(eva_statement_list, list)
self.assertEqual(len(eva_statement_list), 1)
self.assertEqual(eva_statement_list[0].stmt_type, StatementType.INSERT)
insert_stmt = eva_statement_list[0]
# into_table
self.assertIsNotNone(insert_stmt.table)
self.assertIsInstance(insert_stmt.table, TableRef)
self.assertEqual(insert_stmt.table.table_info.table_name, 'MyVideo')
# Column
self.assertIsNotNone(insert_stmt.column_list)
self.assertIsInstance(insert_stmt.column_list, list)
self.assertEqual(len(insert_stmt.column_list), 2)
self.assertEqual(insert_stmt.column_list[0].col_name, 'Frame_ID')
self.assertEqual(insert_stmt.column_list[1].col_name, 'Frame_Path')
# Values
self.assertIsNotNone(insert_stmt.value_list)
self.assertIsInstance(insert_stmt.value_list, list)
self.assertEqual(len(insert_stmt.value_list), 2)
self.assertEqual(insert_stmt.value_list[0].value, 1)
def test_create_udf_statement(self):
parser = Parser()
create_udf_query = """CREATE UDF FastRCNN
INPUT (Frame_Array NDARRAY (3, 256, 256))
OUTPUT (Labels NDARRAY (10), Bbox NDARRAY (10, 4))
TYPE Classification
IMPL 'data/fastrcnn.py';
"""
eva_statement_list = parser.parse(create_udf_query)
self.assertIsInstance(eva_statement_list, list)
self.assertEqual(len(eva_statement_list), 1)
self.assertEqual(eva_statement_list[0].stmt_type,
StatementType.CREATE_UDF)
create_udf_stmt = eva_statement_list[0]
self.assertEqual(create_udf_stmt.name, 'FastRCNN')
self.assertEqual(create_udf_stmt.if_not_exists, False)
self.assertEqual(
create_udf_stmt.inputs[0],
ColumnDefinition('Frame_Array', ParserColumnDataType.NDARRAY,
[3, 256, 256]))
self.assertEqual(
create_udf_stmt.outputs[0],
ColumnDefinition('Labels', ParserColumnDataType.NDARRAY, [10]))
self.assertEqual(
create_udf_stmt.outputs[1],
ColumnDefinition('Bbox', ParserColumnDataType.NDARRAY, [10, 4]))
self.assertEqual(create_udf_stmt.impl_path, Path('data/fastrcnn.py'))
self.assertEqual(create_udf_stmt.udf_type, 'Classification')
class Interpreter(object):
"""
Main class, interacting with user
"""
def __init__(self):
"""
Initializes interpreter
"""
self.environment = Environment()
self.parser = Parser(self.environment)
self.executor = Executor(self.environment)
def interpret(self):
"""
Emulates bash shell
Reads lines, parses them into commands and tokens and executes commands
"""
while True:
try:
sys.stdout.write(f'{self._get_current_directory()}$ ')
sys.stdout.flush()
stdout = self.executor.execute(
self.parser.parse(sys.stdin.readline()))
sys.stdout.write(stdout)
except KeyboardInterrupt:
sys.stdout.write('\n')
def _get_current_directory(self):
home = self.environment.get_variable('HOME')
working_dir = self.environment.get_variable('PWD')
if working_dir.find(home) == 0 and (len(home) == len(working_dir)
or working_dir[len(home)] == '/'):
working_dir = working_dir.replace(home, '~')
return working_dir
def test_select_statement(self):
parser = Parser()
select_query = "SELECT CLASS, REDNESS FROM TAIPAI \
WHERE (CLASS = 'VAN' AND REDNESS < 300 ) OR REDNESS > 500;"
eva_statement_list = parser.parse(select_query)
self.assertIsInstance(eva_statement_list, list)
self.assertEqual(len(eva_statement_list), 1)
self.assertEqual(eva_statement_list[0].stmt_type, StatementType.SELECT)
select_stmt = eva_statement_list[0]
# target List
self.assertIsNotNone(select_stmt.target_list)
self.assertEqual(len(select_stmt.target_list), 2)
self.assertEqual(select_stmt.target_list[0].etype,
ExpressionType.TUPLE_VALUE)
self.assertEqual(select_stmt.target_list[1].etype,
ExpressionType.TUPLE_VALUE)
# from_table
self.assertIsNotNone(select_stmt.from_table)
self.assertIsInstance(select_stmt.from_table, TableRef)
self.assertEqual(select_stmt.from_table.table_info.table_name,
'TAIPAI')
# where_clause
self.assertIsNotNone(select_stmt.where_clause)
def test_select_statement_sample_class(self):
'''Testing sample frequency '''
parser = Parser()
select_query = "SELECT CLASS, REDNESS FROM TAIPAI SAMPLE 5;"
eva_statement_list = parser.parse(select_query)
self.assertIsInstance(eva_statement_list, list)
self.assertEqual(len(eva_statement_list), 1)
self.assertEqual(eva_statement_list[0].stmt_type, StatementType.SELECT)
select_stmt = eva_statement_list[0]
# target List
self.assertIsNotNone(select_stmt.target_list)
self.assertEqual(len(select_stmt.target_list), 2)
self.assertEqual(
select_stmt.target_list[0].etype, ExpressionType.TUPLE_VALUE)
self.assertEqual(
select_stmt.target_list[1].etype, ExpressionType.TUPLE_VALUE)
# from_table
self.assertIsNotNone(select_stmt.from_table)
self.assertIsInstance(select_stmt.from_table, TableRef)
self.assertEqual(
select_stmt.from_table.table.table_name, 'TAIPAI')
# sample_freq
self.assertEqual(select_stmt.from_table.sample_freq,
ConstantValueExpression(5))
def test_create_udf_statement(self):
parser = Parser()
create_udf_query = """CREATE UDF FastRCNN
INPUT (Frame_Array NDARRAY (3, 256, 256))
OUTPUT (Labels NDARRAY (10), Bbox NDARRAY (10, 4))
TYPE Classification
IMPL 'data/fastrcnn.py';
"""
expected_stmt = CreateUDFStatement('FastRCNN', False, [
ColumnDefinition('Frame_Array', ParserColumnDataType.NDARRAY,
[3, 256, 256])
], [
ColumnDefinition('Labels', ParserColumnDataType.NDARRAY, [10]),
ColumnDefinition('Bbox', ParserColumnDataType.NDARRAY, [10, 4])
], Path('data/fastrcnn.py'), 'Classification')
eva_statement_list = parser.parse(create_udf_query)
self.assertIsInstance(eva_statement_list, list)
self.assertEqual(len(eva_statement_list), 1)
self.assertEqual(eva_statement_list[0].stmt_type,
StatementType.CREATE_UDF)
create_udf_stmt = eva_statement_list[0]
self.assertEqual(create_udf_stmt, expected_stmt)
def test_select_union_statement(self):
parser = Parser()
select_union_query = "SELECT CLASS, REDNESS FROM TAIPAI \
UNION ALL SELECT CLASS, REDNESS FROM SHANGHAI;"
eva_statement_list = parser.parse(select_union_query)
select_stmt = eva_statement_list[0]
self.assertIsNotNone(select_stmt.union_link)
self.assertEqual(select_stmt.union_all, True)
second_select_stmt = select_stmt.union_link
self.assertIsNone(second_select_stmt.union_link)
def test_load_data_statement(self):
parser = Parser()
load_data_query = """LOAD DATA INFILE 'data/video.mp4' INTO MyVideo;"""
expected_stmt = LoadDataStatement(TableRef(TableInfo('MyVideo')),
Path('data/video.mp4'))
eva_statement_list = parser.parse(load_data_query)
self.assertIsInstance(eva_statement_list, list)
self.assertEqual(len(eva_statement_list), 1)
self.assertEqual(eva_statement_list[0].stmt_type,
StatementType.LOAD_DATA)
load_data_stmt = eva_statement_list[0]
self.assertEqual(load_data_stmt, expected_stmt)
def handle_request(transport, query):
"""
Reads a request from a client and processes it
If user inputs 'quit' stops the event loop
otherwise just echoes user input
"""
parser = Parser()
eva_statement = parser.parse(query)
insert_statement = eva_statement[0]
LoggingManager().log("Result from the parser: " + str(insert_statement))
convertor = StatementToPlanConvertor()
convertor.visit(insert_statement)
convertor.plan
eva_statement = parser.parse(query)
response_message = ""
for i in range(len(eva_statement)):
LoggingManager().log("Parse Tree: " + str(eva_statement[i]))
logical_plan = StatementToPlanConvertor().visit(eva_statement[i])
physical_plan = PlanGenerator().build(logical_plan)
df = PlanExecutor(physical_plan).execute_plan()
statement_response_message = ','.join(map(str, df.collect()))
response_message += statement_response_message
LoggingManager().log('Response to client: --|' +
str(response_message) +
'|--')
data = response_message.encode('ascii')
transport.write(data)
return response_message
def test_multiple_statement_queries(self):
parser = Parser()
multiple_queries = []
multiple_queries.append("SELECT CLASS FROM TAIPAI \
WHERE CLASS = 'VAN' AND REDNESS < 300 OR REDNESS > 500; \
SELECT REDNESS FROM TAIPAI \
WHERE (CLASS = 'VAN' AND REDNESS = 300)")
for query in multiple_queries:
eva_statement_list = parser.parse(query)
self.assertIsInstance(eva_statement_list, list)
self.assertEqual(len(eva_statement_list), 2)
self.assertIsInstance(eva_statement_list[0], AbstractStatement)
self.assertIsInstance(eva_statement_list[1], AbstractStatement)
class BankOCR:
TEST_FILE = '../data/test_data'
def __init__(self):
self.parser = Parser()
pass
def __del__(self):
parser = None
pass
def run_test(self, iterative=True):
i = 0
with open(self.TEST_FILE, 'r') as fp:
while True:
try:
value = next(fp) + next(fp) + next(fp)
value = value[:-1]
next(fp)
expected = next(fp)[3:-1]
if i <= 13:
actual = self.parser.parse(value, validate_checksum_iterative=iterative, guess_possible_number=False)
else:
actual = self.parser.parse(value, validate_checksum_iterative=iterative, guess_possible_number=True)
if expected == actual:
print(expected + ' == ' + actual)
else:
print(expected + ' != ' + actual)
i += 1
except StopIteration:
break
def main(self):
self.run_test(iterative=True)
self.run_test(iterative=False)
def test_create_statement(self):
parser = Parser()
single_queries = []
single_queries.append("""CREATE TABLE IF NOT EXISTS Persons (
Frame_ID INTEGER UNIQUE,
Frame_Data TEXT(10),
Frame_Value FLOAT(1000, 201),
Frame_Array NDARRAY (5, 100, 2432, 4324, 100)
);""")
for query in single_queries:
eva_statement_list = parser.parse(query)
self.assertIsInstance(eva_statement_list, list)
self.assertEqual(len(eva_statement_list), 1)
self.assertIsInstance(eva_statement_list[0], AbstractStatement)
def test_load_data_statement(self):
parser = Parser()
load_data_query = """LOAD DATA INFILE 'data/video.mp4' INTO MyVideo;"""
eva_statement_list = parser.parse(load_data_query)
self.assertIsInstance(eva_statement_list, list)
self.assertEqual(len(eva_statement_list), 1)
self.assertEqual(eva_statement_list[0].stmt_type,
StatementType.LOAD_DATA)
load_data_stmt = eva_statement_list[0]
# into table
self.assertIsNotNone(load_data_stmt.table)
self.assertIsInstance(load_data_stmt.table, TableRef)
self.assertEqual(load_data_stmt.table.table_info.table_name, 'MyVideo')
self.assertEqual(load_data_stmt.path, Path('data/video.mp4'))
def test_select_statement_limit_class(self):
'''Testing limit clause in select statement
Class: SelectStatement'''
parser = Parser()
select_query = "SELECT CLASS, REDNESS FROM TAIPAI \
WHERE (CLASS = 'VAN' AND REDNESS < 400 ) OR REDNESS > 700 \
ORDER BY CLASS, REDNESS DESC LIMIT 3;"
eva_statement_list = parser.parse(select_query)
self.assertIsInstance(eva_statement_list, list)
self.assertEqual(len(eva_statement_list), 1)
self.assertEqual(eva_statement_list[0].stmt_type, StatementType.SELECT)
select_stmt = eva_statement_list[0]
# target List
self.assertIsNotNone(select_stmt.target_list)
self.assertEqual(len(select_stmt.target_list), 2)
self.assertEqual(select_stmt.target_list[0].etype,
ExpressionType.TUPLE_VALUE)
self.assertEqual(select_stmt.target_list[1].etype,
ExpressionType.TUPLE_VALUE)
# from_table
self.assertIsNotNone(select_stmt.from_table)
self.assertIsInstance(select_stmt.from_table, TableRef)
self.assertEqual(select_stmt.from_table.table_info.table_name,
'TAIPAI')
# where_clause
self.assertIsNotNone(select_stmt.where_clause)
# orderby_clause
self.assertIsNotNone(select_stmt.orderby_list)
self.assertEqual(len(select_stmt.orderby_list), 2)
self.assertEqual(select_stmt.orderby_list[0][0].col_name, 'CLASS')
self.assertEqual(select_stmt.orderby_list[0][1],
ParserOrderBySortType.ASC)
self.assertEqual(select_stmt.orderby_list[1][0].col_name, 'REDNESS')
self.assertEqual(select_stmt.orderby_list[1][1],
ParserOrderBySortType.DESC)
# limit_count
self.assertIsNotNone(select_stmt.limit_count)
self.assertEqual(select_stmt.limit_count, ConstantValueExpression(3))
def do_query(self, query):
"""Takes in SQL query and generates the output"""
# Type exit to stop program
if (query == "exit" or query == "EXIT"):
raise SystemExit
if len(query) == 0:
print("Empty query")
else:
try:
# Connect and Query from Eva
parser = Parser()
eva_statement = parser.parse(query)
select_stmt = eva_statement[0]
print("Result from the parser:")
print(select_stmt)
print('\n')
# Read Input Videos
# Replace with Input Pipeline once finished
input_video = []
for filename in glob.glob('data/sample_video/*.jpg'):
im = Image.open(filename)
# to handle 'too many open files' error
im_copy = im.copy()
input_video.append(im_copy)
im.close()
# Write Output to final folder
# Replace with output pipeline once finished
ouput_frames = random.sample(input_video, 50)
output_folder = "data/sample_output/"
for i in range(len(ouput_frames)):
frame_name = output_folder + "output" + str(i) + ".jpg"
ouput_frames[i].save(frame_name)
print("Refer pop-up for a sample of the output")
ouput_frames[0].show()
except TypeError:
print("SQL Statement improperly formatted. Try again.")
def test_select_statement_orderby_class(self):
'''Testing order by clause in select statement
Class: SelectStatement'''
parser = Parser()
select_query = "SELECT CLASS, REDNESS FROM TAIPAI \
WHERE (CLASS = 'VAN' AND REDNESS < 400 ) OR REDNESS > 700 \
ORDER BY CLASS, REDNESS DESC;"
# if orderby sort_type (ASC/DESC) not provided, should default to ASC
eva_statement_list = parser.parse(select_query)
self.assertIsInstance(eva_statement_list, list)
self.assertEqual(len(eva_statement_list), 1)
self.assertEqual(eva_statement_list[0].stmt_type, StatementType.SELECT)
select_stmt = eva_statement_list[0]
# target List
self.assertIsNotNone(select_stmt.target_list)
self.assertEqual(len(select_stmt.target_list), 2)
self.assertEqual(
select_stmt.target_list[0].etype, ExpressionType.TUPLE_VALUE)
self.assertEqual(
select_stmt.target_list[1].etype, ExpressionType.TUPLE_VALUE)
# from_table
self.assertIsNotNone(select_stmt.from_table)
self.assertIsInstance(select_stmt.from_table, TableRef)
self.assertEqual(
select_stmt.from_table.table.table_name, 'TAIPAI')
# where_clause
self.assertIsNotNone(select_stmt.where_clause)
# orderby_clause
self.assertIsNotNone(select_stmt.orderby_list)
self.assertEqual(len(select_stmt.orderby_list), 2)
self.assertEqual(select_stmt.orderby_list[0][0].col_name, 'CLASS')
self.assertEqual(
select_stmt.orderby_list[0][1], ParserOrderBySortType.ASC)
self.assertEqual(select_stmt.orderby_list[1][0].col_name, 'REDNESS')
self.assertEqual(
select_stmt.orderby_list[1][1], ParserOrderBySortType.DESC)
def test_single_statement_queries(self):
parser = Parser()
single_queries = []
single_queries.append("SELECT CLASS FROM TAIPAI;")
single_queries.append("SELECT CLASS FROM TAIPAI WHERE CLASS = 'VAN';")
single_queries.append("SELECT CLASS,REDNESS FROM TAIPAI \
WHERE CLASS = 'VAN' AND REDNESS > 20;")
single_queries.append("SELECT CLASS FROM TAIPAI \
WHERE (CLASS = 'VAN' AND REDNESS < 300 ) OR REDNESS > 500;")
single_queries.append("SELECT CLASS FROM TAIPAI \
WHERE (CLASS = 'VAN' AND REDNESS < 300 ) OR REDNESS > 500;")
for query in single_queries:
eva_statement_list = parser.parse(query)
self.assertIsInstance(eva_statement_list, list)
self.assertEqual(len(eva_statement_list), 1)
self.assertIsInstance(eva_statement_list[0], AbstractStatement)
def test_insert_statement(self):
parser = Parser()
insert_query = """INSERT INTO MyVideo (Frame_ID, Frame_Path)
VALUES (1, '/mnt/frames/1.png');
"""
expected_stmt = InsertTableStatement(TableRef(TableInfo('MyVideo')), [
TupleValueExpression('Frame_ID'),
TupleValueExpression('Frame_Path')
], [
ConstantValueExpression(1),
ConstantValueExpression('/mnt/frames/1.png')
])
eva_statement_list = parser.parse(insert_query)
self.assertIsInstance(eva_statement_list, list)
self.assertEqual(len(eva_statement_list), 1)
self.assertEqual(eva_statement_list[0].stmt_type, StatementType.INSERT)
insert_stmt = eva_statement_list[0]
self.assertEqual(insert_stmt, expected_stmt)
def test_select_statement_class(self):
''' Testing setting different clauses for Select
Statement class
Class: SelectStatement'''
select_stmt_new = SelectStatement()
parser = Parser()
select_query_new = "SELECT CLASS, REDNESS FROM TAIPAI \
WHERE (CLASS = 'VAN' AND REDNESS < 400 ) OR REDNESS > 700;"
eva_statement_list = parser.parse(select_query_new)
select_stmt = eva_statement_list[0]
select_stmt_new.where_clause = select_stmt.where_clause
select_stmt_new.target_list = select_stmt.target_list
select_stmt_new.from_table = select_stmt.from_table
self.assertEqual(select_stmt_new.where_clause,
select_stmt.where_clause)
self.assertEqual(select_stmt_new.target_list, select_stmt.target_list)
self.assertEqual(select_stmt_new.from_table, select_stmt.from_table)
self.assertEqual(str(select_stmt_new), str(select_stmt))
def handle_request(transport, query):
"""
Reads a request from a client and processes it
If user inputs 'quit' stops the event loop
otherwise just echoes user input
"""
parser = Parser()
eva_statement = parser.parse(query)
for i in range(len(eva_statement)):
LoggingManager().log("Result from the parser: " +
str(eva_statement[i]))
logical_plan = StatementToPlanConvertor().visit(eva_statement[i])
physical_plan = PlanGenerator().build(logical_plan)
PlanExecutor(physical_plan).execute_plan()
table_name = 'MyVideo'
file_url = os.path.join(tempfile.gettempdir(), table_name)
file_url = 'file://' + file_url
df = load_dataframe(file_url)
response_message = ','.join(map(str, df.collect()))
LoggingManager().log("Response received" + str(response_message))
LoggingManager().log('Response to client: --|' + str(response_message) +
'|--')
data = response_message.encode('ascii')
transport.write(data)
return response_message
def main():
if len(sys.argv) == 1:
if not os.path.isfile(os.getcwd() + "/todosconfig.toml"):
print(
sys.argv[0] +
": cannot find todosconfig.toml. Run 'todos init' to create it."
)
exit(1)
with open(os.getcwd() + "/todosconfig.toml", "r") as f:
config = toml.loads(f.read())
tags = config["tags"]
tag_names = tags.keys()
parser = Parser(os.getcwd(), tag_names)
parser.parse()
issues = parser.issues
# Output to text file
text_path = config["text-path"] + "/Tasks.txt"
plugin = TextPlugin(text_path, issues, tag_names)
success = plugin.run()
if not success:
print(sys.argv[0] + ": failed to output to text file.")
# Output to github issues
if any("github" in s for s in config["target"]):
github_repo = config["github-repo"]
if github_repo.endswith("/"):
parts = github_repo.split("/")
github_repo_user = parts[len(parts) - 3]
github_repo_name = partr[len(parts) - 2]
else:
parts = github_repo.split("/")
github_repo_user = parts[len(parts) - 2]
github_repo_name = parts[len(parts) - 1]
plugin = GithubPlugin(config["github-token"], issues,
github_repo_user, github_repo_name)
success = plugin.run()
if not success:
print(sys.argv[0] + ": failed to output to github.")
# Output to trello issues
if any("trello" in s for s in config["target"]):
plugin = TrelloPlugin(config["trello-token"], trello_api_key,
issues, config["trello-board"],
config["trello-tasks-list"],
config["trello-done-list"],
tags) # Change this
success = plugin.run()
if not success:
print(sys.argv[0] + ": failed to output to trello.")
# Init functionality
elif sys.argv[1] == "init":
if os.path.isfile("todosconfig.toml"):
print(sys.argv[0] + ": already been initialized.")
exit(1)
try:
current_dir = os.path.basename(os.path.realpath(os.getcwd()))
project_name = query_string(
"What is the name of the project? [" + current_dir + "] ",
current_dir)
text_path = query_string(
"Where should the output file be placed? [" + os.getcwd() +
"] ", os.getcwd())
github = query_yes_no("Do you want to create github issues?",
False)
github_auth = None
github_repo = None
if github:
try_number = 0
while try_number < 3:
username = query_string(
"Please enter your github username: "******"Please enter your github password: "******"Please enter the link to the github repository: ")
break
else:
print(sys.argv[0] + ": bad credentials, aborting.")
if github_auth is None:
print(sys.argv[0] + ": bad credentials, aborting.")
exit(1)
trello = query_yes_no(
"Do you want to put your tasks on a trello board?", False)
trello_token = None
todo_list = None
done_list = None
if trello:
print("Get your application token from:")
print(
"https://trello.com/1/authorize?key=" + trello_api_key +
"&name=TODOS&expiration=never&response_type=token&scope=read,write"
)
trello_token = query_string("Paste your token here: ")
trello_board = query_string(
"What's the name of the trello board? [" + current_dir +
"] ", current_dir)
# Create the trello board
r = requests.get(
"https://trello.com/1/members/my/boards?key=" +
trello_api_key + "&token=" + trello_token)
if r.status_code < 200 or r.status_code > 299:
print(sys.argv[0] + ": bad credentials, aborting.")
exit(1)
else:
boards = r.json()
if not any(x["name"] == trello_board for x in boards):
r = requests.post("https://trello.com/1/boards",
json={
"key": trello_api_key,
"token": trello_token,
"name": trello_board,
"defaultLists": False
})
board = r.json()
board_id = board["id"]
else:
board_id = [
x["id"] for x in boards
if x["name"] == trello_board
][0]
r = requests.get("https://api.trello.com/1/boards/" +
board_id + "/lists?key=" +
trello_api_key + "&token=" + trello_token)
board_lists = r.json()
# Check if the list exists first
if any(x["name"] == "TODOS Done" for x in board_lists):
done_list = [
x["id"] for x in board_lists
if x["name"] == "TODOS Done"
][0]
else:
r = requests.post("https://trello.com/1/boards/" +
board_id + "/lists",
json={
"key": trello_api_key,
"token": trello_token,
"name": "TODOS Done",
"defaultLists": False
})
if r.status_code < 200 or r.status_code > 299:
print(sys.argv[0] +
": failed to create trello board, aborting.")
exit(1)
else:
list = r.json()
done_list = list["id"]
# Check if the list exists first
if any(x["name"] == "TODOS Tasks" for x in board_lists):
todo_list = [
x["id"] for x in board_lists
if x["name"] == "TODOS Tasks"
][0]
else:
r = requests.post("https://trello.com/1/boards/" +
board_id + "/lists",
json={
"key": trello_api_key,
"token": trello_token,
"name": "TODOS Tasks",
"defaultLists": False
})
if r.status_code < 200 or r.status_code > 299:
print(sys.argv[0] +
": failed to create trello board, aborting.")
exit(1)
else:
list = r.json()
todo_list = list["id"]
except KeyboardInterrupt:
print("")
exit(1)
config = {
"title": project_name,
"tags": {
"TODO": "yellow",
"FIXME": "red",
"NOTE": "green",
},
"target": [],
"text-path": text_path
}
if github:
config["target"].append("github")
config["github-token"] = github_auth
config["github-repo"] = github_repo
if trello:
config["target"].append("trello")
config["trello-token"] = trello_token
config["trello-board"] = board_id
config["trello-tasks-list"] = todo_list
config["trello-done-list"] = done_list
with open(os.getcwd() + "/todosconfig.toml", "w") as f:
f.write(toml.dumps(config))
f = open(".gitignore", "a+")
f.write("\ntodosconfig.toml\nTasks.txt\n.todos\n")
else:
print(sys.argv[0] + ": is not a " + sys.argv[0] + " command.")
def assert_parse(expression, result, strict=True):
p = Parser(expression)
if strict:
assert str(p.parse().value) == result
else:
assert p.parse() == Parser(result).parse()
