def main():
logger = init_logger("main")
logger.info("start program")
p = ProgramArguments()
try:
p.parse()
except ArgPathToFileError:
exit(2)
logger.info(p.path_to_file + p.function_name + p.module_name +
str(p.timeout) + p.init_struct_code + p.step + str(p.repeat))
timer = Timer(p.path_to_file, p.function_name, p.module_name, p.timeout,
p.init_struct_code, p.step, p.repeat)
try:
time_list = timer.measure_time()
except ArgFunNameError:
print("There is no such a fucntion :" + p.function_name)
exit(2)
try:
estimate_complexity(time_list)
except NotEnoughStepsError:
exit(2)
logger.info("end program")
def parse(self):
logger = init_logger("parse")
par = ArgumentParser(
description='Program to estimate python scripts complexity.',
formatter_class=ArgumentDefaultsHelpFormatter,
epilog='')
par.add_argument('-f', '--function', nargs='?', type=str,
default="main",
help="function to be measure", dest="function_name")
par.add_argument('-m', '--module', nargs='?', type=str,
default="module?",
help="module frm witch is function",
dest="module_name")
par.add_argument('-p', '--path',
type=str, default="",
help="path to file containing function",
dest="path_to_file")
par.add_argument('-t', '--timeout', type=int,
default=30,
help="maximum time of execution function in seconds",
dest="timeout")
par.add_argument('-i', '--init', type=str,
default="",
help="initializaion code of varaibles/structurs "
"used by function", dest="init_struct_code")
par.add_argument('-c', '--clean', type=str,
default="",
help="code responsible for cleaning",
dest="clean_code")
par.add_argument('-s', '--step', type=str,
default="",
help="incrementation step",
dest="step")
par.add_argument('-r', '--repeat', type=int,
default=3,
help="number of repeats",
dest="repeat")
# par.print_help()
args = par.parse_args()
if args.path_to_file != "":
try:
with open(args.path_to_file, 'r'):
self.path_to_file = args.path_to_file
except IOError:
logger.info("ERROR cannot open" + str(args.path_to_file))
print("Could not read from file: " + str(args.path_to_file))
raise ArgPathToFileError
self.function_name = args.function_name
self.timeout = args.timeout
self.init_struct_code = args.init_struct_code
self.clean_code = args.clean_code
self.step = args.step
self.repeat = args.repeat
logger.info("End parsing arguments")
def inner(*args, **kwargs):
try:
timer = Timer(s, quit_function)
timer.start()
result = fn(*args, **kwargs)
except KeyboardInterrupt:
logger = init_logger("exit_after_decorator")
logger.info("Timeout reached!")
print("Timeout reached!")
timer.cancel()
return -1
finally:
timer.cancel()
return result
def measure_time(self):
logger = init_logger("measure_time")
logger.info("Begin measure time")
loaded_module = SourceFileLoader(
self.module_name,
self.path_to_module).load_module()
all_functions = inspect.getmembers(loaded_module, inspect.isfunction)
fun_names = []
for i in all_functions:
fun_names.append(i[0])
if self.function_to_measure.split("(")[0] not in fun_names:
logger.info("ArgFunNameError raised because "
"of no function with this name")
logger.debug(self.function_to_measure.split("(")[0])
logger.debug(all_functions)
raise ArgFunNameError
returns_values_list = []
init_struct_code_copy = self.init_struct_code
for i in range(0, self.repeats):
setup = "from {0} import {1}; {2}"\
.format(self.module_name, self.function_to_measure.split("(")[0],
init_struct_code_copy)
@exit_after(self.timeout)
def __inside_measure():
returns_values_list.append(
timeit.timeit(self.function_to_measure, setup=setup,
number=1))
__inside_measure()
logger.debug(init_struct_code_copy)
logger.info("Times loop: " + str(i))
init_struct_code_copy = add_step(init_struct_code_copy, self.step)
logger.info(returns_values_list)
logger.info("End measure time")
return returns_values_list
def add_step(value, step):
logger = init_logger("add_step")
value_list = str.split(value, ";")
logger.debug(value_list)
step_list = str.split(step, ";")
for i in value_list:
if i.find("=") == -1:
value_list.remove(i)
for i in step_list:
if i.find("=") == -1:
step_list.remove(i)
logger.debug(value_list)
for index, elem in enumerate(value_list):
logger.debug(str.split(elem, "="))
start_value = str.split(elem, "=")[1]
old_val = start_value
step_to_add = str.split(step_list[index], "=")[1]
if not any(c.isalpha() for c in step_to_add) \
and not any(c.isalpha() for c in start_value):
if step_list[index].find("-") == -1:
start_value = " " + str(
float(start_value) + float(step_to_add))
else:
start_value = " " + str(
float(start_value) - float(step_to_add))
else:
start_value += step_to_add
start_value += ";"
value_list[index] = str.replace(value_list[index], old_val,
start_value)
logger.info("End adding step values to value result: " +
''.join(value_list))
return ''.join(value_list)
from bs4 import BeautifulSoup
import simplejson as json
import os
import os.path
from pathlib import Path
from datetime import datetime
import stack_rc_classifier_baseline
# init
s = requests.Session()
base_url = 'https://api.stackexchange.com/2.2/'
# site = 'stackoverflow'
site = 'serverfault'
annotation_filename = './outputs/annotated_qa_items_dict.json'
logger = logger_init.init_logger("stackcli_explorer")
def get_number_input(user_input, default=5):
try:
val = int(user_input)
return int(user_input)
except ValueError:
logger.info(
"No.. input is not a number. Using default ({})".format(default))
return default
def find_questions(query_terms, page_nb=1, page_size=30):
logger.info("finding questions with seqrch terms [{}]".format(query_terms))
search_api = 'search/advanced'
os.path.join(os.path.dirname(os.path.realpath(__file__)), "..",
"data", "report_data.csv"),
owerwrite=True)
if result.is_ok():
rv = ResultOk(None)
data_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
"..", "data", "report_data.csv")
result = load_data_file(data_file=data_file)
except Exception as ex:
log.error("Exception caught - {ex}".format(ex=ex))
rv = ResultKo(ex)
log.info(" ({rv}) <<".format(rv=rv))
return rv
if __name__ == "__main__":
init_logger('/tmp',
"etl.log",
log_level=logging.DEBUG,
std_out_log_level=logging.DEBUG)
parser = argparse.ArgumentParser()
args = parser.parse_args()
rv = main(args)
ret_val = os.EX_OK if rv.is_ok() == True else os.EX_USAGE
sys.exit(ret_val)
def estimate_complexity(time_list):
logger = init_logger("complexity")
if len(time_list) <= 2:
print("It is imposible to calculate complexity with this data!")
logger.info("Not enough arguments")
raise NotEnoughStepsError
if len(time_list) == 3:
print("It is the minimum size od data so complexity can be wrong!")
logger.info("minimum number of arguments")
difference = time_list[1] - time_list[0]
const_complex_list = [time_list[0]]
n_complex_list = [time_list[0]]
n_logn_complex_list = [time_list[0]]
n_2_complex_list = [time_list[0]]
n_3_complex_list = [time_list[0]]
for i in range(0, len(time_list) - 1):
const_complex_list.append(time_list[1])
n_complex_list.append(time_list[1] + difference * i)
n_logn_complex_list.append(time_list[1] +
(difference + log10(1 + difference)) * i)
n_2_complex_list.append(time_list[1])
n_3_complex_list.append(time_list[1])
for i in range(2, len(time_list)):
n_2_complex_list.append(n_2_complex_list[i - 1] +
0.66 * i * difference)
n_3_complex_list.append(n_3_complex_list[i - 1] +
i * i * difference * 0.33)
print("Original: \t\t" + str(time_list))
print("calculated O(1): \t" + str(const_complex_list))
print("calculated O(n): \t" + str(n_complex_list))
print("calculated O(nlogn):" + str(n_logn_complex_list))
print("calculated O(n^2): \t" + str(n_2_complex_list))
print("calculated O(n^3): \t" + str(n_3_complex_list))
difference_list = list()
difference_list.append(
total_difference_between_two_lists(time_list, const_complex_list))
difference_list.append(
total_difference_between_two_lists(time_list, n_complex_list))
difference_list.append(
total_difference_between_two_lists(time_list, n_logn_complex_list))
difference_list.append(
total_difference_between_two_lists(time_list, n_2_complex_list))
difference_list.append(
total_difference_between_two_lists(time_list, n_3_complex_list))
min_val = difference_list[0]
min_val_index = 0
for index, elem in enumerate(difference_list):
if elem < min_val:
min_val = elem
min_val_index = index
with open("complexity_tables.txt", "w") as file_to_s:
file_to_s.writelines("N * step time cost\n")
if min_val_index == 0:
print("Complexity is O(1).")
for i in range(1, 100):
file_to_s.writelines(
str(i) + " * step " + str(time_list[1]) + "\n")
if min_val_index == 1:
print("Complexity is O(n).")
for i in range(1, 100):
file_to_s.writelines(
str(i) + " * step " +
str(time_list[1] + difference * i) + "\n")
if min_val_index == 2:
print("Complexity is O(nlogn).")
for i in range(1, 100):
file_to_s.writelines(
str(i) + " * step " +
str(time_list[1] +
(difference + log10(1 + difference)) * i) + "\n")
if min_val_index == 3:
print("Complexity is O(n^2).")
n_2_complex_list_to_save = [time_list[0]]
n_2_complex_list_to_save.append(time_list[1])
for i in range(2, 100):
n_2_complex_list_to_save.append(n_2_complex_list_to_save[i -
1] +
0.66 * i * difference)
for i in range(0, 100):
file_to_s.writelines(
str(i) + " * step " + str(n_2_complex_list_to_save[i]) +
"\n")
if min_val_index == 4:
print("Complexity is O(n^3).")
n_3_complex_list_to_save = [time_list[0]]
n_3_complex_list_to_save.append(time_list[1])
for i in range(2, 100):
n_3_complex_list_to_save.append(n_3_complex_list_to_save[i -
1] +
i * i * difference * 0.33)
for i in range(0, 100):
file_to_s.writelines(
str(i) + " * step " + str(n_3_complex_list_to_save[i]) +
"\n")
logger.info("End calculating complexity")
Compatible with: spaCy v2.0.0+
"""
from __future__ import unicode_literals, print_function
import random
from pathlib import Path
import thinc.extra.datasets
import spacy
from spacy.util import minibatch, compounding
from bs4 import BeautifulSoup
import os
import simplejson as json
import logger_init
logger = logger_init.init_logger("stack_rc_classifier_baseline")
def train(annotation_filename, model=None, output_dir=None, n_iter=20, n_texts=2000, init_tok2vec=None):
if output_dir is not None:
output_dir = Path(output_dir)
if not output_dir.exists():
output_dir.mkdir()
if model is not None:
nlp = spacy.load(model) # load existing spaCy model
logger.info("Loaded model '%s'" % model)
else:
nlp = spacy.blank("en") # create blank Language class
logger.info("Created blank 'en' model")