def ioput_baseline_predict(ff):
with open(ff, 'r') as f:
text = f.read().splitlines()
output_yaml = {}
for i, orig_text in enumerate(text):
rp = RParser()
#Preprocess
pre_p = PreProcessor()
process_text, sub_table = pre_p.process([orig_text.strip()])
sen = process_text[0]
#RParse
verb_parent, method_parent = rp.parse_v_method(sen)
ioput_phrases = rp.parse_input_output(sen, verb_parent, method_parent)
NPs = rp.return_NPs(sen)
#Postprocess
#post_p = PostProcessor(verb_parent, method_parent, ioput_phrases, orig_text.strip(), sub_table)
#summary = post_p.process()
summary = {}
summary['action1'] = {}
for ioput_seq_num, np in enumerate(NPs):
summary['action1']['input_output_{}'.format(ioput_seq_num +
1)] = repr(np)
output_yaml['sen{}'.format(i + 1)] = summary
file_path = ff.split('/')
output_file_name_list = file_path[-1].split('.')
output_file_name_list[1] = 'ioput_baseline'
output_file_name_list[2] = 'yaml'
file_path[-1] = '.'.join(output_file_name_list)
output_file_path = '/'.join(file_path)
with open(output_file_path, 'w') as f:
f.write(yaml.dump(output_yaml, default_flow_style=False))
def pre_process(self):
pre_processor = PreProcessor(self.path, pickle_arb_id_filename,
pickle_j1979_filename, self.use_j1979)
pid_dictionary = pre_processor.import_pid_dict(pid_file)
id_dictionary, j1979_dictionary = pre_processor.generate_arb_id_dictionary(
a_timer, tang_normalize_strategy, pid_dictionary, time_conversion,
freq_analysis_accuracy, freq_synchronous_threshold,
force_pre_processing)
if dump_to_pickle:
if force_pre_processing:
if path.isfile(pickle_arb_id_filename):
remove(pickle_arb_id_filename)
if path.isfile(pickle_j1979_filename):
remove(pickle_j1979_filename)
# Lexical analysis will add additional information to the Arb ID dict. Don't dump if you're going to
# immediately delete and replace pickle_arb_id_filename during Lexical Analysis.
if not force_lexical_analysis:
if not path.isfile(pickle_arb_id_filename) and id_dictionary:
print("\nDumping arb ID dictionary for " +
self.output_vehicle_dir + " to " +
pickle_arb_id_filename)
dump(id_dictionary, open(pickle_arb_id_filename, "wb"))
print("\tComplete...")
if not path.isfile(pickle_j1979_filename) and j1979_dictionary:
print("\nDumping J1979 dictionary for " +
self.output_vehicle_dir + " to " +
pickle_j1979_filename)
dump(j1979_dictionary, open(pickle_j1979_filename, "wb"))
print("\tComplete...")
return id_dictionary, j1979_dictionary, pid_dictionary
def jpg_to_emoji(original_image, work_location, output_file, company_name,
emojis_in_width, emoji_size, do_preprocessing, use_kd_tree):
if (do_preprocessing):
PreProcessor.preprocess_emojis(work_location)
emoji_list = get_filtered_emoji_list(work_location, company_name)
valid_emojis_dict = {
"emojis": emoji_list
}
emoji_mapper = EmojiMapper.EmojiMapper(emoji_dict=valid_emojis_dict,
use_kd_tree=use_kd_tree)
t_before_getting_grid = time.time()
emoji_grid = image_to_emoji_grid(misc.imread(original_image),
emojis_in_width, emoji_mapper)
t_after_getting_grid = time.time()
print("Time to Get Emoji grid: %s" %
str(t_after_getting_grid - t_before_getting_grid))
if (output_file.endswith(".html")):
# Write into local html file.
emoji_grid_to_html_file(emoji_grid, output_file, emoji_size)
elif (output_file.endswith(".png")):
# Write to png
image_array = emoji_grid_to_image(emoji_grid, emoji_size)
misc.imsave(output_file, image_array)
else:
print "UnknownFileName"
t_after_getting_output_file = time.time()
print("Time to Build output file: %s" %
str(t_after_getting_output_file - t_after_getting_grid))
def ioput_baseline_predict(ff):
with open(ff, 'r') as f:
text = f.read().splitlines()
output_yaml = {}
for i, orig_text in enumerate(text):
rp = RParser()
#Preprocess
pre_p = PreProcessor()
process_text, sub_table = pre_p.process([orig_text.strip()])
sen = process_text[0]
#RParse
verb_parent, method_parent = rp.parse_v_method(sen)
ioput_phrases = rp.parse_input_output(sen, verb_parent, method_parent)
NPs = rp.return_NPs(sen)
#Postprocess
#post_p = PostProcessor(verb_parent, method_parent, ioput_phrases, orig_text.strip(), sub_table)
#summary = post_p.process()
summary = {}
summary['action1'] = {}
for ioput_seq_num, np in enumerate(NPs):
summary['action1']['input_output_{}'.format(ioput_seq_num+1)] = repr(np)
output_yaml['sen{}'.format(i+1)] = summary
file_path = ff.split('/')
output_file_name_list = file_path[-1].split('.')
output_file_name_list[1] = 'ioput_baseline'
output_file_name_list[2] = 'yaml'
file_path[-1] = '.'.join(output_file_name_list)
output_file_path = '/'.join(file_path)
with open(output_file_path, 'w') as f:
f.write(yaml.dump(output_yaml, default_flow_style=False))
def test_downsample(self):
# Create a preprocessor object and define a test greyscale array (8-bit)
preprocess = PreProcessor()
testGreyFrame = np.random.randint(255, size=(210, 160))
# Use the PP to half the size of the input
halfFrame = preprocess.halfDownsample(testGreyFrame)
# Ensure the frame has reduced in size by half
self.assertEqual(testGreyFrame.shape[0] / 2, halfFrame.shape[0])
self.assertEqual(testGreyFrame.shape[1] / 2, halfFrame.shape[1])
def transPrepare(self):
ctcaeinfo = AeConverter.ctcaeInfo(self)
valuedic = AeConverter.lrInfo(self)
rules = ctcaeinfo['rules']
paraunit = ctcaeinfo['unit']
preparation = PreProcessor(self.dbConnection, valuedic, rules,
paraunit, self.unittrans)
rulesTransfered = preparation.rulesPrepare()
return rulesTransfered # the format of this list is like [ { rule_grade1:'a string', rule_grade2:'a string',...rule_grade5:'a string'} ,
def run():
data_loader = DataLoader(data_dir_root, data_train_file, data_test_file)
raw_train_df, raw_test_df = data_loader.load_csv_data()
data_loader.print_statistics()
pre_processor = PreProcessor(
raw_train_df,
raw_test_df,
cols_to_consider=cols_to_consider,
# cols_to_consider=raw_train_df.columns[0:-1],
target_feature='SalePrice')
pre_processor.pre_process_data()
print_features_info(pre_processor.raw_train_df,
pre_processor.clean_train_df)
plot_target_feature(pre_processor.raw_train_df,
pre_processor.target_feature)
plot_features_hist(pre_processor.raw_train_df)
plot_correlation_numeric_features(pre_processor.clean_train_df)
train_X, train_y = prepare_data(
pre_processor.clean_train_df,
class_col=pre_processor.target_feature,
reg_encoding_features=[],
one_hot_encoding_features=one_hot_encod_features,
ordinal_encoding_features=features_ordinal_mappings,
no_enc_features=no_enc_features)
test_X, test_y = prepare_data(
pre_processor.clean_test_df,
class_col=pre_processor.target_feature,
reg_encoding_features=[],
one_hot_encoding_features=one_hot_encod_features,
ordinal_encoding_features=features_ordinal_mappings,
no_enc_features=no_enc_features)
evaluator = Predictor(train_X, train_y, test_X, test_y, eval_classifiers,
eval_classifiers_params_grid)
all_predictions, final_prediction = evaluator.build_models(
grid_search=False)
evaluation_df = evaluator.save_predictions_to_df(all_predictions,
final_prediction)
submission_df = evaluator.save_predictions_for_submission(
evaluation_df, id_col=pre_processor.raw_test_df['Id'])
evaluation_df.to_csv("test_evaluation_results.csv", index=False)
submission_df.to_csv("test_submission.csv", index=False)
def test_greyscale(self):
# Create a preprocessor object and define a test RGB array (8-bit)
preprocess = PreProcessor()
testObservation = np.random.randint(255, size=(210, 160, 3))
# Use the PP to convert to greyscale
greyTest = preprocess.toGreyScale(testObservation)
# Ensure the dimsonality has reduced
self.assertEqual(greyTest.shape, (210, 160))
# Ensure the greyscale has been applied correctly
expectedGreyValue = int(
(testObservation[0, 0, 0] + testObservation[0, 0, 1] +
testObservation[0, 0, 2]) / 3)
self.assertEqual(expectedGreyValue, greyTest[0, 0])
def main():
dirList = list()
temp = listdir("./data/Asthma/2010/")
dirList.append(temp)
temp = listdir("./data/Asthma/2011/")
dirList.append(temp)
temp = listdir("./data/Asthma/2012/")
dirList.append(temp)
temp = listdir("./data/Asthma/2013/")
dirList.append(temp)
temp = listdir("./data/Asthma/2014/")
dirList.append(temp)
pre = PreProcessor()
clf = SVM(kernel=GaussianKernel(5.0), C=1.0)
X_train, y_train = pre.loadTrainingSet(
"training_data/Asthma_Sample_Tokenized.csv")
clf.fit(X_train, y_train)
def __init__(self, trainDF):
super().__init__()
prePro = PreProcessor()
self.pf = PlotFunctions()
self.trainDF = trainDF
self.X_train, self.X_test, self.y_train, self.y_test = \
prePro.split_train_test(trainDF['cleaned_sentence'], trainDF['classification'], 0.4)
self.X_test, self.X_cross, self.y_test, self.y_cross = \
prePro.split_train_test(self.X_test, self.y_test, 0.5)
self.all_scores = list()
self.models = {
'MultinomialNB':
naive_bayes.MultinomialNB(alpha=0.767,
class_prior=None,
fit_prior=True),
'ComplementNB':
naive_bayes.ComplementNB(alpha=0.767,
class_prior=None,
fit_prior=True),
'LogisticRegression':
linear_model.LogisticRegression(solver='lbfgs')
}
def main():
preprocesser = PreProcessor()
mm = ModuleManager()
def generate_random_points_on_hyperellipsoid(vol_data,
cor_data,
alpha_vec=np.array(
[0.9, 0.95, 0.975, 0.99]),
n_sample=int(1e4),
dim=30):
header = alpha_vec
result = pd.DataFrame(columns=header)
for i in range(vol_data.shape[0]):
start_time = time.time()
var_estimates = []
vol_mat = np.diag(vol_data.iloc[i, :])
cor_mat = preprocesser.construct_correlation_matrix(
corr_vec=cor_data.iloc[i, :], n=dim)
H = preprocesser.construct_covariance_matrix(vol_matrix=vol_mat,
corr_matrix=cor_mat)
r = np.random.randn(H.shape[0], n_sample)
# u contains random points on the unit hypersphere
u = r / np.linalg.norm(r, axis=0)
for alpha in alpha_vec:
y = np.sqrt(chi2.ppf(q=alpha, df=dim))
# Transform points on the unit hypersphere to the hyperellipsoid
xrandom = sqrtm(H).dot(np.sqrt(y) * u)
# Compute the lowest (equally) weighted average of random points on the hyperellipsoid.
# This is the maximum loss with alpha percent probability, i.e. Value-at-Risk
xrandom_min = np.max(
np.abs(np.array([np.mean(x) for x in xrandom.T])))
var_estimates.append(xrandom_min)
result = pd.merge(result,
pd.DataFrame(np.asarray(var_estimates).reshape(
1, -1),
columns=header),
how='outer')
print((i, time.time() - start_time))
return result
##################################################################################################################
### Multivariate Quantile Computation ###
##################################################################################################################
dim = 30
vol_data = mm.load_data(
'multivariate_analysis/volatilities_garch_norm_DJI30_2000_2001.pkl')
#cor_data = mm.load_data('multivariate_analysis/cor_DCC_mvnorm_DJI30_1994_1995.pkl')
cor_data = mm.load_data(
'multivariate_analysis/pearson/pearson_cor_estimates/cor_knn5_pearson_10_DJI30_2000_2001.pkl'
)
result = generate_random_points_on_hyperellipsoid(vol_data=vol_data,
cor_data=cor_data)
print(result)
#mm.save_data('multivariate_analysis/VaR/var_dcc_mvnorm_1994_1995_nsample_1e6.pkl', result)
#mm.transform_pickle_to_csv('multivariate_analysis/VaR/var_dcc_mvnorm_1994_1995_nsample_1e6.pkl')
mm.save_data(
'multivariate_analysis/VaR/var_knn5_pearson_garch_2000_2001_nsample_1e5_sqrt_chi2.pkl',
result)
mm.transform_pickle_to_csv(
'multivariate_analysis/VaR/var_knn5_pearson_garch_2000_2001_nsample_1e5_sqrt_chi2.pkl'
)
def preProcessCompany(company, start, end):
PP = PreProcessor(company, start=start, end=end)
PP.csv_indices()
targetNetUpdate = 10000 # Number of steps between updating the target network to the value network #----------------------------------------------------------------------- # Trial - Run through a number of episodes, save data and record results #----------------------------------------------------------------------- # Trial scope variables trialSteps = 0 # Total number of steps taken in the current trail epsilon = epsilonMax # Current epsilon value # Initialise the environment env = gym.make(GAME_NAME + 'Deterministic-v4') # Prepares each frame to produce a viable network input preProcessor = PreProcessor() # Stores experiences as a tuple of [s, a, r, s'] memory = ExperienceReplay(replaySize) # Records results and stores them in a csv file resultsRec = ResultsRecorder(GAME_NAME) # Q Value approximation and target networks, set with as many output nodes as viable actions valueNetwork = DeepQnetwork(len(env.unwrapped.get_action_meanings()), alpha) targetNetwork = DeepQnetwork(len(env.unwrapped.get_action_meanings()), alpha) init = tf.global_variables_initializer() # Create tensorflow model saver saver = tf.train.Saver()
from PreProcessor import PreProcessor
from RParser import RParser
__author__ = 'Shaun Rong'
__version__ = '0.1'
__maintainer__ = 'Shaun Rong'
__email__ = '*****@*****.**'
with open('environ.yaml', 'r') as f:
env = yaml.load(f)
stanford_parser_folder = env['stanford_parser_folder']
os.environ['STANFORD_PARSER'] = stanford_parser_folder
os.environ['STANFORD_MODELS'] = stanford_parser_folder
cfuf = PreProcessor()
with open('data/3.raw.txt', 'r') as f:
text = f.read().splitlines()
process_text, sub_table = cfuf.process(text)
sen = process_text[3]
rp = RParser()
verb_parent, method_parent = rp.parse_v_method(sen)
print verb_parent
print method_parent
# glove2word2vec(glove_input_file="../wordVec/glove/glove.6B.50d.txt", word2vec_output_file="WordVectors/gensim_glove_wiki_vectors.txt")
# Load and save the word vectors and index map
model = gensim.models.KeyedVectors.load_word2vec_format("WordVectors/gensim_glove_wiki_vectors.txt", binary=False)
wordVectors = model.syn0
wordsList = model.index2word
wordMap = {wordsList[i]: i for i in range(len(wordsList))}
np.save("IMDBSA/wordMap", wordMap)
np.save("IMDBSA/wordVectors", wordVectors)
# Find the training data
positiveFiles = ['./Data/IMDBData/train/pos/' + f for f in os.listdir('./Data/IMDBData/train/pos/') if os.path.isfile(os.path.join('./Data/IMDBData/train/pos/', f))]
negativeFiles = ['./Data/IMDBData/train/neg/' + f for f in os.listdir('./Data/IMDBData/train/neg/') if os.path.isfile(os.path.join('./Data/IMDBData/train/neg/', f))]
# Initialize the pre-processor and sentiment analyzer
processor = PreProcessor()
analyzer = SentimentAnalyzer(MAX_SEQUENCE_LENGTH, BATCH_SIZE, LSTM_UNITS, LEARNING_RATE, wordMap, wordVectors)
# Load and process the training data
negativeSamples = []
positiveSamples = []
for pf in positiveFiles:
with open(pf, "r", encoding="utf8") as f:
lines = f.readlines()
positiveSamples.extend(processor.cleanTextList(lines))
print("Cleaned positive document: " + pf)
for nf in negativeFiles:
with open(nf, "r", encoding="utf8") as f:
lines = f.readlines()
negativeSamples.extend(processor.cleanTextList(lines))
freq_synchronous_threshold = 0.1
# Threshold parameters used during lexical analysis.
tokenization_bit_distance: float = 0.2
tokenize_padding: bool = True
# Threshold parameters used during semantic analysis
subset_selection_size: float = 0.25
fuzzy_labeling: bool = True
min_correlation_threshold: float = 0.85
# A timer class to record timings throughout the pipeline.
a_timer = PipelineTimer(verbose=True)
# DATA IMPORT AND PRE-PROCESSING #
pre_processor = PreProcessor(can_data_filename, pickle_arb_id_filename,
pickle_j1979_filename)
id_dictionary, j1979_dictionary = pre_processor.generate_arb_id_dictionary(
a_timer, tang_normalize_strategy, time_conversion, freq_analysis_accuracy,
freq_synchronous_threshold, force_pre_processing)
if j1979_dictionary:
plot_j1979(a_timer, j1979_dictionary, force_j1979_plotting)
# LEXICAL ANALYSIS #
print("\n\t\t\t##### BEGINNING LEXICAL ANALYSIS #####")
tokenize_dictionary(a_timer,
id_dictionary,
force_lexical_analysis,
include_padding=tokenize_padding,
merge=True,
max_distance=tokenization_bit_distance)
signal_dictionary = generate_signals(a_timer, id_dictionary,
features, 'tfidf_char', 'test')
# Cross Validation predictions
self.check_model(classifier, xcross_tfidf, self.y_cross,
model_name, features, 'tfidf_char', 'cross')
def get_and_print_all_scores(self):
print('Running for count_vectors')
for i in range(500, 5000, 500):
self.count_vectors(i)
self.tfidf_words(i)
self.tfidf_ngram(i)
self.tfidf_char(i)
imp = Importer()
trainDF = imp.ImportFuncs.read_csv_into_dataframe(
'csv_classification/Multi-class/classified_sentences_all.csv')
prePro = PreProcessor()
trainDF = prePro.clean_dataframe_for_training(trainDF)
print(trainDF.head())
a = MultiClassifier(trainDF)
a.get_and_print_all_scores()
print(a.all_scores)
exp = Exporter()
exp.create_csv_scores(a.all_scores, 'all_scores_cleaned')
import gensim
from Indexer import Index
from PreProcessor import PreProcessor
from QueryProcessor import QueryProcessor
from tkinter import *
from tkinter.messagebox import *
prScores = None
index = None
dataset = None
stemmer = PorterStemmer()
N = 0
print("Initializing PreProcessor...")
preprocessor = PreProcessor()
try:
with open('pgrankscores', 'rb') as inf:
prScores = pickle.load(inf)
except:
print("Please place the 'pgrankscores' pickle file in the same dir as this script or run runCrawler.py first to calculate the pagerank scores...")
exit()
try:
with open('index', 'rb') as indf:
index = pickle.load(indf)
N = index.N
except:
print("Please place the 'index' pickle file in the same dir as this script...")
__author__ = 'Shaun Rong'
__version__ = '0.1'
__maintainer__ = 'Shaun Rong'
__email__ = '*****@*****.**'
with open('../environ.yaml', 'r') as f:
env = yaml.load(f)
stanford_parser_folder = env['stanford_parser_folder']
os.environ['STANFORD_PARSER'] = stanford_parser_folder
os.environ['STANFORD_MODELS'] = stanford_parser_folder
cfuf = PreProcessor()
with open('../data/verb_method_arg/test/paper0105.raw.txt', 'r') as f:
text = f.read().splitlines()
orig_text = text[4]
process_text, sub_table = cfuf.process([orig_text.strip()])
sen = process_text[0]
print sen
parser = stanford.StanfordParser(model_path=env['model_path'])
sentences = parser.raw_parse(sen)
ROOT = 'ROOT'
__author__ = 'Shaun Rong'
__version__ = '0.1'
__maintainer__ = 'Shaun Rong'
__email__ = '*****@*****.**'
with open('environ.yaml', 'r') as f:
env = yaml.load(f)
stanford_parser_folder = env['stanford_parser_folder']
os.environ['STANFORD_PARSER'] = stanford_parser_folder
os.environ['STANFORD_MODELS'] = stanford_parser_folder
cfuf = PreProcessor()
with open('data/3.raw.txt', 'r') as f:
text = f.read().splitlines()
process_text, sub_table = cfuf.process(text)
sen = process_text[3]
rp = RParser()
verb_parent, method_parent = rp.parse_v_method(sen)
print verb_parent
print method_parent
os.environ['STANFORD_PARSER'] = stanford_parser_folder
os.environ['STANFORD_MODELS'] = stanford_parser_folder
tree_parser = stanford.StanfordParser(model_path=env['model_path'])
for ff in train_file:
with open(os.path.join(args.f, "{}.raw.txt".format(ff)), 'r') as f:
text = f.read().splitlines()
with open(os.path.join(args.f, "{}.gold.yaml".format(ff)), 'r') as f:
gold_ticket = yaml.load(f)
for i, orig_text in enumerate(text):
rp = RParser()
#Preprocess
pre_p = PreProcessor()
process_text, sub_table = pre_p.process([orig_text.strip()])
sen = process_text[0]
#RParse
NPs = rp.return_NPs(sen)
gold_NPs = extract_gold_NPs(gold_ticket['sen{}'.format(i+1)], sub_table, tree_parser)
for NP in NPs:
if NP in gold_NPs:
train_summary['input_output'].append(repr(NP))
else:
train_summary['else'].append(repr(NP))
with open('ioput_train_db.yaml', 'w') as f:
f.write(yaml.dump(train_summary, default_flow_style=False))
from PreProcessor import PreProcessor
from Processor_Clustering import Processor_Clustering
from postprocess_lidar import postprocess_lidar
scaling = 100
# define preprocessor to get lidar data
prep = PreProcessor(scaling)
#prep.sendUsingDust()
#prep.receiveUsingDust()
# get range and data
ranges_data = prep.processCSV('/home/thomas/Github/Object_Detection/lidar/lidar_data/2020-12-03-17-26-18/scan.csv')
# convert to euclidean notation
ranges_data_euclidean = prep.convertToEuclidean(ranges_data)
# get the array to apply clustering
ranges_data3 = prep.prepareArray(ranges_data_euclidean)
# apply the clustering algorithm
clustering = Processor_Clustering(ranges_data3,scaling)
class Shadows(GlobalGameData, TkWorldDataShared):
# Shadow table size
shadow_map_size = 0, 0
# Shadow world size
shadow_world_size = 0, 0
# 2D map of each cell
shadow_map = []
def __init__(self):
self.s_shadow_surf = self.tk_surface(self.tk_resolution)
if self.tk_shadow_quality:
self.s_shadow_surf.set_colorkey(self.tk_shadow_mask_color)
self.s_buildShadowDirMap()
def s_buildShadowDirMap(self):
"""
TBD
"""
self.s_shadow_dir_map = []
# Every cell has one line to work as guides for the shadow quadrilateral
# Line meaning align the line clockwise diagonally to cast the shadows
# Finetune these to workout the line inside the wall segment (Doesn't peek out from the wall)
# These are offsets from topleft of the wall
axis_dir = {
0: [0, 31, 31, 31], # Top
90: [31, 32, 31, 0], # Left
180: [32, 1, 0, 1], # Down
270: [1, 0, 1, 32]
} # Right
non_axis_dir = {
45: [0, 32, 32, 0], # TopLeft
315: [0, 0, 32, 32], # TopRight
135: [32, 32, 0, 0], # DownLeft
225: [32, 0, 0, 32]
} # DownRight
# Find the 2d array mid point
x_mid_value = range(sum(self.tk_shadow_minmax_x))
x_mid_value = x_mid_value[len(x_mid_value) / 2]
y_mid_value = range(sum(self.tk_shadow_minmax_y))
y_mid_value = y_mid_value[len(y_mid_value) / 2]
y_range_slide = self.tk_chain(xrange(1, y_mid_value + 1),
xrange(y_mid_value + 1, 0, -1))
for e1, y in enumerate(xrange(sum(self.tk_shadow_minmax_y))):
y_range_value = y_range_slide.next()
x_range_slide = self.tk_chain(xrange(1, x_mid_value + 1),
xrange(x_mid_value + 1, 0, -1))
row = []
for e2, x in enumerate(xrange(sum(self.tk_shadow_minmax_x))):
x_range_value = x_range_slide.next()
# Get Angle to each cell from the middle for shadow cast line
_dir = int(
self.tk_degrees(
self.tk_atan2(x_mid_value - e2, y_mid_value - e1) %
(self.tk_pi * 2)))
if e1 == y_mid_value and e2 == x_mid_value:
# Ignore mid value
d = [0, 0, 0, 0]
elif _dir in axis_dir:
# Axis aligned direction
d = axis_dir[_dir][:]
else:
# Find the closest non-axis angle
non_axis = min(non_axis_dir.keys(),
key=lambda x: abs(x - _dir))
d = non_axis_dir[non_axis][:]
d.append(min(y_range_value, x_range_value))
# Fixes peeking by lowering the extra angle per wall
# as the distance increases
dist = self.tk_hypot(x_mid_value - e2, y_mid_value - e1)
d.append(max(0, 0.16 - 0.01 * dist))
row.append(tuple(d))
self.s_shadow_dir_map.append(tuple(row))
self.s_shadow_dir_map = tuple(self.s_shadow_dir_map)
def s_loadSurfaceMap(self, surface):
"""
Load the ground layer as shadow mask
surfaces -> All macro surfaces
return -> None
"""
fade_surface = self.tk_surface(surface.get_size(), self.tk_srcalpha)
fade_surface.fill(self.tk_shadow_color)
self.s_fade_surf = surface.copy()
self.s_fade_surf.blit(fade_surface, (0, 0))
def s_loadCellWalls(self, cellwalls):
"""
Load world map and convert all cells to 2d binary map (Walls=1 else 0)
cellwalls -> 2d world map
return -> None
"""
final = []
# Per-cell size
self.shadow_map_size = len(cellwalls[0]), len(cellwalls)
# Per-sector size
self.shadow_world_size = self.shadow_map_size[
0] * 32, self.shadow_map_size[1] * 32
# Cells with collisions are marked with 1 else 0
for y in xrange(self.shadow_map_size[1]):
row = []
for x in xrange(self.shadow_map_size[0]):
row.append(1 if cellwalls[y][x].w_collision else 0)
final.append(tuple(row))
self.shadow_map[:] = final
# Note: Remove this
exec(
PreProcessor.parseCode(
"""
def s_applyShadows(self, surface):
x, y = self.w_share["WorldPosition"]
ofsx = x - self.w_share['ShadowOffset'][0]
ofsy = y - self.w_share['ShadowOffset'][1]
#-ifdef/tk_shadow_quality
self.s_shadow_surf.blit(surface, (0, 0))
self.s_shadow_surf.lock()
#-endif
rounded_x, rounded_y = int(x), int(y)
ori_x, ori_y = (self.tk_res_half[0] + rounded_x,
self.tk_res_half[1] + rounded_y)
# Get index of which cell the player is standing on
near_x, near_y = -int(x - 16) >> 5, -int(y - 16) >> 5
min_y, max_y = self.tk_shadow_minmax_y
min_x, max_x = self.tk_shadow_minmax_x
# Shadows are calculated clockwise
for e1, ry in enumerate(xrange(near_y - min_y, near_y + max_y)):
if ry < 1 or ry > self.shadow_map_size[1]-2:
# Most outer rows do not cast shadows (Walls between playable area and the void)
continue
for e2, rx in enumerate(xrange(near_x - min_x, near_x + max_x)):
if rx < 1 or rx > self.shadow_map_size[0]-2:
# Most outer columns do not cast shadows
continue
# Check if the cell can cast shadows
if self.shadow_map[ry][rx]:
# Get the line endpoints
pc = self.s_shadow_dir_map[e1][e2]
# Position of the object(TopLeft)
sox = (ori_x + 32 * rx - 17) - ofsx
soy = (ori_y + 32 * ry - 17) - ofsy
# How far the shadow is casted from the wall/object
length = 80 * pc[4]
ep1 = sox + pc[0], soy + pc[1] # Endpoint 1
# Calculate the angle to endpoints of the cubes
angle_1 = self.tk_atan2(ori_x - (ep1[0] + x), ori_y - (ep1[1] + y)) + pc[5]
end_p_1 = (ep1[0] - self.tk_sin(angle_1) * length,
ep1[1] - self.tk_cos(angle_1) * length)
ep2 = sox + pc[2], soy + pc[3] # Endpoint 2
angle_2 = self.tk_atan2(ori_x - (ep2[0] + x), ori_y - (ep2[1] + y)) - pc[5]
end_p_2 = (ep2[0] - self.tk_sin(angle_2) * length,
ep2[1] - self.tk_cos(angle_2) * length)
# Cast a shadow polygon from the line and color it for colorkeying
self.tk_draw_polygon({surface}, self.tk_shadow_mask_color,
(ep1, end_p_1, end_p_2, ep2))
#-ifdef/tk_shadow_quality
self.s_shadow_surf.unlock()
mapPos = -x, -y
# The shadow map is the chosen layer 'shadowed' and stored in memory.
# This section cuts a correct size of that map and displays it
# If the topleft map corner is in view, clamp it from going below 0
topLeft = max(0, mapPos[0] - self.tk_res_half[0] + 16), max(0, mapPos[1] - self.tk_res_half[1] + 16)
bottomRight = (min(self.shadow_world_size[0] - topLeft[0], self.tk_res_half[0] + mapPos[0] + 16),
min(self.shadow_world_size[1] - topLeft[1], self.tk_res_half[1] + mapPos[1] + 16))
# Area which will be cut from the shadow map
area = (topLeft[0] + (ofsx if topLeft[0] else 0),
topLeft[1] + (ofsy if topLeft[1] else 0),
bottomRight[0] - (ofsx if topLeft[0] else -1 - ofsx),
bottomRight[1] - (ofsy if topLeft[1] else -1 - ofsy))
# The topleft anchor point should always be at topleft corner of the screen
# But when the topleft of the map is in view, it should anchor to that one
dest = (self.tk_res_half[0] + x - 16 + topLeft[0] - (ofsx if not topLeft[0] else 1),
self.tk_res_half[1] + y - 16 + topLeft[1] - (ofsy if not topLeft[1] else 1))
# Blit the shadowed part of the ground
surface.blit(self.s_fade_surf, dest, area=area)
# Blit the visible area
surface.blit(self.s_shadow_surf, (0, 0))
#-endif
""".format(surface='self.s_shadow_surf' if GlobalGameData.
tk_shadow_quality else 'surface'),
tk_shadow_quality=GlobalGameData.tk_shadow_quality))
df = df.append(
{
"train_or_test": path[1],
"review_type": path[2],
"sentence": f.read(),
"review_number": detail[1],
"review_id": detail[0],
},
ignore_index=True,
)
df.to_csv(output_name, index=False)
imdb_raw_df = pd.read_csv(dataset_path.joinpath("imdb_raw.csv"))
"""
list of words that are common to both dataset
> we can play with which word to remove and see the performance of the model
br is a html tag
"""
common_words = [
"br",
# 'film',
# 'movie',
# 'one',
# 'like',
# 'good',
# 'time'
]
imdb_processor = PreProcessor(imdb_raw_df, common_words, "imdb")
imdb_processor.process()
"target": twenty_news_group_train.target,
"train_or_test": "train",
"sentence": twenty_news_group_train.data
})
twenty_news_test_df = pd.DataFrame(
data={
"target": twenty_news_group_test.target,
"train_or_test": "test",
"sentence": twenty_news_group_test.data
})
twenty_news_combined_df = twenty_news_train_df.append(twenty_news_test_df)
twenty_news_combined_df["sentence"] = twenty_news_combined_df[
"sentence"].apply(lambda x: x.replace("\n", " ").replace("\r", "").replace(
"\t", " ").strip())
twenty_news_combined_df.reset_index(inplace=True)
twenty_news_combined_df.rename(columns={"index": "id"}, inplace=True)
twenty_news_combined_df.to_csv(dataset_path.joinpath("twenty_news_raw.csv"),
sep="\t",
index=False)
twenty_news_raw_df = pd.read_csv(dataset_path.joinpath("twenty_news_raw.csv"),
sep="\t")
common_words = [
# TODO: to be determined
]
twenty_news_processor = PreProcessor(twenty_news_raw_df, common_words,
"twenty_news")
twenty_news_processor.process()
from PreProcessor import PreProcessor
__author__ = 'Shaun Rong'
__version__ = '0.1'
__maintainer__ = 'Shaun Rong'
__email__ = '*****@*****.**'
with open('../environ.yaml', 'r') as f:
env = yaml.load(f)
stanford_parser_folder = env['stanford_parser_folder']
os.environ['STANFORD_PARSER'] = stanford_parser_folder
os.environ['STANFORD_MODELS'] = stanford_parser_folder
cfuf = PreProcessor()
with open('../data/train/2.raw.txt', 'r') as f:
text = f.read().splitlines()
orig_text = text[5]
process_text, sub_table = cfuf.process([orig_text.strip()])
sen = process_text[0]
parser = stanford.StanfordDependencyParser(model_path=env['model_path'])
sentences = parser.raw_parse(sen)
for parse in sentences:
for t in parse.triples():
print t
"""
def executeSinglePlSqlFile(data, spec):
f = open(data, 'r')
linesOfCode = len(f.readlines())
f.close()
processor = PreProcessor(spec, data)
tableInfo, assumeConstraintList, assertConstraintList, resultString = processor.start(
)
file = open('wpc/upper_input.sql', "w")
file.write(resultString)
file.close()
# recording startTime
startTime1 = datetime.datetime.now()
input = FileStream('wpc/upper_input.sql')
lexer = PlSqlLexer(input)
stream = CommonTokenStream(lexer)
parser = PlSqlParser(stream)
tree = parser.sql_script()
cfg = MyCFG()
helper = MyHelper(parser)
helper.updateTableDict(tableInfo)
utility = MyUtility(helper)
v = MyVisitor(parser, cfg, utility)
v.visit(tree)
print("\nRaw CFG :", v.rawCFG, "\n")
# for key in v.cfg.nodes:
# if v.cfg.nodes[key].ctx != None:
# print(key, " --> ", v.cfg.nodes[key].ctx.getText())
# print("\n")
res = MyRawCfgToGraph(v.rawCFG, cfg)
res.execute()
# cfg.printPretty()
# print("\n")
utility.generateVariableSet(cfg)
# all properties of each node
# for nodeId in cfg.nodes:
# cfg.nodes[nodeId].printPretty()
ssaString = MySsaStringGenerator(cfg, parser)
ssaString.execute(
) # only for generating DOT file for "before_versioning_graph"
# recording finishTime
finishTime1 = datetime.datetime.now()
# cfg.dotToPng(cfg.dotGraph, "wpc/raw_graph")
#
# hello1 = utility.generateBeforeVersioningDotFile(cfg)
# cfg.dotToPng(hello1, "wpc/before_versioning_graph")
# recording startTime
startTime2 = datetime.datetime.now()
algo = WpcGenerator(cfg, helper, ssaString)
algo.execute()
algo.finalWpcString = algo.finalWpcString.replace(" ", " ")
# done: replace " = " with " == " in algo.finalWpcString
algo.finalWpcString = algo.finalWpcString.replace(" = ", " == ")
print("\n**** Final WPC VC in Well_Bracketted_Format:\n\n",
algo.finalWpcString, "\n")
# print(algo.variablesForZ3)
# algo.finalWpcString = "( ( z ) ^ ( ( ! ( y ) ) ==> ( ( ( 2 ) v ( x ) ) ==> ( y - 2 ) ) ) )" # for testing! Don't UNCOMMENT...
# algo.finalWpcString = "( ( ( z ) ==> ( u ) ) ^ ( ( ! ( y ) ) ==> ( ( ( true ) ) ==> ( y - 2 ) ) ) )" # for testing! Don't UNCOMMENT...
# algo.finalWpcString = "( ( ( z ) ==> ( u ) ) ^ ( ( ! ( y ) ) ==> ( true ) ) ^ ( ( a ) ==> ( b ) ) )" # for testing! Don't UNCOMMENT...
# algo.finalWpcString = "( ( ( ! ( y ) ) ==> ( true ) ) )" # for testing! Don't UNCOMMENT...
# algo.finalWpcString = "( ( ( ! ( y ) ) ^ ( true ) v ( g ) ) )" # for testing! Don't UNCOMMENT...
z3StringConvertor = WpcStringConverter(algo.finalWpcString)
z3StringConvertor.execute()
# z3StringConvertor.convertedWpc is the FINAL VC Generated...
print("\n**** Final WPC VC in Z3 Format:\n\n",
z3StringConvertor.convertedWpc, "\n")
z3FileString = "# This file was generated at runtime on " + str(
datetime.datetime.now()) + "\n"
z3FileString = z3FileString + "from z3 import *\n\n"
z3FileString = z3FileString + "class Z3RuntimeWpcFile():\n"
z3FileString = z3FileString + "\t" + "def __init__(self):\n"
z3FileString = z3FileString + "\t\t" + "self.finalFormula = \"\"\n"
z3FileString = z3FileString + "\t\t" + "self.satisfiability = \"\"\n"
z3FileString = z3FileString + "\t\t" + "self.modelForViolation = \"\"\n\n"
z3FileString = z3FileString + "\t" + "def execute(self):\n"
for i in algo.variablesForZ3:
z3FileString = z3FileString + "\t\t" + i + " = Real(\'" + i + "\')\n"
z3FileString = z3FileString + "\n\t\ts = Solver()\n"
if len(z3StringConvertor.implies_p) > 0:
for i in range(len(z3StringConvertor.implies_p)):
z3FileString = z3FileString + "\t\t" + "s.add(" + z3StringConvertor.implies_p[
i] + ")\n"
if not z3StringConvertor.convertedWpc == z3StringConvertor.implies_p_q[
i]:
z3FileString = z3FileString + "\t\t" + "s.add(" + z3StringConvertor.implies_p_q[
i] + ")\n"
z3FileString = z3FileString + "\t\t" + "s.add( Not( " + z3StringConvertor.convertedWpc + " ) )\n"
# z3FileString = z3FileString + "\n\t\t" + "print()"
z3FileString = z3FileString + "\n\t\t" + "#print(\"\\n%%%%%%%%%% Aggregate Formula %%%%%%%%%%\\n\", s)"
z3FileString = z3FileString + "\n\t\t" + "self.finalFormula = str(s)"
# z3FileString = z3FileString + "\n\t\t" + "print()"
z3FileString = z3FileString + "\n\t\t" + "#print(\"\\n%%%%%%%%%% Satisfiability %%%%%%%%%%\")\n"
z3FileString = z3FileString + "\n\t\t" + "self.satisfiability = str(s.check())"
z3FileString = z3FileString + "\n\t\t" + "if self.satisfiability == \"sat\":"
# z3FileString = z3FileString + "\n\t\t\t" + "print()"
z3FileString = z3FileString + "\n\t\t\t" + "#print(\"\\n-------->> Violation Occurred...\")"
z3FileString = z3FileString + "\n\t\t\t" + "self.satisfiability = \"violation\""
# z3FileString = z3FileString + "\n\t\t\t" + "print()"
z3FileString = z3FileString + "\n\t\t\t" + "#print(\"\\n%%%%%%%%%% An Instance for which Violation Occurred %%%%%%%%%%\\n\", s.model())"
z3FileString = z3FileString + "\n\t\t\t" + "self.modelForViolation = str(s.model())"
z3FileString = z3FileString + "\n\t\t" + "elif self.satisfiability == \"unsat\":"
# z3FileString = z3FileString + "\n\t\t\t" + "print()"
z3FileString = z3FileString + "\n\t\t\t" + "#print(\"\\n-------->> NO Violation Detected so far...\\n\")"
z3FileString = z3FileString + "\n\t\t\t" + "self.satisfiability = \"sat\""
# z3FileString = z3FileString + "\n\t\t\t" + "print()"
# z3FileString = z3FileString + "\n\t\t" + "print()\n"
file = open('wpc/Z3RuntimeWpcFile.py', "w")
file.write(z3FileString)
file.close()
# time.sleep(2)
# import file created on Runtime...
import wpc.Z3RuntimeWpcFile
from wpc.Z3RuntimeWpcFile import Z3RuntimeWpcFile
# Reload after module's creation to avoid old module remain imported from disk...VVI...
wpc.Z3RuntimeWpcFile = reload(wpc.Z3RuntimeWpcFile)
z3Runtime = Z3RuntimeWpcFile()
z3Runtime.execute()
# print(z3Runtime.finalFormula)
# print(z3Runtime.satisfiability)
# print(z3Runtime.modelForViolation)
# recording finishTime
finishTime2 = datetime.datetime.now()
timeDifference = ((finishTime1 - startTime1) +
(finishTime2 - startTime2)).total_seconds()
return linesOfCode, timeDifference, z3StringConvertor.convertedWpc, z3Runtime.satisfiability, z3Runtime.modelForViolation
def generate(self):
if self._dataSet == None:
# Create outdirs
if not os.path.exists(Generator.DIGITS_OUTDIR):
os.makedirs(Generator.DIGITS_OUTDIR)
if not os.path.exists(Generator.TRAINSET_OUTDIR):
os.makedirs(Generator.TRAINSET_OUTDIR)
# Treat all digit images found in source dir
files = Generator.listFiles(Generator.DIGITS_SRCDIR)
outDF = pd.DataFrame(data=[], columns=self._dataColumns+self._classColumns)
for imgFile in files:
outDF1Nb = pd.DataFrame(data=[], columns=self._dataColumns+self._classColumns)
number = Generator.extractNumber(imgFile)
img = Utils.readImage(Generator.DIGITS_SRCDIR + "/" + imgFile);
# double image canvas and center digit
transform = AffineTransform(scale=(2,2), translation=(-img.shape[1]/2,-img.shape[0]/2))
img = warp(img, transform, output_shape=img.shape, mode='edge')
#imgSize = img.shape
# Apply the modifications/augmentations on the digit image
for tilt in Generator.tiltRange:
tiltTform = AffineTransform(shear=tilt, translation=(215*tilt/4, 0))
for angle in Generator.angleRange:
for xtransl in Generator.xTranslRange:
for ytransl in Generator.yTranslRange:
transTform= AffineTransform(translation=(xtransl,ytransl))
for sigma in Generator.blurRange:
for ratio in Generator.zoomRange:
#print(np.mean(img))
zoomTform = AffineTransform(scale=(ratio,ratio), translation=(img.shape[1]/ratio/3-img.shape[1]/3,img.shape[0]/ratio/3-img.shape[0]/3))
tilted = warp(img, tiltTform, output_shape=img.shape, mode='edge')
#print(np.mean(tilted))
rotated = rotate(tilted, angle, mode='edge');
#print(np.mean(rotated))
transtd = warp(rotated, transTform, output_shape=img.shape, mode='edge')
#print(np.mean(transtd))
blured = transtd.copy()
blured[:, :, 0] = gaussian(blured[:, :, 0], sigma, preserve_range = True)
#print(np.mean(blured))
resized =- warp(blured, zoomTform, output_shape=img.shape, mode='edge')
print("resized="+str(np.mean(resized)))
resized = -1*resized ## TODO: why does resize invert all the values?
# Save the new image
newFile = Generator.DIGITS_OUTDIR + "/" + str(number) + \
"_t" + str(tilt) + "_r" + str(angle) + "_t" + str(xtransl) + "_" + str(ytransl) + \
"_b" + str(sigma) + "_s" + str(ratio) + ".png"
print("Writing image file: " + newFile)
#Utils.showImage(resized);
Utils.writeImage(resized, newFile)
print("Treating file: " + newFile)
attributes = pd.DataFrame(PreProcessor.preprocessImage(resized).reshape(1,PreProcessor.TRAIN_WIDTH*PreProcessor.TRAIN_HEIGHT))
attributes["class"] = number
# print(attributes)
# print(type(attributes))
# print(attributes.shape)
attributes.columns=outDF1Nb.columns
outDF1Nb = outDF1Nb.append(attributes) # , ignore_index=True
# Save the partial dataframe for the current digits (in case of crash, allows to restarts only on untreated digits)
outDF1Nb.to_csv(Generator.TRAINSET_OUTDIR + "/trainset_" + number + ".csv");
outDF = outDF.append(outDF1Nb, ignore_index=True)
# Save the whole dataframe for all digits
outDF.to_csv(Generator.TRAINSET_OUTDIR + "/trainset_full.csv")
self._dataSet = outDF
def executeSinglePlSqlFile(data, spec):
f = open(data, 'r')
linesOfCode = len(f.readlines())
f.close()
processor = PreProcessor(spec, data)
tableInfo, assumeConstraint, assertConstraint, resultString = processor.start(
)
file = open('cnf/upper_input.sql', "w")
file.write(resultString)
file.close()
# recording startTime
startTime = datetime.datetime.now()
input = FileStream('cnf/upper_input.sql')
lexer = PlSqlLexer(input)
stream = CommonTokenStream(lexer)
parser = PlSqlParser(stream)
tree = parser.sql_script()
# ast = tree.toStringTree(recog=parser)
# print(str(MyPlSqlVisitor(parser).getRuleName(tree)))
# print("\n\n", signature(tree.toStringTree), "\n")
cfg = MyCFG()
helper = MyHelper(parser)
helper.updateTableDict(tableInfo)
utility = MyUtility(helper)
v = MyVisitor(parser, cfg, utility)
v.visit(tree)
print("\nRaw CFG : ", v.rawCFG)
# for key in v.cfg.nodes:
# if v.cfg.nodes[key].ctx != None:
# print(key, " --> ", v.cfg.nodes[key].ctx.getText())
res = MyRawCfgToGraph(v.rawCFG, cfg)
res.execute()
# cfg.printPretty()
# cfg.dotToPng(cfg.dotGraph, "cnf/raw_graph") # TODO: make dot file in cnf form
utility.generateDomSet(cfg)
# print("Dominator set ended----------->\n\n")
utility.generateSDomSet(cfg)
# print("Strictly Dominator set ended ----------->\n\n")
utility.generatIDom(cfg)
# print("Immediate Dominator ended ----------->\n\n")
utility.generateDFSet(cfg)
utility.insertPhiNode(cfg)
utility.initialiseVersinosedPhiNode(cfg)
utility.versioniseVariable(cfg)
utility.phiDestruction(cfg)
ssaString = MySsaStringGenerator(cfg, parser)
ssaString.execute()
# utility.generateFinalDotGraph(cfg)
# for nodeId in cfg.nodes:
# cfg.nodes[nodeId].printPretty()
# cfg.dotToPng(cfg.dotGraph, "cnf/raw_graph")
#
# hello1 = utility.generateBeforeVersioningDotFile(cfg)
# cfg.dotToPng(hello1, "cnf/before_versioning_graph")
#
# hello4 = utility.generateDestructedPhiNodeWalaDotFile(cfg)
# cfg.dotToPng(hello4, "cnf/destructed_phi_node_wala_graph")
cnfUtility = CnfUtility(helper)
iCnfCfg = cnfUtility.copyCfg(cfg)
reverseCnfCfg = cnfUtility.topologicalSort(iCnfCfg)
cnfUtility.unvisit(iCnfCfg)
cnfUtility.setParentBranching(iCnfCfg)
cnfCfg = cnfUtility.reverseDictOrder(reverseCnfCfg)
cnfUtility.copyParentBranching(cnfCfg, iCnfCfg)
# print("\n\n\n\n\n\t\t\tThe intermediate CNF form is ------------------------------>\n\n\n\n")
# for nodeId in iCnfCfg.nodes:
# iCnfCfg.nodes[nodeId].printPretty()
# print("\n\n\n\n\n\t\t\tThe CNF form is ------------------------------>\n\n\n\n")
cnfVcGenerator = CnfVcGenerator(cnfCfg, parser)
cnfPath = []
for nodeId in cnfCfg.nodes:
cnfPath.append(nodeId)
cnfVcGenerator.generateCnfVc(cnfPath)
# print("\n\n\n\n\t\t\tThe CNF VCs are : ------------------------------->\n\n\n")
# print(cnfVcs)
# for nodeId in cnfCfg.nodes:
# cnfCfg.nodes[nodeId].printPretty()
# cnfVc = cnfUtility.cnfVc(cnfCfg)
#
# print("\n\n\t\tThe CNF VCs are ----------------->\n\n\n")
#
# for str in cnfVc:
# print(str)
varSet, z3Str = cnfUtility.iZ3format(cnfCfg)
# print("\n\n*******************\n\n", z3Str, "\n\n--------------\n\n")
# print(varSet)
#
# print("\n\n")
z3Str = z3Str.replace(" ", " ")
z3Str = z3Str.replace(" == ", " = ")
z3Str = z3Str.replace(" = ", " == ")
print("\n**** Final CNF VC in Well_Bracketted_Format:\n\n", z3Str, "\n")
z3StringConvertor = WpcStringConverter(z3Str)
z3StringConvertor.execute()
# print("\n**** Final CNF VC in Z3 Format:\n", z3StringConvertor.convertedWpc, "\n")
z3FileString = "# This file was generated at runtime on " + str(
datetime.datetime.now()) + "\n"
z3FileString = z3FileString + "from z3 import *\n\n"
z3FileString = z3FileString + "class Z3RuntimeCnfFile():\n"
z3FileString = z3FileString + "\t" + "def __init__(self):\n"
z3FileString = z3FileString + "\t\t" + "self.finalFormula = \"\"\n"
z3FileString = z3FileString + "\t\t" + "self.satisfiability = \"\"\n"
z3FileString = z3FileString + "\t\t" + "self.modelForViolation = \"\"\n\n"
z3FileString = z3FileString + "\t" + "def execute(self):\n"
for i in varSet:
z3FileString = z3FileString + "\t\t" + i + " = Real(\'" + i + "\')\n"
z3FileString = z3FileString + "\n\t\ts = Solver()\n"
if len(z3StringConvertor.implies_p) > 0:
for i in range(len(z3StringConvertor.implies_p)):
z3FileString = z3FileString + "\t\t" + "s.add(" + z3StringConvertor.implies_p[
i] + ")\n"
if not z3StringConvertor.convertedWpc == z3StringConvertor.implies_p_q[
i]:
z3FileString = z3FileString + "\t\t" + "s.add(" + z3StringConvertor.implies_p_q[
i] + ")\n"
# if z3StringConvertor.convertedWpc not in z3StringConvertor.implies_p_q:
# z3FileString = z3FileString + "\t\t" + "s.add(" + z3StringConvertor.convertedWpc + ")\n"
# else:
# z3FileString = z3FileString + "\t\t" + "s.add(" + z3StringConvertor.convertedWpc + ")\n"
z3FileString = z3FileString + "\t\t" + "s.add( Not( " + z3StringConvertor.convertedWpc + " ) )\n"
# z3FileString = z3FileString + "\n\t\t" + "print()"
# z3FileString = z3FileString + "\n\t\t" + "print(\"%%%%%%%%%% Aggregate Formula %%%%%%%%%%\\n\", s)"
z3FileString = z3FileString + "\n\t\t" + "self.finalFormula = str(s)"
# z3FileString = z3FileString + "\n\t\t" + "print()"
# z3FileString = z3FileString + "\n\t\t" + "print(\"%%%%%%%%%% Satisfiability %%%%%%%%%%\")\n"
z3FileString = z3FileString + "\n\t\t" + "self.satisfiability = str(s.check())"
z3FileString = z3FileString + "\n\t\t" + "if self.satisfiability == \"sat\":"
# z3FileString = z3FileString + "\n\t\t\t" + "print()"
# z3FileString = z3FileString + "\n\t\t\t" + "print(\"-------->> Violation Occurred...\")"
z3FileString = z3FileString + "\n\t\t\t" + "self.satisfiability = \"violation\""
# z3FileString = z3FileString + "\n\t\t\t" + "print()"
# z3FileString = z3FileString + "\n\t\t\t" + "print(\"%%%%%%%%%% An Instance for which Violation Occurred %%%%%%%%%%\\n\", s.model())"
z3FileString = z3FileString + "\n\t\t\t" + "self.modelForViolation = str(s.model())"
z3FileString = z3FileString + "\n\t\t" + "elif self.satisfiability == \"unsat\":"
# z3FileString = z3FileString + "\n\t\t\t" + "print()"
# z3FileString = z3FileString + "\n\t\t\t" + "print(\"-------->> NO Violation Detected so far...\")"
z3FileString = z3FileString + "\n\t\t\t" + "self.satisfiability = \"sat\""
# z3FileString = z3FileString + "\n\t\t\t" + "print()"
# z3FileString = z3FileString + "\n\t\t" + "print()\n"
file = open('cnf/Z3RuntimeCnfFile.py', "w")
file.write(z3FileString)
file.close()
import cnf.Z3RuntimeCnfFile
from cnf.Z3RuntimeCnfFile import Z3RuntimeCnfFile
# Reload after module's creation to avoid old module remain imported from disk...VVI...
cnf.Z3RuntimeCnfFile = reload(cnf.Z3RuntimeCnfFile)
z3Runtime = Z3RuntimeCnfFile()
z3Runtime.execute()
finishTime = datetime.datetime.now()
timeDifference = (finishTime - startTime).total_seconds()
return linesOfCode, timeDifference, z3StringConvertor.convertedWpc, z3Runtime.satisfiability, z3Runtime.modelForViolation
from TwitterSentimentAnalyzer import TwitterSentimentAnalyzer
from PreProcessor import PreProcessor
er = TwitterSentimentAnalyzer()
processor = PreProcessor()
tmp = "RT @Cj_Walker1: My family aand I have came to a great decision! With that being said I would like to say I have committed to The Ohio State"
clean = processor.preProcess(tmp, lowercase=True)
cleanString = ""
for s in clean:
cleanString = cleanString + s + " "
cleanStringList = [cleanString]
for i in range(39):
cleanStringList.append(" ")
print(cleanString)
print(processor.cleanText((tmp)))
result = er.Evaluate(cleanStringList)
print(result)
cv2.namedWindow('frame',cv2.WINDOW_NORMAL)
cv2.namedWindow('threshold',cv2.WINDOW_NORMAL)
cv2.resizeWindow('frame', 600,600)
cv2.resizeWindow('threshold', 600,600)
while(True):
# Capture frame-by-frame
if set==0:
ret, frame = cap.read()
else:
url = "http://192.168.1.95:8080/shot.jpg"
imgResp = requests.get(url)
imgArr = np.array((bytearray(imgResp.content)), dtype=np.uint8)
frame = cv2.imdecode(imgArr, -1)
#copy the output
output = frame
thresh = PreProcessor.preProcessImage(frame)
#draw only rectangle-alike countours wchich permieter is bigger than 'some value'
cardImg = CardProcessor.findCards(thresh, toDraw= output)
if len(cardImg) > 0:
print(cardImg)
cv2.imshow("Show Boxes", cardImg)
# Display the resulting frame
cv2.imshow('frame',output)
cv2.imshow('threshold',thresh)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
# When everything done, release the capture
if set==0:
#Utils.showImage(transtd)
blurred = transtd.copy()
blurred[:, :, 0] = gaussian(blurred[:, :, 0], sigma, preserve_range=True)
print("blurred=" + str(np.mean(blurred[:, :, 0:3])))
#Utils.showImage(blurred)
resized = resize(blurred, (ratio * blurred.shape[0], ratio * blurred.shape[1]),
anti_aliasing=True,
preserve_range=True)
print("resized=" + str(np.mean(resized[:, :, 0:3])))
#Utils.showImage(resized)
print("Treating file: " + newFile)
attributes = pd.DataFrame(
PreProcessor.preprocessImage(resized).reshape(1,
TRAIN_WIDTH * TRAIN_HEIGHT))
attributes["class"] = number
print(type(attributes))
print(attributes.shape)
print(attributes)
###################
from DigitPositionDetector import DigitPositionDetector
dpd = DigitPositionDetector(resized)
dpd.detect()
detectedDigits = dpd.getDetectedDigits()
for dd in detectedDigits:
dd.display()
'201804120000', '201804110000', '201804100000', '201804090000',
'201804060000', '201804050000', '201804040000', '201804030000',
'201804020000', '201803290000', '201803280000', '201803270000',
'201803260000'
]
day30End = [
'201804250000', '201804240000', '201804210000', '201804200000',
'201804190000', '201804180000', '201804170000', '201804140000',
'201804130000', '201804120000', '201804110000', '201804100000',
'201804070000', '201804060000', '201804050000', '201804040000',
'201804030000', '201803300000', '201803290000', '201803280000',
'201803270000'
]
#Get Sentiment Score
er = TwitterSentimentAnalyzer()
processor = PreProcessor()
result = []
for i in range(0, 21):
processedTweets = []
rule = gen_rule_payload("MSFT OR Microsoft",
from_date=day30Start[i],
to_date=day30End[i],
results_per_call=80)
tweets = collect_results(rule,
max_results=80,
result_stream_args=premium_search_args)
for tweet in tweets[0:80]:
r = ' '.join(word for word in processor.preProcess(tweet.all_text))
processedTweets.append(r)
total = 0
for i in range(0, 2):
