def __init__(self, no_of_testcases=100, verbose=True, nb=None, bw=None):
self.logger = Logger('Comparer',
'logs\\comparer.log',
is_verbose=verbose)
self.load_html_structure()
if nb is None:
self.nb = NaiveBayes(verbose=False,
test_set_count=no_of_testcases,
no_of_grams=4)
self.nb.ready()
else:
self.nb = nb
self.nb.logger.is_verbose = False
if bw is None:
self.bw = BagOfWordSentiment(verbose=False, no_of_grams=4)
self.bw.ready()
else:
self.bw = bw
self.bw.logger.is_verbose = False
self.no_of_testcases = no_of_testcases
self.nb_correct, self.bw_correct, self.tb_correct = 0, 0, 0
self.nb_wrong, self.bw_wrong, self.tb_wrong = 0, 0, 0
self.nb_accuracy, self.bw_accuracy, self.tb_accuracy = 0, 0, 0
self.counter = 0
self.testcases = dict()
def create_test_set(test_data):
try:
print >> sys.stderr, "*** DOING TEST SET ***"
X_test = test_data.values()
Y_test = test_data.keys()
return X_test, Y_test
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
l = Logger()
l.log_error(
"Exception in create-test-set",
str(traceback.print_exc()) + "\n\n" + fname + " " +
str(exc_tb.tb_lineno))
response = {
'Response': 'FAILED',
'Reason': "Exception in create-test-set process"
}
return HttpResponse(json.dumps(response))
def train_gmm(self):
all_data = self.create_structure()
path = os.path.dirname(os.path.abspath(__file__))
try:
keys = all_data.keys()
n_classes = len(np.unique(keys))
gmm_classifier = mixture.GMM(n_components=n_classes,
covariance_type='full',
init_params='wmc',
min_covar=0.001,
n_init=1,
n_iter=100,
params='wmc',
random_state=None,
thresh=None,
tol=0.001)
for data in all_data.values():
for val in data.values():
f1 = val.get_object(2)
f2 = val.get_object(3)
data = zip(f1, f2)
if len(data) >= n_classes:
gmm_classifier.fit(data)
# save data
path_trainset = os.path.join(path, self.trainset_name)
with open(path_trainset, 'wb') as fid:
cPickle.dump(all_data, fid)
# save the classifier
model_directory = os.path.join(path, self.model_name)
with open(model_directory, 'wb') as fid:
cPickle.dump(gmm_classifier, fid)
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
l = Logger()
l.log_error(
"Exception in GMM-train model",
str(traceback.print_exc()) + "\n\n" + fname + " " +
str(exc_tb.tb_lineno))
response = {
'Response': 'FAILED',
'Reason': "Exception in GMM-train-model process"
}
return HttpResponse(json.dumps(response))
def __init__(self, settings_file=None, *args, **kwargs):
'''
Generate base frame and each page, bind them in a list
'''
self.logger = Logger()
self.baseUrl = '' # NOTE: Need to be overwritten
self.destRootFolder = '' # NOTE: Need to be overwritten
self.destFolder = ''
self.auth = Authenticator()
self.pathParser = PathParser()
# load config from .flash_pvt file
self.load_config_file(settings_file)
def save_results(self):
# TODO: finish save results in logger
inp = input("Do you want to save as json, csv or print: ")
if inp == "json":
Logger().save_to_json(self.most_common_words)
print("Saved to message.json file. ")
elif inp == "csv":
Logger().save_to_csv(self.most_common_words)
print("Saved to message.csv file. ")
elif inp == "print":
print(self.most_common_words)
else:
pass
def do_GET(self):
log = Logger()
# log the received command and the client ip address and port number.
log.log_info(f"{self.command} received from {self.client_address}")
# parse the url
url = urlparse.urlparse(self.path)
# check if the client call is correct.
if url.path == '/geocode':
# send 200 : Ok status.
self.send_response(200)
self.send_header('Content-type', 'json')
self.end_headers()
# check if address query string is passed
if urlparse.parse_qs(url.query).get('address'):
# get the geocodes off the passed address
address = urlparse.parse_qs(url.query)['address'][0].replace(" ", "+")
self._get_geocode(address)
else:
# send 400 : Bad Request status and log the error.
self.send_response(400)
log.log_error("address parameter not passed")
else:
# send 404 : Not Found status and log the error.
self.send_response(404)
log.log_error("Unknown service requested.")
def execute(self, sql=None):
"""
Execute a query and commit conn after
:param sql: query string
:return: None or Exception
"""
try:
cursor = self.conn.cursor()
cursor.execute(sql)
cursor.close()
self.conn.commit()
return None
except Exception as e:
Logger.exception(e)
return e
def query(self, sql=None):
"""
Execute a query and return data
:param sql: SQL query string
:return: array of tupples
"""
try:
cursor = self.conn.cursor()
cursor.execute(sql)
results = cursor.fetchall()
cursor.close()
return results
except Exception as e:
Logger.exception(e)
return e
def __init__(self, no_of_grams=4, verbose=True, no_of_testcases=1000):
self.verbose = verbose
self.logger = Logger('BagOfWordSentiment',
'logs\\bag_of_words.log',
is_verbose=self.verbose)
self.no_of_grams = no_of_grams
self.double_negations, self.double_negations_collection = set(), set()
self.negations, self.negation_collection = set(), set()
self.positive_words, self.positive_word_collection = set(), set()
self.negative_words, self.negative_word_collection = set(), set()
self.no_of_testcases = no_of_testcases
self.positve_test_bag = list()
self.negative_test_bag = list()
def models_if_exist(self):
try:
path = os.path.dirname(os.path.abspath(__file__))
model_path = os.path.join(path, self.model_name)
return os.path.exists(model_path)
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
l = Logger()
l.log_error("Exception in GMM-check-if-models-exist", str(traceback.print_exc()) + "\n\n" + fname + " " + str(exc_tb.tb_lineno))
response = {'Response': 'FAILED', 'Reason': "Exception in GMM-check-if-models-exist process"}
return HttpResponse(json.dumps(response))
def __init__(self, db_pool=None):
self.pool = None
self.conn = None
if db_pool is None:
Logger.warn('can not get database connection pool from env')
Logger.info(
'create a new temporary database connection for this call')
self.conn = Connection(
host=os.environ['DB_HOST'],
user=os.environ['DB_USER'],
password=os.environ['DB_PWD'],
database=os.environ['DB_NAME'],
)
else:
self.pool = db_pool
def __init__(self,
host="172.0.0.1",
user="******",
password="******",
database="test"):
self.host = host
self.user = user
self.password = password
self.database = database
try:
self.conn = mysql.connector.connect(host=self.host,
user=self.user,
password=self.password,
database=self.database)
except Exception as e:
Logger.exception(e)
def make_ellipses(self, ax, native_f1, native_f2, predicted_f1,
predicted_f2):
try:
print >> sys.stderr, "*** MAKE ELLIPSES ***"
x1 = min(native_f1)
x2 = max(native_f1)
y1 = min(native_f2)
y2 = max(native_f2)
centroid_x = (x2 + x1) / 2
centroid_y = (y2 + y1) / 2
x_2 = math.pow((centroid_x - predicted_f1), 2)
y_2 = math.pow((centroid_y - predicted_f2), 2)
distance_from_centroid = math.sqrt(x_2 + y_2)
ellipse = mpl.patches.Ellipse(xy=((x2 + x1) / 2, (y2 + y1) / 2),
width=(x2 - x1) * 1.4,
height=(y2 - y1) * 1.2)
ellipse.set_edgecolor('r')
ellipse.set_facecolor('none')
ellipse.set_clip_box(ax.bbox)
ellipse.set_alpha(0.5)
ax.add_artist(ellipse)
print >> sys.stderr, "*** ELLIPSES DONE ***"
return distance_from_centroid
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
l = Logger()
l.log_error(
"Exception in GMM-make ellipse",
str(traceback.print_exc()) + "\n\n" + fname + " " +
str(exc_tb.tb_lineno))
response = {
'Response': 'FAILED',
'Reason': "Exception in GMM-make-ellipse process"
}
return HttpResponse(json.dumps(response))
def extract_data(audio_file):
print >> sys.stderr, "*** DOING EXTRACT DATA ***"
# need to change speakerfile for the female gender
path = os.path.dirname(os.path.abspath(__file__))
path_fave = path + "/libraries/FAVE_extract/"
config_file = "--outputFormat txt --candidates --speechSoftware praat --formantPredictionMethod default --measurementPointMethod faav --nFormants 3 --minVowelDuration 0.001 --nSmoothing 12 --remeasure --vowelSystem phila --speaker " + path_fave + "/speakerinfo.speakerfile"
textgrid_file_directory = path + "/data/"
output_file_directory = path + "/data/"
wav_file = audio_file
wav_file_cleaned = wav_file.replace('.wav', '.TextGrid')
(dir_name, file_name) = os.path.split(wav_file_cleaned)
textgrid_file = os.path.join(textgrid_file_directory, file_name)
output_file = os.path.join(output_file_directory,
file_name.replace('.TextGrid', '.txt'))
# debug print
command = "python " + path_fave + "bin/extractFormants.py " + config_file + " " + audio_file + " " + textgrid_file + " " + output_file
try:
# run command
proc = Popen(command, shell=True)
proc.wait()
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
l = Logger()
l.log_error(
"Exception in exctract-formants",
str(traceback.print_exc()) + "\n\n" + fname + " " +
str(exc_tb.tb_lineno))
response = {
'Response': 'FAILED',
'Reason': "Exception in extract-formants process"
}
return HttpResponse(json.dumps(response))
def get_native_vowels(self, sentence):
try:
path = os.path.dirname(os.path.abspath(__file__))
label_path = path + self.native_vowels
sentences_path = path + self.native_sentences
s = sentence.lower()
vowels = []
with open(label_path, 'rb') as vowels_file:
reader = csv.reader(vowels_file, delimiter='\n')
all_lines = list(reader)
for line in all_lines:
l = line[0].split(' ')
vowels.append(l)
sentences = []
with open(sentences_path, 'rb') as sentences_file:
reader = csv.reader(sentences_file, delimiter='\n')
all_lines = list(reader)
for line in all_lines:
sen = line[0]
sentences.append(sen)
result = dict(zip(sentences, vowels))
return result[s]
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
l = Logger()
l.log_error(
"Exception in GMM-get-native-vowels-struct",
str(traceback.print_exc()) + "\n\n" + fname + " " +
str(exc_tb.tb_lineno))
response = {
'Response': 'FAILED',
'Reason': "Exception in GMM-get-native-vowels process"
}
return HttpResponse(json.dumps(response))
def create_test_set(test_data):
try:
print>>sys.stderr, "*** DOING TEST SET ***"
X_test = test_data.values()
Y_test = test_data.keys()
return X_test, Y_test
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
l = Logger()
l.log_error("Exception in create-test-set", str(traceback.print_exc()) + "\n\n" + fname + " " + str(exc_tb.tb_lineno))
response = {'Response': 'FAILED', 'Reason': "Exception in create-test-set process"}
return HttpResponse(json.dumps(response))
def __init__(self, verbose=True, training_cases=2500, testcases=500):
self.verbose = verbose
self.training_cases = training_cases
self.testcases = testcases
self.training = list()
self.test = list()
self.frequency = dict()
self.stop_words = self.get_stop_words()
self.positive_words = 0
self.negative_words = 0
self.positive_sentence_count = 0
self.negative_sentence_count = 0
self.total_sentences = 0
self.logger = Logger('NaiveBayers', 'NaiveBayers.log')
self.filenames = [
'res\\benchmark\\yelp_labelled.txt',
'res\\benchmark\\amazon_cells_labelled.txt',
'res\\benchmark\\imdb_labelled.txt'
]
def train_gmm(self):
all_data = self.create_structure()
path = os.path.dirname(os.path.abspath(__file__))
try:
keys = all_data.keys()
n_classes = len(np.unique(keys))
gmm_classifier = mixture.GMM(n_components=n_classes, covariance_type='full',
init_params='wmc', min_covar=0.001, n_init=1,
n_iter=100, params='wmc', random_state=None,
thresh=None, tol=0.001)
for data in all_data.values():
for val in data.values():
f1 = val.get_object(2)
f2 = val.get_object(3)
data = zip(f1, f2)
if len(data) >= n_classes:
gmm_classifier.fit(data)
# save data
path_trainset = os.path.join(path, self.trainset_name)
with open(path_trainset, 'wb') as fid:
cPickle.dump(all_data, fid)
# save the classifier
model_directory = os.path.join(path, self.model_name)
with open(model_directory, 'wb') as fid:
cPickle.dump(gmm_classifier, fid)
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
l = Logger()
l.log_error("Exception in GMM-train model", str(traceback.print_exc()) + "\n\n" + fname + " " + str(exc_tb.tb_lineno))
response = {'Response': 'FAILED', 'Reason': "Exception in GMM-train-model process"}
return HttpResponse(json.dumps(response))
def __init__(self, settings_file=None, *args, **kwargs):
'''
Generate base frame and each page, bind them in a list
'''
self.logger = Logger()
self.baseUrl = '' # NOTE: Need to be overwritten
self.destRootFolder = '' # NOTE: Need to be overwritten
self.destFolder = ''
self.auth = Authenticator()
self.pathParser = PathParser()
# load config from .flash_pvt file
self.load_config_file(settings_file)
def _get_geocode(self, param):
log = Logger()
# log the address received
log.log_info(f"search for {param}")
# get the geocodes using here service.
service = Here()
service.get_geocodes(param)
# if the returned status is nnot 200: ok, call google service.
if service.status != 200:
service = Google()
service.get_geocodes(param)
# build meta data to attach to the result.
meta = {'status': service.status, 'status_desc': str(service.status_desc),
'service_used': service.geocoding_service_used,
'requested_address': param, 'timestamp': service.timestamp}
# compose the final results.
result = {'lat': service.latitude, 'lng': service.longitude,
'meta': meta}
# write the results is json format and in the encoding specified in the config file
self.wfile.write(json.dumps(result).encode(config.reponse_encoding))
def __create_user__(self):
try:
raw_data = json.loads(self.data)
user = User(email=raw_data['email'], password=raw_data['password'])
user.create()
result = self.user_service.create_user(user=user)
if result is None:
raise HttpCreated(data=ResponseData(ex=None,
data={
'id': user.id,
'email': user.email
}))
elif isinstance(result, EmailExisted):
raise HttpFound(data=ResponseData(ex=result, data=None))
else:
raise HttpInternalError(
data=ResponseData(ex=result, data=None))
except Exception as e:
Logger.exception(e)
raise HttpBadRequest(data=ResponseData(ex=e, data=None))
def team_ranker_ovr(data, greater_than, field, all_time_rpg, standard_deviation, average_deviation, playoff_data=None):
logger = Logger(os.path.join(log_prefix, "import_data", "team_ranker_ovr.log"))
logger.log("Calculating overall team ranks: " + field)
start_time = time.time()
final_data = {}
if field != "ovrRank_ovr":
for year, value in data.items():
final_data[year] = []
for ent in value:
if field == "offRank_ovr":
final_data[year].append([ent[0], (ent[1]/all_time_rpg) /
(standard_deviation[str(year)]/average_deviation)])
else:
final_data[year].append([ent[0], (ent[1]/all_time_rpg) *
(standard_deviation[str(year)]/average_deviation)])
else:
for year, value in data.items():
final_data[year] = []
for ent in value:
for team_value in data[year]:
if team_value[0] == ent[0]:
playoff_bump = 1.0
for accomplishment, team_id in playoff_data.items():
if team_id == ent[0]:
if accomplishment == 'ws_champ':
playoff_bump += 0.005
playoff_bump += 0.005
final_data[year].append([ent[0], (ent[1]/(standard_deviation[str(year)]/average_deviation)) *
playoff_bump])
write_to_file(final_data, greater_than, field)
total_time = time_converter(time.time() - start_time)
logger.log("\tTime = " + total_time + '\n\n')
def models_if_exist(self):
try:
path = os.path.dirname(os.path.abspath(__file__))
model_path = os.path.join(path, self.model_name)
return os.path.exists(model_path)
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
l = Logger()
l.log_error(
"Exception in GMM-check-if-models-exist",
str(traceback.print_exc()) + "\n\n" + fname + " " +
str(exc_tb.tb_lineno))
response = {
'Response': 'FAILED',
'Reason': "Exception in GMM-check-if-models-exist process"
}
return HttpResponse(json.dumps(response))
def __init__(self, env, policy, algorithm, plot: bool = True):
"""
:param env: Contains the gym environment the simulations are
performed on
:type env: Environment
:param policy: The policy to improve
:type policy: Policy
:param algorithm: The learning algorithm
:type algorithm: NES or NPG
:param plot: If True the results of Training and Benchmark will
be plotted
:type plot: bool
"""
self.policy = policy
self.env = env
self.algorithm = algorithm
self.plot = plot
self.logger = Logger()
def get_native_vowels(self, sentence):
try:
path = os.path.dirname(os.path.abspath(__file__))
label_path = path + self.native_vowels
sentences_path = path + self.native_sentences
s = sentence.lower()
vowels = []
with open(label_path, 'rb') as vowels_file:
reader = csv.reader(vowels_file, delimiter='\n')
all_lines = list(reader)
for line in all_lines:
l = line[0].split(' ')
vowels.append(l)
sentences = []
with open(sentences_path, 'rb') as sentences_file:
reader = csv.reader(sentences_file, delimiter='\n')
all_lines = list(reader)
for line in all_lines:
sen = line[0]
sentences.append(sen)
result = dict(zip(sentences, vowels))
return result[s]
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
l = Logger()
l.log_error("Exception in GMM-get-native-vowels-struct", str(traceback.print_exc()) + "\n\n" + fname + " " + str(exc_tb.tb_lineno))
response = {'Response': 'FAILED', 'Reason': "Exception in GMM-get-native-vowels process"}
return HttpResponse(json.dumps(response))
def extract_data(audio_file):
print>>sys.stderr, "*** DOING EXTRACT DATA ***"
# need to change speakerfile for the female gender
path = os.path.dirname(os.path.abspath(__file__))
path_fave = path + "/libraries/FAVE_extract/"
config_file = "--outputFormat txt --candidates --speechSoftware praat --formantPredictionMethod default --measurementPointMethod faav --nFormants 3 --minVowelDuration 0.001 --nSmoothing 12 --remeasure --vowelSystem phila --speaker " + path_fave + "/speakerinfo.speakerfile"
textgrid_file_directory = path + "/data/"
output_file_directory = path + "/data/"
wav_file = audio_file
wav_file_cleaned = wav_file.replace('.wav', '.TextGrid')
(dir_name, file_name) = os.path.split(wav_file_cleaned)
textgrid_file = os.path.join(textgrid_file_directory, file_name)
output_file = os.path.join(output_file_directory, file_name.replace('.TextGrid', '.txt'))
# debug print
command = "python " + path_fave + "bin/extractFormants.py " + config_file + " " + audio_file + " " + textgrid_file + " " + output_file
try:
# run command
proc = Popen(command, shell=True)
proc.wait()
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
l = Logger()
l.log_error("Exception in exctract-formants", str(traceback.print_exc()) + "\n\n" + fname + " " + str(exc_tb.tb_lineno))
response = {'Response': 'FAILED', 'Reason': "Exception in extract-formants process"}
return HttpResponse(json.dumps(response))
def get_geocodes(self, address_to_find):
# log the usage of google service.
log = Logger()
log.log_info(f"Calling {self.geocoding_service_used} service")
# create a request object
req = request.Request(self.geocoding_api_url + address_to_find)
try:
# open the URL.
with request.urlopen(req) as api_response:
# get the HTTP status code from the service
self.status = api_response.getcode()
if api_response.status == 200:
self.status_desc = "Ok"
# get the response and load in json format.
json_response = json.loads(api_response.read())
# assign the variables with values returned from the api call.
self.latitude = json_response["results"][0]["geometry"][
"location"]["lat"]
self.longitude = json_response["results"][0]["geometry"][
"location"]["lng"]
self.full_address = json_response["results"][0][
"formatted_address"]
# log the result.
log.log_info(
f"Status= {api_response.status}, lat={self.latitude}, lng={self.longitude}"
)
# close the request.
api_response.close()
except Exception as ex:
# set the status to 500 - internal server error.
self.status_desc = ex
self.status = 500
# log the critical error.
log.log_critical(str(ex))
def make_ellipses(self, ax, native_f1, native_f2, predicted_f1, predicted_f2):
try:
print >> sys.stderr, "*** MAKE ELLIPSES ***"
x1 = min(native_f1)
x2 = max(native_f1)
y1 = min(native_f2)
y2 = max(native_f2)
centroid_x = (x2 + x1) / 2
centroid_y = (y2 + y1) / 2
x_2 = math.pow((centroid_x - predicted_f1), 2)
y_2 = math.pow((centroid_y - predicted_f2), 2)
distance_from_centroid = math.sqrt(x_2 + y_2)
ellipse = mpl.patches.Ellipse(xy=((x2 + x1) / 2, (y2 + y1) / 2), width=(x2 - x1) * 1.4,
height=(y2 - y1) * 1.2)
ellipse.set_edgecolor('r')
ellipse.set_facecolor('none')
ellipse.set_clip_box(ax.bbox)
ellipse.set_alpha(0.5)
ax.add_artist(ellipse)
print >> sys.stderr, "*** ELLIPSES DONE ***"
return distance_from_centroid
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
l = Logger()
l.log_error("Exception in GMM-make ellipse", str(traceback.print_exc()) + "\n\n" + fname + " " + str(exc_tb.tb_lineno))
response = {'Response': 'FAILED', 'Reason': "Exception in GMM-make-ellipse process"}
return HttpResponse(json.dumps(response))
def __init__(self, verbose=True, test_set_count=500, no_of_grams=1):
self.logger = Logger('NaiveBayes',
'logs\\NaiveBayes.log',
is_verbose=verbose)
self.verbose = verbose
self.counts = dict()
self.positive_bag = []
self.negative_bag = []
self.positve_test_bag = []
self.negative_test_bag = []
self.counts["test set"] = test_set_count
self.counts["positive phrases"] = 0
self.counts["negative phrases"] = 0
self.counts["total sentences"] = 0
self.counts["positive sentences"] = 0
self.counts["negative sentences"] = 0
self.no_of_grams = no_of_grams
self.phrase_occurrences = dict()
self.phrase_probabilities = dict()
def main():
cli = CommandLineInterface(sys.argv)
source_type = cli.get_source_type()
part_of_speech = cli.get_part_of_speech()
words_count = cli.get_words_count()
if source_type == 'folder':
parser = FolderParser(part_of_speech, words_count)
elif source_type == 'web':
parser = WebParser(part_of_speech, words_count)
most_common_words = parser.parse_most_common_words(
part_of_speech, words_count, directories=['target_folder'])
Logger().message(debug_message=most_common_words)
print("Saved into a file 'message.log'")
class Test002LoginTestDDT:
logger = Logger.get_logger()
@pytest.mark.regression
@pytest.mark.parametrize("user, pwd, expected", excel_utility.load_excel_data(ConfigParser.get_test_data_file_path(), 'login_data_sheet'))
def test_login_ddt(self,setup, user, pwd, expected):
self.logger.info("**************" + Test002LoginTestDDT.__name__ + "*************")
self.logger.info("************** Verify Home Page title *************")
self.login_page = LoginPage(setup)
self.logger.info(f"Username : {user}")
self.logger.info(f"password : {pwd}")
self.logger.info(f"expected : {expected}")
status_list = list()
self.login_page.set_username(user)
self.login_page.set_password(pwd)
self.login_page.click_login()
time.sleep(5)
title = self.login_page.get_login_page_title()
exp_title = "Dashboard / nopCommerce administration"
if title == exp_title:
if expected == 'pass':
self.logger.info("*********passed***********")
self.login_page.click_logout()
status_list.append("pass")
elif expected == 'fail':
self.logger.info("*******failed*********")
self.login_page.click_logout()
status_list.append("fail")
elif title != exp_title:
if expected == 'fail':
self.logger.info("*********passed***********")
status_list.append("pass")
elif expected == 'pass':
self.logger.info("*********failed***********")
status_list.append("fail")
if 'fail' not in status_list:
self.logger.info("********* Login DDT is passed***********")
assert True
else:
self.logger.info("********* Login DDT is failed***********")
assert False
self.logger.info("************** Completed " + Test002LoginTestDDT.__name__ + "*************")
def logged(func=None, level=logging.DEBUG, name=None, msg=None):
"""
Decorator to automatically the time of execution of a function
in a logfile, and write a message
Parameters
----------
func : the function name
level : the level of the log
name : the name of the log
message : specific message
Examples
--------
>>> @logged(level=logging.INFO)
... def toto(x, y):
... print x + y
>>> toto(3, 4)
7
in toto.txt => INFO:toto:0.000001
"""
if func is None:
return partial(logged, level=level, name=name, msg=msg)
logger = name if name else Logger(func.__name__ + ".log", logging.INFO)
logmsg = msg if msg else func.__name__
@wraps(func)
def wrapper(*args, **kwargs):
start = time.time()
result = func(*args, **kwargs)
end = time.time()
msg = ":".join([str(func.__name__), str(end - start)])
logger.log(level, logmsg)
logger.log(level, msg)
return result
return wrapper
def run(host, port):
log = Logger()
try:
# create http server.
server = http.server.HTTPServer((host, port), RequestHandler)
print(f"Geocoding Proxy Service - v{config.service_version}")
print(f"Server Started on port :{port}")
# log the start of the server.
log.log_info(f"Server Started on port :{str(port)}")
# listen for ever.
server.serve_forever()
except Exception as ex:
# in case of exception, log the incident as critical.
msg = f"Server Start error - {str(ex)}"
log.log_critical(msg)
exit(1)
def main():
try:
args = get_args()
config = process_config(args.config)
except:
print("Missing or invalid arguments")
exit(0)
create_dirs([config.summary_dir, config.checkpoint_dir])
sess = tf.Session()
data = DataGenerator(config)
model = TemplateNet(config)
logger = Logger(sess, config)
trainer = TemplateTrainer(sess, model, data, config, logger)
model.load(sess)
trainer.train()
import datetime
from urllib.request import urlopen, urlretrieve
from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor
from utilities.database.wrappers.baseball_data_connection import DatabaseConnection
from utilities.translate_team_id import translate_team_id
from utilities.time_converter import time_converter
from utilities.logger import Logger
from utilities.anomaly_team import anomaly_team
from utilities.properties import sandbox_mode, log_prefix, import_driver_logger as driver_logger
from import_data.player_data.fielding.cathchers_defense import catcher_defense
data = {}
pages = {}
temp_pages = {}
logger = Logger(os.path.join(log_prefix, "import_data", "fielders.log"))
def fielding_constructor(year):
print('Downloading fielder images and attributes')
driver_logger.log("\tDownloading fielder images and attributes")
start_time = time.time()
global data
data = {}
catcher_info = catcher_defense(year, logger)
logger.log("Downloading fielder " + str(year) + " data || Timestamp: " +
datetime.datetime.today().strftime('%Y-%m-%d %H:%M:%S'))
logger.log("\tAssembling list of players")
table = str(BeautifulSoup(urlopen("https://www.baseball-reference.com/leagues/MLB/" + str(year)
+ "-standard-fielding.shtml"), 'html.parser')).\
split('<table class="sortable stats_table" id')[1].split('<tbody>')[1].split('</tbody>')[0].split('<tr')
from utilities.database.wrappers.baseball_data_connection import DatabaseConnection
from utilities.translate_team_name import translate_team_name
from utilities.logger import Logger
from utilities.time_converter import time_converter
import datetime
import time
import os
from utilities.properties import sandbox_mode, log_prefix, import_driver_logger as driver_logger
logger = Logger(
os.path.join(log_prefix, "import_data", "populate_teams_table.log"))
def populate_teams_table(year):
driver_logger.log('\tPopulating teams table')
print("Populating teams table")
start_time = time.time()
logger.log('Begin populating teams table for ' + str(year) +
' || Timestamp: ' +
datetime.datetime.today().strftime('%Y-%m-%d %H:%M:%S'))
with open(os.path.join("..", "background", "yearTeams.txt"), 'rt') as file:
db = DatabaseConnection(sandbox_mode)
db.write('ALTER TABLE teams DROP INDEX teamId;')
for line in file:
if str(year) in line:
temp_line = line.split(',')[1:-1]
for team in temp_line:
team_id = team.split(';')[0]
db.write('insert into teams (teamId, teamName) values ("' +
team_id + '", "' +
translate_team_name(team_id).replace("'", "\'") +
def __init__(self, parent, controller):
Frame.__init__(self, parent)
self.logger = Logger()
self.grid()
self.controller = controller
def extract_phonemes(audio_file, sentence, predicted_phonemes):
try:
print>>sys.stderr, "*** DOING EXTRACT PHONEMES ***"
path = os.path.dirname(os.path.abspath(__file__))
textgrid_directory = path + "/data"
(dir_name, file_name) = os.path.split(audio_file)
output_filename = os.path.join(textgrid_directory, file_name.replace('.wav', '.txt'))
vowel_stress = []
phonemes = []
with open(output_filename, 'r') as textgrid_file:
reader = csv.reader(textgrid_file, delimiter='\t')
all_lines = list(reader)
print>>sys.stderr, "*** OPENED: " + output_filename + " ***"
i = 0
for line in all_lines:
if i == 0:
i += 1
continue
# vowel, stress
vowel = line[12]
stress = line[13]
vowel_stress.append((vowel, stress))
# phonemes
pre_word_trans = line[39]
word_trans = line[40]
fol_word_trans = line[41]
pre_word_trans = pre_word_trans.replace(' ', '')
if pre_word_trans != "SP" and pre_word_trans not in phonemes:
phonemes.append(pre_word_trans)
word_trans = word_trans.replace(' ', '')
if word_trans != "SP" and word_trans not in phonemes:
phonemes.append(word_trans)
fol_word_trans = fol_word_trans.replace(' ', '')
if fol_word_trans != "SP" and fol_word_trans not in phonemes:
phonemes.append(fol_word_trans)
index = native_sentences.index(sentence)
current_native_phonemes = native_phonemes[index]
# do WER with the CMU Sphinx phonemes but keep the old ones for stress
print>>sys.stderr, "*** WER ***"
test_phonemes = ""
cmu_phonemes_list = str(predicted_phonemes).split(' ')
sentence_list = current_native_phonemes.split(' ')
for s in sentence_list:
for cmu in cmu_phonemes_list[:]:
if cmu in s:
test_phonemes += cmu
cmu_phonemes_list.remove(cmu)
test_phonemes += " "
wer_result, numCor, numSub, numIns, numDel = wer(current_native_phonemes, test_phonemes)
result_wer = "Word Error Rate: {}%".format(wer_result * 100)
return test_phonemes.split(' '), vowel_stress, result_wer
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
l = Logger()
l.log_error("Exception in extract-phonemes", str(traceback.print_exc()) + "\n\n" + fname + " " + str(exc_tb.tb_lineno))
response = {'Response': 'FAILED', 'Reason': "Exception in extract-phonemes process"}
return HttpResponse(json.dumps(response))
class BaseController(object):
def __init__(self, settings_file=None, *args, **kwargs):
'''
Generate base frame and each page, bind them in a list
'''
self.logger = Logger()
self.baseUrl = '' # NOTE: Need to be overwritten
self.destRootFolder = '' # NOTE: Need to be overwritten
self.destFolder = ''
self.auth = Authenticator()
self.pathParser = PathParser()
# load config from .flash_pvt file
self.load_config_file(settings_file)
def setData(self, data=None):
if data is None:
data = self.pathParser.get_builds_list_from_url(self.baseUrl)
self.data = data
def setAuth(self, user, pwd):
## pass auth parameters
self.auth.authenticate(self.baseUrl, user, pwd)
if not self.auth.is_authenticated:
return False
self.setData()
return True
def quit(self):
'''
Halt the program
'''
print('### quit function invoked')
sys.exit(0)
def do_download(self, targets):
if len(self.destFolder) == 0:
self.destFolder = self.destRootFolder
downloader = Downloader()
archives = {}
for target in targets:
archives[target] = downloader.download(self.paths[target], self.destFolder, status_callback=self.printErr)
return archives
def do_flash(self, targets, archives, keep_profile=False):
cmd = './shallow_flash.sh -y'
sp = ''
if _platform == 'darwin':
sp = ' '
if PathParser._IMAGES in targets:
try:
self.temp_dir = tempfile.mkdtemp()
self.logger.log('Create temporary folder:' + self.temp_dir, status_callback=self.printErr)
Decompressor().unzip(archives[PathParser._IMAGES], self.temp_dir, status_callback=self.printErr)
# set the permissions to rwxrwxr-x (509 in python's os.chmod)
os.chmod(self.temp_dir + '/b2g-distro/flash.sh', stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
os.chmod(self.temp_dir + '/b2g-distro/load-config.sh', stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
os.system('cd ' + self.temp_dir + '/b2g-distro; ./flash.sh -f')
finally:
try:
shutil.rmtree(self.temp_dir) # delete directory
except OSError:
self.logger.log('Can not remove temporary folder:' + self.temp_dir, status_callback=self.printErr, level=Logger._LEVEL_WARNING)
flash_img_cmd = 'CUR_DIR=`pwd`; TEMP_DIR=`mktemp -d`; unzip -d $TEMP_DIR ' + archives[PathParser._IMAGES] + '; \\\n' + \
'cd $TEMP_DIR/b2g-distro/; ./flash.sh -f; \\\ncd $CUR_DIR; rm -rf $TEMP_DIR'
self.logger.log('!!NOTE!! Following commands can help you to flash packages into other device WITHOUT download again.\n%s\n' % (flash_img_cmd,))
else:
if PathParser._GAIA in targets:
cmd = cmd + ' -g' + sp + archives[PathParser._GAIA]
if PathParser._GECKO in targets:
cmd = cmd + ' -G' + sp + archives[PathParser._GECKO]
if keep_profile:
self.logger.log('Keep User Profile.')
cmd = cmd + ' --keep_profile'
print('run command: ' + cmd)
os.system(cmd)
self.logger.log('!!NOTE!! Following commands can help you to flash packages into other device WITHOUT download again.\n%s\n' % (cmd,))
self.logger.log('Flash Done.', status_callback=self.printErr)
self.after_flash_action()
self.quit()
def after_flash_action(self):
pass
def printErr(self, message):
raise NotImplementedError
def getPackages(self, src, build_id=''):
'''
input src and build-id, then setup the dest-folder and return the available packages.
'''
#TODO: Async request?
query = self.pathParser.get_available_packages_from_url(base_url=self.baseUrl, build_src=src, build_id=build_id, build_id_format=self.build_id_format)
self.paths = {}
package = []
if PathParser._GAIA in query and PathParser._GECKO in query:
package.append(PathParser._GAIA_GECKO)
if PathParser._GAIA in query:
package.append(PathParser._GAIA)
self.paths[PathParser._GAIA] = query[PathParser._GAIA]
if PathParser._GECKO in query:
package.append(PathParser._GECKO)
self.paths[PathParser._GECKO] = query[PathParser._GECKO]
if PathParser._IMAGES in query:
package.append(PathParser._IMAGES)
self.paths[PathParser._IMAGES] = query[PathParser._IMAGES]
# set up the download dest folder
self.destFolder = self._get_dest_folder_from_build_id(self.destRootFolder, src, build_id)
return package
def getLatestBuildId(self, src):
# TODO: Get from remote and Use in local flash;
# should be an async request?
pass
def load_config_file(self, settings_file=None):
'''
Load ".flash_pvt" as config file.
If there is no file, then copy from ".flash_pvt.template".
'''
if settings_file is None:
settings_file = '.flash_pvt'
if not os.path.exists(settings_file):
self.logger.log('Creating %s from %s' % (settings_file, settings_file + '.template'))
shutil.copy2(settings_file + '.template', settings_file)
self.logger.log('Loading settings from %s' % (settings_file,))
account = {}
with open(settings_file) as f:
config = eval(f.read())
if 'account' in config:
self.account = config['account']
else:
self.account = ''
if 'password' in config:
self.password = config['password']
else:
self.password = ''
if 'download_home' in config:
self.destRootFolder = config['download_home']
else:
self.destRootFolder = 'pvt'
if 'base_url' in config:
self.baseUrl = config['base_url']
else:
self.baseUrl = 'pvhttps://pvtbuilds.mozilla.org/pvt/mozilla.org/b2gotoro/nightly/'
if 'build_id_format' in config:
self.build_id_format = config['build_id_format']
else:
self.build_id_format = '/{year}/{month}/{year}-{month}-{day}-{hour}-{min}-{sec}/'
def _get_dest_folder_from_build_id(self, root_folder, build_src, build_id):
target_folder = ''
if not build_id == '' or build_id == 'latest':
if self.pathParser.verify_build_id(build_id):
sub_folder = re.sub(r'^/', '', self.pathParser.get_path_of_build_id(build_id=build_id, build_id_format=self.build_id_format))
target_folder = os.path.join(root_folder, build_src, sub_folder)
else:
self.logger.log('The build id [' + build_id + '] is not not valid.', status_callback=self.printErr, level=Logger._LEVEL_WARNING)
self.quit()
else:
target_folder = os.path.join(root_folder, build_src, 'latest')
self.logger.log('Set up dest folder to [' + target_folder + '].', status_callback=self.printErr)
return target_folder
def wer(ref, hyp, debug=False):
try:
DEL_PENALTY = 2
SUB_PENALTY = 1
INS_PENALTY = 3
r = ref # .split()
h = hyp # .split()
# costs will holds the costs, like in the Levenshtein distance algorithm
costs = [[0 for inner in range(len(h) + 1)] for outer in range(len(r) + 1)]
# backtrace will hold the operations we've done.
# so we could later backtrace, like the WER algorithm requires us to.
backtrace = [[0 for inner in range(len(h) + 1)] for outer in range(len(r) + 1)]
OP_OK = 0
OP_SUB = 1
OP_INS = 2
OP_DEL = 3
# First column represents the case where we achieve zero
# hypothesis words by deleting all reference words.
for i in range(1, len(r) + 1):
costs[i][0] = DEL_PENALTY * i
backtrace[i][0] = OP_DEL
# First row represents the case where we achieve the hypothesis
# by inserting all hypothesis words into a zero-length reference.
for j in range(1, len(h) + 1):
costs[0][j] = INS_PENALTY * j
backtrace[0][j] = OP_INS
# computation
for i in range(1, len(r) + 1):
for j in range(1, len(h) + 1):
if r[i - 1] == h[j - 1]:
costs[i][j] = costs[i - 1][j - 1]
backtrace[i][j] = OP_OK
else:
substitutionCost = costs[i - 1][j - 1] + SUB_PENALTY # penalty is always 1
insertionCost = costs[i][j - 1] + INS_PENALTY # penalty is always 1
deletionCost = costs[i - 1][j] + DEL_PENALTY # penalty is always 1
costs[i][j] = min(substitutionCost, insertionCost, deletionCost)
if costs[i][j] == substitutionCost:
backtrace[i][j] = OP_SUB
elif costs[i][j] == insertionCost:
backtrace[i][j] = OP_INS
else:
backtrace[i][j] = OP_DEL
# back trace though the best route:
i = len(r)
j = len(h)
numSub = 0
numDel = 0
numIns = 0
numCor = 0
if debug:
print("OP\tREF\tHYP")
lines = []
while i > 0 or j > 0:
if backtrace[i][j] == OP_OK:
numCor += 1
i -= 1
j -= 1
if debug:
lines.append("OK\t" + r[i] + "\t" + h[j])
elif backtrace[i][j] == OP_SUB:
numSub += 1
i -= 1
j -= 1
if debug:
lines.append("SUB\t" + r[i] + "\t" + h[j])
elif backtrace[i][j] == OP_INS:
numIns += 1
j -= 1
if debug:
lines.append("INS\t" + "****" + "\t" + h[j])
elif backtrace[i][j] == OP_DEL:
numDel += 1
i -= 1
if debug:
lines.append("DEL\t" + r[i] + "\t" + "****")
if debug:
lines = reversed(lines)
for line in lines:
print(line)
print("#cor " + str(numCor))
print("#sub " + str(numSub))
print("#del " + str(numDel))
print("#ins " + str(numIns))
return (numSub + numDel + numIns) / (float)(len(r))
wer_result = round((numSub + numDel + numIns) / (float)(len(r)), 3)
return wer_result, numCor, numSub, numIns, numDel
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
l = Logger()
l.log_error("Exception in WER", traceback.print_exc() + "\n\n" + fname + " " + str(exc_tb.tb_lineno))
response = {'Response': 'FAILED', 'Reason': "Exception in WER process"}
return HttpResponse(json.dumps(response))
def get_pitch_contour(audio_file, sentence):
try:
print>>sys.stderr, "*** DOING PITCH CONTOUR ***"
path = os.path.dirname(os.path.abspath(__file__))
path_script = path + "/libraries/pitch_contour/pitch_intensity_formants.praat"
(dir_name, file_name) = os.path.split(audio_file)
output_name = file_name.replace(".wav", ".csv")
output_folder = path + "/data/" + output_name
sentence = sentence.lower()
sentence = sentence.replace(' ', '_')
min_pitch = '65'
native_csv = path + "/data/native/male/" + sentence + ".csv"
# see script file for the usage
command = '/usr/bin/praat ' + path_script + " " + audio_file + " " + output_folder + " " + 'wav' + " " + '10' + " " + min_pitch + " " + '500' + " " + '11025'
print>>sys.stderr, command
proc = Popen(command, shell=True)
proc.wait()
# native
print>>sys.stderr, "*** READING NATIVE CSV ***"
native_pitch = []
with open(native_csv, 'r') as native_file:
reader = csv.reader(native_file, delimiter=',')
all_lines = list(reader)
for line in all_lines:
if line[1] == 'pitch':
continue
if line[1] == '?':
native_pitch.append('0')
else:
native_pitch.append(line[1])
# user
print>>sys.stderr, "*** READING USER CSV ***"
user_pitch = []
with open(output_folder, 'r') as user_file:
reader = csv.reader(user_file, delimiter=',')
all_lines = list(reader)
for line in all_lines:
if line[1] == 'pitch':
continue
if line[1] == '?':
user_pitch.append('0')
else:
user_pitch.append(line[1])
print>>sys.stderr, "*** PADDING ***"
# Padding with 0s on the end
if len(native_pitch) != len(user_pitch):
copy_native_pitch = native_pitch
index = 0
for val in copy_native_pitch:
if val == 0 or val == '0':
del native_pitch[index]
index += 1
else:
break
copy_user_pitch = user_pitch
index = 0
for val in copy_user_pitch:
if val == 0 or val == '0':
del user_pitch[index]
index += 1
else:
break
length_native = len(native_pitch)
length_user = len(user_pitch)
if length_native > length_user:
diff = length_native - length_user
temp = ['0'] * diff
user_pitch += temp
elif length_user > length_native:
diff = length_user - length_native
temp = ['0'] * diff
native_pitch += temp
# Create scatter image
print>>sys.stderr, "*** CREATING FIGURE ***"
time = []
val = 0
for i in range(len(native_pitch)):
val += 0.1
time.append(val)
# Normalized Data
normalized_native = []
normalized_native_floats = [float(x) for x in native_pitch]
for val in normalized_native_floats:
dd = (val - min(normalized_native_floats)) / (max(normalized_native_floats) - min(normalized_native_floats))
normalized_native.append(dd)
normalized_user = []
normalized_user_floats = [float(x) for x in user_pitch]
for val in normalized_user_floats:
dd = (val - min(normalized_user_floats)) / (max(normalized_user_floats) - min(normalized_user_floats))
normalized_user.append(dd)
return normalized_native, normalized_user
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
l = Logger()
l.log_error("Exception in get-pitch-contour", str(traceback.print_exc()) + "\n\n" + fname + " " + str(exc_tb.tb_lineno))
response = {'Response': 'FAILED', 'Reason': "Exception in get-pitch-contour process"}
return HttpResponse(json.dumps(response))
def create_test_data(filename):
try:
print>>sys.stderr, "*** DOING TEST DATA ***"
path = os.path.dirname(os.path.abspath(__file__))
path_data = path + "/data/"
txt_file = path_data + filename.replace('.wav', '_norm.txt')
csv_file = path_data + filename.replace('.wav', '.csv')
# use 'with' if the program isn't going to immediately terminate
# so you don't leave files open
# the 'b' is necessary on Windows
# it prevents \x1a, Ctrl-z, from ending the stream prematurely
# and also stops Python converting to / from different line terminators
# On other platforms, it has no effect
with open(txt_file, "rb") as opened_txt:
in_txt = csv.reader(opened_txt, delimiter='\t')
with open(csv_file, 'wb') as opened_csv:
out_csv = csv.writer(opened_csv)
out_csv.writerows(in_txt)
all_data = dict()
with open(csv_file, 'r') as tabbed_file:
reader = csv.reader(tabbed_file, delimiter="\t")
all_lines = list(reader)
not_included = 0
for line in all_lines:
if not_included <= 2:
not_included += 1
continue
l = line[0].split(',')
data = GmmStructure()
data.set_object(0, l[1])
data.set_object(1, l[2])
try:
if l[3] == '':
f1_val = 0.0
else:
f1_val = float(l[3])
if l[4] == '':
f2_val = 0.0
else:
f2_val = float(l[4])
data.set_object(2, f1_val)
data.set_object(3, f2_val)
except:
print "Error: ", sys.exc_info()
if l[0] in all_data:
# append the new number to the existing array at this slot
obj = all_data[l[0]]
# we use it only for phoneme prediction
obj.concat_object(0, data.norm_F1)
obj.concat_object(1, data.norm_F2)
all_data[l[0]] = obj
else:
# create a new array in this slot
all_data[l[0]] = data
return all_data
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
l = Logger()
l.log_error("Exception in create-test-data", str(traceback.print_exc()) + "\n\n" + fname + " " + str(exc_tb.tb_lineno))
response = {'Response': 'FAILED', 'Reason': "Exception in create-test-data process"}
return HttpResponse(json.dumps(response))
def fe_solve(exo_io, runid, control, X, connect, elements, fixnodes,
tractions, nforces, nproc=1):
""" 2D and 3D Finite Element Code
Currently configured to run either plane strain in 2D or general 3D but
could easily be modified for plane stress or axisymmetry.
Parameters
----------
control : array_like, (i,)
control[0] -> time integration scheme
control[1] -> number of steps
control[2] -> Newton tolerance
control[3] -> maximum Newton iterations
control[4] -> relax
control[5] -> starting time
control[6] -> termination time
control[7] -> time step multiplier
X : array like, (i, j,)
Nodal coordinates
X[i, j] -> jth coordinate of ith node for i=1...nnode, j=1...ncoord
connect : array_like, (i, j,)
Nodal connections
connect[i, j] -> jth node on on the ith element
elements : array_like, (i,)
Element class for each element
fixnodes : array_like, (i, j,)
List of prescribed displacements at nodes
fixnodes[i, 0] -> Node number
fixnodes[i, 1] -> Displacement component (x: 0, y: 1, or z: 2)
fixnodes[i, 2] -> Value of the displacement
tractions : array_like, (i, j,)
List of element tractions
tractions[i, 0] -> Element number
tractions[i, 1] -> face number
tractions[i, 2:] -> Components of traction as a function of time
Returns
-------
retval : init
0 on completion
failure otherwise
Notes
-----
Original code was adapted from [1].
References
----------
1. solidmechanics.org
"""
# Local Variables
# ---------------
# du : array_like, (i,)
# Nodal displacements.
# Let wij be jth displacement component at ith node. Then du
# contains [w00, w01, w10, w11, ...] for 2D
# and [w00, w01, w02, w10, w11, w12, ...) for 3D
# dw : array_like, (i,)
# Correction to nodal displacements.
# K : array_like, (i, j,)
# Global stiffness matrix. Stored as
# [K_1111 K_1112 K_1121 K_1122...
# K_1211 K_1212 K_1221 K_1222...
# K_2111 K_2112 K_2121 K_2122...]
# for 2D problems and similarly for 3D problems
# F : array_like, (i, )
# Force vector.
# Currently only includes contribution from tractions acting on
# element faces (body forces are neglected)
# R : array_like, (i, )
# Volume contribution to residual
# b : array_like (i, )
# RHS of equation system
# number of processors
nproc = np.amin([mp.cpu_count(), nproc, elements.size])
# Set up timing and logging
t0 = time.time()
logger = Logger(runid)
# Problem dimensions
dim = elements[0].ndof
nelems = elements.shape[0]
nnode = X.shape[0]
ndof = elements[0].ndof
ncoord = elements[0].ncoord
# Setup kinematic variables
u = np.zeros((2, nnode * ndof))
v = np.zeros((2, nnode * ndof))
a = np.zeros((2, nnode * ndof))
# Simulation setup
tint, nsteps, tol, maxit, relax, tstart, tterm, dtmult = control
nsteps, maxit = int(nsteps), int(maxit)
t = tstart
dt = (tterm - tstart) / float(nsteps) * dtmult
# Newmark parameters
b = [.5, 0.]
# Global mass, find only once
M = global_mass(X, elements, connect, nproc)
# Determine initial accelerations
du = np.diff(u, axis=0)[0]
K = global_stiffness(0., 1., X, elements, connect, du, nproc)
findstiff = False
F = global_traction(
0., 1., X, elements, connect, tractions, nforces, du)
apply_dirichlet_bcs(ndof, nnode, 0., fixnodes, u[0], du, 1., K, F, M)
a[0] = np.linalg.solve(M, -np.dot(K, u[0]) + F)
logger.write_intro("Explicit", runid, nsteps, tol, maxit, relax, tstart,
tterm, ndof, nelems, nnode))
for step in range(nsteps):
err1 = 1.
t += dt
logger.write(
"Step {0:.5f}, Time: {1}, Time step: {2}".format(step + 1, t, dt))
# --- Update the state of each element to end of step
du = np.diff(u, axis=0)[0]
update_element_states(dt, X, elements, connect, du)
# --- Get global quantities
K = global_stiffness(
t, dt, X, elements, connect, du, nproc)
def create_structure(self):
try:
all_data = dict()
path = os.path.dirname(os.path.abspath(__file__))
formants_files = os.path.join(path, self.formants_files_directory)
os.chdir(formants_files)
for filename in os.listdir("."):
if ".DS_Store" in filename or "_norm" not in filename:
continue
cleaned_filename = filename.replace(".txt", "")
cleaned_filename = cleaned_filename.replace('_norm', '')
last_index = cleaned_filename.rfind("_")
cleaned_filename = cleaned_filename[:last_index]
training_data = dict()
with open(filename, 'r') as tabbed_file:
reader = csv.reader(tabbed_file, delimiter="\n")
all_lines = list(reader)
not_included = 0
for line in all_lines:
if not_included <= 2:
not_included += 1
continue
l = line[0].split('\t')
data = GmmStructure()
data.set_object(0, l[1])
data.set_object(1, l[2])
try:
if l[3] == '':
f1_val = 0.0
else:
f1_val = float(l[3])
if l[4] == '':
f2_val = 0.0
else:
f2_val = float(l[4])
data.set_object(2, f1_val)
data.set_object(3, f2_val)
except:
print "Error: ", sys.exc_info()
if l[0] in training_data:
# append the new number to the existing array at this slot
obj = training_data.get(l[0])
# we use it only for phoneme prediction
obj.concat_object(0, data.norm_F1)
obj.concat_object(1, data.norm_F2)
training_data[l[0]] = obj
else:
# create a new array in this slot
training_data[l[0]] = data
if cleaned_filename in all_data:
curr = all_data.get(cleaned_filename)
vowels = curr.keys()
for key, value in training_data.items():
if key in vowels: # the vowel is present - otherwise mistake
old_gmm_struct = curr.get(key)
old_gmm_struct.concat_object(0, value.norm_F1)
old_gmm_struct.concat_object(1, value.norm_F2)
curr[key] = old_gmm_struct
else:
curr[key] = value
else:
all_data[cleaned_filename] = training_data
return all_data
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
l = Logger()
l.log_error("Exception in GMM-create-struct", str(traceback.print_exc()) + "\n\n" + fname + " " + str(exc_tb.tb_lineno))
response = {'Response': 'FAILED', 'Reason': "Exception in GMM-creation-structure process"}
return HttpResponse(json.dumps(response))
def test_gmm(self, X_test, Y_test, plot_filename, sentence):
# region LOAD SETS
path = os.path.dirname(os.path.abspath(__file__))
path += '/'
model_name = path + self.model_name
trainset_name = path + self.trainset_name
print >> sys.stderr, "*** Model name: " + model_name + "***"
print >> sys.stderr, "*** Trainset name: " + trainset_name + "***"
# load it again
with open(model_name, 'rb') as model:
gmm_classifier = cPickle.load(model)
with open(trainset_name, 'rb') as traindata:
all_data = cPickle.load(traindata)
print >> sys.stderr, "*** LOADED Model name: " + model_name + "***"
print >> sys.stderr, "*** LOADED Trainset name: " + trainset_name + "***"
print >> sys.stderr, "*** ITEMS N: " + str(len(all_data.items())) + " ***"
all_vowels = []
for key, val in all_data.items():
for v in val.keys():
all_vowels.append(v)
labels = np.unique(all_vowels)
print >> sys.stderr, "*** LABELS ***"
int_labels = np.arange(len(labels))
print >> sys.stderr, "*** INT LABELS ***"
map_int_label = dict(zip(int_labels, labels))
print >> sys.stderr, "*** MAP INT LABELS ***"
map_label_int = dict(zip(labels, int_labels))
print >> sys.stderr, "*** MAP LABELS INT ***"
# results
key_sentence = sentence.lower()
key_sentence = key_sentence.replace(' ', '_')
train_dict = all_data.get(key_sentence)
X_train = train_dict.values()
print >> sys.stderr, "*** X_TRAIN ***"
Y_train = train_dict.keys()
print >> sys.stderr, "*** Y_TRAIN ***"
# endregion
try:
# region PLOT PARAMETERS
print >> sys.stderr, "*** PREPARING FOR PLOTTING GMM ***"
plt.figure()
plt.subplots_adjust(wspace=0.4, hspace=0.5)
colors = ['b', 'g', 'c', 'm', 'y', 'k']
predicted_formants = []
current_trend_formants_data = dict()
# endregion
# 3 rows when we have 5 vowels
if len(X_test) > 4:
rows = 3
else:
rows = 2
# region PRINT PREDICTED VOWELS
print >> sys.stderr, "*** PRINT PREDICTED VOWELS ***"
columns = 2
index = 1
for val in X_test:
f1 = val.norm_F1
f2 = val.norm_F2
data = zip(f1, f2)
gmm_predict = gmm_classifier.predict(data)
current_trend_formants_data[index] = data # save data for trend graph + index of subplot
gmm_l = gmm_predict.tolist()
predicted_formants.append(gmm_l[0])
# print the predicted-vowels based on the formants
l = gmm_l[0] # TODO: investigate on how to have the highest probability only
plt.subplot(rows, columns, index)
plt.scatter(f1, f2, s=80, c='r', marker='+', label=r"$ {} $".format(map_int_label[l]))
index += 1
score = gmm_classifier.score(data)
print >> sys.stderr, "*** LOG-PROBABILITY: " + str(score) + " ***"
# endregion
# region STRUCT FOR RETRIEVING THE ACTUAL LABEL
print >> sys.stderr, "*** STRUCT FOR RETRIEVING THE ACTUAL LABEL ***"
predicted_labels = []
for pf in predicted_formants:
predicted_labels.append(map_int_label[pf])
native_vowels = self.get_native_vowels(sentence)
uniq_predicted_labels = np.unique(predicted_labels)
# endregion
# TODO: saving data for creating trend chart
current_trend_data = zip(predicted_labels, current_trend_formants_data)
# region ACCURACY
# try:
# print >> sys.stderr, "\n"
#
# pred_uniq = []
# for i in uniq_predicted_labels.ravel():
# pred_uniq.append(map_label_int[i])
#
# print >> sys.stderr, "*** PRED UNIQ ***"
#
# native_lab = []
# for i in np.array(native_vowels):
# native_lab.append(map_label_int[i])
#
# print >> sys.stderr, "*** NATIVE LAB ***"
#
# print >> sys.stderr, "*** PREDICTED LABELS: " + np.array_str(np.asarray(pred_uniq)) + " ***"
# print >> sys.stderr, "*** NATIVE LABELS: " + np.array_str(np.asarray(native_lab)) + " ***"
#
# #test_accuracy = np.mean(pred_uniq == native_lab) * 100
#
# print >> sys.stderr, "*** ACCURACY: " + str(0) + " ***"
# print >> sys.stderr, "\n"
# except:
# print >> sys.stderr, "*** EXCEPTION ***"
# print >> sys.stderr, "\n"
# pass
# endregion
new_trend_data = []
# region PRINT NATIVE VOWELS FORMANTS
print >> sys.stderr, "*** PRINT NATIVE VOWELS FORMANTS ***"
i = 0
duplicate = []
native_data = dict(zip(Y_train, X_train))
index = 1
for n in native_vowels:
if n in duplicate:
continue
found = False
for pred in current_trend_data:
if n in pred[0]:
plot_index = pred[1]
predicted_data = current_trend_formants_data[plot_index]
found = True
if found is False:
plot_index = index
predicted_data = current_trend_formants_data[index]
print >> sys.stderr, "*** READY TO CREATE THE PLOT ***"
struct = native_data[n]
native_f1 = struct.get_object(2)
native_f2 = struct.get_object(3)
ax = plt.subplot(rows, columns, plot_index)
plt.tight_layout()
ax.scatter(native_f1, native_f2, s=40, c=colors[i], marker='.', label=r"$ {} $".format(n))
axes = plt.gca()
axes.set_xlim([min(native_f1) - 500, max(native_f1) + 500])
axes.set_ylim([min(native_f2) - 500, max(native_f2) + 500])
ax.set_xlabel('F1')
ax.set_ylabel('F2')
ax.set_title("Vowel: " + n)
# ellipse inside graph
distance_from_centroid = self.make_ellipses(ax, native_f1, native_f2, predicted_data[0][0], predicted_data[0][1])
# American date format
date_obj = datetime.datetime.utcnow()
date_str = date_obj.strftime('%m-%d-%Y')
new_trend_data.append((current_trend_data[index - 1][0], n, distance_from_centroid, date_str))
duplicate.append(n)
i += 1
index += 1
# endregion
print >> sys.stderr, "*** SAVE THE PLOT ***"
plt.savefig(plot_filename, bbox_inches='tight', transparent=True)
with open(plot_filename, "rb") as imageFile:
return base64.b64encode(imageFile.read()), new_trend_data
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
l = Logger()
l.log_error("Exception in GMM-test-model", str(traceback.print_exc()) + "\n\n" + fname + " " + str(exc_tb.tb_lineno))
response = {'Response': 'FAILED', 'Reason': "Exception in GMM-test-model process"}
return HttpResponse(json.dumps(response))
