def hatanaka_decompress(local_file):
"""Hatanaka decompresses a local file using the CRX2RNX program
Outputs data to new file with correct name under same directory as input
Input:
local_file path to Hatanaka compressed RINEX file
Returns:
new_name name of created decompressed RINEX file
"""
# Check if CRX2RNX is in /tmp - where Lambda instances are throttled
if os.path.isfile('/tmp/CRX2RNX'):
CRX2RNX = Executable('/tmp/CRX2RNX', True)
else:
CRX2RNX = Executable('lib/executables/CRX2RNX')
rinex_data = CRX2RNX.run('{} -'.format(local_file))
if CRX2RNX.returncode > 0:
raise Exception('CRX2RNX failed with error code {}: {}'.format(
CRX2RNX.returncode, CRX2RNX.stderr))
# RINEX 3 file extension changes from crx to rnx when decompressed
new_name = local_file.replace('.crx', '.rnx')
# Hatanaka compressed RINEX 2 files are suffixed with d, replace with o
if new_name == local_file:
new_name = local_file[:-1] + 'o'
with open(new_name, 'w') as out_file:
out_file.write(rinex_data)
return new_name
def __extract_command(self, name, dictionary):
command = dictionary.pop(name, None)
if command is not None:
self[name] = Executable(command=command.get('command'),
options=command.get('options', []),
arguments=command.get('arguments'),
filters=command.get('filters'))
def lambda_handler(event, context):
user_str = 'aaa'
exe_file = 'hello'
try:
# API Gateway GET method
if event['httpMethod'] == 'GET':
user_str = event['queryStringParameters']['str']
exe_file = event['queryStringParameters']['bin']
# API Gateway POST method
elif event['httpMethod'] == 'POST':
data = json.loads(event['body'])
user_str = data['str']
exe_file = data['bin']
except KeyError:
# direct invocation
user_str = event['str']
exe_file = event['bin']
user_str = '\"' + user_str + '\"'
exe = Executable('executables/' + exe_file)
result = exe.run(user_str)
print('OUT: {}\nERR: {}\nRET: {}'.format(exe.stdout, exe.stderr,
exe.returncode))
out = {
"headers": {
"content-type": "application/json",
"Access-Control-Allow-Origin": "*"
},
"body": exe.stdout,
"statusCode": 200
}
return out
def main():
current_app.logger.info("Received request")
msg = '---HEADERS---\n{}\n--BODY--\n{}\n-----\n'.format(
request.headers, request.get_data())
current_app.logger.info(msg)
# Get request body
reqStr = request.get_data().decode("utf-8")
current_app.logger.info(reqStr)
# Call executable binary
exe = Executable(os.path.join(MYPATH, 'hello'))
result = exe.run(reqStr)
current_app.logger.info('OUT: {}\nERR: {}\nRET: {}'.format(
exe.stdout, exe.stderr, exe.returncode))
return result, 200
MYPATH = os.path.dirname(os.path.realpath(__file__))
def main():
current_app.logger.info("Received request")
msg = '---HEADERS---\n{}\n--BODY--\n{}\n-----\n'.format(
request.headers, request.get_data())
current_app.logger.info(msg)
# Get request body
reqStr = request.get_data().decode("utf-8")
current_app.logger.info(reqStr)
# Call executable binary
exe = Executable(os.path.join(MYPATH, 'hello'))
result = exe.run(reqStr)
current_app.logger.info('OUT: {}\nERR: {}\nRET: {}'.format(
exe.stdout, exe.stderr, exe.returncode))
return result, 200
if __name__ == "__main__":
reqStr = "haha"
exe = Executable('./hello')
result = exe.run(reqStr)
print('OUT: {}\nERR: {}\nRET: {}'.format(exe.stdout, exe.stderr,
exe.returncode))
print(result)
def lambda_handler(event, context):
# Get the file object and bucket names from the event
bucket = event['Records'][0]['s3']['bucket']['name']
key = urllib.unquote_plus(
event['Records'][0]['s3']['object']['key']).decode('utf8')
print('Quality Check: Key {}'.format(key))
status, file_type, data_type, year, day = key.split('/')[:5]
if data_type == 'nav':
nav_file = os.path.basename(key)
if nav_file[:4].lower() == 'brdc' and nav_file[-4:] == 'n.gz':
triggerQCFromNav(year, day, context.function_name, bucket)
else:
print('Do not Quality Check using non-Broadcast Navigation data')
return
# Use AWS request ID from context object for unique directory
session_id = context.aws_request_id
local_path = '/tmp/{}'.format(session_id)
if not os.path.exists(local_path):
os.makedirs(local_path)
try:
response = S3.get_object(Bucket=bucket, Key=key)
except Exception as err:
# This should only fail more than once if permissions are incorrect
print('Error: Failed to get object {} from bucket {}.'.format(
key, bucket))
raise err
# Decompress Observation file and store locally
filename, extension = os.path.splitext(os.path.basename(key))
local_file = os.path.join(local_path, filename)
file_data = zlib.decompress(response['Body'].read(), 15+32)
with open(local_file, 'wb') as out_file:
out_file.write(file_data)
# Parse RINEX file
rinex_obs = RINEXData(local_file)
# Attempt to get Broadcast Navigation file from archive
nav_file = getBRDCNavFile(bucket, rinex_obs.start_time, local_path)
if nav_file == None:
print('Daily BRDC file does not yet exist for {}/{}'.format(
year, day))
return
# Hatanaka decompress RINEX file if needed
if rinex_obs.compressed == True:
rinex_obs.local_file = hatanaka_decompress(rinex_obs.local_file)
# Generate an Anubis XML config file
anubis_config, result_file = generateQCConfig(
rinex_obs, nav_file, local_path)
# Run Anubis with the generated config file as input
anubis = Executable('lib/executables/anubis-2.0.1')
anubis_log = anubis.run('-x {}'.format(anubis_config))
if anubis.returncode > 0:
print('Anubis errored with return code {}: {}\n{}'.format(
anubis.returncode, anubis.stderr, anubis.stdout))
return
# Parse results of Anubis
parseQCResult(result_file, key)
# Delete tmp working space and Anubis copy to resolve Lambda disk
# space allocation issue
shutil.rmtree(local_path)
return
def setUp(self):
self.command_mock = CommandMock.create()
self.executable = Executable()
class Handler:
def __init__(self, builder):
self.builder = builder
indexation_parameters = {
"has_stemming": False,
"has_stop_words_removal": False,
"has_compression": False,
"files_list": [],
"ignore_case": False,
"date_weight": 1,
"title_weight": 1,
"use_weights": False,
"memory_limit": 50
}
query_parameters = {
"algorithm": "NAIVE",
"results_number": 5,
"query": "",
"similar_words_number": 5
}
backend = Executable()
@staticmethod
def get_filelist_from_folderpath(folderpath):
filenameslist = []
# Get all file names in folder at first level only
for (dirpath, dirnames, filenames) in walk(folderpath):
filenameslist.extend(filenames)
break
# Concatenate them with folder path
filelist = [folderpath + "/" + filename for filename in filenameslist]
return filelist
def get_document_from_DOCID_and_filepath(docid, filepath):
documentViews = []
with open(filepath) as file:
document_string = ""
for line in file:
document_string = "{}\n{}".format(document_string, line)
if "</DOC>" in line:
documentView = et.fromstring(document_string)
for el in documentView.findall('DOC'):
print('-------------------')
for ch in el.getchildren():
print('{:>15}: {:<30}'.format(ch.tag, ch.text))
documentViews.append(documentView)
document_string = ""
def toggle_stemming(self, button):
print("Toggle stemming to " + str(button.get_active()))
self.indexation_parameters["has_stemming"] = button.get_active()
def toggle_stop_words_removal(self, button):
print("Toggle stop words removal to " + str(button.get_active()))
self.indexation_parameters[
"has_stop_words_removal"] = button.get_active()
def toggle_ignore_case(self, button):
print("Toggle ignore case to " + str(button.get_active()))
self.indexation_parameters["ignore_case"] = button.get_active()
def toggle_compression(self, button):
print("Toggle compression to " + str(button.get_active()))
self.indexation_parameters["has_compression"] = button.get_active()
def memory_limit_changed(self, spinbutton):
print("Memory Limit changed to " + str(spinbutton.get_value_as_int()))
self.indexation_parameters[
"memory_limit"] = spinbutton.get_value_as_int()
def toggle_weights_use(self, button):
print("Toggle weights use to " + str(button.get_active()))
self.indexation_parameters["use_weights"] = button.get_active()
title_weight_grid = self.builder.get_object("title_weight_grid")
title_weight_grid.set_visible(button.get_active())
date_weight_grid = self.builder.get_object("date_weight_grid")
date_weight_grid.set_visible(button.get_active())
def title_weight_changed(self, spinbutton):
print("Title weight changed to " + str(spinbutton.get_value_as_int()))
self.indexation_parameters[
"title_weight"] = spinbutton.get_value_as_int()
def date_weight_changed(self, spinbutton):
print("Date weight changed to " + str(spinbutton.get_value_as_int()))
self.indexation_parameters[
"date_weight"] = spinbutton.get_value_as_int()
def open_file_chooser(self, button):
print("Open file chooser")
main_window = self.builder.get_object("main_window")
dialog = Gtk.FileChooserDialog(
"Please choose a folder", main_window,
Gtk.FileChooserAction.SELECT_FOLDER,
(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL, "Select",
Gtk.ResponseType.OK))
dialog.set_default_size(800, 400)
response = dialog.run()
if response == Gtk.ResponseType.OK:
folder_path = dialog.get_filename()
print("Select clicked")
print("Folder selected: " + folder_path)
choose_folder_label = self.builder.get_object(
"choose_folder_label")
choose_folder_label.set_text(folder_path)
# Get all files in that folder
self.indexation_parameters[
"files_list"] = self.get_filelist_from_folderpath(folder_path)
print(self.indexation_parameters["files_list"])
elif response == Gtk.ResponseType.CANCEL:
print("Cancel clicked")
dialog.destroy()
def call_backend_indexation(self):
print("Indexation started !")
self.backend.indexing(
files=self.indexation_parameters["files_list"],
ignore_stop_words=self.
indexation_parameters["has_stop_words_removal"],
stemming=self.indexation_parameters["has_stemming"],
use_vbytes=self.indexation_parameters["has_compression"],
ignore_case=self.indexation_parameters["ignore_case"],
date_weight=self.indexation_parameters["date_weight"],
title_weight=self.indexation_parameters["title_weight"],
use_weights=self.indexation_parameters["use_weights"],
memory_limit=self.indexation_parameters["memory_limit"])
GLib.idle_add(self.on_indexation_complete)
def start_indexation(self, button):
print("Start indexation")
if len(self.indexation_parameters["files_list"]) <= 0:
print("No file has been specified !")
return
button.set_sensitive(False)
loading_box = self.builder.get_object("loading_box")
loading_box.set_visible(True)
indexation_statistics_box = self.builder.get_object(
"indexation_statistics_box")
query_box = self.builder.get_object("query_box")
results_box = self.builder.get_object("results_box")
similar_words_box = self.builder.get_object("similar_words_box")
similar_words_box.set_visible(False)
indexation_statistics_box.set_visible(False)
query_box.set_visible(False)
results_box.set_visible(False)
thread = threading.Thread(target=self.call_backend_indexation)
thread.daemon = True
thread.start()
def on_indexation_complete(self):
print("Indexation complete !")
# When indexation is finished Change this to get vocabulary from inverted file
# Eventualy manage exception if vocabular inexistent
vocabulary = self.backend.inv_file.get_terms()
liststore = Gtk.ListStore(str)
for s in vocabulary:
liststore.append([s])
completion = Gtk.EntryCompletion()
completion.set_model(liststore)
completion.set_text_column(0)
entry = self.builder.get_object("search_entry")
entry.set_completion(completion)
loading_box = self.builder.get_object("loading_box")
indexation_statistics_box = self.builder.get_object(
"indexation_statistics_box")
query_box = self.builder.get_object("query_box")
start_indexation_button = self.builder.get_object(
"start_indexation_button")
loading_box.set_visible(False)
indexation_stats = StatsControl.last_indexing()
indexation_start_time_tofill = self.builder.get_object(
"indexation_start_time_tofill")
indexation_start_time_tofill.set_text("{:%H:%M:%S.%f}".format(
indexation_stats.start_time))
indexation_end_time_tofill = self.builder.get_object(
"indexation_end_time_tofill")
indexation_end_time_tofill.set_text("{:%H:%M:%S.%f}".format(
indexation_stats.finish_time))
indexation_total_time_tofill = self.builder.get_object(
"indexation_total_time_tofill")
indexation_total_time_tofill.set_text("{}".format(
indexation_stats.total_time))
indexation_file_size_tofill = self.builder.get_object(
"indexation_file_size_tofill")
indexation_file_size_tofill.set_text(str(indexation_stats.file_size))
indexation_statistics_box.set_visible(True)
query_box.set_visible(True)
start_indexation_button.set_sensitive(True)
def algo_combo_changed(self, combobox):
print("Algo combo changed to " + combobox.get_active_text())
self.query_parameters["algorithm"] = combobox.get_active_text()
def results_number_changed(self, spinbutton):
print("Results number changed to " +
str(spinbutton.get_value_as_int()))
self.query_parameters["results_number"] = spinbutton.get_value_as_int()
def similar_words_number_changed(self, spinbutton):
print("Similar words number changed to " +
str(spinbutton.get_value_as_int()))
self.query_parameters[
"similar_words_number"] = spinbutton.get_value_as_int()
def search_changed(self, searchentry):
query = searchentry.get_text()
print("Search changed to " + query)
self.query_parameters["query"] = query
start_query_button = self.builder.get_object("start_query_button")
display_similar_words_button = self.builder.get_object(
"display_similar_words_button")
if query == "":
start_query_button.set_sensitive(False)
display_similar_words_button.set_sensitive(False)
elif len(query.split()) == 1:
start_query_button.set_sensitive(True)
display_similar_words_button.set_sensitive(True)
else:
start_query_button.set_sensitive(True)
display_similar_words_button.set_sensitive(False)
def call_backend_query(self):
print("Query started !")
results = self.backend.query(
query=self.query_parameters["query"],
algorithm=self.query_parameters["algorithm"],
number_of_results=self.query_parameters["results_number"])
GLib.idle_add(self.on_query_complete, results)
def start_query(self, button):
print("Start query")
if self.query_parameters["query"] == "":
print("No query has been specified !")
return
button.set_sensitive(False)
loading_box = self.builder.get_object("loading_box")
loading_box.set_visible(True)
print("Dict : {}".format(self.query_parameters))
thread = threading.Thread(target=self.call_backend_query)
thread.daemon = True
thread.start()
def on_query_complete(self, results):
print("Query complete !")
query_stats = StatsControl.last_query()
loading_box = self.builder.get_object("loading_box")
loading_box.set_visible(False)
start_time_tofill = self.builder.get_object("start_time_tofill")
start_time_tofill.set_text("{:%H:%M:%S.%f}".format(
query_stats.start_time))
end_time_tofill = self.builder.get_object("end_time_tofill")
end_time_tofill.set_text("{:%H:%M:%S.%f}".format(
query_stats.finish_time))
total_time_tofill = self.builder.get_object("total_time_tofill")
total_time_tofill.set_text("{}".format(query_stats.total_time))
pl_accesses_tofill = self.builder.get_object("pl_accesses_tofill")
pl_accesses_tofill.set_text(str(query_stats.pl_accesses))
disk_accesses_tofill = self.builder.get_object("disk_accesses_tofill")
disk_accesses_tofill.set_text(str(query_stats.memory_accesses))
results_text = "\t Score |\tDOCID |\t File path \n"
for result in results:
results_text += ("\t{:8.5f} |\t{:8} |\t{}".format(
result[1], result[0], result[2])) + "\n"
print("results" + results_text)
results_textview = self.builder.get_object("results_textview")
results_textview_buffer = results_textview.get_buffer()
results_textview_buffer.set_text(results_text)
results_box = self.builder.get_object("results_box")
results_box.set_visible(True)
start_query_button = self.builder.get_object("start_query_button")
start_query_button.set_sensitive(True)
def call_backend_similar_search(self):
print("Similar search started !")
results = self.backend.random_indexing(
choice_key=self.query_parameters["query"],
top_results=self.query_parameters["similar_words_number"])
GLib.idle_add(self.on_similar_search_complete, results)
def start_similar_search(self, button):
print("Start query")
query = self.query_parameters["query"]
# Just in case, check if the query is only a single word, otherwise random indexing querying doesn't work
if len(query.split()) > 1 or query == "":
print("Can only find similar words to a single word !")
return
button.set_sensitive(False)
similar_words_box = self.builder.get_object("similar_words_box")
similar_words_box.set_visible(False)
thread = threading.Thread(target=self.call_backend_similar_search)
thread.daemon = True
thread.start()
def on_similar_search_complete(self, results):
print("Query complete !")
display_similar_words_button = self.builder.get_object(
"display_similar_words_button")
display_similar_words_button.set_sensitive(True)
results_text = str(results)
similar_words_textview = self.builder.get_object(
"similar_words_textview")
similar_words_textview_buffer = similar_words_textview.get_buffer()
similar_words_textview_buffer.set_text(results_text)
similar_words_box = self.builder.get_object("similar_words_box")
similar_words_box.set_visible(True)
filenameslist = []
# Get all file names in folder at first level only
for (dirpath, dirnames, filenames) in walk(folderpath):
filenameslist.extend(filenames)
break
# Concatenate them with folder path
filelist = [folderpath + "/" + filename for filename in filenameslist]
return filelist
file_paths = get_filelist_from_folderpath("latests")
exe = Executable()
algorithm = DEFAULT_ALGORITHM
number_of_results = DEFAULT_NUMBER_OF_RESULTS
memorylimit = 200
exe.indexing(file_paths, memory_limit=memorylimit)
print(SC.last_indexing())
try:
in_res = int(input("Number of results desired ? ").strip())
number_of_results = in_res
except ValueError:
print("Non-int value entered using default {}".format(
def test_exec_random_indexing(self):
self.exec_var = Executable()
self.file_path = ["../latimes/la010189"]
self.exec_var.indexing(self.file_path)
self.exec_var.random_indexing("washington", 10)
def setUp(self):
self.exec_var = Executable()
self.file_path = ["../latimes/la010189"]
