def get(self, name, flag):
try:
if flag == flag:
for config,cfgval in configuration_counts.copy().items():
if config != name:
configuration_counts.pop(config)
con = configuration_counts
return Response(
response = json.dumps(run_main(con,flag),default=str),
status=200,
mimetype="application/json"
)
elif flag != flag:
for config,cfgval in configuration_counts.copy().items():
if config != name:
configuration_counts.pop(config)
con = configuration_counts
return Response(
response = json.dumps(run_main(con,flag),default=str),
status=200,
mimetype="application/json"
)
except Exception as e:
return Response(
response = json.dumps(
{"message":"Check Your URL Please!!"},default=str),
status=404,
mimetype="application/json"
)
def get(self, name, flag):
try:
if flag == flag:
configuration_counts = cfg.configuration_count
dict1 = {}
for config, cfgval in list(configuration_counts.items()):
if config == name:
con = configuration_counts[name]
dict1[name] = con
return Response(
response=json.dumps(run_main(dict1, flag), default=str),
status=200,
mimetype="application/json",
)
elif flag != flag:
for config, cfgval in configuration_counts.copy().items():
if config != name:
configuration_counts.pop(config)
con = configuration_counts
return Response(response=json.dumps(run_main(con, flag),
default=str),
status=200,
mimetype="application/json")
except Exception as e:
print(e)
return Response(
response=json.dumps(
{
"status": 404,
"message": "Check Your URL Please!!"
},
default=str),
status=404,
mimetype="application/json",
)
def get(self, name, flag):
try:
configuration_counts = cfg.configuration_count
dict1 = {}
for config, cfgval in list(configuration_counts.items()):
if config == name:
con = configuration_counts[name]
dict1[name] = con
message = mn.message
json_response = {
"message": message,
"status": mn.status,
"records": run_main(dict1, flag),
}
return Response(
response=json.dumps(json_response, default=str),
mimetype="application/json",
)
# return Response(
# response=json.dumps(run_main(dict1, flag), default=str),
# status=200,
# mimetype="application/json",
# )
except Exception as e:
print(e)
return Response(
response=json.dumps(
{"status": 404, "message": "Check Your URL Please!!"}, default=str
),
status=404,
mimetype="application/json",
)
def upload_pdf():
if request.method == "POST":
if request.files:
if "filesize" in request.cookies:
if not allowed_pdf_filesize(request.cookies["filesize"]):
print("Filesize exceeded maximum limit")
flash('File size exceeds maximum limit', 'danger')
return redirect(request.url)
pdf = request.files["pdf"]
if pdf.filename == "":
print("No filename")
flash('No file uploaded or invalid file name', 'danger')
return redirect(request.url)
if allowed_pdf(pdf.filename):
global filename
filename = secure_filename(pdf.filename)
pdf.save(os.path.join(
app.config["PDF_UPLOADS"], filename))
print("PDF saved")
file_path = os.path.join(
app.config["PDF_UPLOADS"], filename)
print(file_path)
print(filename)
main.run_main(file_path, filename)
return render_template('/download_csv.html', data="static/csv-json-txt/" + filename[:-4] + "_" + "student_data.json")
else:
print("That file extension is not allowed")
flash('Please upload a PDF.', 'danger')
return redirect(request.url)
return render_template("/upload_pdf.html")
def get(self, flag):
try:
if flag == flag:
run = run_main(configuration_counts, flag)
return Response(
response=json.dumps(run, default=str),
status=200,
mimetype="application/json",
)
elif flag != flag:
run = run_main(configuration_counts, flag)
return Response(
response=json.dumps(run, default=str),
status=200,
mimetype="application/json",
)
except Exception as e:
print(e)
return Response(
response=json.dumps(e, default=str),
status=404,
mimetype="application/json",
)
def main_plot():
"""The view for rendering the scatter chart"""
fs_function = []
number = int(session['form']['number'])
dataset_name = session['form']['dataset']
method = session['form']['method']
score_name = session['form']['score']
if session['form'].__contains__('pearson'):
fs_function.append(session['form']['pearson'])
if session['form'].__contains__('fisher'):
fs_function.append(session['form']['fisher'])
if session['form'].__contains__('greedy'):
fs_function.append(session['form']['greedy'])
img = run_main(method, fs_function, score_name, number,dataset_name)
return send_file(img, mimetype='image/png', cache_timeout=0)
def get(self, flag):
try:
run = run_main(configuration_counts, flag)
message = mn.message
json_response = {"message": message, "status": mn.status, "records": run}
return Response(
response=json.dumps(json_response, default=str),
mimetype="application/json",
)
except Exception as e:
print(e)
return Response(
response=json.dumps(e, default=str),
status=404,
mimetype="application/json",
)
def test_run_main(spark_session: SparkSession) -> None:
configuration = Configuration(
source_folder='test_data_source',
db_name='schwacke_test',
host='localhost',
port=27017,
current_date=str(datetime.now().date()),
current_timestamp=str(int(datetime.now().timestamp())),
)
mode_folder = Path(Path.cwd(), configuration.source_folder)
for path in Path(mode_folder, 'archive').iterdir():
if path.is_file():
copy(str(path), str(Path(mode_folder)))
tmp_table, mongo_collection = run_main(spark_session, configuration)
try:
result_from_file = tmp_table.read_from_file()
result_from_db = mongo_collection.read_from_mongodb()
for dict in result_from_db:
dict.pop('_id')
with Path(mode_folder, 'control_output.json').open(
'r'
) as file_result:
control_result = [loads(row) for row in file_result]
finally:
rmtree(
Path(
tmp_table.tmp_table_folder,
f'{tmp_table.configuration.current_date}'
f'_{tmp_table.configuration.current_timestamp}',
),
ignore_errors=True,
)
mongo_collection.drop_collection_mongodb()
assert result_from_db == control_result
assert result_from_file == control_result
__author__ = 'Jrudascas'
import warnings
warnings.filterwarnings("always")
import main as main
import definitions as d
import time
t = time.time()
main.run_main(d.path_input, d.path_output)
print("Total duration: " + str(time.time() - t) + " sec")
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import tensorflow as tf
from main import run_main
if __name__ == "__main__":
assets = os.path.join(tf.resource_loader.get_data_files_path(),
'assets.zip')
run_main(assets)
def __main():
"""Main function for setup.py. Contains and handles the menu loop."""
print """
--------------------------------------------------------------------------------
Setup for PiPark, 2014
================================================================================
"""
# continually loop menu unitl user chooses 'q' to quit
setup_image_location = "./images/setup.jpeg"
while True:
user_choice = __menu_choice()
if user_choice == '1':
# initialise camera and start pi camera preview
camera = imageread.setup_camera()
camera.start_preview()
print "INFO: Setup image save location:", str(setup_image_location)
print "INFO: Camera preview initiated."
print ""
# when user presses enter, take a new setup image. Then ask the user to confirm or
# reject the image. If image is rejected, take a new image.
while True:
raw_input("When ready, press ENTER to capture setup image.")
camera.capture(setup_image_location)
user_input = raw_input("Accept image (y/n)? > ")
if user_input.lower() in ('y', "yes"): break
# picture saved, end preview
camera.close()
print ""
print "INFO: Setup image has been saved to:", str(setup_image_location)
print "INFO: Camera preview closed."
elif user_choice == '2':
# check setup image exists, if not print error directing to option 1
try:
setup_image = Image.open(setup_image_location)
except:
print "ERROR: Setup image does not exist. Select option 1 to"
print "create a new setup image."
# setup image exists, so open GUI window allowing selection of spaces and
# reference areas
print """
When the window opens use the mouse to mark parking spaces and 3 reference
areas. Press T to toggle between marking parking spaces and reference areas and
the numbers 1 - 0 to mark new areas.
"""
raw_input("\nPress ENTER to continue...\n")
setup_selectarea.main(setup_image)
elif user_choice == '3':
# attempt to import the setup data and ensure 'boxes' is a list. If fail,
# return to main menu prompt.
try:
import setup_data
boxes = setup_data.boxes
if not isinstance(boxes, list): raise ValueError()
except:
print "ERROR: Setup data does not exist. Please run options 1 and 2 first."
continue
# attempt to import the server senddata module. If fail, return to main menu
# prompt.
try:
import senddata
except:
print "ERROR: Could not import send data file."
continue
# deregister all areas associated with this pi (start fresh)
out = senddata.deregister_pi()
try:
out['error']
print "ERROR: Error in connecting to server. Please update settings.py."
continue
except:
pass
# register each box on the server
for box in boxes:
if box[1] == 0:
senddata.register_area(box[0])
print "INFO: Registering area", box[0], "on server database."
print "\nRegistration complete."
elif user_choice == '4':
print "This will complete the setup and run the main PiPark program."
user_input = raw_input("Continue and run the main program? (y/n) >")
if user_input.lower in ('y', 'yes'):
main.run_main()
break
elif user_choice.lower() in ('h', "help"):
print """
Welcome to PiPark setup, here's a quick run-through of the steps to successfully
setup your PiPark unit.
1) Ensure that the PiPark unit is mounted suitable above the cark park so that
the PiPark's camera is able to look down with a clear view upon the spaces
that are required to be monitored.
2) At the main menu prompt of this setup program enter option '1' to fine tune
the direction of the camera so that it is clearly able to view the spaces.
When you have finalised the position of the camera press ENTER to capture a
setup image. If everything is correct and you do not wish to recapture the
setup image, type 'y' or 'yes' at the prompt to continue the setup program.
If you wish to recapture the setup image, type 'n' or 'no' at the prompt.
Note: The setup image does not require the car park to be empty.
3) Once back at the main menu prompt, enter option '2' to open up the setup
GUI, which will open a new window allowing you to mark parking space areas
and reference areas on the setup image that was captures as part of step 2.
When the window has opened use the mouse to mark start and end corners of
rectangles, which will represent the areas to be processed by the main
program. To toggle between marking reference areas and parking space areas
press 'T'. Blue areas represent parking spaces, and red areas represent
reference areas.
Currently up to ten areas can be marked, using the number keys from 1 - 0.
There must be exactly 3 reference areas (RED) marked on the setup image
and at least 1 parking space.
When all areas have been marked on the image press 'O' to output the marked
areas' co-ordinates to a file and close the window to continue completing
the setup.
Controls Summary:
T -- Toggle between reference areas (RED) and parking spaces (BLUE).
C -- Clear all marked areas.
O -- Output the reference areas to a file.
1 to 0 -- Change area ID numbers.
4) The final step to completing the setup is to choose option '3' from the
main menu prompt. When selected the parking spaces will be registered to
the server, and can now be viewed on the website.
5) The setup is now complete; at the main menu prompt type 'q' or 'quit' to
terminate the setup program. Alternatively, choose option 3 to immediately
run the main PiPark program.
"""
elif user_choice.lower() in ('q', "quit"):
print ""
break
else:
print "\nERROR: Invalid menu choice.\n"
continue
if comm.Get_rank() == 0:
timer.tic()
num_labels_list = list(
itertools.product(c.num_labels, range(c.num_splits)))
no_viz = False
pool.map(mpi_run_main_args,
[n + (
no_viz,
c,
) for n in num_labels_list])
pool.close()
if comm.Get_rank() == 0:
print 'TOTAL TIME:'
timer.toc()
main.run_main(configs=c)
else:
assert False, 'Use MPI instead!'
if use_multiprocessing_pool:
pool = multiprocessing_utility.LoggingPool(processes=pool_size)
pool.map(launch_subprocess_args, num_labels_list)
else:
for i in num_labels_list:
launch_subprocess_args(i)
comm = MPI.COMM_WORLD
if comm.Get_rank() == 0:
print 'TOTAL TIME:'
timer.toc()
main.run_main()
__author__ = 'Aubrey'
import sys
from operator import add
from pyspark import SparkContext
from pyspark import SparkConf
from main import run_main
if __name__ == "__main__":
try:
sc
except NameError:
try:
spark_conf = SparkConf()
spark_conf.set("spark.executor.memory", "8g")
spark_conf.set("spark.eventLog.enable", "True")
spark_conf.set("spark.logConf", "true")
spark_conf.set("spark.default.parallelism", "5")
spark_conf.set("spark.task.maxFailure", "4")
# spark_conf.set("spark.storage.memoryFraction", ".2")
sc = SparkContext(conf=spark_conf, appName="CASP-ML")
except:
print "sc doesn't exist AND cannot create new SparkContext!"
print "Did you pass globals() to execfile()?"
else:
sc
num_jobs = 10
run_main(num_jobs, sc)
def body(self):
# execute st calls for body section
# a header for this section
sub_title = 'Application Experiment for Advanced Deep Learning'
st.markdown(
f"<h3 style='text-align: center; color: black;font-family:courier;'>{sub_title}</h3>",
unsafe_allow_html=True)
# display some overview graphs
my_expander = st.beta_expander(
"Project Methods and Results (Click to Hide or Show)",
expanded=False)
with my_expander:
st.markdown(
"<h2 style='text-align: center; color: black;'> * * * </h2>",
unsafe_allow_html=True)
narrative_introduction()
st.markdown(
"<h2 style='text-align: center; color: black;'> * * * </h2>",
unsafe_allow_html=True)
st.header('Model Overview')
lstm_math()
st.markdown(
"<h2 style='text-align: center; color: black;'> - - </h2>",
unsafe_allow_html=True)
arima_math()
st.markdown(
"<h2 style='text-align: center; color: black;'> - - </h2>",
unsafe_allow_html=True)
prophet_math()
st.markdown(
"<h2 style='text-align: center; color: black;'> * * * </h2>",
unsafe_allow_html=True)
st.header('Data Preparation')
d_col1, d_col2 = st.beta_columns(2)
d_col1.write(
'Scale prices and interpolate sentiment scores with spline.')
d_col1.image(self.data_1)
d_col2.write(
'Data with varying degrees of missing values and date ranges.')
d_col2.image(self.data_2)
st.markdown(
"<h2 style='text-align: center; color: black;'> * * * </h2>",
unsafe_allow_html=True)
st.header('Data Sequences')
st.write('Sliding Sequences of Prices')
st.image(self.seq_1)
st.write('Sliding Sequences of Prices and Sentiment Data')
st.image(self.seq_2)
st.markdown(
"<h2 style='text-align: center; color: black;'> * * * </h2>",
unsafe_allow_html=True)
st.header('Results')
st.write('Baseline Experiments')
res1 = st.beta_columns(2)
res1[0].write('ARIMA train-predict results.')
res1[0].image(self.results1)
res1[1].write('FB Prophet train-predict results.')
res1[1].image(self.results2)
st.write('LSTM Experiments')
res2 = st.beta_columns(2)
res2[0].write('LSTM 1 - Stock Price Only')
res2[0].image(self.results31)
res2[0].image(self.results32)
res2[1].write('LSTM 2 - Stock Price With Sentiment')
res2[1].image(self.results41)
res2[1].image(self.results42)
st.markdown(
"<h2 style='text-align: center; color: black;'> * * * </h2>",
unsafe_allow_html=True)
st.header('LSTM Notes')
st.write(
'Set default epochs to 100 with multiple early stopping conditions.'
)
st.write(
'High variance in stock prices make prediction accuracy difficult.'
)
st.markdown(
"<h2 style='text-align: center; color: black;'> * * * </h2>",
unsafe_allow_html=True)
st.header('Analysis of Price and Sentiment Variance')
df = pd.read_csv(self.sample_chart)
make_scatter(
df=df,
title=
'High-level Results for Error by Sentiment and Price Variance')
make_histogram(filename=self.sample_chart)
st.markdown(
"<h2 style='text-align: center; color: black;'> * * * </h2>",
unsafe_allow_html=True)
narrative_conclusion()
st.markdown(
"<h2 style='text-align: center; color: black;'> * * * </h2>",
unsafe_allow_html=True)
# makes a sidebar selection in index
experiment_mode = exp_mode()
debug_type = debug_mode()
demo_run_mode = None
tickers = []
if experiment_mode == 'demo':
tickers = ['Amazon']
run_modes_selection = default_runs()
st.write('Default Run Modes set to:', run_modes_selection)
if run_modes_selection == 'Baseline (ARIMA and FB Prophet)':
demo_run_mode = ['arima', 'prophet']
elif run_modes_selection == 'Featured (LSTMs)':
demo_run_mode = ['lstm1', 'lstm2']
else:
# dummy data directories and paths
try:
historical_news_filename = 'news.json'
class_data_folder = os.sep.join(
[os.environ['PWD'], 'data', 'class_data'])
historical_news_path = os.sep.join(
[class_data_folder, historical_news_filename])
tickers = get_stock_tickers(historical_news_path)
except:
pass
st.write('Experiment Configuration:')
st.write('Experiment Mode:', experiment_mode)
st.write('Debug Mode:', debug_type)
st.write('Tickers:', len(tickers))
run_exp = st.button('Run!')
results_df = None
if run_exp:
if not debug_type:
st.write('Running with Multiprocessor')
with st.spinner('Running The Experiment!...'):
if demo_run_mode:
results_df = run_main(experiment_mode=experiment_mode,
tickers=tickers,
debug_mode=debug_type,
demo_run_mode=demo_run_mode)
else:
results_df = run_main(experiment_mode=experiment_mode,
tickers=tickers,
debug_mode=debug_type)
st.success('Yay! Made Predictions.')
st.write('Check data/ and figures/ for results')
if results_df is not None:
st.dataframe(results_df)
make_scatter(df=results_df, title='Your Experiment Results')
st.balloons()
def __main():
setup_image_location = "./images/setup.jpeg"
while True:
user_choice = __menu_choice()
if user_choice == '1':
camera = imageread.setup_camera()
camera.start_preview()
print "INFO: Setup image save location:", str(setup_image_location)
print "INFO: Camera preview initiated."
print ""
while True:
raw_input("When ready, press ENTER to capture setup image.")
camera.capture(setup_image_location)
user_input = raw_input("Accept image (y/n)? > ")
if user_input.lower() in ('y', "yes"): break
# picture saved, end preview
camera.close()
print ""
print "INFO: Setup image has been saved to:", str(setup_image_location)
print "INFO: Camera preview closed."
elif user_choice == '2':
try:
setup_image = Image.open(setup_image_location)
except:
print "ERROR: Setup image does not exist. Select option 1 to"
print "create a new setup image."
raw_input("\nPress ENTER to continue...\n")
setup_selectarea.main(setup_image)
elif user_choice == '3':
try:
import setup_data
boxes = setup_data.boxes
if not isinstance(boxes, list): raise ValueError()
except:
print "ERROR: Setup data does not exist. Please run options 1 and 2 first."
continue
# attempt to import the server senddata module. If fail, return to main menu
# prompt.
try:
import senddata
except:
print "ERROR: Could not import send data file."
continue
# deregister all areas associated with this pi (start fresh)
out = senddata.deregister_pi()
try:
out['error']
print "ERROR: Error in connecting to server. Please update settings.py."
continue
except:
pass
# register each box on the server
for box in boxes:
if box[1] == 0:
senddata.register_area(box[0])
print "INFO: Registering area", box[0], "on server database."
print "\nRegistration complete."
elif user_choice == '4':
print "This will complete the setup and run the main PiPark program."
user_input = raw_input("Continue and run the main program? (y/n) >")
if user_input.lower in ('y', 'yes'):
main.run_main()
break
elif user_choice.lower() in ('h', "help"):
elif user_choice.lower() in ('q', "quit"):
print ""
break
else:
print "\nERROR: Invalid menu choice.\n"
def read_data():
main.run_main()
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import main
if __name__ == '__main__':
main.run_main("forward_selection", ["pearson", "fisher", "greedy"], "auc")
# -*- coding: utf-8 -*-
from main import run_main
from subprocess import call
import os
import time
start_time = time.clock()
colmap_dir = '/user_data/colmap/build/src/exe/'
print(colmap_dir)
# First Run
print('Iteration 1')
project_directory = run_main('./configFiles/configMuscatAWSLoop1.yml')
# COLMAP
print('Starting Semi-Dense Reconstruction')
call([
colmap_dir + '/feature_extractor', '--database_path',
project_directory + '/database.db', '--image_path',
project_directory + '/Selected/small', '--SiftGPUExtraction.index', '0'
])
call([
colmap_dir + '/exhaustive_matcher', '--database_path',
project_directory + '/database.db', '--SiftMatching.use_gpu', '1'
])
if not os.path.exists(project_directory + '/sparse'):
os.makedirs(project_directory + '/sparse')
comm = MPI.COMM_WORLD
for c in batch_configs.config_list:
if results_exist(c):
if comm.Get_rank() == 0:
print 'Skipping: ' + c.results_file
continue
if comm.Get_rank() == 0:
timer.tic()
num_labels_list = list(itertools.product(c.num_labels, range(c.num_splits)))
no_viz = False
pool.map(mpi_run_main_args, [n + (no_viz, c, ) for n in num_labels_list])
pool.close()
if comm.Get_rank() == 0:
print 'TOTAL TIME:'
timer.toc()
main.run_main(configs=c)
else:
if use_multiprocessing_pool:
pool = multiprocessing_utility.LoggingPool(processes=pool_size)
pool.map(launch_subprocess_args, num_labels_list)
else:
for i in num_labels_list:
launch_subprocess_args(i)
comm = MPI.COMM_WORLD
if comm.Get_rank() == 0:
print 'TOTAL TIME:'
timer.toc()
main.run_main()
import main main.run_main()
features_list_group = []
if os.path.isdir(path_input_group):
for subject in sorted(os.listdir(path_input_group)):
path_input_subject = os.path.join(path_input_group, subject)
if os.path.isdir(path_input_subject):
path_output_study = os.path.join(
d.path_output, u.to_extract_foldername(d.path_input))
if not (os.path.exists(path_output_study)):
os.mkdir(path_output_study)
path_output_group = os.path.join(path_output_study, group)
if not (os.path.exists(path_output_group)):
os.mkdir(path_output_group)
path_output_subject = os.path.join(path_output_group, subject)
if not (os.path.exists(path_output_subject)):
os.mkdir(path_output_subject)
features_list = m.run_main(path_input_subject,
path_output_subject + '/')
features_list_group.append(features_list)
np.savetxt(path_output_group + 'features.out',
np.array(features_list_group),
delimiter=' ',
fmt='%s')
def gen_sub(body: GenSub):
meeting = body.meeting
output = run_main(meeting)
return {
"path": output,
}
]
args['comparisons_per_hit'] = 1 #len(args['onboarding_tasks'])
args['block_on_onboarding'] = True
args['onboarding_threshold'] = 1.0
# HIT options
# add 1 for onboarding task HIT
# Manager workers by creating HITS until num_conversations is reached, including
# onboarding tasks
#args['num_conversations'] = 10 + (len(args['model_comparisons']) * args['pairs_per_matchup'] * args['annotations_per_pair'])
args['num_conversations'] = math.ceil(
(len(args['model_comparisons']) * args['pairs_per_matchup'] *
args['annotations_per_pair']) * 1.33)
args['assignment_duration_in_seconds'] = 1800
args['reward'] = 0.15 # in dollars
args['bonus_reward'] = 0.85 # in dollars
args['max_hits_per_worker'] = 100
# Additional args that can be set - here we show the default values.
# For a full list, refer to run.py & the ParlAI/parlai/params.py
# args['seed'] = 42
args['verbose'] = False
args['is_debug'] = False
return args
if __name__ == '__main__':
args = set_args()
run_main(args, task_config)
# sys.exit()
# Each item in the list 'list_doc_score' will contain an ordered dictionary with the doc scores for each topic in the form {docid : docscore} .
list_doc_score = []
# Loop through all the topics in the topic_list.
for topic in topic_list:
# Get directory path for the seed document of the current topic.
seedDoc_filepath = os.path.join(seedDoc_folder, topic)
# Implement CAL with Supervised Machine Learning for the current topic & receive the prediction scores for the documents of the current topic.
list_doc_score.append(
run_main(projDir, seedDoc_filepath, qrels_filepath, tfidf_dict,
vocab_idx_dict, topic_docid_label, clf, tfidf_sp,
out_file_features, docid_idx_dict, topic,
qrels_content_filepath, idf_dict, vocab_idx_dict_2,
cur_idx, total_words, idf_word_dict,
dict_initialScoreRankingResults))
LambdaParam_list = [
0, 0.4, 0.5, 1
] # [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1]
list_doc_score_copies = [None] * len(LambdaParam_list)
# Assign for each item of the list a copy of the list_doc_score.
for i in range(len(LambdaParam_list)):
list_doc_score_copies[i] = list_doc_score
# Do a Lambda parameter sweep and write results.
for idx, LambdaParam in enumerate(LambdaParam_list):
