def load_datasets(*sampSizes):
x = []
y = []
p = []
for ss in sampSizes:
log.print_debug("Opening Cropped dataset " + str(ss))
cropped_dataset_folder = path.join(CROP_FOLDER, str(ss))
for filename in os.listdir(cropped_dataset_folder):
try:
img_path = path.join(cropped_dataset_folder, filename)
img_patient = filename.split("_")[0]
img_class = CATEGORIES.index(str(filename.split("_")[1]))
img = Image.open(img_path).convert('RGB')
img_array = np.asarray(img, np.uint8)
if filter.check_valid(img_array):
to_append_img = np.asarray(
img.resize((int(INPUT_SIZE), int(INPUT_SIZE)),
Image.LANCZOS))
x.append(to_append_img)
y.append(img_class)
p.append(img_patient)
else:
log.print_warning("Img " + filename +
" not loaded: too much white")
except Exception as e:
log.print_error("Cannot load image " + filename)
return x, y, p
def place_order(self, data, i):
'''
Funtion to place an order with ST and TP parameters
'''
log.print_green('Placing an order...')
print data
try:
self._instr = self.instruments_dict.get(
data.get('instrument').upper())[0].get('instrument')
self._unit = self.instruments_dict.get(
data.get('instrument').upper())[3].get('tradeUnit')
for x in range(1, (i + 1)):
self._action = data.get('trade')[x].keys()[0]
if not data.get('trade')[x][self._action]['TP']:
self._tp = 0
else:
self._tp = data.get('trade')[x][self._action]['TP']
self._sl = data.get('trade')[x][self._action]['SL']
log.print_warning(self._instr, self._unit, self._action,
self._tp, self._sl)
self.ordr = order.MyOanda(self._instr, self._unit,
self._action, self._sl, self._tp)
try:
self._art_data.add_oanda(self.ordr.create_order())
except OandaError:
log.print_error('Placing a order failed')
return
except AttributeError, TypeError:
log.print_error('AttributeError, data NoneType')
return
def parse_all_tasks():
"""
Parse all tasks from user specified folders.
Return list with tuples (tasks, number) where tasks is list of
datamodel.Task objects and number is number of tasks per student from
this list.
"""
tasks = []
tasks_paths = common.get_config().get_tasks_paths()
for (task_path, points, number) in tasks_paths:
tasks_from_folder = parse_tasks(task_path)
# something got wrong
if tasks_from_folder == None:
logger.print_error("there is something wrong with folder " +
task_path)
return None
# too little tasks in folder
if len(tasks_from_folder) < number:
logger.print_error("too little tasks in folder " +
task_path + ", need " + str(number))
return None
# set max points for every task in folder
for task in tasks_from_folder:
task.set_max_points(points)
# add tuple (tasks_from_folder, number) to the end list
tasks += [(tasks_from_folder, number)]
return tasks
def place_order(self, data, i):
'''
Funtion to place an order with ST and TP parameters
'''
log.print_green('Placing an order...')
print data
try:
self._instr = self.instruments_dict.get(data.get('instrument').upper())[0].get('instrument')
self._unit = self.instruments_dict.get(data.get('instrument').upper())[3].get('tradeUnit')
for x in range(1,(i+1)):
self._action = data.get('trade')[x].keys()[0]
if not data.get('trade')[x][self._action]['TP']:
self._tp = 0
else:
self._tp = data.get('trade')[x][self._action]['TP']
self._sl = data.get('trade')[x][self._action]['SL']
log.print_warning(self._instr, self._unit, self._action, self._tp, self._sl)
self.ordr = order.MyOanda(self._instr, self._unit, self._action, self._sl, self._tp)
try:
self._art_data.add_oanda(self.ordr.create_order())
except OandaError:
log.print_error('Placing a order failed')
return
except AttributeError, TypeError:
log.print_error('AttributeError, data NoneType')
return
def write_points(student, points):
path = os.path.join(common.get_config().get_output_path(), \
student.get_login() + "_points.txt")
try:
with open(path, "w", encoding="u8") as f:
print(str(round(sum(points), 2)), file=f)
except IOError:
logger.print_error("cannot write file to the output folder")
def setup_mqtt():
try:
m = Mqtt(MQTT_CLIENT_NAME, MQTT_BROKER)
m.connect()
except OSError as ose:
logger.print_error("MQTT Setup Failed!")
raise
return m
def setup_wifi():
try:
w = Wifi()
w.set_hostname(MY_HOST)
except OSError as ose:
logger.print_error("WIFI Setup Failed!")
raise
return w
def import_instruments(self):
'''
Importing list of indtrument from instruments.yaml file
'''
try:
with open(instruments, 'r') as f:
self.instruments_dict = yaml.load(f)
log.print_green('Instruments imported!')
except:
log.print_error('Instruments import failed')
def import_instruments(self):
'''
Importing list of indtrument from instruments.yaml file
'''
try:
with open(instruments, 'r') as f:
self.instruments_dict = yaml.load(f)
log.print_green('Instruments imported!')
except:
log.print_error('Instruments import failed')
def import_test_page(self):
'''
Importing test_page from mainpage.yaml file
'''
try:
with open(self.test_page, 'r') as f:
self.test_page_cont = f.read()
log.print_green('Test page imported!')
except:
log.print_error('Test page import failed')
def open_connection(self):
try:
self._open_connection(user='******',
password='******',
database='dantalian',
host='mysql')
except Exception as e:
print_error(__name__, str(e))
print_error(__name__, "Critical: Shutting down")
sys.exit(11)
def save_latex_file(latex_str, id_):
"""
Save latex file to the output folder.
"""
path = os.path.join(common.get_config().get_output_path(),
'{:0>4}'.format(str(id_)) + ".tex")
try:
with open(path, "w", encoding="u8") as f:
print(latex_str, file=f)
except IOError:
logger.print_error("cannot write file to the output folder")
def save_to_yaml_all(self, data):
self._time = time.ctime().split(' ')
self._time.remove('')
self._dir_name = "./%s-%s/" % (self._time[2], self._time[1])
self._file_name = 'all_trade_ideas.yaml'
try:
with open(self._file_name, "ab") as f:
yaml.dump(data, f)
log.print_green('All data has been saved to: ', self._file_name)
except IOError:
log.print_error('Error during saving: ', self._file_name)
def setup_wifi():
try:
w = Wifi()
w.set_hostname(MY_HOST)
w.connect(WIFI_SSID, WIFI_PASS)
w.info()
except OSError as ose:
logger.print_error("WIFI Setup Failed!")
raise
return w
def get_output(task, prog_path):
"""
Run program specified by prog_path and return tuple (output, error_output).
"""
# check task language and set relevant interpreter
if task.get_language() == "python2":
interpreter = common.get_config().get_python2path()
elif task.get_language() == "python3":
interpreter = common.get_config().get_python3path()
elif task.get_language() == "ruby":
interpreter = common.get_config().get_ruby_path()
else:
logger.print_error("unknown task language")
return ("", "")
# run subprocess
try:
p = subprocess.Popen([interpreter, prog_path],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except OSError:
logger.print_error("cannot run interpreter for language " + \
task.get_language())
return ("", "")
# send input to subprocess in another thread
outputs = ["", ""]
t = Thread(target=subprocess_communicate,
args=(p, task.get_input(), outputs))
t.start()
# wait until subprocess ends or time out
timeout = common.get_config().get_evaluator_timeout()
start = datetime.datetime.now()
while p.poll() == None:
sleep(0.1)
now = datetime.datetime.now()
if (now - start).seconds > timeout:
p.kill()
logger.print_warning("subprocess with interpreter " + \
task.get_language() + " " + \
"has been killed")
break
# wait until communication thread ends or time out (5 sec)
timeout = 5
start = datetime.datetime.now()
while t.is_alive():
sleep(0.1)
now = datetime.datetime.now()
if (now - start).seconds > timeout:
logger.print_warning("subprocess communication thread is " + \
"still running, interpreter: " + \
task.get_language())
break
return (outputs[0], outputs[1])
def save_to_yaml_all(self, data):
self._time = time.ctime().split(' ')
self._time.remove('')
self._dir_name = "./%s-%s/" % (self._time[2], self._time[1])
self._file_name = 'all_trade_ideas.yaml'
try:
with open(self._file_name, "ab") as f:
yaml.dump(data, f)
log.print_green('All data has been saved to: ',
self._file_name)
except IOError:
log.print_error('Error during saving: ', self._file_name)
def open_slide(slide_path):
start_time = time.time()
try:
slide = openslide.open_slide(slide_path)
except Exception:
slide = None
elapsed_time = time.time() - start_time
if slide is None:
log.print_error("Cannot open slide " + slide_path +
" || Time Elapsed: " + str(elapsed_time))
else:
log.print_debug("Opened slide " + slide_path + " || Shape: " +
str(get_slide_size(slide)) + " || Time Elapsed: " +
str(elapsed_time))
return slide
def __call(self,action,datas={}):
"""
execute HTTP request to the API
@param string action
@param dict datas
@return answer body
"""
params = urllib.urlencode(datas)
req = urllib2.Request(self.URL+action, params, headers={'Authorization': self.token, 'User-agent': 'Mozilla/5.0'})
response = urllib2.urlopen(req).read()
if '{"error"' in response:
logger.print_error( json.loads(response)['error'] )
exit()
return response
def eval_fulltext(student, task, num, formdata, points):
"""
Evaluate student's answer on fulltext task.
"""
# write task to the output folder
path = os.path.join(common.get_config().get_output_path(), \
student.get_login() + "_task_" + \
'{:0>2}'.format(str(num+1)) + ".txt")
try:
answer = formdata["a_" + str(num)][0]
except (KeyError, IndexError):
answer = ""
if common.get_config().get_evaluator_strictness() == 0:
pattern = re.compile(r'\s+')
cleaned_answer = re.sub(pattern, '', answer)
else:
cleaned_answer = answer.strip()
final_points = 0
for correct_answer, coef in task.get_answers():
if common.get_config().get_evaluator_strictness() == 0:
pattern = re.compile(r'\s+')
correct_answer = re.sub(pattern, '', correct_answer)
else:
correct_answer = correct_answer.strip()
if string_equals(cleaned_answer, correct_answer):
p = task.get_max_points() * coef
if p.is_integer():
p = int(p)
else:
p = 0
final_points = max(final_points, p)
try:
with open(path, "w", encoding="u8") as f:
print(QUESTION, file=f)
print(task.get_question().strip(), file=f)
for correct_answer, coef in task.get_answers():
print(CORRECT_ANSWER, file=f)
print(correct_answer.strip() + " (" + str(coef) + ")", file=f)
print(ANSWER, file=f)
print(answer.strip(), file=f)
print(POINTS, file=f)
print(final_points, file=f)
except IOError:
logger.print_error("cannot write file to the output folder")
# set points
points[num] = final_points
def compile_model(x_train, y_train, x_test, y_test, drop_rate):
if tf.test.is_built_with_cuda:
if tf.test.is_gpu_available(cuda_only=False,
min_cuda_compute_capability=None):
log.print_error("MAYBE GPU IS USED")
else:
log.print_warning("NO GPU IS USED")
else:
log.print_warning("THIS VERSION OF TENSORFLOW DOES NOT USES CUDA")
input_tensor = tf.keras.Input(shape=x_train[0].shape)
bayesian_model = models.Model(
input_tensor, bayesian_cnn(inputs=input_tensor, drop_rate=drop_rate))
opt = tf.keras.optimizers.Adam(lr=LEARNING_RATE, decay=DECAY)
bayesian_model.compile(loss='sparse_categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
model_name = str(x_train[0].shape[0]) + "_" + str(N_EPOCH) + "_" + str(BATCH_SIZE) + "_" + str(LEARNING_RATE) \
+ "_" + str(DECAY) + "_" + str(drop_rate) + "_" + str(USE_BIAS) + "_" + str(DENSE_SIZE) + "_" \
+ str(SEPARABLE_CONVOLUTION) + "_local.h5"
bayesian_model.summary()
# Save model skeleton
if not os.path.isdir(SUMMARY_FOLDER):
os.makedirs(SUMMARY_FOLDER)
summary_path = os.path.join(SUMMARY_FOLDER, model_name + ".txt")
with open(summary_path, 'w') as f:
with redirect_stdout(f):
bayesian_model.summary()
bayesian_train = bayesian_model.fit(x_train,
y_train,
batch_size=BATCH_SIZE,
epochs=N_EPOCH,
validation_data=(x_test, y_test),
shuffle=True)
# Save model and weights
if not os.path.isdir(MODEL_FOLDER):
os.makedirs(MODEL_FOLDER)
model_path = os.path.join(MODEL_FOLDER, model_name)
bayesian_model.save_weights(model_path)
log.print_info('Saved trained model at %s ' % model_path)
# Score trained model.
scores = bayesian_model.evaluate(x_test, y_test, verbose=1)
log.print_info('Test loss : ' + str(scores[0]))
log.print_info('Test accuracy : ' + str(scores[1]))
def __call(self, action, datas={}):
"""
execute HTTP request to the API
@param string action
@param dict datas
@return answer body
"""
params = urllib.urlencode(datas)
req = urllib2.Request(self.URL + action,
params,
headers={
'Authorization': self.token,
'User-agent': 'Mozilla/5.0'
})
response = urllib2.urlopen(req).read()
if '{"error"' in response:
logger.print_error(json.loads(response)['error'])
exit()
return response
def read_students_list():
"""
Read students list from CSV file and return list with datamodel.Student
objects.
"""
students = []
csv.register_dialect('students', quoting=csv.QUOTE_NONE)
try:
with open(common.get_config().get_students_csv_file_path(),
encoding="u8",
newline='') as f:
reader = csv.reader(f, 'students')
for row in reader:
if len(row) != 2:
logger.print_error("incorrect students list CSV file")
return None
else:
student = datamodel.Student()
student.set_login(row[0])
student.set_name(row[1])
students += [student]
except IOError:
logger.print_error("cannot open students list " +
common.get_config().get_students_csv_file_path())
return None
if len(students) < 1:
logger.print_error("none students has been found")
else:
logger.print_msg("students list successfully loaded")
return students
def write_choice_task(student, task, formdata, num, points):
"""
Write choice or multichoice task to the output folder (with student's
answer and hit points)
"""
path = os.path.join(common.get_config().get_output_path(), \
student.get_login() + "_task_" + \
'{:0>2}'.format(str(num+1)) + ".txt")
try:
with open(path, "w", encoding="u8") as f:
print(QUESTION, file=f)
print(task.get_question().strip(), file=f)
print(CHOICES, file=f)
i = 1
for (choice, is_correct) in task.get_answers():
print("[" + str(i) + "] " + str(is_correct), file=f)
print(choice, file=f)
i += 1
print(ANSWER, file=f)
answer = ""
try:
i = 0
for ans in formdata["a_" + str(num)]:
if ans.isdigit():
answer += str(int(ans)+1)
else:
answer += ans
if i < len(formdata["a_" + str(num)]) - 1:
answer += ","
i += 1
except KeyError:
answer = ""
print(answer, file=f)
print(POINTS, file=f)
print(round(points, 2), file=f)
except IOError:
logger.print_error("cannot write file to the output folder")
def get_rc_result_helper(info):
date, RcNo = info
if RcNo == '1':
print_debug('Trying %s', date)
res = requests.post(
'http://race.kra.co.kr/raceScore/ScoretableDetailList.do', {
'meet': '1',
'realRcDate': date,
'realRcNo': RcNo
}).content.decode('euc-kr')
soup = BeautifulSoup(res, 'html.parser')
table = soup.find_all('div', class_='tableType2')
if len(table) < 2:
print_error('Failed to get table (%s, %s)', *info)
return None
table1 = table[0].table
table2 = table[1].table
tmp = {}
for horse1 in table1.find_all('tr')[1:]:
datas = horse1.find_all('td')
item = [date, RcNo]
item += list(map(lambda x: x.text.strip(), datas))
item += [
Crawler.get_index(datas[2]),
Crawler.get_index(datas[8]),
Crawler.get_index(datas[9]),
Crawler.get_index(datas[10])
]
tmp[int(datas[1].text)] = item
for horse2 in table2.find_all('tr')[1:]:
datas = horse2.find_all('td')
item = list(map(lambda x: x.text.strip(), datas[3:]))
tmp[int(datas[1].text)] += item
return sorted(list(tmp.values()), key=lambda x: int(x[0][0]))
def eval_program(student, task, num, formdata, points_list):
"""
Evaluate student's answer on program task.
"""
# first get student's input program part
try:
student_program_part = formdata["a_" + str(num)][0].replace("\r\n", \
"\n")
except (KeyError, IndexError):
student_program_part = ""
# create file with program
try:
# prepare folder for program script
dir_path = os.path.join(common.get_config().get_output_path(),
"programs")
if not os.path.isdir(dir_path):
os.mkdir(dir_path)
prog_path = os.path.join(dir_path, task.get_id() + "_" + \
student.get_login() + "." + \
task.get_language())
# create program as string
program = ""
for (program_part, indent) in task.get_program_parts():
if program_part != None:
program += program_part.replace("\r\n", "\n")
else:
program += indent_all_rows(student_program_part, indent)
program += "\n"
# write program to the file
with open(prog_path, "w", encoding="u8", newline="") as f:
f.write(program)
except IOError:
logger.print_error("cannot write file to the output folder")
points_list[num] = 0
return
final_points = 0
if task.get_language() == "unknown":
python2_task = copy.deepcopy(task)
python2_task.set_language("python2")
python3_task = copy.deepcopy(task)
python3_task.set_language("python3")
ruby_task = copy.deepcopy(task)
ruby_task.set_language("ruby")
tasks_for_eval = [python2_task, python3_task, ruby_task]
else:
tasks_for_eval = [task]
for task_for_eval in tasks_for_eval:
# run program and get output
(output, error_output) = get_output(task_for_eval, prog_path)
# compare output with the correct output
correct_output = task_for_eval.get_correct_output()
cleaned_output = output
if common.get_config().get_evaluator_strictness() == 0:
pattern = re.compile(r'\s+')
correct_output = re.sub(pattern, '', correct_output)
cleaned_output = re.sub(pattern, '', output)
if cleaned_output == correct_output:
points = task_for_eval.get_max_points()
else:
points = 0
final_points = max(points, final_points)
if final_points == task_for_eval.get_max_points():
break
# write task with results to the output folder
path = os.path.join(common.get_config().get_output_path(),
student.get_login() + "_task_" + \
'{:0>2}'.format(str(num+1)) + ".txt")
try:
with open(path, "w", encoding="u8") as f:
print(QUESTION, file=f)
print(task.get_question().strip(), file=f)
print(ANSWER, file=f)
answer = student_program_part
print(answer, file=f)
print(PROGRAM, file=f)
print(prog_path, file=f)
print(INPUT, file=f)
prog_input = ""
if task.get_input() != None:
prog_input = task.get_input().strip()
print(prog_input, file=f)
print(CORRECT_OUTPUT, file=f)
print(task.get_correct_output().strip(), file=f)
print(OUTPUT, file=f)
print(output.strip(), file=f)
print(ERROR_OUTPUT, file=f)
print(error_output, file=f)
print(POINTS, file=f)
print(round(final_points, 2), file=f)
except IOError:
logger.print_error("cannot write file to the output folder")
# write points and return
points_list[num] = final_points
def processDownloadTransmission(self,torrent_datas):
torrent_datas = torrent_replace_announce( torrent_datas, self.config.get('transmission','tracker') )
logger.print_info('Connexion au server transmission ... ', eol='')
tc = transmissionrpc.Client( self.config.get('transmission','host') ,
port=self.config.get('transmission','port') ,
user=self.config.get('transmission','username'),
password=self.config.get('transmission','password'))
logger.print_ok()
logger.print_info("Upload du torrent ... ",eol='')
torrent = tc.add_torrent(base64.b64encode(torrent_datas))
logger.print_ok()
torrent = tc.get_torrent(torrent.id)
while torrent.progress < 100:
sys.stdout.write('\r %.2f%% [%-100s] ' % ( torrent.progress, "="*int(torrent.progress)+">" ))
sys.stdout.flush()
torrent = tc.get_torrent(torrent.id)
time.sleep(1)
print '\r 100%% [%s] '%('='*100)
logger.print_success( 'Download complet' )
tc.stop_torrent(torrent)
try:
app = MainApp()
app.run()
except NoSectionError,e:
logger.print_error( "Invalid config.ini ( %s )" % e )
def processDownloadTransmission(self,torrent_datas):
torrent_datas = torrent_replace_announce( torrent_datas, self.config.get('transmission','tracker') )
logger.print_info('Connexion au server transmission ... ', eol='')
tc = transmissionrpc.Client( self.config.get('transmission','host') ,
port=self.config.get('transmission','port') ,
user=self.config.get('transmission','username'),
password=self.config.get('transmission','password'))
logger.print_ok()
logger.print_info("Upload du torrent ... ",eol='')
torrent = tc.add_torrent(base64.b64encode(torrent_datas))
logger.print_ok()
torrent = tc.get_torrent(torrent.id)
while torrent.progress < 100:
sys.stdout.write('\r %.2f%% [%-100s] ' % ( torrent.progress, "="*int(torrent.progress)+">" ))
sys.stdout.flush()
torrent = tc.get_torrent(torrent.id)
time.sleep(1)
print '\r 100%% [%s] '%('='*100)
logger.print_success( 'Download complet' )
tc.stop_torrent(torrent)
try:
app = MainApp()
app.run()
except NoSectionError,e:
logger.print_error( "Invalid config.ini ( %s )" % e )
