console = Console()
usage = "You shouldn't be running this file."
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
console.setVerbosity(3) # only error, success, log
script = 'predict.py'
dataset_filename = './neuralnet/corpus/carnegie_mellon.csv'
maxgpa = 5.0
maxtest = 2400
dataset_filename = str(dataset_filename)
maxgpa = float(maxgpa)
maxtest = int(maxtest)
if dataset_filename[-4:] != ".csv":
console.error("Filetype not recognized as CSV.")
print(usage)
exit(0)
# Data sets
DATA_TRAINING = dataset_filename
DATA_TEST = dataset_filename
''' We are expecting features that are floats (gpa, sat, act) and outcomes that are integers (0 for reject, 1 for accept) '''
# Load datasets using tf contrib libraries
training_set = tf.contrib.learn.datasets.base.load_csv_without_header(
filename=DATA_TRAINING, target_dtype=np.int, features_dtype=np.float)
test_set = tf.contrib.learn.datasets.base.load_csv_without_header(
filename=DATA_TEST, target_dtype=np.int, features_dtype=np.float)
##
with open('jobs.json') as f:
job_data = json.load(f)
console.info("Crawling %d career pages." % len(job_data))
i = 0
for job_entry in job_data:
try:
url = job_entry['link']
page = requests.get(url)
tree = html.fromstring(page.content)
links = tree.xpath('//a')
job_postings = []
for link in links:
job_title = link.text_content().strip().lstrip()
if 'intern' in job_title: # only test if intern position
res = requests.post(
'http://127.0.0.1:8000/predict', json={'title': job_title})
prediction = res.text.strip().lstrip()
if prediction in ['IT/Software Development', 'Engineering']:
job_postings.append(job_title)
job_entry['positions'] = job_postings
except Exception as e:
console.error(e)
i = i + 1
if i % 20 == 0:
console.log("Processed %d pages." % i)
console.success("Finished crawling.")
with open('jobs.json', 'w') as f:
json.dump(job_data, f)
console.success("Dumped data.")
import time, traceback
from streaming_event_compliance.objects.logging.server_logging import ServerLogging
from streaming_event_compliance.objects.exceptions.exception import ThreadException, ReadFileException
from console_logging.console import Console
import sys
# resource.setrlimit(resource.RLIMIT_NOFILE, (2000, -1))
console = Console()
console.setVerbosity(5)
if __name__ == '__main__':
func_name = sys._getframe().f_code.co_name
try:
ServerLogging().log_info(func_name, "Created all db tables")
db.create_all()
except Exception as ec:
console.error('Error: Database connection!' + str(ec.__class__) + traceback.format_exc())
ServerLogging().log_error(func_name, "Database connection error!")
exit(1)
from streaming_event_compliance.objects.variable.globalvar import gVars
from streaming_event_compliance.services import setup
from streaming_event_compliance.services.build_automata import build_automata
from streaming_event_compliance.database import dbtools
dbtools.empty_tables()
setup.init_automata()
if gVars.auto_status == 0:
start = time.clock()
console.secure("Start time: ", start)
try:
ServerLogging().log_info(func_name, "Building automata...")
######################################
#### Setup and Run Scheduled Jobs ####
######################################
"""
msg = Figlet(font='slant')
print(msg.renderText('JIRA Workflow'))
try:
jira = jira_session(get_user(), get_pwd()) # Start a new JIRA Session
jira.async_do(options['async_workers'])
except JIRAError as error:
console.error(Fore.LIGHTRED_EX + "An error occurred while logging into JIRA: \n" +
error.status_code + "\n" +
error.response
)
finally:
# Run scheduled jobs every Tuesday and Thursday at 7AM
schedule.every().monday.at("06:50").do(pre_run_jobs)
schedule.every().monday.at("07:00").do(scheduled_jobs)
schedule.every().friday.at("06:50").do(pre_run_jobs)
schedule.every().friday.at("07:00").do(scheduled_jobs)
# Enable this if doing testing and or development
# schedule.every(1).minutes.do(pre_run_jobs)
# schedule.every(2).minutes.do(scheduled_jobs)
while True:
import numpy as np
import os
import sys
from console_logging.console import Console
from sys import argv
usage = "\nUsage:\npython neuralnet/main.py path/to/dataset.csv path/to/crossvalidation_dataset.csv #MAX_GPA #MAX_TEST_SCORE\n\nExample:\tpython main.py harvard.csv 6.0 2400\n\nThe dataset should have one column of GPA and one column of applicable test scores, no headers."
console = Console()
console.setVerbosity(3) # only logs success and error
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
try:
script, dataset_filename, test_filename, maxgpa, maxtest = argv
except:
console.error(str(sys.exc_info()[0]))
print(usage)
exit(0)
dataset_filename = str(dataset_filename)
maxgpa = float(maxgpa)
maxtest = int(maxtest)
if dataset_filename[-4:] != ".csv":
console.error("Filetype not recognized as CSV.")
print(usage)
exit(0)
# Data sets
DATA_TRAINING = dataset_filename
DATA_TEST = test_filename