def event_analysis():
"""
Event analysis process. It fetches all the event in the database and analyse the description & website and
then create all the related features
"""
event_analysis = EventAnalysis()
# Store all available website and avoid parsing a website several times
websites = dict(dict())
# Contains the list of key-word with tree tagger
description_tree_tagger = dict()
website_tree_tagger = dict()
events = Event.objects.all()
if len(events) == 0:
return
nb_core = cpu_count()
nb_events = len(events)
nb_events_thread = nb_events / nb_core
events_thread = []
for i in range(nb_core - 1):
events_thread.append(events[i * nb_events_thread:(i + 1) *
nb_events_thread])
events_thread.append(events[(nb_core - 1) * nb_events_thread:])
# Fulfill the corpus
start_threads(nb_core, event_analysis_fulfill_corpus, events_thread,
event_analysis, websites, description_tree_tagger,
website_tree_tagger)
#Monothread - event_analysis_fulfill_corpus(event_analysis, websites, description_tree_tagger, website_tree_tagger, events)
event_analysis.set_corpus_complete()
# We compute the tf-idf of the key word in the description and in the website if exists
start_threads(nb_core, event_analysis_compute_tf_idf, events_thread,
event_analysis, websites, description_tree_tagger,
website_tree_tagger)
#Monothread - event_analysis_compute_tf_idf(event_analysis, websites, description_tree_tagger, website_tree_tagger, events)
# We fetch the k most important tags by event
job_queue = JobQueue()
job_queue.start()
start_threads(
nb_core,
event_analysis_fetch_k_most_important_features_and_push_database,
events_thread, job_queue, event_analysis, websites)
job_queue.finish()
#Monothread - event_analysis_fetch_k_most_important_features_and_push_database(None, event_analysis, websites, events)
compute_statistics(events, description_tree_tagger, website_tree_tagger)
def create_job_queue():
data = json.loads(request.data, encoding="utf-8")
# Checks
CONFIG_CREATE_JOB_QUEUE.validate(data)
if data["jobQueueName"] in job_queues:
abort(400, "Job queue already exists.")
for ce in data["computeEnvironmentOrder"]:
if ce["computeEnvironment"] not in compute_environments:
abort(400, f"Compute environment {ce} does not exist")
if compute_environments[ce["computeEnvironment"]].state != ComputeEnvironment.STATE_ENABLED:
abort(400, f"Compute environment {ce} is not enabled.")
if not (0 < len(data["computeEnvironmentOrder"]) < 3):
abort(400, f"Invalid number ({len(data['computeEnvironmentOrder'])}) of compute environments selected")
orders = set()
for ce in data["computeEnvironmentOrder"]:
if ce["order"] in orders:
abort(400, f"Two compute environments have the same order.")
orders.add(ce["order"])
# Action
new_job_queue = JobQueue(**data)
job_queues[data["jobQueueName"]] = new_job_queue
return jsonify(new_job_queue.describe(everything=False))
def main():
for test in glob("tests/*[!*.a]"):
print(f"Running {test}")
with open(test, "r") as test_fh:
test_input = test_fh.read().strip()
sys.stdin = io.StringIO(test_input)
with open(f"{test}.a") as test_answer_fh:
test_answer = test_answer_fh.read().strip()
try:
job_queue = JobQueue()
job_queue.solve(write=False)
test_output = "\n".join([
f"{assigned_worker} {start_time}"
for assigned_worker, start_time in zip(
job_queue.assigned_workers, job_queue.start_times)
])
assert test_output.strip() == test_answer
except AssertionError:
print(
f"AssertionError at {test}:\n input: {test_input}\n expected output: {test_answer}\n actual output: {test_output}"
)
break
def __init__(self,
cert_file,
addr,
logger,
err_logger=None,
timeout=10,
idle_timeout=3600,
white_list=()):
self.logger = logger
self.logger_err = err_logger or self.logger
self.engine = SSLSocketEngine(io_poll.get_poll(),
cert_file=cert_file,
is_blocking=True)
self.engine.set_logger(logger)
self.engine.set_timeout(rw_timeout=timeout,
idle_timeout=idle_timeout)
self.inf_sock = None
self.addr = addr
self.jobqueue = JobQueue(logger)
self.is_running = False
self.ip_dict = dict()
for ip in white_list:
self.ip_dict[ip] = None
self.rpc_handles = RPC_ServerHandle()
def __init__(self,
addr,
client_keys,
logger,
err_logger=None,
timeout=10,
idle_timeout=3600,
block_size=128):
""" client_keys : dict(ip=>key) or str """
self.logger = logger
self.logger_err = err_logger or self.logger
self.engine = TCPSocketEngine(io_poll.get_poll(), is_blocking=True)
self.engine.set_logger(logger)
self.engine.set_timeout(rw_timeout=timeout,
idle_timeout=idle_timeout)
assert isinstance(client_keys, dict)
self.client_keys = client_keys
self.inf_sock = None
self.addr = addr
self.jobqueue = JobQueue(logger)
self.is_running = False
self.ip_dict = dict()
self.block_size = block_size
self.rpc_handles = RPC_ServerHandle()
from job_queue import JobQueue
app = Flask(__name__, static_url_path='/static')
app.config.from_envvar('POGOJIG_SETTINGS')
class UploadForm(FlaskForm):
upload_file = FileField(validators=[DataRequired()])
class ResetForm(FlaskForm):
pass
job_queue = JobQueue(app.config['JOB_QUEUE_DB'])
def tempfile_path(namespace):
""" Return a path for a per-session temporary file identified by the given namespace. Create the session tempfile
dir if necessary. The application tempfile dir is controlled via the upload_path config value and not managed by
this function. """
sess_tmp = path.join(app.config['UPLOAD_PATH'], session['session_id'])
os.makedirs(sess_tmp, exist_ok=True)
return path.join(sess_tmp, namespace)
def require_session_id(fun):
@wraps(fun)
def wrapper(*args, **kwargs):
if 'session_id' not in session:
####################################################################
parser = argparse.ArgumentParser(description="Automate Todoist workflows.")
parser.add_argument("--loglevel", dest="loglevel", nargs=1, help="set a log level")
args = parser.parse_args()
# If the user specified a log level, use it.
if args.loglevel is not None:
loglevel, *rest = args.loglevel
ch.setLevel(loglevel.upper())
# Register the console handler with the logger.
logger.addHandler(ch)
# Setup.
user = todoist.login_with_api_token(API_TOKEN)
logger.info("Logged in to Todoist.")
q = JobQueue(logger=logger)
# Load the config.
with open(CONFIG_DIR / "config.yml") as f:
conf = yaml.load(f, Loader=yaml.SafeLoader)
# Add the environment variables to the config dict.
conf["email_addr"] = EMAIL_ADDR
conf["email_pw"] = EMAIL_PW
conf["api_token"] = API_TOKEN
logger.debug("Loaded config file.")
###############################################################################
# Add jobs from the jobs.py file.
###############################################################################
# Add each job to the queue, but first bind user and conf variables.
import itertools
from job_queue import JobQueue
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('queue', help='job queue sqlite3 database file')
parser.add_argument('--loglevel', '-l', default='info')
args = parser.parse_args()
numeric_level = getattr(logging, args.loglevel.upper(), None)
if not isinstance(numeric_level, int):
raise ValueError('Invalid log level: %s' % loglevel)
logging.basicConfig(level=numeric_level)
job_queue = JobQueue(args.queue)
signal.signal(signal.SIGALRM, lambda *args: None) # Ignore incoming alarm signals while processing jobs
signal.setitimer(signal.ITIMER_REAL, 0.001, 1)
while signal.sigwait([signal.SIGALRM, signal.SIGINT]) == signal.SIGALRM:
logging.debug('Checking for jobs')
for job in job_queue.job_iter('render'):
logging.info(f'Processing {job.type} job {job.id} session {job["session_id"]} from {job.client} submitted {job.created}')
with job:
job.result = subprocess.call(['sudo', '/usr/local/sbin/pogojig_generate.sh', job['session_id']])
logging.info(f'Finishied processing {job.type} job {job.id}')
logging.info('Caught SIGINT. Exiting.')
from flask import Flask, request, jsonify
from werkzeug.exceptions import HTTPException
from interface import PotholeEvent
from job_queue import JobQueue
import json
import glob
from utility import fakes_pothole_obj
import os
app = Flask(__name__)
job_queue = JobQueue(verbose=True)
@app.route('/')
def hello_world():
return 'Hello, World!'
@app.route('/test/<hash>', methods=['GET'])
def test(hash):
for i in range(5):
job_queue.add(hash)
print("End test main")
return jsonify(hash=hash, success=True)
@app.route('/analyze', methods=['POST'])
@app.route('/analyze/<path:json_filename>', methods=['GET'])
def analyze(json_filename=None):
# if GET and File
if json_filename is not None:
