def run():
rj = ReadJson(SONGS_FILENAME, MIN_VIEWS)
rj.read_song_texts()
print("Songs count: {}".format(len(rj.song_texts)))
preprocessor = Preprocessor(rj.get_song_texts())
preprocessor.run()
corp_and_dict = ElementsForLDA(preprocessor.documents)
corp_and_dict.save_to_file()
def __init__(self, event_str):
rj = ReadJson()
self.event_str = event_str
self.prep_path = rj.get_preproc_path()
self.fail_path = rj.get_failures_path()
self.thumb_path = rj.get_thumbs_path()
self.ingest_path = rj.get_ingestion_path()
self.recipient = rj.get_recipient()
self.sender = rj.get_sender()
self.smtphost = rj.get_smtp_host()
self.st_file = rj.get_foreign_stations_filename()
self.event_folder = os.path.join(self.prep_path, self.event_str)
self.event_str_full = self.event_str + '_UT'
def map_sensor_to_attribute(self, base_importer):
self._questions('Are attributes at the same level or a level below sensor tag: (Y/N) ')
attr_at_level = input()
self._questions('Enter Tag name of the first object: ')
root_tag = input()
attr_to_sensor_mapping = {}
if attr_at_level == 'y' or attr_at_level == 'Y':
r_json = ReadJson(None, None)
r_json.map_sensors_to_attributes(base_importer.dataset, '@SiteCode', '@SpeciesCode',
attr_to_sensor_mapping, root_tag)
# print(values)
else:
# Need to loop through every sensor and map attributes manually to them
pass
return attr_to_sensor_mapping
def main():
try:
VerifyLinux()
rj = ReadJson()
cam_path = rj.get_rsync_path()
month_nr = rj.get_months_number()
months_list = MonthsString(month_nr).create_strings()
cameras_path_list = CamerasPathList(cam_path).create_list()
for camera in cameras_path_list:
for month in months_list:
print(camera, month)
copyfits = ArchiveFITS(camera, month).copy_fits()
copyjpg = ArchiveFITS(camera, month).copy_jpg()
except Exception as e:
msg = "Main exception - main() -- "
log.error("{0}{1}".format(msg, e))
import os
import shutil
from glob import glob
import multiprocessing
from datetime import datetime
from datetime import timedelta
from database import Queries
from database import DataFile
from read_json import ReadJson
from multiprocessing import Pool
from utilities import LoggingClass
from database import MySQLDatabase
log = LoggingClass('', True).get_logger()
rj = ReadJson()
dbuser = rj.get_db_user()
dbpwd = rj.get_db_pwd()
dbname = rj.get_db_name()
dbhost = rj.get_db_host()
dbport = rj.get_db_port()
thr_nr = rj.get_threads_number()
db = MySQLDatabase(dbuser, dbpwd, dbname, dbhost, dbport)
Session = db.mysql_session()
def _query_filename(filename):
try:
rows = Queries(Session, DataFile, filename).match_filename()
from flask import Flask, jsonify, request from flask_pymongo import PyMongo from bson.json_util import dumps from bson.objectid import ObjectId from werkzeug.security import generate_password_hash, check_password_hash from read_json import ReadJson from prediction_read_json import PredictionReadJson from schedule_json import ScheduleOfficers from flask_mail import Mail, Message #officers recommendation json file called off1 = ReadJson() tzone = off1.readjsontzoneOne() tzTwo = off1.readjsontzoneTwo() tzThree = off1.readjsontzoneThree() tzFour = off1.readjsontzoneFour() accpre = off1.readjsonAccidentPrediction() #severity prediction functions call prediction_results = PredictionReadJson() sv_rs_count = prediction_results.severityreadjson() sv_summary = prediction_results.severitySummary() #reason prediction function call rs_rs_count = prediction_results.reasonreadjson() rs_summary = prediction_results.reasonSummary() #frequency prediction function call fp_rs_count = prediction_results.frequncyreadjson() fp_summary = prediction_results.frequencySummary() #officer schedule time zone one schedulejsontzoneOne function call sch = ScheduleOfficers() scOne = sch.schedulejsontzoneOne() scTwo = sch.schedulejsontzTwo() scThree = sch.schedulejsontzThree()
__author__ = "Elisa Londero"
__email__ = "*****@*****.**"
__date__ = "June 2018"
import os
from database import db_query
from utilities import md5Checksum
from datetime import datetime
from datetime import timedelta
from utilities import LoggingClass
from read_json import ReadJson
from database import MySQLDatabase
from utilities import SendEmail
rj = ReadJson()
log = LoggingClass('',True).get_logger()
class StorageFilesList(object):
def __init__(self):
self.months = int(rj.get_months())
self.strg_fldr = rj.get_storage_folder()
def create_storage_list(self):
files_list = []
for root,subdirs,files in os.walk(self.strg_fldr):
for filename in files:
file_path = os.path.join(root, filename)
files_list.append(file_path)
return files_list
import shutil
from glob import glob
from os.path import isdir
from database import Queries
from database import DataFile
from read_json import ReadJson
from utilities import SendEmail
from utilities import FitsAddKey
from utilities import TarHandling
from utilities import ReadStations
from utilities import FolderSize
from utilities import LoggingClass
from database import MySQLDatabase
from shutil import ignore_patterns
rj = ReadJson()
rsync_path = rj.get_rsync_path()
log = LoggingClass('',True).get_logger()
class SelectEventString(object):
def __init__(self):
self.dbhost = rj.get_db_host()
self.dbuser = rj.get_db_user()
self.dbpwd = rj.get_db_pwd()
self.dbport = rj.get_db_port()
self.dbname = rj.get_db_name()
self.db = MySQLDatabase(self.dbuser, self.dbpwd, self.dbname, self.dbhost, self.dbport)
self.Session = self.db.mysql_session()
def _check_event_string(self):