def __init__(self):
# Initialize
self.model = None
self.optimizer = None
self.scheduler = None
self.solver = None
self.opts, self.args = parser.parse_args()
opts = self.opts
timestamp = datetime.datetime.now().strftime('%Y%m%d%H%M')
self.name = ('%s N-%d ep-%d lr-%f geom-%s' %
(timestamp, opts.train_size
or 0, opts.num_epochs, opts.lr, opts.geometry))
# Set logger
if self.opts.no_prefix:
log_name = self.name + '.log'
else:
log_name = 'out.log'
self.opts.outdir = os.path.join(self.opts.outdir or '', self.name)
if self.opts.logfile:
logger.Logger(self.opts.logfile)
elif self.opts.outdir:
logger.Logger(self.opts.outdir, log_name)
print(sys.argv)
print(opts)
print('PID %d' % (os.getpid(), ))
def startup(settingsFile, debugMode=False):
setting = settings.Settings(settingsFile)
if setting.check_all_settings() is False:
logSettings.log(logger.LogLevel.INFO, 'Failed to load settings')
return
""" tiBot starting, that checks database, twitter and imgur """
if debugMode:
logSettings = logger.Logger(setting.logFileName,
setting.logFolder,
maxSize=setting.logSize,
printLevel=logger.LogLevel.INFO)
else:
logSettings = logger.Logger(setting.logFileName,
setting.logFolder,
maxSize=setting.logSize,
printLevel=setting.LOGGER_PRINT_LEVEL)
logSettings.log(logger.LogLevel.INFO,
'tiBot is starting up',
forcePrint=True)
db = check_database(logSettings, setting.database)
if db is None:
logSettings.log(logger.LogLevel.CRITICAL,
'Database is not set up correctly. Exiting..')
return
twit = twitter.Twitter(logSettings, setting)
twit.authenticated = twit.authenticate()
if twit.authenticated is False:
logSettings.log(logger.LogLevel.CRITICAL,
'Twitter is not authenticated. Exiting..')
return
imgr = imgur.Imgur(logSettings, setting)
imgr.authenticated = imgr.authenticate()
if imgr.authenticated is False:
logSettings.log(logger.LogLevel.CRITICAL,
'Imgur is not authenticated. Exiting..')
return
if debugMode:
logSettings.log(
logger.LogLevel.INFO,
'Everything is set up, starting tiBot in debug mode..')
debug.debug(logSettings, setting, db, twit, imgr)
else:
logSettings.log(logger.LogLevel.INFO,
'Everything is set up, starting tiBot..')
main.start(logSettings, setting, db, twit, imgr)
def train(self, max_iter, test_episodes, output_dir, output_iters):
log_file = os.path.join(output_dir, "log.txt")
self._logger = logger.Logger()
self._logger.configure_output_file(log_file)
model_file = os.path.join(output_dir, "model.ckpt")
iter = 0
total_train_return = 0
total_train_path_count = 0
test_return = 0
test_path_count = 0
start_time = time.time()
while (iter < max_iter):
train_return, train_path_count, new_sample_count = self._rollout_train(
self._samples_per_iter)
total_train_return += train_path_count * train_return
total_train_path_count += train_path_count
avg_train_return = total_train_return / total_train_path_count
total_samples = self.get_total_samples()
wall_time = time.time() - start_time
wall_time /= 60 * 60 # store time in hours
self._logger.log_tabular("Iteration", iter)
self._logger.log_tabular("Wall_Time", wall_time)
self._logger.log_tabular("Samples", total_samples)
self._logger.log_tabular("Train_Return", avg_train_return)
self._logger.log_tabular("Train_Paths", total_train_path_count)
self._logger.log_tabular("Test_Return", test_return)
self._logger.log_tabular("Test_Paths", test_path_count)
if (self._need_normalizer_update() and iter == 0):
self._update_normalizers()
self._update(iter, new_sample_count)
if (self._need_normalizer_update()):
self._update_normalizers()
if (iter % output_iters == 0):
test_return, test_path_count = self._rollout_test(
test_episodes, print_info=False)
self._logger.log_tabular("Test_Return", test_return)
self._logger.log_tabular("Test_Paths", test_path_count)
self.save_model(model_file)
self._logger.print_tabular()
self._logger.dump_tabular()
#total_train_return = 0
#total_train_path_count = 0
else:
self._logger.print_tabular()
iter += 1
return
def __init__(self, urls):
self.urls = urls
self.headers = config.DOWNLOADER_HEADERS
self.timeout = config.DOWNLOADER_TIMEOUT
self.size = config.DOWNLOADER_SIZE
self.proxy_count = config.DOWNLOADER_PROXY_COUNT
self.dbredis = dbredis.DBRedis()
self.logger = logger.Logger()
def __init__(self):
self.dbredis = dbredis.DBRedis()
self.logger = logger.Logger()
self.url = config.URL
with open(config.DATA_FILE, 'r') as datafile:
self.datas = [
data.replace('\n', '') for data in datafile.readlines()
]
def __init__(self):
self.host = config.REDIS_HOST
self.port = config.REDIS_PORT
self.redis = redis.StrictRedis(host=self.host, port=int(self.port), db=0, decode_responses=True)
self.proxy_name = config.REDIS_PROXY_NAME
self.start_name = config.REDIS_START_NAME
self.index_name = config.REDIS_INDEX_NAME
self.start_count = config.START_COUNT
self.index_count = config.INDEX_COUNT
self.logger = logger.Logger()
def __init__(self, key, headers={}, queries={}, verbose=False):
#required
self.key = key
#options
self.headers = headers
self.queries = queries
self.verbose = verbose
#config
self.logger = logger.Logger()
def startup(args):
log = logger.Logger('yiasa_log', 'logs')
log.log(logger.LogLevel.INFO, 'Yiasabot is starting. DebugMode: %s' % args.debug, force_print=True)
db = check_database(log)
if db is None:
return
if args.debug:
debug.debug(log, db)
else:
main.start(log, db, args)
def __init__(self, timestamp):
self.timestamp = timestamp
self.host = config.POSTGRE_HOST
self.port = config.POSTGRE_PORT
self.user = config.POSTGRE_USER
self.passwd = config.POSTGRE_PASSWD
self.dbname = config.POSTGRE_DBNAME
self.conn = psycopg2.connect(host=self.host,
port=self.port,
user=self.user,
password=self.passwd,
dbname=self.dbname)
self.logger = logger.Logger()
def __init__(self, sock, addr, command_class):
#start a thread
super().__init__(None, daemon=True)
#if the class logger has not been initialized yet, create it
try:
self.__class__.logger
except:
self.__class__.logger = logger.Logger("ClientHandler")
self.active_connections.append(self)
self.s = sock
self.addr = addr
self.commands = command_class(self)
self.sessid = None
self.connected = True
def __init__(self, args):
super(Trainer, self).__init__(args)
#### 0. Setup
self.save_dir = tools.set_save_dir(args)
with open(os.path.join(self.save_dir, "args.json"), "w") as j:
json.dump(vars(args), j)
#### 1. Models
model = getattr(models, args.model)(args)
print(
"Model param nums: ",
sum(p.numel() for p in model.parameters() if p.requires_grad),
)
self.model = model.cuda()
#### 2. Opt
self.optimizer = opts.get_optimizer(args, self.model)
self.scheduler = None
if self.args.lr_scheduler is not None:
self.scheduler = opts.get_scheduler(args, self.optimizer)
#### 3. Data
if args.augment is not None:
augmentation = getattr(augmentations, args.augment)
else:
augmentation = None
self.train_loader, self.val_loader = inputs.get_dataloader(
args, transform=augmentation)
#### 4. Logger
self.writer = writer.Writer(log_dir=self.save_dir)
self.logger = logger.Logger()
self.logger.open(os.path.join(self.save_dir, "log.train.txt"),
mode="a")
self.logger.write("\n>> Pytorch version: {}".format(torch.__version__))
self.logger.write("\n>> Args: {}".format(args))
# Validator
self.validator = Validator(
args,
is_trainval=True,
writer=self.writer,
val_loader=self.val_loader,
)
def __init__(self,
command_class,
host=socket.gethostbyname(socket.gethostname()),
port=5000,
max_clients=5):
try:
self.__class__.logger
except:
self.__class__.logger = logger.Logger("Server")
self.hostname = host
self.port = port
self.command_class = command_class
self.max_clients = max_clients
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.__running = True
self.start()
def __init__(self, args):
self.args = args
#### 0. Setup
self.save_dir = tools.set_save_dir(args)
with open(os.path.join(self.save_dir, "args.json"), "w") as j:
json.dump(vars(args), j)
#### 1. Data
# TODO: augmentation
augmentation = getattr(augmentations, args.augment)
self.train_loader, self.val_loader = my_input.get_dataloader(
args, transform=augmentation)
#### 2. Model
model = models.PENetClassifier(**vars(args))
model.load_pretrained(PRETRAINED_WEIGHTS, "0")
self.model = model.cuda()
#### 3. Opt
self.optimizer = opts.get_optimizer(args, self.model)
self.scheduler = None
if self.args.lr_scheduler is not None:
self.scheduler = opts.get_scheduler(args, self.optimizer)
#### 4. Logger
self.writer = writer.Writer(log_dir=self.save_dir)
self.logger = logger.Logger()
self.logger.open(os.path.join(self.save_dir, "log.train.txt"),
mode="a")
self.logger.write("\n>> Pytorch version: {}".format(torch.__version__))
self.logger.write("\n>> Args: {}".format(args))
# self.visualizer = visualizer.Visualizer(
# args, "train", self.save_dir, self.writer
# )
# Validator
self.validator = Validator(
args,
is_trainval=True,
writer=self.writer,
val_loader=self.val_loader,
)
# -*- coding: utf-8 -*-
import os
import sys
ROOT = os.getcwd()
sys.path.append(ROOT)
from util import logger
if __name__ == "__main__":
log = logger.Logger()
try:
print(10 / 0)
except Exception:
error = traceback.format_exc()
log.error(message=error)
dest="NoTimer",
action='store_true',
help="no timestamp should be displayed")
(options, args) = parser.parse_args()
return options
def detect_os():
operation_system = platform.system()
LOGGER.info("Detected " + operation_system + " as Operation System")
return operation_system
options = parse_start_arguments()
LOGGER = logger.Logger(options.NoTimer)
os = detect_os()
def start_unity_build_command():
LOGGER.info("Start Unity Build")
try:
build_command = options.UnityPath + " -projectPath " + options.ProjectPath + \
" -logfile " + options.LogPath + \
" -buildTarget " + options.Target + \
" -quit " \
"-batchmode " \
"-nographics " \
"-executeMethod " + options.ExecutionMethod
if os != "Windows":
process = subprocess.Popen(build_command,
def __init__(self, credential_path):
self.logger = logger.Logger(name=__name__)
self.twitter_client = self.build_twitter_client(credential_path)
args = my_args
np.random.seed(args.seed)
outputpath = os.path.join(args.output, args.id)
if not os.path.exists(args.output):
os.mkdir(args.output)
if not os.path.exists(outputpath):
os.mkdir(outputpath)
files = glob.glob(os.path.join(outputpath, '*'))
for f in files:
os.remove(f)
sys.stdout = logger.Logger(os.path.join(outputpath, 'out.log'))
print('-' * 50)
print('Run ID:', args.id)
print('Output directory:', os.path.join(args.output, args.id))
print('Solver:', args.solver)
print('Rounds:', args.rounds)
print('Maximum tries:', args.limit)
print('-' * 50)
if args.complexity:
complexityAnalysis(limit=args.limit,
matches=args.matches,
outputpath=outputpath,
rounds=args.rounds,
solver=args.solver,
import json
import time
from datetime import datetime as dt
from util import logger as Logger
from google.cloud import language
from google.cloud.language_v1 import enums
from google.cloud.language_v1 import types
logger = Logger.Logger(__name__)
def estimate_sentiment(client, sentence):
document = types.Document(content=sentence,
type=enums.Document.Type.PLAIN_TEXT,
language="ja_JP")
sentiment = client.analyze_sentiment(document=document).document_sentiment
logger.info("score: {}, magnitude: {}".format(sentiment.score,
sentiment.magnitude))
return sentiment
def build_client():
return language.LanguageServiceClient()
client = build_client()
with open("../zombie_2018-11-30.json.senti", encoding="utf8") as f:
lines = f.readlines()
result_set = []
try:
from util.doc_process import get_postgredb_query
from util.doc_process import filter_html_stopwords_pos
from util.doc_process import filter_tags
from util.doc_process import get_nids_sentences
import subject_queue
from util import simhash
import datetime
import os
import traceback
from multiprocessing import Pool
import jieba
from util import logger
real_dir_path = os.path.split(os.path.realpath(__file__))[0]
logger_9965 = logger.Logger('9965', os.path.join(real_dir_path, 'log/log_9965.txt'))
logger_9966 = logger.Logger('9966', os.path.join(real_dir_path, 'log/log_9966.txt'))
channel_for_multi_vp = ('科技', '外媒', '社会', '财经', '体育', '国际',
'娱乐', '养生', '育儿', '股票', '互联网', '健康',
'影视', '军事', '历史', '点集', '自媒体')
exclude_chnl = ['搞笑', '趣图', '美女', '萌宠', '时尚', '美食', '美文', '奇闻', '美食',
'旅游', '汽车', '游戏', '科学', '故事', '探索']
insert_sentence_hash = "insert into news_sentence_hash_cache (nid, sentence, sentence_id, hash_val, first_16, second_16, third_16, fourth_16, ctime, first2_16, second2_16, third2_16, fourth2_16) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
query_sen_sql = "select ns.nid, ns.hash_val from news_sentence_hash_cache ns inner join newslist_v2 nl on ns.nid=nl.nid " \
"where (first_16=%s or second_16=%s or third_16=%s or fourth_16=%s) and " \
"(first2_16=%s or second2_16=%s or third2_16=%s or fourth2_16=%s) and " \
"nl.state=0 group by ns.nid, ns.hash_val "
def __init__(self):
self.timestamp = time.strftime("%Y%m%d_%H%M%S",
time.localtime(time.time()))
self.dbredis = dbredis.DBRedis()
self.dbpostgresql = dbpostgresql.DBPostgresql(timestamp=self.timestamp)
self.logger = logger.Logger()
def run_auto(nav):
log = logger.Logger('Autonomous')
log.i("Starting autonomous path")
nav.default_speed = 0.65
nav.actuator('turret', {'action': 'home'})
# Request camera data
nav.actuator('webcamDetect', {})
time.sleep(0.5)
data = nav.sense('ringsSeen')
if len(data) < 1:
log.e("No ring data!")
return
rings_seen = data[0]
log.d("Got ring data: %d", rings_seen)
# Wobble 1
nav.move(-6, 0) # Move off the wall so we don't hit it
if rings_seen != 0:
# Go around the ring stack
nav.move(-28, -12)
if rings_seen == 0:
nav.move(-59, -15)
nav.turn(30)
elif rings_seen == 1:
nav.move(-84, 0)
nav.turn(0)
elif rings_seen == 4:
nav.move(-105, -13, speed=0.9)
nav.turn(25)
nav.actuator('shooter', {'action': 'start', 'speed': SHOOT_SPEED})
place_wobble(nav)
if rings_seen == 4:
nav.move(-60, 0, reverse=False, speed=0.9)
# Line up for shooting
nav.move(-58.5, 0, reverse=False)
nav.actuator('turret', {'action': 'rotate', 'angle': 0.16})
nav.turn(90)
if rings_seen == 0:
time.sleep(3)
else:
time.sleep(1.5)
shoot_rings(nav, 3)
if rings_seen == 1:
# Intake [experimental]
nav.actuator('turret', {'action': 'home'})
nav.actuator('intake', {'action': 'intake'})
nav.turn(10)
nav.move(-41, 5, reverse=False, speed=0.5)
time.sleep(1)
nav.actuator('intake', {'action': 'stop'})
# nav.actuator('turret', {'action': 'home'})
nav.move(-58.5, 1)
nav.turn(3)
shoot_rings(nav, 1)
nav.turn(90)
elif rings_seen == 4:
nav.actuator('turret', {'action': 'home'})
# nav.turn(-3)
nav.actuator('intake', {'action': 'intake', 'speed': -0.75})
nav.move(-46, 5, reverse=False, speed=0.85)
nav.actuator('intake', {'action': 'intake', 'speed': 0.82})
time.sleep(0.15)
nav.move(-37, 12, reverse=False, speed=0.45)
time.sleep(1)
nav.move(-58, 2)
nav.turn(0)
nav.actuator('intake', {'action': 'stop'})
shoot_rings(nav, 3)
nav.actuator('wobble', {'action': 'down'})
nav.actuator('shooter', {'action': 'stop'})
#if rings_seen == 4:
# second wobble
# nav.move(-41, 26)
#nav.move(-27.5, 16)
#nav.turn(-195)
#else:
# second wobble
if rings_seen == 0 or rings_seen == 1:
nav.move(-27.5, 17)
nav.turn(192)
time.sleep(0.75)
elif rings_seen == 4:
nav.move(-27, 18)
time.sleep(0.25)
pick_wobble(nav)
if rings_seen == 0:
nav.move(-55, -10)
nav.turn(50)
elif rings_seen == 1:
nav.move(-74, -1)
nav.turn(-9)
elif rings_seen == 4:
# nav.move(-47, 18)
nav.move(-103, -13, speed=0.9)
# nav.turn(-323)
place_wobble(nav)
nav.move(-76, 0, reverse=(rings_seen != 4))
log.i("Path complete")
# -*- coding: utf-8 -*-
# @Time : 16/12/23 上午10:37
# @Author : liulei
# @Brief : 定时任务
# @File : timed_task.py
# @Software: PyCharm Community Edition
from util.doc_process import get_postgredb
from redis_process import nid_queue
from graphlab_kmeans.kmeans_for_update import chnl_k_dict
import datetime
from datetime import timedelta
import os
from util import logger
real_dir_path = os.path.split(os.path.realpath(__file__))[0]
logger_9981 = logger.Logger('process9981', os.path.join(real_dir_path, 'log/log_9981.txt'))
#定义取用户点击的循环周期
period = 3
click_sql = "select c.uid, c.nid, c.ctime, c.itime from newsrecommendclick c \
inner join newslist_v2 nl on c.nid=nl.nid \
INNER JOIN channellist_v2 cl on nl.chid = cl.id \
where cname in ({0}) and c.itime > '{1}' order by c.itime "
#where cname in ({0}) and c.ctime > now() - INTERVAL '{1} second' "
#last_time = (datetime.datetime.now() - timedelta(seconds=3)).strftime('%Y-%m-%d %H:%M:%S.%f')
last_time = datetime.datetime.now() - timedelta(seconds=3)
channels = ', '.join("\'" + ch+"\'" for ch in chnl_k_dict.keys())
def get_clicks_5m():
# -*- coding: utf-8 -*-
# @Time : 16/12/23 上午10:37
# @Author : liulei
# @Brief : 定时任务
# @File : timed_task.py
# @Software: PyCharm Community Edition
from util.doc_process import get_postgredb
from redis_process import nid_queue
from data_process import channel_for_topic_dict
import datetime
from datetime import timedelta
from util import logger
import os
real_dir_path = os.path.split(os.path.realpath(__file__))[0]
logger_9989 = logger.Logger('process9989_3', os.path.join(real_dir_path, 'log/log_9989_3.txt'))
#定义取用户点击的循环周期
period = 3
#click_sql = "select uid, nid, ctime from newsrecommendclick where ctime > now() - INTERVAL '5 minute'"
click_sql = "select c.uid, c.nid, c.ctime, c.itime from newsrecommendclick c \
inner join newslist_v2 nl on c.nid=nl.nid \
INNER JOIN channellist_v2 cl on nl.chid = cl.id \
where cname in ({0}) and c.itime > '{1}' order by c.itime"
#where cname in ({0}) and c.ctime > now() - INTERVAL '{1} second' and c.stime>0"
#last_time = (datetime.datetime.now() - timedelta(seconds=3)).strftime('%Y-%m-%d %H:%M:%S.%f')
last_time = datetime.datetime.now() - timedelta(seconds=3)
channels = ', '.join("\'" + ch+"\'" for ch in channel_for_topic_dict.keys())
def get_clicks_5m():
logger_9989.info('news epoch...')
import os
import traceback
from util import logger
import pandas as pd
from util.doc_process import get_postgredb_query
from util.doc_process import get_postgredb
import datetime
import math
import json
from heapq import nlargest
from operator import itemgetter
TEST_FLAG = False
real_dir_path = os.path.split(os.path.realpath(__file__))[0] #文件所在路径
log_cf = logger.Logger('log_cf', os.path.join(real_dir_path, 'log', 'log.txt'))
log_cf_clear_data = logger.Logger('log_cf_clear_data', os.path.join(real_dir_path, 'log', 'log_clear_data.txt'))
################################################################################
#@brief: 获取最新的topic版本
################################################################################
def get_newest_topic_v():
topic_sql = "select model_v from user_topics_v2 group by model_v"
conn, cursor = get_postgredb_query()
cursor.execute(topic_sql)
rows = cursor.fetchall()
topic_vs = []
for row in rows:
topic_vs.append(row[0])
conn.close()
import pickle
FIXED_PARAMETERS, config = params.load_parameters()
modname = FIXED_PARAMETERS["model_name"]
if not os.path.exists(FIXED_PARAMETERS["log_path"]):
os.makedirs(FIXED_PARAMETERS["log_path"])
if not os.path.exists(config.tbpath):
os.makedirs(config.tbpath)
config.tbpath = FIXED_PARAMETERS["log_path"]
if config.test:
logpath = os.path.join(FIXED_PARAMETERS["log_path"], modname) + "_test.log"
else:
logpath = os.path.join(FIXED_PARAMETERS["log_path"], modname) + ".log"
logger = logger.Logger(logpath)
model = FIXED_PARAMETERS["model_type"]
module = importlib.import_module(".".join(['models', model]))
MyModel = getattr(module, 'MyModel')
# Logging parameter settings at each launch of training script
# This will help ensure nothing goes awry in reloading a model and we consistenyl use the same hyperparameter settings.
logger.Log("FIXED_PARAMETERS\n %s" % FIXED_PARAMETERS)
######################### LOAD DATA #############################
if config.debug_model:
# training_snli, dev_snli, test_snli, training_mnli, dev_matched, dev_mismatched, test_matched, test_mismatched = [],[],[],[],[],[], [], []
test_matched = load_nli_data(FIXED_PARAMETERS["dev_matched"],
class Commands:
class Endpoints:
endpoints = {}
@classmethod
def route(cls, route):
def wrapper(func):
cls.endpoints[route] = func
return func
return wrapper
logged_in = {}
#init logger
logger = logger.Logger("Commands", file=True, loglevel=logging.INFO, fileformatter=logging.Formatter("[%(asctime)s] %(user)s(%(uname)s) requested %(msg)s"))
def __init__(self, socket):
#read in dataset
self.dataset = pd.read_csv(path.join(path.dirname(__file__), "data/kepler.csv"))
self.socket = socket
@staticmethod
def get_endpoint_counts():
return {k: v.counter for k,v in Commands.Endpoints.endpoints.items()}
#return all koi object with a confirmed disposition
@Endpoints.route("confirmed")
@count_function
def get_confirmed(self):
return pickle.dumps(self.dataset[self.dataset['koi_disposition'] == 'CONFIRMED'])
#get koi objects by search query on the name. May be incomplete name
@Endpoints.route("kepler_name")
@count_function
def get_kepler_name(self, name):
return pickle.dumps(self.dataset[self.dataset['kepler_name'].str.contains(name, na=False, regex=False)])
#login the user and send back the session id
@Endpoints.route("login")
@count_function
def login(self, uname, fullname, email):
sessid = str(uuid.uuid4())
self.logged_in[sessid] = dict(username = uname, fullname = fullname, email = email, socket=self.socket)
return sessid
#log the user out, and remove the session from logged in
@Endpoints.route("logout")
@count_function
def logout(self, session_id):
if session_id in self.logged_in.keys():
self.logged_in.pop(session_id)
return 200
return 404
#filter by koi score
@Endpoints.route("koi_score")
@count_function
def get_koi_score(self, score, operand='lt'):
try:
#operands to be used
ops = {
'lt': operator.lt,
'le': operator.le,
'eq': operator.eq,
'ge': operator.ge,
'gt': operator.gt
}
score = float(score)
#if score out of range, raise error
if score > 1 or score < 0:
raise ValueError()
#return the filtered data as a pickled pandas dataframe
return pickle.dumps(self.dataset[ops[operand.lower()](self.dataset['koi_score'], score)])
except Exception as e:
logging.error(e)
return 400 #will mainly trigger if operand is not defined
#get a countplot of the koi dispositions
@Endpoints.route("countplot")
@count_function
def countplot(self):
#generate a graph, and save as a temporary image
temp_id = uuid.uuid4()
dirpath = path.dirname(__file__) + f"/temp"
fp = dirpath + f"/{temp_id}.png"
dataset = self.dataset.copy()
dataset['koi_disposition'].dropna(axis=0, inplace=True)
sns.countplot(data=dataset, x='koi_disposition')
if not path.exists(dirpath):
os.mkdir(dirpath)
del dataset
#reset pyplot and save the graph
plt.savefig(fp)
plt.close('all')
#keep trying to open the image, may cause errors because pyplot is still saving it
while True:
try:
with open(fp, 'rb') as f:
data = f.readlines()
f.close()
if data: #once the image is read, exit loop
break
except:
pass
#try to remove the temp image to save memory on disk
while True:
try:
os.remove(fp)
print(os.path.exists(fp))
if not path.exists(fp):
break
except Exception as e:
print(e)
#return image bytes
return data
#get all possible column filters for scatterplot
@Endpoints.route("column_names")
@count_function
def get_columns(self):
return list(self.dataset.columns)
#plot 2 columns in a scatterplot to analyze correlation
@Endpoints.route("scatterplot")
@count_function
def scatterplot(self, x='koi_teq', y='koi_srad'):
#same method of working as countplot
temp_id = uuid.uuid4()
dirpath = path.dirname(__file__) + f"/temp"
fp = dirpath + f"/{temp_id}.png"
dataset = self.dataset[[x, y]].dropna(axis=0, inplace=False)
plt.scatter(dataset[x], dataset[y])
plt.xlabel(x)
plt.ylabel(y)
if not path.exists(dirpath):
os.mkdir(dirpath)
plt.savefig(fp)
plt.close('all')
while True:
try:
with open(fp, 'rb') as f:
data = f.readlines()
f.close()
if data:
break
except:
pass
while True:
try:
os.remove(fp)
print(os.path.exists(fp))
if not path.exists(fp):
break
except Exception as e:
print(e)
return data
