from Data.DataManager import DataManager
from ScriptToolkit import ScriptToolkit
from Preprocessing.DataReader import DataReader
from Preprocessing import ProcessorFactory
from Model.ConditionalRandomField import CRF
if __name__ == '__main__':
# create data manager
DM = DataManager()
DM.change_pwd()
DM.source_data_file = 'CorpusLabelData_SalesModule.txt'
DM.remove(DM.log_wrong_sentences)
# create datums
DR = DataReader(source_data_file=DM.source_data_file)
DR.standard_read()
# create toolkits
ST = ScriptToolkit(DM)
features = ScriptToolkit.get_demo_features()
# analysis
sent_accuracys, train_times, test_times = [], [], []
cycle_times = 30
for i in range(cycle_times):
# data preprocessing
crf_processor = ProcessorFactory.CRFProcessorFactory().produce(
source_data_file=DM.source_data_file,
train_file=DM.train_file,
test_file=DM.test_file)
crf_processor.get_train_data(DR.Datums)
import Model
from Data.DataManager import DataManager
import numpy as np
import pandas as pd
# Dash and plotly
import dash
import plotly.express as px
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output, State
import plotly.graph_objects as go
### Variables ###
dm = DataManager()
options = [{'label' : stock, 'value' : stock, 'disabled' : not available} for (stock,available) in
zip(dm.retrieveInformation(columns="Kortnamn"),dm.priceAvailable())]
d = dm.getOptionsAvailable()
stock_info = dm.retrieveInformation()
app = dash.Dash(__name__, external_stylesheets=["Datascience\Projects\Effective frontier\Assets\Layout.css"])
app.layout = html.Div( children=[
### Top part ###
html.Div(children = [
html.H1("Stocks",
style={
'textAlign': 'center',
'color':"#ffffff"
from Data.DataManager import DataManager
from ScriptToolkit import ScriptToolkit
from Preprocessing.DataReader import DataReader
from Preprocessing import ProcessorFactory
from Model.ConditionalRandomField import CRF
if __name__ == '__main__':
# create data manager
DM = DataManager()
DM.change_pwd()
DM.source_data_file = 'CorpusLabelData_MergedFilter.txt'
DM.remove(DM.log_wrong_sentences)
# create datums
DR = DataReader(source_data_file=DM.source_data_file)
DR.standard_read()
# create toolkits
ST = ScriptToolkit(DM)
features = ScriptToolkit.get_demo_features()
# analysis
sent_accuracys = []
cycle_times = 1
for i in range(cycle_times):
# data preprocessing
# crf_processor = ProcessorFactory.CRFProcessorFactory().produce(source_data_file=DM.source_data_file,
# train_file=DM.train_file, test_file=DM.test_file)
# crf_processor.get_train_data(DR.Datums)
null2o_processor = ProcessorFactory.ReplaceNullWithOPreprocessorFactory(
).produce(train_file=DM.train_file, test_file=DM.test_file)
from Preprocessing import Preprocessor
from Data.DataManager import DataManager
class Provider(object):
def produce(self, **kw):
pass
class CRFProcessorFactory(Provider):
def produce(self, **kw):
return Preprocessor.CRFPreprocessor(**kw)
class LSTMProcessorFactory(Provider):
def produce(self, **kw):
return Preprocessor.LSTMPreprocessor(**kw)
if __name__ == '__main__':
DM = DataManager()
DM.change_pwd()
DM.source_data_file = 'CorpusLabelData_MergedFilter_Update.txt'
crf_factory = CRFProcessorFactory()
crf_processor = crf_factory.produce(source_data_file=DM.source_data_file,
train_file=DM.train_file,
test_file=DM.test_file)
crf_processor.preprocess()
crf_processor.get_train_data()
from Data.DataManager import DataManager
from Preprocessing import ProcessorFactory
from Model.ConditionalRandomField import CRF
from ScriptToolkit import ScriptToolkit
from Preprocessing.DataReader import DataReader
if __name__ == '__main__':
# create data manager
DM = DataManager()
DM.change_pwd()
DM.remove(DM.log_wrong_sentences)
# create toolkits
ST = ScriptToolkit(DM)
features = ScriptToolkit.get_demo_features()
# analysis
cycle_times = 1
sent_accuracys, sent_accuracys_f = [], []
for i in range(cycle_times):
DM.source_data_file = 'CorpusLabelData_MergedFilter.txt' # change source data as old corpus
# create datums
DR = DataReader(source_data_file=DM.source_data_file)
DR.standard_read()
# data preprocessing
crf_processor = ProcessorFactory.CRFProcessorFactory().produce(
source_data_file=DM.source_data_file,
train_file=DM.train_file,
test_file=DM.test_file)
crf_processor.get_train_data(DR.Datums)
# -*- coding:utf-8 -*-
from sys import argv
from Data.DataManager import DataManager
from Scripts.ScriptToolkit import ScriptToolkit
from Preprocessing.DataReader import DataReader
from Preprocessing import ProcessorFactory
from Model.ConditionalRandomField import CRF
if __name__ == '__main__':
# create data manager
DM = DataManager()
DM.change_pwd()
DM.source_data_file = argv[
argv.index("-source") +
1] if "-source" in argv else 'CorpusLabelData_MergedFilter.txt'
DM.remove(DM.log_wrong_sentences)
# create datums
DR = DataReader(source_data_file=DM.source_data_file)
DR.standard_read()
# create toolkits
ST = ScriptToolkit(DM)
features = ScriptToolkit.get_demo_features()
# analysis
sent_accuracys, train_times, test_times = [], [], []
cycle_times = int(argv[argv.index("-iter") + 1]) if "-iter" in argv else 10
for i in range(cycle_times):
print "This is the %d-th experiments." % (i + 1)
from Scripts.ScriptToolkit import ScriptToolkit
from Preprocessing.DataReader import DataReader
from Data.DataManager import DataManager
if __name__ == '__main__':
# create data manager
DM = DataManager()
DM.change_pwd()
DM.source_data_file = 'CorpusLabelData_SalesModule.txt'
# create datums
DR = DataReader(source_data_file=DM.source_data_file)
DR.standard_read()
# create toolkits
ST = ScriptToolkit(DM)
# analysis
sentences_amount, tokens_amount, tokens_distribution, glabels_distribution = ScriptToolkit.StatisticDatums(
DR.Datums)
# result display
print 'Sentences Amount is %d' % (sentences_amount)
print 'Tokens Amount is %d' % (tokens_amount)
print 'Tokens distribution is:'
print tokens_distribution
print 'glabels_distribution is:'
print glabels_distribution
import os
import csv
import matplotlib.pyplot as plt
from Core.Stock import Stock
from Data.DataManager import DataManager
dm = DataManager()
data_manager = dm.get_instance()
issuer_list = data_manager.get_issuers()
for issuer in issuer_list:
print("\n{} {} {}\n".format(issuer.id, issuer.ticker, issuer.name))
stock_list = data_manager.get_stocks_by_issuer_id(issuer.id)
high_stocks = []
for stock in stock_list:
print("{} {} {} {} {} {} {} {} {}".format(stock.id, stock.ticker, stock.per, stock.trade_date, stock.open, stock.high, stock.low, stock.close, stock.volume))
rpl = Stock.nth_repl_all(stock.high, '.', '', 2)
high_stocks.append(float(rpl))
plt.plot(high_stocks)
plt.show()
print("\nCompleted")
import os
from Data.DataManager import DataManager
from Preprocessing import ProcessorFactory
from Model.ConditionalRandomField import CRF
from Scripts.ScriptToolkit import ScriptToolkit
from Preprocessing.DataReader import DataReader
if __name__ == '__main__':
# create data manager
DM = DataManager()
DM.change_pwd()
DM.source_data_file = 'CorpusLabelData_SalesModule.txt'
DM.remove(DM.features_train)
DM.remove(DM.features_test)
# create datums
DR = DataReader(source_data_file=DM.source_data_file)
DR.standard_read()
# create toolkits
ST = ScriptToolkit(DM)
features = ScriptToolkit.get_demo_features()
# feature setting
features['printFeatures'] = '1'
feature_sets = [features]
# feature_sets = ScriptToolkit.get_custom_features('custom_features.txt')
train_times = []
sent_accuracys = []
for features in feature_sets:
def verify_by_sentence(self, sentence):
pass
def verify(self, file=''):
if not file:
file = self.test_file
command = self.command_line + ' -loadClassifier ' + self.model_filename + ' -testFile ' + file
cmd = shlex.split(command)
sout, serr = java(cmd, classpath=self.path_to_jar, stdout=PIPE, stderr=PIPE)
print '--------------------TEST------------------------'
print serr
return sout, serr
def train_and_verify(self, train_file='', test_file=''):
if not train_file: train_file = self.train_file
if not test_file: test_file = self.test_file
sout_train, serr_train = self.train(train_file)
sout_test, serr_test = self.verify(test_file)
return sout_train, serr_train, sout_test, serr_test
if __name__ == '__main__':
DM = DataManager()
DM.change_pwd()
crf_test = CRF(path_to_jar=DM.path_to_jar, prop_file=DM.prop_file, model_file=DM.model_file,
train_file=DM.train_file, test_file=DM.test_file)
crf_test.feature_config()
crf_test.train()
crf_test.verify()
from mesa.visualization.ModularVisualization import ModularServer
from Data.DataManager import DataManager
from Market.Shelf import Shelf
from agents.Checkout import Checkout
from agents.ShelfAgent import ShelfAgent
from model.MarketModel import MarketModel
from model.handlers import CustomersHandler, ColorChangeHandler, AgentCountsHandler
article_dictionary = {}
width = 50
height = 50
market = DataManager.initialize_market()
def agent_portrayal(agent):
if type(agent) is ShelfAgent:
portrayal = {
"Shape": "rect",
"Color": "blue",
"Filled": "true",
"Layer": 0,
"w": 0.9,
"h": 0.9
}
elif type(agent) is Checkout:
portrayal = {