def load_data_locally(self):
try:
self.heroes = DataManager().load_hero_data()
except IOError:
info = QtGui.QMessageBox(
'Datafile not found',
'I couldn\'t load the data file from within the current directory, do you want me to download it from the website?',
QtGui.QMessageBox.Information, QtGui.QMessageBox.Ok,
QtGui.QMessageBox.Cancel, QtGui.QMessageBox.NoButton, self)
if info.exec_() == QtGui.QMessageBox.Ok:
self.download_data()
else:
self.buttons = []
layout = QtGui.QGridLayout()
layout.addWidget(self.create_searchfield(), 0, 13, 1, 3)
for i, hero in enumerate(self.heroes):
pb = HeroButton(hero, self)
pb.left_click.connect(
self.parentWidget().parentWidget().hero_selected)
pb.right_click.connect(
self.parentWidget().parentWidget().show_hero_info)
layout.addWidget(pb, 1 + i / HeroPoolView.heroes_per_row,
i % HeroPoolView.heroes_per_row)
self.buttons.append(pb)
self.setLayout(layout)
def __init__(self, argv):
self.config = yaml.load(open(".peergovrc", "r").read())
self.basedir = self.getConfig('basedir')
self.datadir = self.getConfig('datadir')
self.authdir = self.getConfig('authdir')
self.user = self.getConfig(
'userfpr'
) #TODO: confirm that fingerprint actually exists in key ring
self.port = self.getConfig('port')
self.authorizations = {}
self.currentAuthorization = None #authorization to vote on currentTopic
self.voting = SchulzeVoting.SchulzeVoting()
self.manager = DataManager()
self.manager.datadir = self.datadir
self.cctx = pyme.core.Context() #crypto context
self.ensureDirExists(self.basedir)
self.ensureDirExists(self.datadir)
self.ensureDirExists(self.authdir)
for root, dirs, files in os.walk(self.datadir):
for dir in dirs:
self.loadTopic(root + "/" + dir)
for file in files:
self.loadData(root, file)
for root, dirs, files in os.walk(self.authdir):
for file in files:
self.loadAuth(root, file)
if hasGui:
self.initGui()
self.peermanager = PeerManager(argv, peergov=self)
def __init__(self, ground_ip):
self.status_vector = dict()
self.command_vector = dict()
self.ground_ip = ground_ip
self.info_logger = InfoLogger()
self.data_logger = DataLogger()
self.adcs_logger = AdcsLogger()
#@TODO where antenna to start
#self.adcs_logger.write_info(' {}, {}, {}, {}'.format(0, 0, 0, 0))
self.elink = elinkmanager.ELinkManager(self, self.ground_ip)
self.thread_elink = None
self.data_manager = DataManager(self, self.info_logger,
self.data_logger)
self.thread_data_manager = None
self.dmc = dmc.DMC(self)
self.thread_dmc = None
self.heat = heat.HEAT(self)
self.thread_heat = None
self.adc = adc.ADC(self)
self.thread_adc = None
self.tx = tx.TX(self)
self.thread_tx = None
self.counterdown = CounterDown(self)
self.paths = paths.Paths()
GPIO.setmode(GPIO.BOARD)
Master.__instance = self
def test_integracion(self):
DM = DataManager(os.environ['DB_URI'], 'test',
'test_shops_integration')
shop = {
"nombre": "App Store",
"descripcion": "Tienda de electronica",
"direccion": "avd Barcelona",
"info": "applesupport",
"productos": []
}
product = {
"nombre_p": "Iphone 6",
"descripcion_p": "Iphone 6 64GB",
"precio": 700,
"cantidad": 5,
"online": ""
}
self.assertEqual(DM.save_one(shop), True)
self.assertEqual(DM.insert_product(shop['nombre'], product), True)
self.assertEqual(DM.find_product(shop['nombre'], product['nombre_p']),
product)
product = {
"nombre_p": "Iphone 6",
"descripcion_p": "Iphone 6 64GB Plata"
}
self.assertEqual(DM.update_product(shop['nombre'], product), True)
self.assertEqual(
DM.delete_product(shop['nombre'], product['nombre_p']), True)
DM.getCollection().drop()
def pre_prosseccing(self):
dm = DataManager()
data = dm.select_all_db('eco_news_data')
data = data['제목']
with open(self.model_path + '/model.documents', 'wb') as f:
pickle.dump(data, f)
# # 수정된 job_title에서 posting_id 가지고 오기
# posting_ids = data['posting_id']
# posting_list = posting_ids.to_list()
#
# # posting_id에 따라 description_data set 만들기
# des_data = [data['job_description'][id] for id in posting_ids]
# title_data = [data['job_title'][id] for id in posting_ids]
# id_list = [i for i in range(len(posting_list))]
# df = pd.DataFrame({'id': posting_list, 'job_title': title_data, 'job_description': des_data, 'posting_id':posting_list})
# df.to_csv('data/doc2vec_test_data/0702/merge_0629_adj.csv', mode='w', encoding='utf-8')
# 수정된 description set 불러와 데이터 전처리 수행
# data = dm.load_csv(file='data/doc2vec_test_data/0702/merge_0629_adj.csv', encoding='utf-8')
sentences = self.data_text_cleansing(data)
data_words = list(self.sent_to_words(sentences))
# data_words_nostops = self.remove_stopwords(data_words)
# data_lemmatized = self.lematization(data_words)
# print(data_lemmatized)
# bigram = self.make_bigram(data_lemmatized)
## 형태소 분석을 먼저 수행한 후 bigram을 만들어야 함
data_lemmatized = self.lematization(data_words)
trigram = self.make_bigram(data_lemmatized, trigram_check=1)
with open(self.model_path + '/model.corpus', 'wb') as f:
pickle.dump(trigram, f)
return trigram
def Application():
import interrupt
e = interrupt.GetInterruptEvent()
while not e.wait(0.5):
managers = []
mgr1 = DataManager()
mgr2 = RequestManager()
mgr1.connect_to(mgr2)
mgr2.connect_to(mgr1)
managers.append(mgr1)
managers.append(mgr2)
for mgr in managers:
mgr.start()
print 'In main run loops...'
while not e.wait(0.5):
for mgr in managers:
if not mgr.is_alive():
print mgr.name, 'crashed. Restarting...'
pass
print 'Stopping managers...'
for mgr in managers:
mgr.stop()
def run(data_config, n_epochs, num_kernels, kernel_size, dense_hidden_size,
use_maxpool, use_torch_conv_layer):
"""
Runs a training regime for a CNN.
"""
train_set = DataPartition(data_config, './data', 'train')
test_set = DataPartition(data_config, './data', 'test')
manager = DataManager(train_set, test_set)
loss = nlog_softmax_loss
learning_rate = .001
image_width = 64
net = create_cnn(num_kernels=num_kernels,
kernel_size=kernel_size,
output_classes=2,
image_width=image_width,
dense_hidden_size=dense_hidden_size,
use_maxpool=use_maxpool,
use_torch_conv_layer=use_torch_conv_layer)
optimizer = optim.Adam(net.parameters(), lr=learning_rate)
best_net, monitor = minibatch_training(net,
manager,
batch_size=32,
n_epochs=n_epochs,
optimizer=optimizer,
loss=loss)
classifier = Classifier(best_net, num_kernels, kernel_size,
dense_hidden_size, manager.categories, image_width)
return classifier, monitor
def runAcquisition(self, T_val=None):
packetIdx: int = 0
command: int = 0 # 0 equals request a packet
# set time value (TE/TI)
if T_val is not None:
self.setTval(T_val)
# Get/construct package to be send
tmp_sequence_pack = ComMngr.constructSequencePacket(
self.operation) # uses self.operation.sequencebytestream
tmp_scanparam_pack = ComMngr.constructScanParameterPacket(
self.operation) # uses self.operation.scanparameters.f_Ex
tmp_package = {
**tmp_sequence_pack,
**tmp_sequence_pack,
**tmp_scanparam_pack
}
fields = [command, packetIdx, 0, self.version, tmp_package]
response = ComMngr.sendPacket(fields)
if response is None:
self.parent.OpMngr.setOutput(
"Console not connected. Nothing received.")
self.haveResult = False
return
self.haveResult = True
# get the actual data
tmp_data = np.frombuffer(response[4]['acq'], np.complex64)
print("Size of received data: {}".format(len(tmp_data)))
self.dataobject: DataManager = DataManager(tmp_data, self.f_Ex,
self.numSamples)
def set_library_window_values(force=False):
log.debug("set_library_window_values Called forced={0}", force)
home_window = HomeWindow()
already_set = home_window.getProperty("view_item.0.name")
if not force and already_set:
return
for index in range(0, 20):
home_window.clearProperty("view_item.%i.name" % index)
home_window.clearProperty("view_item.%i.id" % index)
home_window.clearProperty("view_item.%i.type" % index)
home_window.clearProperty("view_item.%i.thumb" % index)
data_manager = DataManager()
url = "{server}/emby/Users/{userid}/Views"
result = data_manager.GetContent(url)
if result is None:
return
result = result.get("Items")
server = downloadUtils.getServer()
index = 0
for item in result:
type = item.get("CollectionType")
if type in ["movies", "boxsets", "music", "tvshows"]:
name = item.get("Name")
id = item.get("Id")
# plugin.video.jellyfincon-
prop_name = "view_item.%i.name" % index
home_window.setProperty(prop_name, name)
log.debug(
"set_library_window_values: plugin.video.jellyfincon-{0}={1}",
prop_name, name)
prop_name = "view_item.%i.id" % index
home_window.setProperty(prop_name, id)
log.debug(
"set_library_window_values: plugin.video.jellyfincon-{0}={1}",
prop_name, id)
prop_name = "view_item.%i.type" % index
home_window.setProperty(prop_name, type)
log.debug(
"set_library_window_values: plugin.video.jellyfincon-{0}={1}",
prop_name, type)
thumb = downloadUtils.getArtwork(item, "Primary", server=server)
prop_name = "view_item.%i.thumb" % index
home_window.setProperty(prop_name, thumb)
log.debug(
"set_library_window_values: plugin.video.jellyfincon-{0}={1}",
prop_name, thumb)
index += 1
def __init__(self, width, height):
self.width = width
self.height = height
self.world = World(100, 100)
self.dm = DataManager()
newObject = self.dm.createInstance("mobile rectangle", "sims")
self.world.addToWorld(newObject, 20, 20, 10, 5)
def __init__(self, master):
Frame.__init__(self, master)
Frame.configure(self)
db = DataManager(nom_fichier_bdd)
self.liste_film = db.recup_films_reponses()
self.question_freq_absolue = questions_maximisant_esperance(
self.liste_film)
self.liste_questions_posee = list()
self.liste_question_repondue = list()
self.liste_reponse = list()
self.liste_questions_posee.append(10)
self.liste_questions_posee.append(11)
self.question_ideale = question_ideale(self.liste_film,
self.liste_questions_posee)
self.liste_filtre = self.liste_film
message_jeu = Label(self,
text="Jeu",
font=("Courrier", 30, "bold"),
foreground="purple")
message_jeu.grid(row=1, column=1)
self.question = Label(self, text=self.question_ideale[1], pady=10)
self.question.grid(row=2, column=1, pady=20)
self.bouton_oui = Button(self,
text="Oui",
foreground="black",
background="green",
width=10,
height=3,
command=self.repondre_oui)
self.bouton_oui.grid(row=3, column=1, padx=20)
self.bouton_jsp = Button(self,
text="Je ne sais pas",
width=10,
height=3,
command=self.repondre_jsp)
self.bouton_jsp.grid(row=4, column=1, padx=20)
self.bouton_non = Button(self,
text="Non",
foreground="black",
background="red",
width=10,
height=3,
command=self.repondre_non)
self.bouton_non.grid(row=5, column=1, padx=20)
self.bouton_retour = Button(
self,
text="Retour",
foreground="purple",
background="pink",
command=lambda: master.switch_frame(StartPage))
self.bouton_retour.grid(row=0, column=0, padx=1, pady=5)
def test_metodos(self):
DM = DataManager(os.environ['DB_URI'], 'test', 'test_shops')
shop = {
"nombre": "Cascabeles",
"descripcion": "Tienda de musica",
"direccion": "avd Barcelona",
"info": "111111111"
}
shop2 = {
"nombre": "Tecmies",
"descripcion": "Tienda de moviles",
"direccion": "CC Arcangel",
"info": "angeljimez"
}
shop3 = {
"nombre": "Arteaga",
"descripcion": "Tienda de deportes",
"direccion": "avd Barcelona",
"info": ""
}
#shops =[shop, shop2, shop3]
self.assertEqual(DM.save_one(shop), True)
self.assertEqual(DM.save_one(shop2), True)
self.assertEqual(DM.save_one(shop3), True)
#self.assertEqual(DM.find_all(),shops)
shop4 = DM.find_shop(shop['nombre'])
self.assertEqual(shop4['nombre'], shop['nombre'])
self.assertEqual(len(DM.find_all()),
3) # poner los documentos de la bd
self.assertEqual(DM.save_one(shop), False)
self.assertEqual(len(DM.find_all()), 3)
shop = {
"nombre": "Cascabeles",
"descripcion": "Tienda de musica y arte",
"direccion": "avd Barcelona",
"info": "111111111"
}
self.assertEqual(DM.update_one(shop), True)
shop4 = DM.find_shop(shop['nombre'])
self.assertEqual(shop4, shop)
self.assertEqual(DM.isShop(shop['nombre']), True)
self.assertEqual(DM.delete_one(shop['nombre']), True)
self.assertEqual(DM.delete_one(shop2['nombre']), True)
self.assertEqual(DM.delete_one(shop3['nombre']), True)
self.assertEqual(DM.find_all(), False)
DM.getCollection().drop()
def get_stop_words(self):
print(' -> Getting stop word list...')
file = 'stopwords_list.csv'
stop_words_list = []
if os.path.isfile(self.data_path+file):
print(' -> Stop Words File is found')
dm = DataManager()
df = dm.load_csv(file=self.data_path + file, encoding='utf-8')
stop_words_list = df['Stopwords'].tolist()
else:
print(' -> Stop Words File is not found')
return stop_words_list
def create():
"""Create the API endpoints."""
cors = CORS(allow_all_origins=True,allow_all_methods=True, allow_all_headers=True)
app = falcon.API(middleware=[cors.middleware, MultipartMiddleware()])
dm = DataManager(InMemoryStorage(), InMemoryStorage(), data_folder, lcdURL, lcdPort)
app.add_route('/annotatedFiles', AnnotatedFiles(dm))
app.add_route('/annotatedFiles/json', AnnotatedFilesJSON(dm))
app.add_route('/publications', GetPublications(dm))
app.add_route('/expression', CubeExpressions(dm))
app.add_route('/corpus', CubeCorpora(dm))
app.add_route('/genre', CubeGenres(dm))
app.add_route('/function', CubeFunctions(dm))
app.add_route('/corpus/groups', GroupCorpora(dm))
app.add_route('/expression/groups', GroupExpressions(dm))
app.add_route('/function/groups', GroupFunctions(dm))
app.add_route('/genre/groups', GroupGenres(dm))
app.add_route('/groups', Groups(dm))
app.add_route('/obs/filtered', FilteredObservations(dm))
app.add_route('/obs/filtered/query', QueryFilteredObservations(dm))
app.add_route('/obs/filtered/result', FilteredResultObservations(dm))
app.add_route('/obs/filtered/preview', FilteredObservationsPreview(dm))
app.add_route('/infer', Infer(dm))
app.add_route('/cc/filtered', CCFiltered(dm))
app.add_route('/normalize', Normalize(dm))
app.add_route('/obs/norm2', CreateNormalizedCube(dm))
app.add_route('/obs/norm/query', QueryNormalizedCube(dm))
app.add_route('/obs/norm/defs', NormalizedCubeDefinitions(dm))
app.add_route('/obs/excluded', ExcludedObservations(dm))
app.add_route('/pub/excluded', ExcludedPublications(dm))
app.add_route('/image', Image(image_folder))
app.add_route('/lcd/status', CheckForLCDConnection(dm))
main_logger.info('App2 is running.')
return app
def get_including_words(self):
print(' -> Getting including word list...')
file = 'including_words_list.csv'
including_words_list = []
if os.path.isfile(self.data_path+file):
print(' -> Including Words File is found')
dm = DataManager()
df = dm.load_csv(file=self.data_path+file, encoding='utf-8')
including_words_list = df['Includingwords'].tolist()
else:
print(' -> Including Words File is not found')
print(including_words_list)
return including_words_list
def get_including_words(self, path):
file = 'including_words_list.csv'
including_words_list = []
if os.path.isfile(path + '/' + file):
print(' ..Including Words File is found..')
dm = DataManager()
df = dm.load_csv(file=path + 'including_words_list.csv',
encoding='utf-8')
including_words_list = df['Includingwords'].tolist()
else:
print(' ..Including Words File is not found..')
print(including_words_list)
return including_words_list
def get_stop_words(self, path):
file = 'stopwords_list.csv'
stop_words_list = []
if os.path.isfile(path + '/' + file):
print(' ..Stop Words File is found..')
dm = DataManager()
df = dm.load_csv(
file='data/doc2vec_test_data/0702/stopwords_list.csv',
encoding='utf-8')
stop_words_list = df['Stopwords'].tolist()
else:
print(' ..Stop Words File is not found..')
return stop_words_list
def __init__(self, master, borne_inf, borne_sup):
Frame.__init__(self, master)
Frame.configure(self)
self.db = DataManager(nom_fichier_bdd)
self.borne_inf = borne_inf
self.borne_sup = borne_sup
self.nb_filmmax = borne_sup - borne_inf
self.nb_filmfait = 0
self.questions_non_repondue = self.db.recup_questions_non_repondues(
borne_inf, borne_sup)
self.cles_films = list()
self.cles_questions = list()
for (cle1, cle2) in self.questions_non_repondue.keys():
self.cles_films.append(cle1)
self.cles_questions.append(cle2)
self.i_film = 0
self.i_question = 0
self.message_jeu = Label(self,
text=self.questions_non_repondue.get(
(self.cles_films[0],
self.cles_questions[0])),
font=("Courrier", 12, "bold"),
foreground="black")
self.message_jeu.grid(row=1, column=1)
self.bouton_oui = Button(self,
text="Oui",
foreground="black",
background="green",
width=10,
height=3,
command=self.repondre_oui)
self.bouton_oui.grid(row=5, column=1, padx=20)
self.bouton_non = Button(self,
text="Non",
foreground="black",
background="red",
width=10,
height=3,
command=self.repondre_non)
self.bouton_non.grid(row=6, column=1, padx=20)
self.bouton_retour = Button(
self,
text="Retour",
foreground="purple",
background="pink",
command=lambda: master.switch_frame(StartPage))
self.bouton_retour.grid(row=0, column=0, padx=1, pady=5)
def playListOfItems(id_list, monitor):
log.debug("Loading all items in the list")
data_manager = DataManager()
items = []
for id in id_list:
url = "{server}/emby/Users/{userid}/Items/" + id + "?format=json"
result = data_manager.GetContent(url)
if result is None:
log.debug("Playfile item was None, so can not play!")
return
items.append(result)
return playAllFiles(items, monitor)
def __init__(self, globalSession, name):
self.name = name
self.globalSession = globalSession
self.settings = self.globalSession.settings
self.dataManager = DataManager(self.settings, self.globalSession)
self.fitted = False
self.acquisitionDone = False
self.dataInMemory = False
# initialize file manager
self.fileManager = FileManager(self.settings)
def getLarmor(self):
print('Centering frequency by additional acquisition.')
# make sure there is an operation and an f_Ex
# f_Ex = frequency of excitation; f_Larmor = maximum of response
if self.AcqMngr is not None:
if hasattr(self.AcqMngr, 'operation'):
if self.AcqMngr.operation is not None:
self.f_Ex = self.AcqMngr.operation.scanparameters[
nmspc.f_Ex][0]
if self.operation is None:
self.operation = self.AcqMngr.operation
if not hasattr(self, 'f_Ex') and self.operation is not None:
self.f_Ex = self.operation.scanparameters[nmspc.f_Ex][0]
if self.operation is None:
self.operation = defaultoperations['FID Spectrum']
self.f_Ex = self.operation.scanparameters[nmspc.f_Ex][0]
# do an acquisition
packetIdx: int = 0
command: int = 0 # 0 equals request a packet
version = (1 << 16) | (1 << 8) | 1 # needs a version to work
self.numSamples = self.operation.numSamplesPerTimeValue
tmp_sequence_pack = ComMngr.constructSequencePacket(
self.operation) # uses self.operation.sequencebytestream
tmp_scanparam_pack = ComMngr.constructScanParameterPacket(
self.operation) # uses self.operation.scanparameters.f_Ex
tmp_package = {
**tmp_sequence_pack,
**tmp_sequence_pack,
**tmp_scanparam_pack
}
fields = [command, packetIdx, 0, version, tmp_package]
response = ComMngr.sendPacket(fields)
if response is None:
print("Nothing received. Frequency centering abandoned.")
return
# get the actual data
tmp_data = np.frombuffer(response[4]['acq'], np.complex64)
print("Size of received data: {}".format(len(tmp_data)))
dataobject: DataManager = DataManager(tmp_data, self.f_Ex,
self.numSamples)
[_, _, self.f_Larmor, _] = dataobject.get_peakparameters()
self.setLarmor()
def toggle_watched(params):
log.debug("toggle_watched: {0}", params)
item_id = params.get("item_id", None)
if item_id is None:
return
url = "{server}/emby/Users/{userid}/Items/" + item_id + "?format=json"
data_manager = DataManager()
result = data_manager.GetContent(url)
log.debug("toggle_watched item info: {0}", result)
user_data = result.get("UserData", None)
if user_data is None:
return
if user_data.get("Played", False) is False:
markWatched(item_id)
else:
markUnwatched(item_id)
def __init__(self):
self.status_vector = dict()
self.command_vector = dict()
self.info_logger = InfoLogger()
self.data_logger = DataLogger()
self.data_manager = DataManager(self, self.info_logger,
self.data_logger)
self.thread_data_manager = None
self.heat = heat.HEAT_HAI(self)
self.thread_heat = None
self.counterdown = CounterDown(self)
self.paths = paths.Paths()
self.pin_powerB = pins.Pins(
).pin_powerB # @TODO change it in boot/config.txt
GPIO.setmode(GPIO.BOARD)
#GPIO.setup(self.pin_powerB, GPIO.OUT)
Master.__instance = self
def get_next_episode(item):
if item.get("Type", "na") != "Episode":
log.debug("Not an episode, can not get next")
return None
parendId = item.get("ParentId", "na")
item_index = item.get("IndexNumber", -1)
if parendId == "na":
log.debug("No parent id, can not get next")
return None
if item_index == -1:
log.debug("No episode number, can not get next")
return None
url = ( '{server}/emby/Users/{userid}/Items?' +
'?Recursive=true' +
'&ParentId=' + parendId +
'&IsVirtualUnaired=false' +
'&IsMissing=False' +
'&IncludeItemTypes=Episode' +
'&ImageTypeLimit=1' +
'&format=json')
data_manager = DataManager()
items_result = data_manager.GetContent(url)
log.debug("get_next_episode, sibling list: {0}", items_result)
if items_result is None:
log.debug("get_next_episode no results")
return None
item_list = items_result.get("Items", [])
for item in item_list:
index = item.get("IndexNumber", -1)
# find the very next episode in the season
if index == item_index + 1:
log.debug("get_next_episode, found next episode: {0}", item)
return item
return None
def pre_prosseccing(self):
dm = DataManager()
data = dm.load_csv(file=self.data_path + self.data_file_name+'.csv', encoding='utf-8')
with open(self.data_path + self.data_file_name+'.documents', 'wb') as f:
pickle.dump(data, f)
with open(self.data_path + self.data_file_name+'_tm.documents', 'wb') as f:
pickle.dump(data['job_description'], f)
# # 수정된 job_title에서 posting_id 가지고 오기
# posting_ids = data['posting_id']
# posting_list = posting_ids.to_list()
#
# # posting_id에 따라 description_data set 만들기
# des_data = [data['job_description'][id] for id in posting_ids]
# title_data = [data['job_title'][id] for id in posting_ids]
# id_list = [i for i in range(len(posting_list))]
# df = pd.DataFrame({'id': posting_list, 'job_title': title_data, 'job_description': des_data, 'posting_id':posting_list})
# df.to_csv('data/doc2vec_test_data/0702/merge_0629_adj.csv', mode='w', encoding='utf-8')
# 수정된 description set 불러와 데이터 전처리 수행
# data = dm.load_csv(file='data/doc2vec_test_data/0702/merge_0629_adj.csv', encoding='utf-8')
sentences = self.data_text_cleansing(data)
data_words = list(self.sent_to_words(sentences))
data_words_nostops = self.remove_stopwords(data_words)
data_lemmatized = self.lematization(data_words_nostops)
bigram = self.make_ngram(data_lemmatized, n=2)
# bigram = self.make_bigram(data_words_nostops)
# data_lemmatized = self.lematization(bigram)
# for i in range(len(bigram)):
# print(f'[{i}] : {bigram[i]}')
data_lemmatized_filter = self.word_filtering(bigram)
for i in range(len(data_lemmatized_filter)):
print(f'[{i}] : {data_lemmatized_filter[i]}')
# # uniquewords = self.make_unique_words(data_lemmatized)
with open(self.data_path + self.data_file_name+'.corpus', 'wb') as f:
pickle.dump(data_lemmatized_filter, f)
print('=== end preprocessing ===')
return data['id'], data_lemmatized_filter
def __init__(self):
self.dataset_name = ('train', 'valid', 'test')
self.datamanager = DataManager(FLAGS)
self.data = {}
for tmp in self.dataset_name:
self.data[tmp] = self.datamanager.load_data(
FLAGS.data_dir, '%s.txt' % tmp)
vocab, embed, vocab_dict = self.datamanager.build_vocab(
'%s/%s' % (FLAGS.data_dir, FLAGS.path_wordvec),
self.data['train'] + self.data['valid'] + self.data['test'])
print('model parameters: %s' % str(FLAGS))
print("Use cuda: %s" % use_cuda)
for name in self.dataset_name:
print('Dataset Statictis: %s data: %s' %
(name, len(self.data[name])))
self.model = bridgeModel(FLAGS.dim_word,
FLAGS.dim_hidden,
FLAGS.n_layer,
FLAGS.n_label,
FLAGS.n_aspect,
batch_size=FLAGS.batch_size,
max_length_sen=FLAGS.max_length_sen,
learning_rate=FLAGS.learning_rate,
lr_word_vector=FLAGS.lr_word_vector,
weight_decay=FLAGS.weight_decay,
vocab=vocab,
embed=embed,
embed_dropout_rate=FLAGS.embed_dropout,
cell_dropout_rate=FLAGS.cell_dropout,
final_dropout_rate=FLAGS.final_dropout,
bidirectional=FLAGS.bidirectional,
optim_type=FLAGS.optim_type,
rnn_type=FLAGS.rnn_type,
lambda1=FLAGS.lambda1,
use_cuda=use_cuda)
def pre_prosseccing(self):
dm = DataManager()
data = dm.load_csv(
file='analysis/doc2vec_test_data/0702/job_title.csv',
encoding='utf-8')
print(data['job Title'].size)
sentences = self.data_text_cleansing(data, 'job Title')
data_words = list(self.sent_to_words(sentences))
data_words_nostops = self.remove_stopwords(data_words)
# bigram = self.make_bigram(data_words_nostops)
# data_lemmatized = self.lematization(data_words_nostops)
# data_words_nostops_sorted = [sorted(i) for i in data_words_nostops]
data_words_nostops = [' '.join(i) for i in data_words_nostops]
print(data_words_nostops)
df = pd.DataFrame({
'id': data['id'],
'Job_Title': data_words_nostops,
'posting_id': data['posting_id']
})
# df = df.append(data_words_nostops)
df.to_csv('analysis/doc2vec_test_data/0702/job_title_adj_extend.csv',
mode='w',
encoding='utf-8')
def cache_artwork(self, progress):
log.debug("cache_artwork")
# is the web server enabled
web_query = {"setting": "services.webserver"}
result = json_rpc('Settings.GetSettingValue').execute(web_query)
xbmc_webserver_enabled = result['result']['value']
if not xbmc_webserver_enabled:
log.debug("Kodi web server not enabled, can not cache images")
return
# get the port
web_port = {"setting": "services.webserverport"}
result = json_rpc('Settings.GetSettingValue').execute(web_port)
xbmc_port = result['result']['value']
log.debug("xbmc_port: {0}", xbmc_port)
# get the user
web_user = {"setting": "services.webserverusername"}
result = json_rpc('Settings.GetSettingValue').execute(web_user)
xbmc_username = result['result']['value']
log.debug("xbmc_username: {0}", xbmc_username)
# get the password
web_pass = {"setting": "services.webserverpassword"}
result = json_rpc('Settings.GetSettingValue').execute(web_pass)
xbmc_password = result['result']['value']
params = {"properties": ["url"]}
json_result = json_rpc('Textures.GetTextures').execute(params)
textures = json_result.get("result", {}).get("textures", [])
if self.stop_all_activity:
return
texture_urls = set()
for texture in textures:
url = texture.get("url")
url = urllib.unquote(url)
url = url.replace("image://", "")
url = url[0:-1]
texture_urls.add(url)
del textures
del json_result
if self.stop_all_activity:
return
url = ('{server}/emby/Users/{userid}/Items?' + '&Recursive=true' +
'&IncludeItemTypes=Movie,Series,Episode,BoxSet' +
'&ImageTypeLimit=1' + '&format=json')
data_manager = DataManager()
results = data_manager.GetContent(url)
if results is None:
results = []
if isinstance(results, dict):
results = results.get("Items")
server = downloadUtils.getServer()
missing_texture_urls = set()
if self.stop_all_activity:
return
image_types = [
"thumb", "poster", "banner", "clearlogo", "tvshow.poster",
"tvshow.banner", "tvshow.landscape"
]
for item in results:
art = getArt(item, server)
for image_type in art:
image_url = art[image_type]
if image_url not in texture_urls and not image_url.endswith(
"&Tag=") and len(image_url) > 0:
missing_texture_urls.add(image_url)
if self.stop_all_activity:
return
log.debug("texture_urls: {0}", texture_urls)
log.debug("missing_texture_urls: {0}", missing_texture_urls)
log.debug("Number of existing textures: {0}", len(texture_urls))
log.debug("Number of missing textures: {0}", len(missing_texture_urls))
kodi_http_server = "localhost:" + str(xbmc_port)
headers = {}
if xbmc_password:
auth = "%s:%s" % (xbmc_username, xbmc_password)
headers = {'Authorization': 'Basic %s' % base64.b64encode(auth)}
total = len(missing_texture_urls)
index = 1
count_done = 0
for get_url in missing_texture_urls:
log.debug("texture_url: {0}", get_url)
url = double_urlencode(get_url)
kodi_texture_url = ("/image/image://%s" % url)
log.debug("kodi_texture_url: {0}", kodi_texture_url)
percentage = int((float(index) / float(total)) * 100)
message = "%s of %s" % (index, total)
progress.update(percentage, "%s" % (message))
conn = httplib.HTTPConnection(kodi_http_server, timeout=20)
conn.request(method="GET", url=kodi_texture_url, headers=headers)
data = conn.getresponse()
if data.status == 200:
count_done += 1
log.debug("Get Image Result: {0}", data.status)
index += 1
if "iscanceled" in dir(progress) and progress.iscanceled():
break
if self.stop_all_activity:
break
result_report = []
result_report.append(
i18n('existing_textures') + str(len(texture_urls)))
result_report.append(
i18n('missing_textures') + str(len(missing_texture_urls)))
result_report.append(i18n('loaded_textures') + str(count_done))
return result_report
warnings.filterwarnings('ignore')
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
# Parameters to select the LSTM architecture
semana = 2
n_LSTM_hidden_layers = 2
n_cells = 200
n_features = 5
# Value to graph a moving average of observations. Just for graphical purposes
window_moving_average = 5
# Correction of time zone (LOTOS-EUROS time zone is UTC-0, Colombia is UTC-5)
UTC = 5
days_forecast = 1
dataM = DataManager(path="data/", filter_items=["pm25", "temperature", "wind"])
n_input_steps = 24 * 7 * semana
n_output_steps = 24 * 3
stations = pd.read_csv('./data/Lista_estaciones.csv', delimiter=',')
stations_SIATA = np.array(stations['PM25'].values).astype('str')
stations_Meteo = np.array(stations['Meteo'].values).astype('str')
mls = CnnMeteo(n_input_steps,
n_features,
n_output_steps,
drop=True,
n_LSTM_hidden_layers=n_LSTM_hidden_layers,
n_cells=n_cells)
csv_writer_1 = CsvWriter()
y_true.extend(batched_data['labels'])
loss += outputs
st, ed = ed, ed + FLAGS.batch_size
times += 1
pred_vector = np.argmax(np.array(pred_matrix), axis=1)
c_m = confusion_matrix(np.array(y_true),
pred_vector,
labels=range(FLAGS.n_label))
loss /= times
accuracy = np.sum([c_m[i][i] for i in range(FLAGS.n_label)]) / np.sum(c_m)
return loss, accuracy, c_m
if __name__ == "__main__":
dataset_name = ['train', 'dev', 'test']
datamanager = DataManager(FLAGS)
data = {}
for tmp in dataset_name:
data[tmp] = datamanager.load_data(FLAGS.data_dir, '%s.json' % tmp)
vocab, embed, vocab_dict = datamanager.build_vocab(
'%s/%s' % (FLAGS.data_dir, FLAGS.wordvec_name), data['train'])
print('model parameters: %s' % str(FLAGS))
print("Use cuda: %s" % use_cuda)
print('train data: %s, dev data: %s, test data: %s' %
(len(data['train']), len(data['dev']), len(data['test'])))
model = rnnCapsule(FLAGS.word_dim,
FLAGS.hidden_dim,
FLAGS.n_layer,
FLAGS.n_label,
