def provide(self, road_id): parts = DataHandling.readRoads(road_id) widths = self.formatter.formatWidthData(DataHandling.readWidths(road_id)) traffic = self.formatter.formatTrafficData(DataHandling.readTraffic(road_id)) merged_data = SegmentDataMerger().merge(traffic, self.provideFormattedBridges(road_id), widths) return self.addLaneDensity(merged_data)
def __init__(self, grouping_size, mode): self.grouping_size = grouping_size self.road_ids = [] if mode == 'short': self.road_ids = ['N1', 'N2', 'N3', 'N4', 'N5'] elif mode == 'all': self.road_ids = DataHandling.readRoadIds() elif mode == 'short-z': self.road_ids = DataHandling.readRoadIds(filename='zroads.csv') else: for ch in list(mode): self.road_ids.extend(DataHandling.readRoadIds(typeChar=ch))
def __init__(self, parent=None): pyqtgraph.setConfigOption('background', 'w') super(Dynamic_Gui, self).__init__(parent) self.setupUi(self) self.grFFTx.plotItem.showGrid(True, True, 0.7) self.grFFTy.plotItem.showGrid(True, True, 0.7) self.grFFTz.plotItem.showGrid(True, True, 0.7) self.grAccX.plotItem.showGrid(True, True, 0.7) self.grAccY.plotItem.showGrid(True, True, 0.7) self.grAccZ.plotItem.showGrid(True, True, 0.7) self.maxFFTx = 0 self.maxFFTy = 0 self.maxFFTz = 0 self.maxAccX = 0 self.maxAccY = 0 self.maxAccZ = 0 self.textEdit = "Hello World!" #read available com ports self.data = DataHandling.DataHandling() self.comboBox.clear() self.comboBox.addItem("Select...") for p in self.data.portList(): self.comboBox.addItem(str(p)) self.comboBox.currentIndexChanged.connect(self._connectComPort) # self.comboBox.highlighted.connect(self._connectComPort) #connect the buttons self.btnOpenPort.clicked.connect(self._btnOpenPort) self.btnStartStream.clicked.connect(self._btnStartStream) self.btnStopStream.clicked.connect(self._btnStopStream) # self.actionClose.triggered.connect(QtGui.qApp.quit) self.actionData_Folder.triggered.connect(self._actionFolderSave)
def generateMethod2Material(auction_list, date_start='2014-04-01', date_end='2014-04-07'): for auction_name in auction_list: auction = read_csv(working_dir + reading_dir + auction_name + '_alliance.csv') # trim date range to a specific week (a week that does not encounter realm connection) # need not to generate week num column auction = DataHandling.getTimeRangeData(auction, start=date_start, end=date_end) # create temp_auction dataframe in order to do outlier detection temp_auction = DataHandling.removeUselessColumns( auction, remain_columns=['Item ID', 'AH MarketPrice']) auction = DataHandling.removeUselessColumns( auction, remain_columns=['Item ID', 'Avg Daily Posted']) # remove outlier of item price temp_auction = temp_auction.groupby(['Item ID'], as_index=False) temp_auction = temp_auction.apply(DataPreprocess.removeOutliers) auction = auction.merge(temp_auction, on=['Item ID'], right_index=True, left_index=True) # calculate profit auction = DataHandling.calItemsProfit(auction) auction.drop_duplicates(inplace=True) #merge all datafrme together grouped_auction = auction.groupby(['Item ID'], as_index=False) # get "item weekly quantity" item_q_weekly_mean = grouped_auction['Avg Daily Posted'].mean() # get "totoal profit of a week" item_profit_weekly_mean = grouped_auction['Profit'].mean() realm_weekly_profit = item_profit_weekly_mean['Profit'].sum() item_q_weekly_mean['Realm Profit'] = realm_weekly_profit item_q_weekly_mean.to_csv('../method2/' + date_start + '_' + auction_name, index=False) print 'finished', auction_name, '...'
import importlib import h5py import DataHandling as DH importlib.reload(DH) DH.transform_2_spectra( filename='../AttoStreakSimulations/TF_training_densespace.hdf5', transfer='reformed_spectra_densesapce.hdf5') h5_file = h5py.File('reformed_spectra_densesapce.hdf5', 'r') print(h5_file.keys()) for key in list(h5_file.keys()): print('shape of {} is {}'.format(key, h5_file[key].shape)) h5_file.close()
def testFileList(self): filesInThisPath = DataHandling.getFileNames('.', path = False) self.assertTrue('__init__.py' in filesInThisPath, 'DataHandling: Did not find find files')
def testOpensFile(self): filu = DataHandling.openFile( '__init__.py', 'r') DataHandling.closeFile(filu)
def provideRawBridges(self, road_id): return DataHandling.readBridges(road_id)
import JobAppScraper import SearchResultsScraper import DataHandling Bot = SearchResultsScraper.ZipRecruiterBot() Search_terms = ["Python Automation"] Locations = ["Hayward", "San Francisco"] for term in Search_terms: for location in Locations: # SQL Initialization Code db = DataHandling.Database(term, location) db.create_table() Bot.set_search(term, location) Bot.load_all_search_results() all_job_postings = Bot.return_all_job_postings() for link in all_job_postings: try: job = JobAppScraper.JobPosting(link) table_name = db.table_name current_search_term = term current_location = location current_title = job.job_title current_company = job.company current_job_text = job.job_text db.write_to_table(table_name, current_search_term, current_location, current_title, current_company, current_job_text) print("There is an opening for a {} at company {}".format(
import csv import Classifier import DataHandling import numpy as np import math """ Data import and handling """ #train data train_data_dir = 'Data/train.csv' train_data = DataHandling.ImportData(train_data_dir) train_data = DataHandling.CleanData(train_data) n = len(train_data) print('Number of data available: ', n) train_data = np.array(train_data) n_train = math.ceil(n * 0.95) X_train = train_data[:n_train, :-1] Y_train = train_data[:n_train, -1] X_eval = train_data[n_train:, :-1] Y_eval = train_data[n_train:, -1] # test data test_data_dir = 'Data/test.csv' test_data = DataHandling.ImportData(test_data_dir) test_data = DataHandling.CleanDataTest(test_data) n_test = len(test_data) print('Number of test data: ', n_test)
window.FindElement('_tab1_').Update(visible=True) window.FindElement('_tab2_').Update(visible=False) window.FindElement('_tab3_').Update(visible=False) elif event == 'Reset': #resetting values on the tab for key in key_list: window[key]('') elif event == 'Add': #adding data values to database try: values['_dttransac_'] = datetime.strptime(values['_dttransac_'], '%Y-%m-%d %H:%M:%S') except: sg.popup('Select date through the \'Select a Date\' button.') else: try: DataHandling.add_record(values['_username_'], values['_category_'], values['_amt_'], values['_dttransac_'], values['_memo_']) sg.popup('Values Added!') for key in key_list: window[key]('') except: sg.popup( 'Please enter all the right values before proceeding. Amount should be a number and shouldn\'t be left blank' ) for key in key_list: window[key]('') elif event == 'Login': # login with username and password check = DataHandling.check_userpass(values['_username_'], values['_pass_']) if check == True: window['-COL1-'].Update(visible=False)
import time import DataHandling import Drawing import APIconfig start_time = time.time() refresh = True # If true, reprocess all history data, station data, and retrieve information from API's, also redraw base for drawing. print("Getting railmap..." ) # set to true whenever things (eg. resolution) are changed railmap = DataHandling.get_railmap_data( APIconfig.ns_app_key, refresh=refresh) # Get the Railmap from NS's API print("Getting history, stations...") history, stations = DataHandling.get_csv_data( APIconfig.public_travel_information_key, railmap, refresh=refresh) # Prepare journey history, and station list print("Writing frames...") Drawing.draw(history, railmap) # Draw all the frames, render the video with FFmpeg print("Finished: %s seconds" % (time.time() - start_time))
x_data['Fare'].fillna(-1, inplace=True) x_data['Fare'] = np.ceil(x_data['Fare']).astype(int) #final x_data = x_data[[ 'Pclass', 'Age', 'SibSp', 'Parch', 'Fare', 'Female', 'Male', 'C', 'Q', 'S' ]] test_id = data['PassengerId'] if train: return x_data, y_data else: return x_data, test_id x_train, y_train = load_data(train_path, True) x_test, test_id = load_data(test_path, False) clf = ExtraTreesRegressor(n_estimators=1000, random_state=50, max_features='auto', n_jobs=-1) clf.fit(x_train, y_train) pred = clf.predict(x_test) pred[pred > 0.5] = 1 pred[pred <= 0.5] = 0 DataHandling.WriteResults('Data/Julia_submission.csv', test_id, pred)
def method1(auction_list, date_start='2014-03-13', date_end='2014-10-12'): for fraction in fractionlist: auction_profit = DataFrame( columns=['Realm', 'Fraction', 'Week Num', 'Profit']) for auction_name in auction_list: auction = read_csv(working_dir + auction_name + fraction + '.csv') # time range auction = DataHandling.getTimeRangeData(auction, start=date_start, end=date_end) # add 'Week Num' column auction['Week Num'] = auction['PMktPrice Date'].apply( DataHandling.toWeekNumber) # temp_auction is used for doing remove outliers of weekly item price temp_auction = DataHandling.removeUselessColumns( auction, remain_columns=['Item ID', 'Week Num', 'AH MarketPrice']) auction = DataHandling.removeUselessColumns( auction, remain_columns=['Item ID', 'Week Num', 'AH Quantity']) temp_auction = DataHandling.removeOutlierOfDailyPrice(temp_auction) auction = auction.merge(temp_auction, on=['Item ID', 'Week Num'], right_index=True, left_index=True) # get profit of each item, the calItemsProfit will generate 'Profit' column auction = DataHandling.calItemsProfit(auction) # drop duplicates auction.drop_duplicates(inplace=True) # group auction by 'week number' and 'item id' auction = auction.ix[:, [ 'Week Num', 'Item ID', 'AH Quantity', 'Profit', 'AH MarketPrice' ]] grouped_auction = auction.groupby(['Week Num', 'Item ID'], as_index=False) # get "mean of item weekly quantity" item_q_weekly_mean = grouped_auction['AH Quantity'].mean() # get "mean of item weekly profit" item_profit_weekly_mean = grouped_auction['Profit'].mean() # get "mean of item weekly price" item_price_weekly_mean = grouped_auction['AH MarketPrice'].mean() # generate auction detail file, columns contain (Week Num, Item ID, Avg Daily Posted, AH MarketPrice) auction_detail = item_q_weekly_mean.merge(item_price_weekly_mean) auction_detail.to_csv('../corr_result/auction_detail/' + auction_name + fraction, index=False) # get "realm's weekly profit" realm_weekly_profit = item_profit_weekly_mean.groupby( ['Week Num'])['Profit'].sum() realm_weekly_profit = DataFrame(realm_weekly_profit, columns=['Profit']).reset_index() auction = item_q_weekly_mean.merge(realm_weekly_profit) # gathering data for building auction_profit dataframe. realm_weekly_profit['Realm'] = auction_name realm_weekly_profit['Fraction'] = fraction[1:] auction_profit = auction_profit.append(realm_weekly_profit, ignore_index=True) # get all item id show up in that realm # put the code here note outside for loop because items show up in each realm may be different temp_id_list = auction['Item ID'] temp_id_list = temp_id_list.drop_duplicates() id_list = temp_id_list.tolist() # get correlation coefficient between "item's mean daily posted" and "auction weekly profit" for each item. # item shows up at least 10 of 30 weeks will be recorded. item_corr_df = DataFrame(columns=['Item ID', 'Corr']) for item in id_list: temp_item = auction[auction['Item ID'] == item] occurence_count = len(temp_item) if occurence_count >= 15: corr = np.corrcoef(x=temp_item['AH Quantity'], y=temp_item['Profit']) new_corr = DataFrame([{ 'Item ID': item, 'Corr': corr[0][1] }]) item_corr_df = item_corr_df.append(new_corr, ignore_index=True) item_corr_df.to_csv('../corr_result/' + auction_name + fraction, index=False) print 'finish', auction_name + fraction, '...' auction_profit.to_csv('../corr_result/' + fraction[1:] + '_profit.csv', index=False, header=False, mode='a')
import DataHandling DataHandling.parseWidthInfo() DataHandling.parseTrafficInfo()
def __init__(self, controller, multPlayer=False, score1=0, player=1): #--------------------------------------------- #***Configuraciones de la ventana y Tkinter*** #--------------------------------------------- #Se designa una variable self.master para inicializar la ventana principal del juego. self.master = Tk() #Se designa un widget de Canvas para el juego y otro para el HUD. self.master.geometry("1100x700") self.master.configure(bg="black") #self.master.iconbitmap("assets/favicon.ico") self.w = Canvas(self.master, width=1100, height=650, bg="gray10", highlightbackground="black") self.hud = Canvas(self.master, width=1100, height=50, bg="#e8e8e8", highlightbackground="black") self.dibujar = Fractal(self.w) r = lambda: random.randint(0, 300) self.dibujar.Fractal(None, r(), r(), 'white', 1, 3) self.controller = controller self.settings = DataHandling("settings.json").load_data() self.multPlayer = multPlayer self.score1 = score1 self.player = player #Título de la ventana. self.master.title("Juego") #Se restringe el tamaño de la ventana al especificado en el widget. self.master.resizable(width=0, height=0) self.flag = True ''' La función verificarCoordenadas se usar para evitar que se creen elementos del juego sobre otros elementos ya creados. ''' def verificarCoordenadas(rango): #Este bucle infinito se usa para establecer coordenadas que no estén ocupadas por otros objetivos. while (True): x1 = random.randint(100, 1000) y1 = random.randint(50, 400) disponible = self.w.find_overlapping(x1 - rango, y1 - rango, x1 + rango, y1 + rango) if (len(disponible) == 0): return x1, y1 ''' La función generarObjetivos crea una lista en la que almacenará listas con las coordenadas, el color y los puntos de los objetivos, para asignar colores se hace uso de una tupla con los códigos hexadecimales y se utiliza un número aleatorio para seleccionar. Las coordenadas se generan con valores aleatorios en un rango específico y los puntajes se definen sumando 4 al número aleatorio del color y multiplicando el resultado por el valor que tenga el multiplicador dependiendo de la dificultad establecida. Se retorna una matriz con todos los self.objetivos representados por una fila. ''' def generarObjetivos(cantidad): objetivos = [] colores = ("#f48c42", "#bbf441", "#41d0f4", "#f441dc") for i in range(0, cantidad): x1, y1 = verificarCoordenadas(15) valor = random.randint(0, 100) if (valor >= 0 and valor < 50): color = 0 elif (valor >= 50 and valor < 75): color = 1 elif (valor >= 75 and valor < 90): color = 2 elif (valor >= 90): color = 3 puntos = (color + 4) * self.mult objetivos.append([ self.w.create_oval(x1, y1, x1 + 30, y1 + 30, fill=colores[color], tags='target'), puntos ]) return objetivos ''' La función generarObstaculos funciona de manera similar a generarObjetivos, con la diferencia que cada coordenada tiene un valor diferente para que varíe su forma, se verifica que los obstáculos no estén encima de otros obstáculos u objetivos. ''' def generarObstaculos(cantidad): obstaculos = [] for i in range(0, cantidad): while (True): x1 = random.randint(100, 1000) y1 = random.randint(50, 400) xa = x1 - random.randint(10, 50) ya = y1 - random.randint(10, 50) xb = x1 + random.randint(10, 50) yb = y1 + random.randint(10, 50) disponible = self.w.find_overlapping(xa, ya, xb, yb) if (len(disponible) == 0): break obstaculos.append( self.w.create_rectangle(xa, ya, xb, yb, fill="black", tags='obstacle')) return obstaculos ''' La función generarPowerUps genera los objetivos que suman tiempo y generará un power up aleatorio ''' def generarPowerUps(cantidad): powerUps = [] tipos = ( ("#fff9af", "#ff9900", "timer"), ("#800080", "#ff9900", "+Retrocesos"), ("#fede12", "#ff9900", "Puntos_x2"), ("#ff6666", "#ff9900", "+Rango"), ("#8c0000", "#ff9900", "-Obstaculo")) #(Relleno, borde, tag) for i in range(0, cantidad): x1, y1 = verificarCoordenadas(30) powerUps.append( self.w.create_oval(x1, y1, x1 + 30, y1 + 30, fill=tipos[0][0], outline=tipos[0][1], width=3, tags=tipos[0][2])) MegapowerUp = random.randint(1, len(tipos) - 1) x2, y2 = verificarCoordenadas(30) powerUps.append( self.w.create_oval(x2, y2, x2 + 30, y2 + 30, fill=tipos[MegapowerUp][0], outline=tipos[MegapowerUp][1], width=3, tags=tipos[MegapowerUp][2])) return powerUps #Se crea un limitador de retrocesos, un multiplicador de puntos y la cantidad de obstáculos dependiendo de la dificultad del juego. self.dif = self.settings["difficulty"]["value"] if (self.dif == 1): self.retrocesosMax = 5 self.cantidadObstaculos = 5 self.mult = 11 self.cantidadPowerUps = 6 elif (self.dif == 2): self.retrocesosMax = 3 self.cantidadObstaculos = 8 self.mult = 13 self.cantidadPowerUps = 4 elif (self.dif == 3): self.retrocesosMax = 1 self.cantidadObstaculos = 12 self.mult = 15 self.cantidadPowerUps = 2 else: self.retrocesosMax = 0 self.cantidadObstaculos = 14 self.mult = 18 self.cantidadPowerUps = 0 #Se define la cantidad de objetivos a generar. self.cantidadObjetivos = 6 #Se crean los objetivos usando la cantidad especificada anteriormente. self.objetivos = generarObjetivos(self.cantidadObjetivos) #Se generan los obstáculos del juego. self.obstaculos = generarObstaculos(self.cantidadObstaculos) #Se generan los power ups. self.powerUps = generarPowerUps(self.cantidadPowerUps) #Se asigna el tiempo que tendrá el jugador para alcanzar todos los objetivos. self.tiempo = 30 self.tipoT = 0 #Se define el .puntaje máximo para terminar el juego al alcanzarlo. self.puntajeMax = 0 for valor in self.objetivos: self.puntajeMax += valor[1] #Si es 1 el siguiente objetivo alcanzado dará doble puntaje self.doblePuntaje = 0 self.w.pack() self.hud.pack() #Se define el puntaje con valor inicial de cero y el contador del turno. self.puntaje = 0 self.movimiento = 0 #Se muestran tanto el puntaje como el número del movimiento y el temporizador en el HUD. self.marcadorP = self.hud.create_text(30, 22, text="Puntaje: 000" + str(self.puntaje), anchor="w", font=("", "12", "bold")) self.hud.create_line(161, 0, 161, 50, width=3) self.hud.create_line(922, 0, 922, 50, width=3) self.marcadorM = self.hud.create_text(951, 22, text="Movimiento #00" + str(self.movimiento + 1), anchor="w", font=("", "12", "bold")) self.timer = self.hud.create_text((408, 22), text="Texto", font=("", "12", "bold"), fill="#0048b5") self.hud.create_line(460, 0, 460, 50, width=3) self.textoPU = self.hud.create_text(530, 22, text="???", font=("", "12", "bold"), fill="black") self.hud.create_line(600, 0, 600, 50, width=3) #Rango máximo y mínimo para generar un fractal nuevo. self.minimo = 25 self.maximo = 40 self.retrocesos = 0 #Se muestra la cantidad de retrocesos en el HUD. self.marcadorR = self.hud.create_text(195, 22, text="Retrocesos: " + str(self.retrocesos) + "/" + str(self.retrocesosMax), anchor="w", font=("", "12", "bold")) self.hud.create_line(350, 0, 350, 50, width=3) #Se definen los puntos de inicio de la hitbox. self.xInicial, self.yInicial = 550, 610 #Se crea una lista que almacenará las acciones realizadas en cada turno. self.circulos = [[ self.w.create_line(550, 650, 550, 610), self.xInicial, self.yInicial, self.w.create_oval(self.xInicial - 2, self.yInicial - 2, self.xInicial + 2, self.yInicial + 2, fill="black"), self.movimiento, [] ]] #Se reproduce la música de fondo. if (self.settings["sounds"]["value"] == 1): try: winsound.PlaySound("assets/sound.wav", winsound.SND_ASYNC) except NameError: pass