class Coupon: couponDeirectory = '' couponDatas = [] def __init__(self,couponDeirectory): self.couponDeirectory = couponDeirectory self.csvReader = CSVReader() self.__generateCouponDatas() def __generateCouponDatas(self): filesDirectory = self.couponDeirectory for dirname, dirnames, filenames in os.walk(filesDirectory): # print path to all filenames. for filename in filenames: fileNamePath = os.path.join(dirname, filename) fileNamePath = os.path.normpath(fileNamePath) filename = filename.split('.')[0] fileDatas = self.__getCSVData(fileNamePath) couponData = [] couponData.append(filename) couponData.append(fileDatas) self.couponDatas.append(couponData) def __getCSVData(self,fileNamePath): self.csvReader.setFileName(fileNamePath) return self.csvReader.readCsv() def getCouponDatas(self): return self.couponDatas
def main(filename="./sample.csv"): """ Main method for testing """ reader = CSVReader() generator = KeywordGenerator(reader.read_file(filename)) for word in generator.generate(): print(word + ",", end="", flush=True)
def extract(self): logger.info('extracting csvs from shared drive') csv_reader = CSVReader(Config.directory) csv_reader.targets = [Target(*params) for params in Config.targets] received, shipped, history, picked = csv_reader.read() received.to_csv('data/received.csv') shipped.to_csv('data/shipped.csv') history.to_csv('data/history.csv') picked.to_csv('data/picked.csv') return received, shipped, history, picked
def model_test(): print("here") sentiments = np.load('sentiments.npy', allow_pickle=True) texts = np.load('texts.npy', allow_pickle=True) all_texts = np.load('text_cache.npy', allow_pickle=True) neutral = [] _, X_test, _, Y_test = train_test_split(texts, sentiments, test_size=0.01) print("here ",len(Y_test)) airline_data = CSVReader.dataframe_from_file("Tweets.csv",['airline_sentiment','text']) airline_text = np.array(airline_data.text) airline_sentiment = np.array(airline_data.airline_sentiment) count = 0 for i in range(len(airline_text)): if(count > 1000): break if(airline_sentiment[i] == "neutral"): neutral = np.append(neutral,airline_text[i]) count+=1 X_test = np.append(X_test,neutral) Y_test = np.append(Y_test,[0]*len(neutral)) Y_test[Y_test==-1] = 4 Y_test[Y_test==-2] = 3 # categ_test = to_categorical(Y_test,num_classes=5) tokenizer = Tokenizer(num_words=300000) tokenizer.fit_on_texts(all_texts) model = load_model("savedModel2/saved-model3-60.h5") result = model.predict_on_batch(pad_sequences(tokenizer.texts_to_sequences(X_test),maxlen=75)) result = np.argmax(result,axis=-1) # cat_result = to_categorical(result,num_classes=5) print("f1 ",precision_score(Y_test,result, average=None))
def read(self): if not os.path.exists(self.filepath): raise Exception("File not found {}".format(self.filepath)) self.filetype = os.path.splitext(self.filepath)[1] if self.filetype == '.xls': # read excel sheets, these are expected to be there, TODO: error if not ingredient_reader = ExcelReader(self.filepath, 'ingredients') step_reader = ExcelReader(self.filepath, 'steps') data_reader = ExcelReader(self.filepath, 'data') # get as dataframes ingredients = ingredient_reader.read_xl() self.ingredients = ingredients.fillna(0) steps = step_reader.read_xl() self.steps = steps.fillna(0) self.data = data_reader.read_xl() elif self.filetype == '.txt': reader = TextReader(self.filepath) self.data = reader.read() elif self.filetype == '.csv': reader = CSVReader(self.filepath) self.data = reader.read() else: raise Exception("Incorrect file type {}".format(self.filetype)) return self.data
def _packet_in_handler(self, ev): msg = ev.msg datapath = msg.datapath ofproto = datapath.ofproto parser = datapath.ofproto_parser # get Datapath ID to identify OpenFlow switches. dpid = datapath.id self.mac_to_port.setdefault(dpid, {}) # analyse the received packets using the packet library. pkt = packet.Packet(msg.data) eth_pkt = pkt.get_protocol(ethernet.ethernet) dst = eth_pkt.dst src = eth_pkt.src # get the received port number from packet_in message. in_port = msg.match['in_port'] self.logger.info("packet in %s %s %s %s", dpid, src, dst, in_port) # learn a mac address to avoid FLOOD next time. self.mac_to_port.setdefault(dpid, {}) self.mac_to_port[dpid][src] = in_port # read policies from csv file policies = CSVReader().read() out_port = 0 if (dst in policies[src]) or dst == 'ff:ff:ff:ff:ff:ff': self.logger.info("packet is permitted") if dst in self.mac_to_port[dpid]: out_port = self.mac_to_port[dpid][dst] self.logger.info("packet is knowingly forwarded to port %s", out_port) else: out_port = ofproto.OFPP_FLOOD self.logger.info("packet is flooded.") actions = [parser.OFPActionOutput(out_port)] # install a flow to avoid packet_in next time. if out_port != ofproto.OFPP_FLOOD: self.logger.info("installing flood...") match = parser.OFPMatch(in_port=in_port, eth_dst=dst) self.add_flow(datapath, 1, match, actions) # construct packet_out message and send it. out = parser.OFPPacketOut(datapath=datapath, buffer_id=ofproto.OFP_NO_BUFFER, in_port=in_port, actions=actions, data=msg.data) datapath.send_msg(out) else: self.logger.info("packet has no policy.")
def run_script(): user_id = np.random.randint(low=1000000000, high=9999999999, size=DATASET_SIZE) category = np.random.choice(CATEGORY, DATASET_SIZE) file1_data = {'user_id': user_id, 'category': category} # creating file1.csv CSVReader().create_csv_file(file_name='file1.csv', data=file1_data, columns_name=['user_id', 'category']) age = np.random.randint(low=10, high=100, size=DATASET_SIZE) height = np.random.uniform(low=3.5, high=6.5, size=DATASET_SIZE) file2_data = {'user_id': user_id, 'age': age, 'height': height} #creating file2.csv CSVReader().create_csv_file(file_name='file2.csv', data=file2_data, columns_name=['user_id', 'age', 'height'])
def testSquare(self) -> None: value1_index,result_index,test_data = CSVReader.callCreateCSVReader("TwoFieldsCSV", "../csv_files/", "Unit Test Square.csv" ) #pprint(test_data) #print() for row in test_data: value1 = int(row[value1_index]) result = int(row[result_index]) self.assertEqual(self.calculator.sqr(value1), result) self.assertEqual(self.calculator.result, result)
def testDivision(self) -> None: value1_index,value2_index,result_index,test_data = CSVReader.callCreateCSVReader("ThreeFieldsCSV", "../csv_files/", "Unit Test Division.csv" ) #pprint(test_data) #print() for row in test_data: value1 = int(row[value1_index]) value2 = int(row[value2_index]) result = float(row[result_index]) self.assertEqual(self.calculator.divide(value1, value2), result) self.assertEqual(self.calculator.result, result)
def testFileNotExist(self): expectedValue = [] self.assertEqual(CSVReader("rafiar.csv").readCsv, expectedValue)
def testFileExists(self): expectedValue = [[ 'rafiar', 'rahmansyah', 'borneo', 'tangerang', 'pegipegi' ], ['nafiar', 'rahmansyah', 'borneo', 'tangerang', 'tokopedia']] self.assertEqual(CSVReader("rafiar.csv").readCsv, expectedValue)
def __init__(self): #self.emotion_data = CSVReader.dataframe_from_file("VentDataset/emotions.csv", ['id', 'emotion_category_id']) # self.emotion_data = self.emotion_data[self.emotion_data.enabled == 'TRUE'] self.vent_data = CSVReader.dataframe_from_file("VentDataset/vents.csv", ['emotion_id', 'text']) self.textPreProcessing = TextPreprocessing()
def train_network(network, device): csv_reader = CSVReader() # Load the training data from the csv files training_loader = csv_reader.get_training_loader( Config.batch_size, Config.shuffle, Config.load_data_from_file, Config.preprocess_data) validation_loader = csv_reader.get_validation_loader( Config.batch_size, Config.shuffle, Config.load_data_from_file, Config.preprocess_data) # Converts data into 3D renderings if (Config.use_image_generator): from skeleton_visualizer import SkeletonVisualizer # You need to give the visualizer 2 skeleton instances to display skeleton1 = training_loader.dataset.features[0 + 12].numpy() skeleton2 = training_loader.dataset.features[6261 + 12].numpy() render = SkeletonVisualizer(skeleton1, skeleton2) return optimizer = optim.SGD(net.parameters(), lr=0.01) criterion_ce = nn.CrossEntropyLoss(size_average=True) num_classes = RecognitionDataset.num_classifications class_names = RecognitionDataset.classification_names # This plotter is a class than I have created visdom_env_name = 'Plots' if (Config.use_visdom): plotter = visdom_utils.VisdomLinePlotter(env_name=visdom_env_name) confusion_logger_training = torchnet.logger.VisdomLogger( 'heatmap', env=visdom_env_name, opts={ 'title': 'Training Confusion Matrix', 'columnnames': class_names, 'rownames': class_names }) confusion_logger_validation = torchnet.logger.VisdomLogger( 'heatmap', env=visdom_env_name, opts={ 'title': 'Validation Confusion Matrix', 'columnnames': class_names, 'rownames': class_names }) confusion_graph_training = torchnet.meter.ConfusionMeter(num_classes) confusion_graph_validation = torchnet.meter.ConfusionMeter(num_classes) batch_count = 1 total_confusion = np.zeros((num_classes, num_classes)) for epoch in range(Config.max_num_epochs): total = 0 total_correct = 0 network.train() if (Config.use_visdom): confusion_graph_training.reset() confusion_graph_validation.reset() for data in training_loader: features, labels = data #print((features.min(), features.max())) loss_ce, target_np, predicted_np = run_network( network, optimizer, criterion_ce, device, features, labels, batch_count, True) confusion = np.swapaxes( confusion_matrix(target_np, predicted_np, labels=np.arange(0, num_classes)), 0, 1) total_confusion = np.add(total_confusion, confusion) # The predicted and target are flopped in order to make the data on the correct axis #confusion_graph_training.add(torch.from_numpy(target_np), torch.from_numpy(predicted_np)) if (Config.use_visdom): confusion_graph_training.add(torch.from_numpy(predicted_np), torch.from_numpy(target_np)) total_correct += (predicted_np == target_np).sum() total += features.size(0) batch_count += 1 # Calculate the network's accuracy every epoch validation_correct, validation_total = get_validation_accuracy( network, optimizer, criterion_ce, device, validation_loader, confusion_graph_validation) accuracy_validation = 100 * validation_correct / validation_total accuracy_training = 100 * total_correct / total print( 'loss={} TRAINING: #total={} #correct={} VALIDATION: #total={} #correct={}' .format(loss_ce, total, total_correct, validation_total, validation_correct)) # Plot our results to visdom if (Config.use_visdom): plotter.plot('loss', 'train', 'Class Loss', epoch, loss_ce.item()) plotter.plot('accuracy', 'train', 'Class Accuracy', epoch, accuracy_training) plotter.plot('accuracy', 'validation', 'Class Accuracy', epoch, accuracy_validation) confusion_logger_training.log(confusion_graph_training.value()) confusion_logger_validation.log(confusion_graph_validation.value())
def __init__(self,couponDeirectory): self.couponDeirectory = couponDeirectory self.csvReader = CSVReader() self.__generateCouponDatas()
"everything is great, i have lost some weight", "awesome, really cool", "should I play cards", "I am full and inshape", "is it okay to be that hungry at night?" ]), maxlen=75)) print("result: ", np.argmax(result, axis=-1), "\n") if __name__ == "__main__": embeddings = np.load('text_embedding.npy', allow_pickle=True) sentiments = np.load('sentiments.npy', allow_pickle=True) texts = np.load('texts.npy', allow_pickle=True) all_texts = np.load('text_cache.npy', allow_pickle=True) _, X_test, _, Y_test = train_test_split(texts, sentiments, test_size=0.01) airline_data = CSVReader.dataframe_from_file("Tweets.csv", ['airline_sentiment', 'text']) airline_text = np.array(airline_data.text) airline_sentiment = np.array(airline_data.airline_sentiment) count = 0 for i in range(len(airline_text)): if (count > 1000): break if (airline_sentiment[i] == "neutral"): X_test = np.append(X_test, airline_text[i]) Y_test = np.append(Y_test, [0]) count += 1 models = [] models = np.append(models, load_model("ensemble_bgru.h5")) models = np.append(models, load_model("ensemble_gru.h5")) models = np.append(models, load_model("ensemble_gru.h5")) models = np.append(models, load_model("ensemble_lstm.h5"))
from hand_selector import HandSelector from body_box_extractor import BodyBoxExtractor from bspline_preprocessor import BSplinePreprocessor from dtw_processor import DTWProcessor from task_runner import TaskRunner ROOT_DIRECTORY = join(dirname(realpath(sys.argv[0])), '..', '..') left_hand = False right_hand = True smoothing = 0.01 sampling_rate = 100 hand_presenter = HandPresenter("Hand Presenter") bspline_presenter = BSplinePresenter("B-Spline Presenter") body_box_presenter = BodyBoxPresenter("Body Box Presenter") dtw_presenter = DTWPresenter("DTW Presenter") csv_reader = CSVReader(ROOT_DIRECTORY) hand_selector = HandSelector("Hand Selector", left_hand, right_hand, [hand_presenter]) body_box_extractor = BodyBoxExtractor("Body Box Extractor", [body_box_presenter]) bspline_preprocessor = BSplinePreprocessor("B-Spline Preprocessor", smoothing, sampling_rate, [bspline_presenter]) dtw_processor = DTWProcessor("DTW Processor", 2, [dtw_presenter]) task_runner = TaskRunner(csv_reader, [hand_selector, body_box_extractor, bspline_preprocessor], dtw_processor) input_files = ['res/HKG_001_a_0002 Aaron 41/HKG_001_a_0002 Aaron 41.csv', 'res/HKG_001_a_0001 Aaron 22/HKG_001_a_0001 Aaron 22.csv'] for input_file in input_files: csv_reader.set_file(input_file) task_runner.add_data() task_runner.process()
def main(): # store downloaded CSV files download_path = f'{pathlib.Path().absolute()}\\download\\' print(f'path: {download_path}') # iframe url zacks.com iframe_url = 'https://screener-api.zacks.com/?scr_type=stock&c_id=zacks&c_key=0675466c5b74cfac34f6be7dc37d4fe6a008e212e2ef73bdcd7e9f1f9a9bd377&ecv=2ITM2QTOyQDO&ref=screening#' # path to web-browser wb_path = "C:\\Program Files\\Mozilla Firefox\\firefox.exe" # path to web-driver firefox_dPath = f'{pathlib.Path().absolute()}\\drivers\\Firefox\\geckodriver.exe' if len(sys.argv) >= 2: order = "no-sorting" output_fn = "all-no-sorting.csv" # test if user provided a sorting method # ---------------------------------------------------------------------------------- # if sys.argv[1] != "ascending" and sys.argv[1] != "descending": print(f'Default sorting for final report: {order}') elif sys.argv[1] == "ascending" or sys.argv[1] == "descending": order = sys.argv[1] # user should provide a sorting name print(f'Sorting for final report: {order}') # ---------------------------------------------------------------------------------- # # sort report # ---------------------------------------------------------------------------------- # if order == "ascending": output_fn = "all-ascending.csv" if order == "descending": output_fn = "all-descending.csv" # ---------------------------------------------------------------------------------- # # Init FireFox Run # ---------------------------------------------------------------------------------- # run_firefox = RunFirefox(download_path, iframe_url, wb_path, firefox_dPath) # ---------------------------------------------------------------------------------- # # init CSV Reader # ---------------------------------------------------------------------------------- # csv_dir = "download/" csv_dir_out = "output/" csv_reader = CSVReader(csv_dir, csv_dir_out) # init CSV reader csv_reader.clean_download( ) # remove downloaded reports before getting new files # ---------------------------------------------------------------------------------- # # get csv files that have tickers and its values # ---------------------------------------------------------------------------------- # s0 = 10 # time in seconds --> system time page_load_time = 180 # time in seconds to wait for page to load wait_time = 1 # Explicit wait time driver_time = 60 # implicitly wait time --> driver time # Run firefox for zacks.com run_firefox.run_firefox(s0, page_load_time, wait_time, driver_time) # using a url of the iframe # ---------------------------------------------------------------------------------- # # working with csv files # ---------------------------------------------------------------------------------- # print(f'Building final report for zacks.com tickets.') csv_reader.empty_reports("output/all-ascending.csv") # empty doc csv_reader.empty_reports("output/all-descending.csv") # empty doc csv_reader.empty_reports("output/all-no-sorting.csv") # empty doc csv_reader.empty_reports("output.html") # empty doc print(f'Sorting final report: {order}') csv_reader.write_csv( order ) # arg: ascending, descending, no-sorting; overwrites existing file csv_reader.clean_download() # remove downloaded reports # load final report into dictionary and this report will not include the header from CSV file final_report = csv_reader.read_final_report(output_fn) num_of_records = len(final_report) print(f'Report is reade.') # ---------------------------------------------------------------------------------- # # Using Selenium WebDriver # ---------------------------------------------------------------------------------- # s0 = 2 # time in seconds --> system time page_load_time = 180 # time in seconds to wait for page to load wait_time = 1 # Explicit wait time driver_time = 60 # implicitly wait time --> driver time # singlethread: get charts # singlethread(final_report, s0, page_load_time, wait_time, driver_time) # multithreading: get charts # multithreading(final_report, csv_reader, num_of_records, s0, page_load_time, wait_time, driver_time) # ---------------------------------------------------------------------------------- # # Using BeautifulSoup and urllib3 # ---------------------------------------------------------------------------------- # # singlethread: get charts pc = PageCrawler() pc.run(final_report) # multithreading: get charts # ---------------------------------------------------------------------------------- # else: print( "Please, provide one of the sorting names: ascending, descending, no-sorting as an argument." )
import sys from os import listdir from os.path import join,dirname,realpath,isdir import json from csv_reader import CSVReader from body_box_extractor import BodyBoxExtractor ROOT_DIRECTORY = join(dirname(realpath(sys.argv[0])), '..', '..', 'res') data_directories = [f for f in listdir(ROOT_DIRECTORY) if isdir(join(ROOT_DIRECTORY, f))] csv_reader = CSVReader(ROOT_DIRECTORY) body_box_extractor = BodyBoxExtractor('', None) check_results = {} for data_directory in data_directories: try: type_json_file = open(join(ROOT_DIRECTORY, data_directory, 'type.json')) type_data = json.load(type_json_file) csv_reader.set_file(join(data_directory, data_directory+'.csv')) skeleton_data = {'raw_data': csv_reader.read()} data = body_box_extractor.process(skeleton_data) check_results[data_directory] = type_data check_results[data_directory]['has_shoulder'] = all(abs(data['body_box']['shoulder_left']['3d']['mean']) > 0.001) and all(abs(data['body_box']['shoulder_left']['3d']['mean']) > 0.001) check_results[data_directory]['has_hip'] = all(abs(data['body_box']['hip_left']['3d']['mean']) > 0.001) and all(abs(data['body_box']['hip_right']['3d']['mean']) > 0.001) except IOError: continue selected_signs = [data_name for data_name,data in check_results.items() if data['has_hip'] and data['oneOrTwo'] is 1 and data['handRecognized']] print sorted(selected_signs)
app = Flask(__name__) utils = Application() @app.route("/") def main(): return "Welcome!" @app.route("/getCityHotels", methods=['POST']) @check_rate_limit('api_key') def get_hotels(): try: request_body = json.loads(request.data) city_id = request_body.get('city_id') sort_type_value = request_body.get('sort_type') meta_data = utils.get_city_hotels(city_id, sort_type=sort_type_value) format_data = utils.format_data(meta_data, config.CSV_DATA.headers) response = json.dumps(format_data) return response except KeyError: response = "Key not found for {}".format(city_id) abort(make_response(str(response), 404)) except Exception as e: response = e.args abort(make_response(str(response), 400)) if __name__ == "__main__": config.CSV_DATA = CSVReader() # reads CSV while while starting the server app.run()
from data_reconfiguration import reconfigure_contestants_data as rc from data_reconfiguration import reconfigure_question_data as rcq from data_reconfiguration import reconfigure_game_contestant_location_data as rc_game_player_loc if __name__ == '__main__': # Parse command line arguments input_config, output_config = ap.argument_parser() # Retrieve csv locations from the config contestants_csv_location = input_config.get('files', 'contestants') game_contestant_csv_location = input_config.get('files', 'locations') questions_csv_location = input_config.get('files', 'questions') trend_csv_location = input_config.get('files', 'trend') final_results_csv_location = input_config.get('files', 'final_results') # Request data-frame from CSVs contestants_df = CSVReader.get_dataframe(contestants_csv_location) game_contestant_df = CSVReader.get_dataframe(game_contestant_csv_location) questions_df = CSVReader.get_dataframe(questions_csv_location) trend_df = CSVReader.get_dataframe(trend_csv_location) final_results_df = CSVReader.get_dataframe(final_results_csv_location) # Reconfiguration rc.generate_contestant_and_occupation(contestants_df, input_config, output_config) df_player_loc = rc_game_player_loc.generate_sql_statements( game_contestant_df, input_config, output_config, rc) # Reconfiguration of questions rcq.generate_sql_statements(questions_df, trend_df, final_results_df, df_player_loc, input_config, output_config)
def convert_data(rows): def convert(t): return datetime.strptime(t, '%I:%M%p').strftime('%H:%M') start_of_day = '09:00' end_of_day = '17:00' array = [] for row in rows: current_row = [] for column in row: if column != '': column = convert(column) current_row.append(column) if '' in current_row: if current_row[0] == '': end_of_day = current_row[1] else: start_of_day = current_row[0] else: array.append(current_row) return start_of_day, end_of_day, array data = CSVReader('times') start, end, data = convert_data(data) meetings = Meetings(start, end, data) print(meetings) conflicts, out_of_hours = meetings.report_anomalies() print('conflicting:', conflicts) print('out of hours:', out_of_hours)
from tensorflow.keras.models import load_model, Sequential, Model from models import Models from sklearn.metrics import confusion_matrix, f1_score, precision_score, recall_score from text_preprocessing import TextPreprocessing import json import os from csv_reader import CSVReader from datetime import datetime from tensorflow.keras import backend as K from sklearn.metrics import classification_report textPreProcessing = TextPreprocessing() if __name__ == "__main__": embeddings = np.load('text_embedding.npy', allow_pickle=True) texts = [] sentiments = [] good = CSVReader.dataframe_from_txt("WordNetAffectEmotionLists/joy.txt") good = good.texts texts = texts = np.append(texts, np.array(good)) sentiments = np.append(sentiments, [1] * len(good)) surprise = CSVReader.dataframe_from_txt( "WordNetAffectEmotionLists/surprise.txt") surprise = surprise.texts texts = np.append(texts, np.array(surprise)) sentiments = np.append(sentiments, [2] * len(surprise)) sad = CSVReader.dataframe_from_txt("WordNetAffectEmotionLists/sadness.txt") sad = sad.texts texts = np.append(texts, np.array(sad)) sentiments = np.append(sentiments, [-1] * len(sad)) fear = CSVReader.dataframe_from_txt("WordNetAffectEmotionLists/fear.txt") fear = fear.texts texts = np.append(texts, np.array(fear))
from autoencoder import TimeDistributedAutoEncoder from csv_reader import CSVReader from data_generator import DataGenerator if __name__ == "__main__": conf = docopt(__doc__, version='LSTM Anomaly Detector 0.1') ae = TimeDistributedAutoEncoder(conf) p = Plot(conf, ae) # determine whether to pull in fake data or a csv file if conf['synthetic']: #print 'generating synthetic data....' source = DataGenerator(conf, p) else: #print 'reading from csv file...' source = CSVReader(conf, p) # pull in the data (x_train, y_train), (x_test, y_test) = source.split_data() #print 'X_train.shape %s | Y_train.shape: %s' % (x_train.shape, y_train.shape) #print 'X_test.shape %s | Y_test.shape: %s' % (x_test.shape, y_test.shape) # build an LSTM or a regular autoencoder #print 'building %s autoencoder...' % conf['--model_type'] # Add the required layers model_type = conf['--model_type'].strip().lower() if model_type == 'lstm': # Deep autoencoder # ae.add_lstm_autoencoder([int(conf['--input_dim']), int(conf['--hidden_dim']) # , int(conf['--hidden_dim'])/2, int(conf['--hidden_dim']) / 4]
import sys from csv_reader import CSVReader from csv_writer import CSVWriter from trustpilot_url_encrypt import TrustPilotURLEncryption from config import Config if len(sys.argv) < 2: print('Requires at least one filename') exit() for filename in sys.argv[1:]: input_file = filename output_file = filename.replace('.csv', '_links.csv') print('reading', input_file) config = Config('config.yaml') reader = CSVReader(input_file) writer = CSVWriter(output_file) # write the header only once in the output file write_header = True # go through the source csv for row in reader(): # build the record for trust pilot we get something like record = {} for field in ('name', 'email', 'ref'): record[field] = row[config['fields'][field]] url = TrustPilotURLEncryption.encrypt(record) new_row = [] for key in reader.header: new_row.append(row[key]) new_row.append(url) if write_header:
from linear_regression import LinearRegression from train_test_split import TrainTestSplit import matplotlib.pyplot as plt from csv_reader import CSVReader f = CSVReader() f.read(csv="autos_prepared.csv") x_train, x_test, y_train, y_test = TrainTestSplit().split(x=f.data["powerPS"], y=f.data["price"]) model = LinearRegression() model.train(x=x_train, y=y_train) print("Fehler |", model.error) print("Durchschnittsfehler |", model.avg_error) print("a |", model.a) print("b |", model.b) print("Bestimmtheitsmaß |", model.score(xs=x_test, ys=y_test)) predicted = model.predict(x_test) plt.scatter(x_train, y_train) plt.scatter(x_test, y_test, color="green") plt.plot(x_test, predicted, color="red") plt.show()
import sys from os import listdir from os.path import join, dirname, realpath, isdir import json from csv_reader import CSVReader from body_box_extractor import BodyBoxExtractor ROOT_DIRECTORY = join(dirname(realpath(sys.argv[0])), '..', '..', 'res') data_directories = [ f for f in listdir(ROOT_DIRECTORY) if isdir(join(ROOT_DIRECTORY, f)) ] csv_reader = CSVReader(ROOT_DIRECTORY) body_box_extractor = BodyBoxExtractor('', None) check_results = {} for data_directory in data_directories: try: type_json_file = open(join(ROOT_DIRECTORY, data_directory, 'type.json')) type_data = json.load(type_json_file) csv_reader.set_file(join(data_directory, data_directory + '.csv')) skeleton_data = {'raw_data': csv_reader.read()} data = body_box_extractor.process(skeleton_data) check_results[data_directory] = type_data check_results[data_directory]['has_shoulder'] = all( abs(data['body_box']['shoulder_left']['3d']['mean']) > 0.001 ) and all( abs(data['body_box']['shoulder_left']['3d']['mean']) > 0.001) check_results[data_directory]['has_hip'] = all(