def main(): # output some information print("=====================================================") print("Welcome to Letterboxd Friend Ranker!") print("Instructions: This program compares you and") print("your friend's film taste. Once all the data has") print("been scraped and scores have been computed,") print("a report will be generated. The lower the avg.") print("difference, the better. If you and a friend do") print("not share at least 30 watched films, a score will") print("not be computed.") print("=====================================================\n") # prompt for info file_name = "" username = input("Enter your Letterboxd username: "******"Scraping friend data...\n") friends = sc.scraper(username) print("Computing scores...\n") # compute commonality for each friend results = commonality(current_user, friends) # write report print("Generating report...\n") gr.generate_report(results, current_user) print("Done! View the report in the current directory!")
def main(file): clean = input(""" Clean up old report files?\n This will NOT delete but move old files (i.e. previously sourced data) to a directory at your current location (If you don\'t do this and choose option 1 at hte next prompt your new report data will mix mixed with the old)\n Usage: if you have previously ran this application and are running it again with the intention of specifying a new date-range. i.e. you are going to choose option 1 at the next prompt. Type y or n and press <Enter> [y/n] """) if clean == 'y': clean_up() print("----------------NEXT CHOICE-----------------") choice = choose() if choice == 1: prefix = input("What kind or report are you running: 'metered' customers or 'control' customers : ") endpoints = parser(file) generate_script(endpoints) run_report() generate_report(prefix) elif choice == 2: prefix = input("What kind or report are you running: 'metered' customers or 'control' customers : ") generate_report(prefix)
def sort_camera_trap_images(unsorted_dir): json_name = os.path.dirname(unsorted_dir) + ".json" snapcat_database_dir = os.path.join(unsorted_dir, json_name) snapcat_json = json_database.JSONDatabase(snapcat_database_dir) burst.create_bursts(snapcat_json, unsorted_dir) segmentation.segment_images(snapcat_json) label_images.label_images(snapcat_json) user_label_image.user_label_images_burst(snapcat_json) generate_report.generate_report(snapcat_json, unsorted_dir)
def main(app_name): print "=== HPA Test ===" print "--- Start to Run K8sHPA Test x %d and Federator.ai Test x %d ---" % ( number_k8shpa, number_alamedahpa) start_time = time.time() i = 0 test_case_list = get_test_case_list() for test_case in test_case_list: run_scenario(test_case, app_name, i) generate_report(["table"]) end_time = time.time() duration = (end_time - start_time) / 60 print "It takes %d minutes" % duration
def reports(): # Begin report generation groups = dfd.groupby('var_type') from plot import plot_dists for name, group in groups: plot_dists(group.sstop - group.sstart, name, rpath) generate_report(report_dict) type_count = dfd.groupby('var_type').agg(lambda x: x.shape[0]).loc[:, ['chr']] var_percent = type_count.ix[:,0]/float(dfd.shape[0])*100 type_count['var_percent'] = var_percent print(type_count) report_dict['type_counts'] = type_count.to_html()
def run(self): while True: tbd_url = self.frontier.get_tbd_url() if not tbd_url: self.logger.info("Frontier is empty. Stopping Crawler.") generate_report() break resp = download(tbd_url, self.config, self.logger) self.logger.info(f"Downloaded {tbd_url}, status <{resp.status}>, " f"using cache {self.config.cache_server}.") scraped_urls = scraper(tbd_url, resp) for scraped_url in scraped_urls: self.frontier.add_url(scraped_url) self.frontier.mark_url_complete(tbd_url) time.sleep(self.config.time_delay)
def sort_camera_trap_images(unsorted_dir): json_name = os.path.basename(os.path.dirname(unsorted_dir)) + ".json" snapcat_database_dir = os.path.join(unsorted_dir, json_name) snapcat_json = json_database.JSONDatabase(snapcat_database_dir) burst.create_bursts(snapcat_json, unsorted_dir) #segmentation.segment_images( snapcat_json ) #label_images( snapcat_json ) # TODO make sure this is working well to deliver to Island conservation. # make sure the label is saved in the dataset user_label_image.user_label_images_burst(snapcat_json) generate_report.generate_report(snapcat_json, unsorted_dir)
def run(daysago): """A simple program that reports on your daily desktop app usage.""" cronstatus = setup_cron() if not cronstatus: click.echo('Cronjob already setup') else: click.echo(click.style('Cronjob setup successfull', fg='green')) date = datetime.now() - timedelta(days=daysago) date = date.strftime("%m-%d-%Y") report = generate_report(date) if not report: click.echo(click.style('No data for this date', fg='red')) return click.echo(click.style('Report generated sucessfully', fg='green')) click.echo(f'Stored to {report}') open_report(report)
def main(): config = ConfigParser.RawConfigParser() config.read(sys.argv[1]) gpath = config.get('input', 'make_ref') size_limit = config.getfloat('params', 'max_size') files = glob.glob(gpath + "tab/*.txt") studies_include = config.get('params', 'studies_include') studies_exclude = config.get('params', 'studies_exclude').split(",") vartype_f = config.get('params', 'var_type') if studies_include == '' or studies_include == None: studies_include = [] else: studies_include = studies_include.split(",") filtered = [] start = timeit.default_timer() pool = mp.Pool(8) files = files[0:20] studies = [i.split("/")[-1].rstrip(".txt") for i in files] for i in files: study = i.split("/")[-1].rstrip(".txt") if study in studies_exclude: pass else: if (len(studies_include) == 0) or (study in studies_include): reader = pd.read_csv(i, sep="\t", index_col=0, dtype={'chr': 'S5'}) pool.apply_async(filter_by_size, [reader, study], {'max_size': size_limit}, callback=lambda x: filtered.append(x)) else: pass # Remove duplicated elements ###### Step takes around 7 minutes ################### pool.close() pool.join() df = pd.concat(filtered) print(vartype_f) stop = timeit.default_timer() print('Time to load in files and parse: {0!s}'.format(stop - start)) p_studies = set(df.study) non_passed = [] for i in studies: if i not in p_studies: non_passed.append(i) print(('Studies that had no variants that did ' 'not pass size filtering:{0}').format("\t".join(non_passed))) ############## HACK for now until we find out what is going on # # Get rid of the contigs for now df = df.ix[df.contig.isnull(), :] # The GRc37 to 38 multiple mapping isn't resolved need to discuss how to # deal with this df = df.ix[np.logical_not(df.index.duplicated()), :] # :TODO if sstart and sstop are the same, no # matter if it was originally annotated as inner_start # or inner stop it will be collapsed # For now since, ignore fuzzy dfd = df.drop_duplicates(['chr', 'var_type', 'sstart', 'sstop'], inplace=False) new_unique_index = np.arange(dfd.shape[0]) dfd.loc[:, 'uID'] = new_unique_index print('new index created') # This step takes forever start = timeit.default_timer() groups = df.groupby('chr') unique_mapping = [] pool = mp.Pool(8) for name, group in groups: pool.apply_async(generate_unique_mapping, args=(dfd.ix[dfd.chr == name, :], group), callback=lambda x: unique_mapping.append(x)) ''' tgroup = dfd.ix[dfd['chr'] == name,] pool.apply_async(generate_unique_mapping_numba, args = (group.sstart.values, group.sstop.values, tgroup.sstart.values, tgroup.sstop.values, tgroup.index.values), callback=lambda x: unique_mapping.append(pd.Series(x, index = group.index))) ''' pool.close() pool.join() ns = pd.concat(unique_mapping) print('Time to generate mapping: {0!s}'.format((stop - start))) df['uID'] = ns report_dict = {} nstudies = config.getint('params', 'nstudies') start = timeit.default_timer() output = np.zeros(dfd.uID.shape[0], dtype=bool) embed() std_filter = groupby_study_numba(df.uID.values, df.study, output, nstudies=nstudies) print(np.sum(std_filter)) dfd = dfd.ix[std_filter, :] df = df.ix[df.uID.isin(dfd.uID), :] dfd.to_csv(gpath + 'filtered_no_dupes.txt', sep="\t") df.to_csv(gpath + 'study_filtered_all.txt', sep="\t") print('Time to run: {0!s}'.format(stop - start)) groups = dfd.groupby('var_type') from plot import plot_dists generate_report(report_dict) rpath = config.get('output', 'report_dir') for name, group in groups: plot_dists(group.sstop - group.sstart, name, rpath) type_count = dfd.groupby('var_type').agg(lambda x: x.shape[0]).loc[:, ['chr']] var_percent = type_count.ix[:, 0] / float(dfd.shape[0]) * 100 type_count['var_percent'] = var_percent type_count['var_percent'].round(2) report_dict['var_type_pivot'] = type_count.to_html() report_dict['studies'] = [] report_dict['var_types'] = [name for name, _ in groups] generate_report(report_dict)
crater_analysis_list = [] for file in files: crater_analysis = DataCraterAnalysis() mesh_name = file.stem print(f"processing {mesh_name}") output_name = mesh_name + ".csv" mesh, mesh_points, fit_parameters = plane_fit.prepare_and_fit( str(file)) # Mesh export (stl) mesh.export(out_dir.joinpath(f"{mesh_name}_transformed.stl")) crater_analysis.project_name = project_name crater_analysis.name = mesh_name crater_analysis.points = mesh_points crater_analysis.fit = fit_parameters for plane in [[0, 1, 0], [1, 0, 0]]: cs = slice_mesh(mesh, direction=plane) crater_analysis.cross_section = pd.concat( [crater_analysis.cross_section, cs], axis=1) crater_analysis.cross_section.to_csv( out_dir.joinpath(f"{mesh_name}_cs.csv"), index=False) # DataClass export (pkl) pickle.dump(crater_analysis, open(out_dir.joinpath(mesh_name + ".pkl"), "wb")) # CS export (csv) crater_analysis_list.append(crater_analysis) plot_contour(crater_analysis) plot_slices(crater_analysis_list) generate_report.generate_report(crater_analysis_list) # data = pd.read_csv('file', header=[0,1])
def detect_video(yolo, video_path, output_path=""): import cv2 vid = cv2.VideoCapture(video_path) if not vid.isOpened(): raise IOError("Couldn't open webcam or video") video_FourCC = int(vid.get(cv2.CAP_PROP_FOURCC)) video_fps = vid.get(cv2.CAP_PROP_FPS) video_size = (int(vid.get(cv2.CAP_PROP_FRAME_WIDTH)), int(vid.get(cv2.CAP_PROP_FRAME_HEIGHT))) isOutput = True if output_path != "" else False if isOutput: print("!!! TYPE:", type(output_path), type(video_FourCC), type(video_fps), type(video_size)) out = cv2.VideoWriter(output_path, video_FourCC, video_fps, video_size) accum_time = 0 curr_fps = 0 fps = "FPS: ??" prev_time = timer() time_log = [] label_log = [] while True: return_value, frame = vid.read() image = Image.fromarray(frame) image, label_list = yolo.detect_image(image) result = np.asarray(image) curr_time = timer() exec_time = curr_time - prev_time prev_time = curr_time accum_time = accum_time + exec_time curr_fps = curr_fps + 1 label_log.append(label_list) time_log.append('Timestamp: {:%Y-%m-%d %H:%M:%S}'.format( datetime.datetime.now())) if accum_time > 1: accum_time = accum_time - 1 fps = "FPS: " + str(curr_fps) curr_fps = 0 cv2.putText(result, text=fps, org=(3, 15), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.50, color=(255, 0, 0), thickness=2) cv2.namedWindow("result", cv2.WINDOW_NORMAL) cv2.imshow("result", result) if isOutput: out.write(result) if cv2.waitKey(1) & 0xFF == ord('q'): break time.sleep(1) output_df = pd.DataFrame({'label': label_log, 'timestamp': time_log}) output_df.to_csv('out.csv') time.sleep(3) generate_report.generate_report() yolo.close_session()
from generate_sample_data import generate_sample_data from generate_report import generate_report if __name__ == '__main__': try: data = generate_sample_data(1000) generate_report(data) except Exception as err: print('error', err)
from data_preparation import read_data,remove_outliers,get_dummies,split_data from train_model import train from logger import get_logger from generate_report import generate_report # Logger logger = get_logger(__name__) if __name__ == "__main__": # Lendo dados df = read_data() # Removendo Outliers df = remove_outliers(df) # Get Dummies df = get_dummies(df) # Divindo dados X_train, X_test, y_train, y_test = split_data(df) # Treinando Modelo regLinear,score = train(X_train,y_train) # Avaliando Modelo generate_report(regLinear,X_test,y_test,score)
@author: Filo """ from scrappers import Crag from config import db_file import sqlite3 import generate_report if __name__ == "__main__": from generate_report import reports_dict crags_to_crawl = [] for crags in reports_dict.values(): crags_to_crawl += crags conn = sqlite3.connect(db_file) c = conn.cursor() for crag_id in crags_to_crawl: scrapper = Crag(id=crag_id) new_jobs = scrapper.scrap() scrapper.save(c) for job in new_jobs: job.scrap() job.save(c) conn.commit() conn.close() generate_report.generate_report(reports_dict)
def main(): config = ConfigParser.RawConfigParser() config.read(sys.argv[1]) gpath = config.get('input', 'make_ref') size_limit = config.getfloat('params', 'max_size') files = glob.glob(gpath + "tab/*.txt") studies_include = config.get('params', 'studies_include') studies_exclude = config.get('params', 'studies_exclude').split(",") vartype_f = config.get('params', 'var_type') if studies_include == '' or studies_include == None: studies_include = [] else: studies_include = studies_include.split(",") filtered = [] start = timeit.default_timer() pool = mp.Pool(8) files = files[0:20] studies = [i.split("/")[-1].rstrip(".txt") for i in files] for i in files: study = i.split("/")[-1].rstrip(".txt") if study in studies_exclude: pass else: if (len(studies_include) == 0) or (study in studies_include): reader = pd.read_csv(i, sep="\t", index_col=0, dtype={'chr':'S5'}) pool.apply_async(filter_by_size, [reader, study], {'max_size': size_limit}, callback = lambda x: filtered.append(x)) else: pass # Remove duplicated elements ###### Step takes around 7 minutes ################### pool.close() pool.join() df = pd.concat(filtered) print(vartype_f) stop = timeit.default_timer() print('Time to load in files and parse: {0!s}'.format(stop-start)) p_studies = set(df.study) non_passed = [] for i in studies: if i not in p_studies: non_passed.append(i) print(('Studies that had no variants that did ' 'not pass size filtering:{0}').format("\t".join(non_passed))) ############## HACK for now until we find out what is going on # # Get rid of the contigs for now df = df.ix[df.contig.isnull(), :] # The GRc37 to 38 multiple mapping isn't resolved need to discuss how to # deal with this df = df.ix[np.logical_not(df.index.duplicated()),:] # :TODO if sstart and sstop are the same, no # matter if it was originally annotated as inner_start # or inner stop it will be collapsed # For now since, ignore fuzzy dfd = df.drop_duplicates(['chr', 'var_type', 'sstart', 'sstop'], inplace=False) new_unique_index = np.arange(dfd.shape[0]) dfd.loc[:,'uID'] = new_unique_index print('new index created') # This step takes forever start = timeit.default_timer() groups = df.groupby('chr') unique_mapping = [] pool = mp.Pool(8) for name, group in groups: pool.apply_async(generate_unique_mapping, args = (dfd.ix[dfd.chr == name,:], group), callback=lambda x: unique_mapping.append(x)) ''' tgroup = dfd.ix[dfd['chr'] == name,] pool.apply_async(generate_unique_mapping_numba, args = (group.sstart.values, group.sstop.values, tgroup.sstart.values, tgroup.sstop.values, tgroup.index.values), callback=lambda x: unique_mapping.append(pd.Series(x, index = group.index))) ''' pool.close() pool.join() ns = pd.concat(unique_mapping) print('Time to generate mapping: {0!s}'.format((stop-start))) df['uID'] = ns report_dict = {} nstudies = config.getint('params', 'nstudies') start = timeit.default_timer() output = np.zeros(dfd.uID.shape[0], dtype=bool) embed() std_filter = groupby_study_numba(df.uID.values, df.study, output, nstudies=nstudies) print(np.sum(std_filter)) dfd = dfd.ix[std_filter,:] df = df.ix[df.uID.isin(dfd.uID),:] dfd.to_csv(gpath + 'filtered_no_dupes.txt', sep="\t") df.to_csv(gpath + 'study_filtered_all.txt', sep="\t") print('Time to run: {0!s}'.format(stop - start)) groups = dfd.groupby('var_type') from plot import plot_dists generate_report(report_dict) rpath = config.get('output', 'report_dir') for name, group in groups: plot_dists(group.sstop - group.sstart, name, rpath) type_count = dfd.groupby('var_type').agg(lambda x: x.shape[0]).loc[:, ['chr']] var_percent = type_count.ix[:,0]/float(dfd.shape[0])*100 type_count['var_percent'] = var_percent type_count['var_percent'].round(2) report_dict['var_type_pivot'] = type_count.to_html() report_dict['studies'] = [] report_dict['var_types'] = [name for name, _ in groups] generate_report(report_dict)