def Main(): port = int(sys.argv[1]) print "Starting IslandsMUD DEVELOPMENTAL server" factory = ChatProtocolFactory() reactor.listenTCP(port, factory) # Create the players directory incrementally, useful if this is the first # time the MUD is run. functions.mkdir_p('./players') reactor.run()
def plt_cm(y_true, predictions): """Plotting confusion matrices""" headers = ['track_id', 'genre', 'album_id', 'set', 'artist_id'] Dataset = pd.read_csv(join("splits", "MSD-I_dataset.tsv"), header=0, names=headers + [''], sep='\t', usecols=headers) #Convert from one hot encoded to column vector cm = confusion_matrix(y_true, predictions) classes = sorted(Dataset.genre.unique()) fig, axs = plt.subplots(nrows=1, ncols=2, figsize=(15, 8)) fig.subplots_adjust(hspace=0, wspace=0.5) fig.suptitle(Net_Name.upper(), x=0.1, y=0.93, size=15) i = 0 for ax in axs: ax.set_title("W/o Normalisation") if i == 1: cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] ax.set_title("With Normalisation") im = ax.imshow(cm, cmap='Blues') ax.figure.colorbar(im, ax=ax, shrink=0.7) # We want to show all ticks... ax.set(xticks=np.arange(cm.shape[1]), yticks=np.arange(cm.shape[0]), yticklabels=classes, xlabel='Predicted label', ylabel='True label') ax.set_xticklabels(classes, rotation=90) i += 1 fig.tight_layout() if cfg.save_test.upper() == "Y": mkdir_p(join("Results", "CMs", "GCN")) fig.savefig(join("Results", "CMs", "GCN", Net_Name + ".png")) i += 1
def save_dwellings_csv(self, dwellings, save_folder_name, save_regions): '''Save dwellings in csv format''' if isinstance(save_folder_name, str): self.folder = save_folder_name else: if isinstance(self.level, str): self.folder = self.level else: self.folder = 'level_'.format(self.level) functions.mkdir_p(self.folder) save_as = ('{}/{}-{}').format(self.folder, self.name, self.level) dwellings.Dwellings = dwellings.Dwellings.round(1) dwellings.to_csv(save_as + '-dwelings.csv', index=False) if save_regions == True: regions = dwellings[['id','Region','Taxonomy','Dwellings']] regions = regions.groupby(['id','Region']).sum() regions.Dwellings = regions.Dwellings.round(1) regions.to_csv(save_as + '-regions.csv') print '''----- end -----
print USAGE_MESSAGE sys.exit(2) for opt, arg in opts: if opt == "-o": output_dir = arg if opt == "-f": url_file = arg if not output_dir and not url_file: print USAGE_MESSAGE sys.exit(2) else: if output_dir: if os.path.isdir(output_dir): print "Found directory: %s" % output_dir else: mkdir_p(output_dir) print "Created directory: %s" % output_dir if url_file: if os.path.isfile(url_file): print "Removing existing file: %s" % url_file os.remove(url_file) # Get URLs from config for d in urls_to_crawl: files_processed = 0 files_written = 0 errors_encountered = 0 seed_url = d["url"] urls_to_visit = [seed_url] all_urls = [seed_url]
def main(argv): input_file = None output_dir = None failed_urls = [] errors_encountered = 0 try: opts, args = getopt.getopt(argv, "i:o:") except getopt.GetoptError: print USAGE_MESSAGE sys.exit(2) for opt, arg in opts: if opt == "-i": input_file = arg if opt == "-o": output_dir = arg if not input_file or not output_dir: print USAGE_MESSAGE sys.exit(2) if os.path.isdir(output_dir): print "Found directory: %s" % output_dir else: mkdir_p(output_dir) print "Created directory: %s" % output_dir with open(input_file) as f: urls = f.readlines() print "Found %d URLs" % len(urls) browser = get_selenium_browser() for url in urls: url = url.strip() if not url: continue try: print "\nProcessing URL: %s" % url print "Requesting URL with Python Requests: ", url get_response = requests.get(url, headers=REQUEST_HEADERS, timeout=60) content_type = get_response.headers.get('content-type') encoding = get_response.encoding page_source = get_response.text final_url = get_response.url if 'text/html' in content_type and not "<body" in page_source: print "No <body> tag found in page source. Requesting URL with Selenium: ", final_url try: browser.get(final_url) page_source = browser.page_source except: print "First Selenium request failed. Trying one last time." browser.get(final_url) page_source = browser.page_source else: if 'text/html' in content_type and not "<body" in page_source: print "No <body> tag found in page source. Requesting URL with Selenium one last time." browser.get(final_url) page_source = browser.page_source final_url = browser.current_url print "Found final URL: %s" % final_url encoded_data, encoding_used = get_encoded_data( page_source, encoding) filepath = get_filepath(final_url, encoding_used, output_dir) with open(filepath, 'w') as f: f.write(encoded_data) print "Wrote file: %s with encoding: %s" % (filepath, encoding_used) except: errors_encountered += 1 failed_urls.append(url) try: traceback_info = '\n'.join( traceback.format_exception(*(sys.exc_info()))) except: traceback_info = '' print "*** ERROR PROCESSING: %s ***\nTraceback: %s\n" % ( url, traceback_info) print "\nOperational Errors: %d\n" % errors_encountered if failed_urls: print "The following %d URLs failed:" % len(failed_urls) for url in failed_urls: print url
def main(argv): input_file = None output_dir = None failed_urls = [] errors_encountered = 0 try: opts, args = getopt.getopt(argv, "i:o:") except getopt.GetoptError: print USAGE_MESSAGE sys.exit(2) for opt, arg in opts: if opt == "-i": input_file = arg if opt == "-o": output_dir = arg if not input_file or not output_dir: print USAGE_MESSAGE sys.exit(2) if os.path.isdir(output_dir): print "Found directory: %s" % output_dir else: mkdir_p(output_dir) print "Created directory: %s" % output_dir with open(input_file) as f: urls = f.readlines() print "Found %d URLs" % len(urls) browser = get_selenium_browser() for url in urls: url = url.strip() if not url: continue try: print "\nProcessing URL: %s" % url print "Requesting URL with Python Requests: ", url get_response = requests.get(url, headers=REQUEST_HEADERS, timeout=60) content_type = get_response.headers.get("content-type") encoding = get_response.encoding page_source = get_response.text final_url = get_response.url if "text/html" in content_type and not "<body" in page_source: print "No <body> tag found in page source. Requesting URL with Selenium: ", final_url try: browser.get(final_url) page_source = browser.page_source except: print "First Selenium request failed. Trying one last time." browser.get(final_url) page_source = browser.page_source else: if "text/html" in content_type and not "<body" in page_source: print "No <body> tag found in page source. Requesting URL with Selenium one last time." browser.get(final_url) page_source = browser.page_source final_url = browser.current_url print "Found final URL: %s" % final_url encoded_data, encoding_used = get_encoded_data(page_source, encoding) filepath = get_filepath(final_url, encoding_used, output_dir) with open(filepath, "w") as f: f.write(encoded_data) print "Wrote file: %s with encoding: %s" % (filepath, encoding_used) except: errors_encountered += 1 failed_urls.append(url) try: traceback_info = "\n".join(traceback.format_exception(*(sys.exc_info()))) except: traceback_info = "" print "*** ERROR PROCESSING: %s ***\nTraceback: %s\n" % (url, traceback_info) print "\nOperational Errors: %d\n" % errors_encountered if failed_urls: print "The following %d URLs failed:" % len(failed_urls) for url in failed_urls: print url