def Main():
port = int(sys.argv[1])
print "Starting IslandsMUD DEVELOPMENTAL server"
factory = ChatProtocolFactory()
reactor.listenTCP(port, factory)
# Create the players directory incrementally, useful if this is the first
# time the MUD is run.
functions.mkdir_p('./players')
reactor.run()
def plt_cm(y_true, predictions):
"""Plotting confusion matrices"""
headers = ['track_id', 'genre', 'album_id', 'set', 'artist_id']
Dataset = pd.read_csv(join("splits", "MSD-I_dataset.tsv"),
header=0,
names=headers + [''],
sep='\t',
usecols=headers)
#Convert from one hot encoded to column vector
cm = confusion_matrix(y_true, predictions)
classes = sorted(Dataset.genre.unique())
fig, axs = plt.subplots(nrows=1, ncols=2, figsize=(15, 8))
fig.subplots_adjust(hspace=0, wspace=0.5)
fig.suptitle(Net_Name.upper(), x=0.1, y=0.93, size=15)
i = 0
for ax in axs:
ax.set_title("W/o Normalisation")
if i == 1:
cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
ax.set_title("With Normalisation")
im = ax.imshow(cm, cmap='Blues')
ax.figure.colorbar(im, ax=ax, shrink=0.7)
# We want to show all ticks...
ax.set(xticks=np.arange(cm.shape[1]),
yticks=np.arange(cm.shape[0]),
yticklabels=classes,
xlabel='Predicted label',
ylabel='True label')
ax.set_xticklabels(classes, rotation=90)
i += 1
fig.tight_layout()
if cfg.save_test.upper() == "Y":
mkdir_p(join("Results", "CMs", "GCN"))
fig.savefig(join("Results", "CMs", "GCN", Net_Name + ".png"))
i += 1
def save_dwellings_csv(self, dwellings, save_folder_name, save_regions):
'''Save dwellings in csv format'''
if isinstance(save_folder_name, str):
self.folder = save_folder_name
else:
if isinstance(self.level, str):
self.folder = self.level
else:
self.folder = 'level_'.format(self.level)
functions.mkdir_p(self.folder)
save_as = ('{}/{}-{}').format(self.folder, self.name, self.level)
dwellings.Dwellings = dwellings.Dwellings.round(1)
dwellings.to_csv(save_as + '-dwelings.csv', index=False)
if save_regions == True:
regions = dwellings[['id','Region','Taxonomy','Dwellings']]
regions = regions.groupby(['id','Region']).sum()
regions.Dwellings = regions.Dwellings.round(1)
regions.to_csv(save_as + '-regions.csv')
print '''----- end -----
print USAGE_MESSAGE
sys.exit(2)
for opt, arg in opts:
if opt == "-o":
output_dir = arg
if opt == "-f":
url_file = arg
if not output_dir and not url_file:
print USAGE_MESSAGE
sys.exit(2)
else:
if output_dir:
if os.path.isdir(output_dir):
print "Found directory: %s" % output_dir
else:
mkdir_p(output_dir)
print "Created directory: %s" % output_dir
if url_file:
if os.path.isfile(url_file):
print "Removing existing file: %s" % url_file
os.remove(url_file)
# Get URLs from config
for d in urls_to_crawl:
files_processed = 0
files_written = 0
errors_encountered = 0
seed_url = d["url"]
urls_to_visit = [seed_url]
all_urls = [seed_url]
def main(argv):
input_file = None
output_dir = None
failed_urls = []
errors_encountered = 0
try:
opts, args = getopt.getopt(argv, "i:o:")
except getopt.GetoptError:
print USAGE_MESSAGE
sys.exit(2)
for opt, arg in opts:
if opt == "-i":
input_file = arg
if opt == "-o":
output_dir = arg
if not input_file or not output_dir:
print USAGE_MESSAGE
sys.exit(2)
if os.path.isdir(output_dir):
print "Found directory: %s" % output_dir
else:
mkdir_p(output_dir)
print "Created directory: %s" % output_dir
with open(input_file) as f:
urls = f.readlines()
print "Found %d URLs" % len(urls)
browser = get_selenium_browser()
for url in urls:
url = url.strip()
if not url:
continue
try:
print "\nProcessing URL: %s" % url
print "Requesting URL with Python Requests: ", url
get_response = requests.get(url,
headers=REQUEST_HEADERS,
timeout=60)
content_type = get_response.headers.get('content-type')
encoding = get_response.encoding
page_source = get_response.text
final_url = get_response.url
if 'text/html' in content_type and not "<body" in page_source:
print "No <body> tag found in page source. Requesting URL with Selenium: ", final_url
try:
browser.get(final_url)
page_source = browser.page_source
except:
print "First Selenium request failed. Trying one last time."
browser.get(final_url)
page_source = browser.page_source
else:
if 'text/html' in content_type and not "<body" in page_source:
print "No <body> tag found in page source. Requesting URL with Selenium one last time."
browser.get(final_url)
page_source = browser.page_source
final_url = browser.current_url
print "Found final URL: %s" % final_url
encoded_data, encoding_used = get_encoded_data(
page_source, encoding)
filepath = get_filepath(final_url, encoding_used, output_dir)
with open(filepath, 'w') as f:
f.write(encoded_data)
print "Wrote file: %s with encoding: %s" % (filepath,
encoding_used)
except:
errors_encountered += 1
failed_urls.append(url)
try:
traceback_info = '\n'.join(
traceback.format_exception(*(sys.exc_info())))
except:
traceback_info = ''
print "*** ERROR PROCESSING: %s ***\nTraceback: %s\n" % (
url, traceback_info)
print "\nOperational Errors: %d\n" % errors_encountered
if failed_urls:
print "The following %d URLs failed:" % len(failed_urls)
for url in failed_urls:
print url
def main(argv):
input_file = None
output_dir = None
failed_urls = []
errors_encountered = 0
try:
opts, args = getopt.getopt(argv, "i:o:")
except getopt.GetoptError:
print USAGE_MESSAGE
sys.exit(2)
for opt, arg in opts:
if opt == "-i":
input_file = arg
if opt == "-o":
output_dir = arg
if not input_file or not output_dir:
print USAGE_MESSAGE
sys.exit(2)
if os.path.isdir(output_dir):
print "Found directory: %s" % output_dir
else:
mkdir_p(output_dir)
print "Created directory: %s" % output_dir
with open(input_file) as f:
urls = f.readlines()
print "Found %d URLs" % len(urls)
browser = get_selenium_browser()
for url in urls:
url = url.strip()
if not url:
continue
try:
print "\nProcessing URL: %s" % url
print "Requesting URL with Python Requests: ", url
get_response = requests.get(url, headers=REQUEST_HEADERS, timeout=60)
content_type = get_response.headers.get("content-type")
encoding = get_response.encoding
page_source = get_response.text
final_url = get_response.url
if "text/html" in content_type and not "<body" in page_source:
print "No <body> tag found in page source. Requesting URL with Selenium: ", final_url
try:
browser.get(final_url)
page_source = browser.page_source
except:
print "First Selenium request failed. Trying one last time."
browser.get(final_url)
page_source = browser.page_source
else:
if "text/html" in content_type and not "<body" in page_source:
print "No <body> tag found in page source. Requesting URL with Selenium one last time."
browser.get(final_url)
page_source = browser.page_source
final_url = browser.current_url
print "Found final URL: %s" % final_url
encoded_data, encoding_used = get_encoded_data(page_source, encoding)
filepath = get_filepath(final_url, encoding_used, output_dir)
with open(filepath, "w") as f:
f.write(encoded_data)
print "Wrote file: %s with encoding: %s" % (filepath, encoding_used)
except:
errors_encountered += 1
failed_urls.append(url)
try:
traceback_info = "\n".join(traceback.format_exception(*(sys.exc_info())))
except:
traceback_info = ""
print "*** ERROR PROCESSING: %s ***\nTraceback: %s\n" % (url, traceback_info)
print "\nOperational Errors: %d\n" % errors_encountered
if failed_urls:
print "The following %d URLs failed:" % len(failed_urls)
for url in failed_urls:
print url
