def art_uk(file_name): with g_open(file_name, 'rt', encoding = 'utf-8') as f0: for f0_line in f0: f0_dict = json.loads(f0_line) if f0_dict['title'] == 'イギリス': article_uk = f0_dict['text'] return article_uk
def load_mnist(path=join(getcwd(), 'Data'), kind='train'): labels_path = join(path, f'{kind}-labels-idx1-ubyte.gz') images_path = join(path, f'{kind}-images-idx3-ubyte.gz') with g_open(labels_path, 'rb') as lbpath: labels = np.frombuffer(lbpath.read(), dtype=np.uint8, offset=8) with g_open(images_path, 'rb') as imgpath: images = np.frombuffer(imgpath.read(), dtype=np.uint8, offset=16).reshape(len(labels), 784) size = 1000 index_label2 = np.where((labels == 2)) x_label2 = images[index_label2][:size, :].astype('float64') index_label4 = np.where((labels == 4)) x_label4 = images[index_label4][:size, :].astype('float64') index_label6 = np.where((labels == 6)) x_label6 = images[index_label6][:size, :].astype('float64') y_label2 = np.array([2] * 1000) y_label4 = np.array([4] * 1000) y_label6 = np.array([6] * 1000) label2_data = np.append(x_label2, y_label2.reshape(1000, 1), axis=1) label4_data = np.append(x_label4, y_label4.reshape(1000, 1), axis=1) label6_data = np.append(x_label6, y_label6.reshape(1000, 1), axis=1) all_data = np.append(np.append(label2_data, label4_data, axis=0), label6_data, axis=0) data_x = all_data[:, :-1] data_y = all_data[:, -1] train_x, test_x, train_y, test_y = train_test_split(data_x, data_y, test_size=0.2, random_state=SEED) scaler = StandardScaler() train_x = scaler.fit_transform(train_x) test_x = scaler.transform(test_x) return train_x, test_x, train_y, test_y
def load_mnist(path=join(getcwd(), 'Data'), kind='train'): labels_path = join(path, f'{kind}-labels-idx1-ubyte.gz') images_path = join(path, f'{kind}-images-idx3-ubyte.gz') with g_open(labels_path, 'rb') as lbpath: labels = np.frombuffer(lbpath.read(), dtype=np.uint8, offset=8) with g_open(images_path, 'rb') as imgpath: images = np.frombuffer(imgpath.read(), dtype=np.uint8, offset=16).reshape(len(labels), 784) index_label2 = np.where((labels == 2)) x_label2 = images[index_label2][:1000, :].astype('float64') y_label2 = labels[index_label2][:1000].astype('float64') index_label4 = np.where((labels == 4)) x_label4 = images[index_label4][:1000, :].astype('float64') y_label4 = labels[index_label4][:1000].astype('float64') # converting labels of classes 2 and 4 into +1 and -1, respectively y_label2 = y_label2 / 2.0 y_label4 = y_label4 / -4.0 x_label_24 = np.vstack((x_label2, x_label4)) y_label_24 = np.concatenate((y_label2, y_label4)) # Normalize the data scaler = StandardScaler() scaler.fit(x_label_24) x_label_24 = scaler.transform(x_label_24) data = train_test_split(x_label_24, y_label_24, test_size=0.3, random_state=SEED) x_train24, x_test24, y_train24, y_test24 = data return x_train24, y_train24, x_test24, y_test24
def memos(event, context): print("Loading Function...") bucket = event['Records'][0]['s3']['bucket']['name'] key = unquote_plus(event['Records'][0]['s3']['object']['key']).decode('utf8') S3.download_file(bucket, key, '/tmp/%s' % (key.split('/')[-1])) # If queue already exists it will just fetch the url of the queue. queue_url = SQS.create_queue(QueueName=QUEUE_NAME, Attributes={'MessageRetentionPeriod': MESSAGE_RETENTION_PERIOD})['QueueUrl'] with g_open('/tmp/%s' % (key.split('/')[-1]), 'r+') as f: batch_of_mess = [] for identifier, content in enumerate(f.readlines()): # Gets rid of odd unicode characters that SQS does not like and the message would fail to send. batch_of_mess.append({'Id': str(identifier), 'MessageBody': ''.join(l for l in content if l in printable)}) # Maximum size of a batch is 256 KB and/or 10 messages. if getsizeof(batch_of_mess) >= 225 or len(batch_of_mess) == 10: message = SQS.send_message_batch(QueueUrl=queue_url, Entries=batch_of_mess) # SQS does not throw up an error if a message fails to send. if 'Failed' in message: print(message) del batch_of_mess[:] # Takes the remainder of the messages and sends them to SQS. if batch_of_mess: last_message = SQS.send_message_batch(QueueUrl=queue_url, Entries=batch_of_mess) if 'Failed' in last_message: print(last_message) print("Done!")