#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
"""
from pypchutils.generic import create_logger
from pyspark.sql import SparkSession, DataFrame
from pyspark.sql import types as T, functions as F
from pyspark_demo.commons.rate_processor import RateProcessor
logger = create_logger(__name__, level="info")
def gen_test_data(spark: SparkSession, verbose: int = 1) -> DataFrame:
"""
"""
# Create a Spark data frame
schema = T.StructType([
T.StructField("date", T.StringType(), True),
T.StructField("user_id", T.IntegerType(), True),
T.StructField("user_name", T.StringType(), True),
T.StructField("total_orders", T.IntegerType(), True),
T.StructField("total_amount", T.FloatType(), True),
])
data = [
("2020-01-01", 1, "AA", 111, 111.11),
("2020-01-01", 2, "BB", 222, 222.22),
("2020-04-04", 1, "AA", 444, 444.44),
("2020-04-01", 3, "CC", 333, 333.33),
]
data = spark.createDataFrame(data, schema=schema)
Tutorial: https://towardsdatascience.com/an-easy-introduction-to-pytorch-for-neural-networks-3ea08516bff2
# Usage
export PYTHONPATH=$(pwd)
"""
import json
import os
import pandas as pd
from pypchutils.generic import create_logger
import torch
import torch.nn as nn
from torch.nn import functional as F
import torchvision
import torchvision.transforms as transforms
logger = create_logger(__name__)
class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
self.conv1 = nn.Conv2d(1, 64, kernel_size=(3, 3), padding=1)
self.conv2 = nn.Conv2d(64, 64, kernel_size=(3, 3), padding=1)
self.max_pool = nn.MaxPool2d(2, 2)
self.global_pool = nn.AvgPool2d(7)
self.fc1 = nn.Linear(64, 64)
self.fc2 = nn.Linear(64, 10)
def forward(self, x):
"""Set up the model by stacking all the layers together"""
x = F.relu(self.conv1(x))
