# To get consistent results we switch from a random matrix initialization to something deterministic
# In[24]:
u = np.arange(100000).reshape((100, 1000))
s = np.arange(100000).reshape((1000, 100))
w = np.arange(10000).reshape((100, 100))
# In[25]:
prog = dml(script).input('U', u).input('S', s).input('W', w).output('res')
prog = dml(script).output('res')
res = ml.execute(prog).get('res')
print(res)
# If everything runs fine you should get *6244089899151.321* as result. Feel free to submit your DML script to the grader now!
#
# ### Submission
# In[26]:
get_ipython().system(u'rm -f rklib.py')
get_ipython().system(u'wget https://raw.githubusercontent.com/romeokienzler/developerWorks/master/coursera/ai/rklib.py')
# In[27]:
import os
import numpy as np
from pyspark.sql.functions import col, max
import systemml # pip3 install systemml
from systemml import MLContext, dml
from pyspark.context import SparkContext
from pyspark.sql import SQLContext
sc = SparkContext()
sqlContext = SQLContext(sc)
ml = MLContext(sc)
# train_df = sqlContext.read.load('data/train_256.parquet')
val_df = sqlContext.read.load('data/val_256.parquet')
X_val = val_df.select("__INDEX", "sample")
ml.setStatistics(True).setStatisticsMaxHeavyHitters(30).setExplain(True)
script = dml("resnet_prediction_parfor_rowwisecropping.dml").input(
X=X_val).output("Y")
Y = ml.execute(script).get("Y").toDF()
Y.show()