def find_col_min(column):
result = service.get_col_min(column)
if result:
helper.write_history_csv(datetime.now(), "get_col_min",
'print(spark_df.agg({{{name}: "min"}}))'.format(name=column),"{}".format(column))
return result
else:
return Response("{'error':'invalid operation '}", status=500, mimetype='application/json')
def ffill():
result = service.ffill()
if result:
helper.write_history_csv(datetime.now(), "ffill",
'df=spark_df.toPandas().ffill(axis ="rows")')
return result
else:
return Response("{'error':'invalid operation '}", status=500, mimetype='application/json')
def replace(colname,tovalue,fromval):
result = service.replace(colname,tovalue,fromval)
if result:
helper.write_history_csv(datetime.now(), "replace",
'spark_df=spark_df.withColumn({n}, f.regexp_replace({n2}, {f}, {t}))'.format(n=colname,n2=colname,f=fromval,t=tovalue), "{}|{}|{}".format(colname,fromval,tovalue))
return result
else:
return Response("{'error':'invalid operation '}", status=500, mimetype='application/json')
def to_int(colname):
result = service.to_int(colname)
if result:
helper.write_history_csv(datetime.now(), "to_int",
'spark_df=spark_df.withColumn({}, self.spark_df[{}].cast(IntegerType()))'.format(colname,colname), "{}".format(colname))
return result
else:
return Response("{'error':'invalid operation '}", status=500, mimetype='application/json')
def drop_col(column):
result = service.drop_column(column)
if result:
helper.write_history_csv(datetime.now(), "drop_column",
'spark_df=spark_df.drop({name})'.format(name=column), "{}".format(column))
return result
else:
return Response("{'error':'invalid operation '}", status=500, mimetype='application/json')
def get_head(num):
# service.read_original_file()
result = service.get_head(num)
if result:
helper.write_history_csv(datetime.now(), "get_head", 'print(spark_df.head(int({n})))'.format(n=num),"{}".format(num))
return result
else:
return Response("{'error':'invalid operation '}", status=500, mimetype='application/json')
def get_first():
# service.read_original_file()
result = service.get_first()
if result:
helper.write_history_csv(datetime.now(), "get_first", 'print(spark_df.first())')
return result
else:
return Response("{'error':'invalid operation '}", status=500, mimetype='application/json')
def lower_col(column):
result = service.lower_column(column)
if result:
helper.write_history_csv(datetime.now(), "drop_column",
"spark_df = spark_df.withColumn('temp', f.lower(f.col({})))."
"drop({}).withColumnRenamed('temp', {})".format(column,column, column), "{}".format(column))
return result
else:
return Response("{'error':'invalid operation '}", status=500, mimetype='application/json')
def rename_col(old_column_name, new_col_name):
result = service.rename_column(old_column_name, new_col_name)
if result:
helper.write_history_csv(datetime.now(), "rename_column",
'spark_df=spark_df.withColumnRenamed({o}, {n})'.format(o=old_column_name,n=new_col_name),"{}|{}".format(old_column_name,new_col_name))
return result
else:
return Response("{'error':'invalid operation '}", status=500, mimetype='application/json')
def sort_col(column, condition):
result = service.order_col(column, condition)
if result:
helper.write_history_csv(datetime.now(), "order_col",
'spark_df=spark_df.orderBy({col}, ascending={con})'.format(col=column,con=condition),"{}|{}".format(column,condition))
return result
else:
return Response("{'error':'invalid operation '}", status=500, mimetype='application/json')
