def append(rows):
"""
Append a row at the end of a dataframe
:param rows: List of values or tuples to be appended
:return: Spark DataFrame
"""
df = self
if is_list_of_tuples(rows):
columns = [str(i) for i in range(df.cols.count())]
if not is_list_of_tuples(rows):
rows = [tuple(rows)]
new_row = op.Create.df(columns, rows)
df_result = df.union(new_row)
elif is_list_of_dataframes(rows) or is_dataframe(rows):
row = val_to_list(rows)
row.insert(0, df)
df_result = append_df(row, like="rows")
else:
RaiseIt.type_error(rows, ["list of tuples", "list of dataframes"])
df_result = df_result.preserve_meta(self, Actions.NEST.value,
df.cols.names())
return df_result
def gbt(df, columns, input_col, **kwargs):
"""
Runs a gradient boosting tree classifier for input DataFrame.
:param df: Pyspark dataframe to analyze.
:param columns: List of columns to select for prediction.
:param input_col: Column to predict.
:return: DataFrame with gradient boosting tree and prediction run.
"""
if not is_dataframe(df):
raise TypeError("Spark dataframe expected")
columns = parse_columns(df, columns)
if not is_str(input_col):
raise TypeError("Error, input column must be a string")
data = df.select(columns)
feats = data.columns
feats.remove(input_col)
df = string_to_index(df, input_cols=input_col)
df = vector_assembler(df, input_cols=feats, output_col="features")
model = GBTClassifier(**kwargs)
df = df.cols.rename(name_col(input_col, "index_to_string"), "label")
gbt_model = model.fit(df)
df_model = gbt_model.transform(df)
return df_model, gbt_model
def decision_tree(df, columns, input_col, **kargs):
"""
Runs a decision tree classifier for input DataFrame.
:param df: Pyspark dataframe to analyze.
:param columns: List of columns to select for prediction.
:param input_col: Column to predict.
:return: DataFrame with decision tree and prediction run.
"""
if not is_dataframe(df):
raise TypeError("Spark dataframe expected")
columns = parse_columns(df, columns)
if not is_str(input_col):
raise TypeError("Error, input column must be a string")
data = df.select(columns)
feats = data.columns
feats.remove(input_col)
df = string_to_index(df, input_cols=input_col)
df = vector_assembler(df, input_cols=feats)
model = DecisionTreeClassifier(**kargs)
df = df.cols.rename(name_col(input_col, "index"), "label")
dt_model = model.fit(df)
df_model = dt_model.transform(df)
return df_model, dt_model
def __init__(self, df, col_name):
"""
:param df: Spark Dataframe
:param col_name: column name
"""
if not is_dataframe(df):
raise TypeError("Spark Dataframe expected")
self.df = df
self.col_name = one_list_to_val(parse_columns(df, col_name))
def n_gram(df, input_col, n=2):
"""
Converts the input array of strings inside of a Spark DF into an array of n-grams.
:param df: Pyspark dataframe to analyze
:param input_col: Column to analyzer.
:param n: number of elements per n-gram >=1.
:return: Spark DataFrame with n-grams calculated.
"""
is_dataframe(df)
tokenizer = feature.Tokenizer().setInputCol(input_col) | feature.StopWordsRemover()
count = feature.CountVectorizer()
gram = feature.NGram(n=n) | feature.CountVectorizer()
tf = tokenizer | (count, gram) | feature.VectorAssembler()
tfidf = tf | feature.IDF().setOutputCol('features')
tfidf_model = tfidf.fit(df)
df_model = tfidf_model.transform(df)
return df_model, tfidf_model
def logistic_regression_text(df, input_col):
"""
Runs a logistic regression for input (text) DataFrame.
:param df: Pyspark dataframe to analyze
:param input_col: Column to predict
:return: DataFrame with logistic regression and prediction run.
"""
if not is_dataframe(df):
raise TypeError("Spark dataframe expected")
pl = feature.Tokenizer().setInputCol(input_col) | feature.CountVectorizer()
ml = pl | classification.LogisticRegression()
ml_model = ml.fit(df)
df_model = ml_model.transform(df)
return df_model, ml_model
def __init__(self, df, col_name, threshold):
"""
:param df: Spark Dataframe
:param col_name:
"""
if not is_dataframe(df):
raise TypeError("Spark Dataframe expected")
self.df = df
if not is_numeric(threshold):
raise TypeError("Numeric expected")
self.threshold = threshold
self.col_name = one_list_to_val(parse_columns(df, col_name))
def __init__(self, df, col_name, threshold, relative_error=RELATIVE_ERROR):
"""
:param df:
:param col_name:
:param threshold:
"""
if not is_dataframe(df):
raise TypeError("Spark Dataframe expected")
if not is_numeric(threshold):
raise TypeError("Numeric expected")
if not is_numeric(relative_error):
raise TypeError("Numeric expected")
self.df = df
self.threshold = threshold
self.relative_error = relative_error
self.col_name = one_list_to_val(parse_columns(df, col_name))
def parse_columns(df,
cols_args,
get_args=False,
is_regex=None,
filter_by_column_dtypes=None,
accepts_missing_cols=False,
invert=False):
"""
Return a list of columns and check that columns exists in the dataframe
Accept '*' as parameter in which case return a list of all columns in the dataframe.
Also accept a regex.
If a list of tuples return to list. The first element is the columns name the others element are params.
This params can be used to create custom transformation functions. You can find and example in cols().cast()
:param df: Dataframe in which the columns are going to be checked
:param cols_args: Accepts * as param to return all the string columns in the dataframe
:param get_args:
:param is_regex: Use True is col_attrs is a regex
:param filter_by_column_dtypes: A data type for which a columns list is going be filtered
:param accepts_missing_cols: if true not check if column exist in the dataframe
:param invert: Invert the final selection. For example if you want to select not integers
:return: A list of columns string names
"""
if not is_dataframe(df):
RaiseIt.type_error(df, "Dataframe")
attrs = None
# if columns value is * get all dataframes columns
if is_regex is True:
r = re.compile(cols_args[0])
cols = list(filter(r.match, df.columns))
elif cols_args == "*" or cols_args is None:
cols = df.columns
# In case we have a list of tuples we use the first element of the tuple is taken as the column name
# and the rest as params. We can use the param in a custom function as follow
# def func(attrs): attrs return (1,2) and (3,4)
# return attrs[0] + 1
# df.cols().apply([('col_1',1,2),('cols_2', 3 ,4)], func)
# Verify if we have a list with tuples
elif is_tuple(cols_args) or is_list_of_tuples(cols_args):
cols_args = val_to_list(cols_args)
# Extract a specific position in the tuple
cols = [(i[0:1][0]) for i in cols_args]
attrs = [(i[1:]) for i in cols_args]
else:
# if not a list convert to list
cols = val_to_list(cols_args)
# Get col name from index
cols = [c if is_str(c) else df.columns[c] for c in cols]
# Check for missing columns
if accepts_missing_cols is False:
check_for_missing_columns(df, cols)
# Filter by column data type
filter_by_column_dtypes = val_to_list(filter_by_column_dtypes)
if is_list_of_list(filter_by_column_dtypes):
filter_by_column_dtypes = [
item for sublist in filter_by_column_dtypes for item in sublist
]
columns_residual = None
# If necessary filter the columns by data type
if filter_by_column_dtypes:
# Get columns for every data type
columns_filtered = filter_col_name_by_dtypes(df,
filter_by_column_dtypes)
# Intersect the columns filtered per data type from the whole dataframe with the columns passed to the function
final_columns = list(OrderedSet(cols).intersection(columns_filtered))
# This columns match filtered data type
columns_residual = list(
OrderedSet(cols) - OrderedSet(columns_filtered))
else:
final_columns = cols
cols_params = []
if invert:
final_columns = list(
OrderedSet(df.cols.names()) - OrderedSet(final_columns))
if get_args is True:
cols_params = final_columns, attrs
elif get_args is False:
cols_params = final_columns
else:
RaiseIt.value_error(get_args, ["True", "False"])
if columns_residual:
logger.print("%s %s %s", ",".join(escape_columns(columns_residual)),
"column(s) was not processed because is/are not",
",".join(filter_by_column_dtypes))
# if because of filtering we got 0 columns return None
if len(cols_params) == 0:
cols_params = None
logger.print("Outputting 0 columns after filtering. Is this expected?")
return cols_params
def run(self,
df,
func_request=None,
func_response=None,
return_type="json",
calls=60,
period=60,
max_tries=8):
"""
Read a the url key from a mongo collection an make a request to a service
:param df: Dataframe to me loaded to the enricher collection.
:param func_request: help to create a custom request
:param func_response: help to create a custom response
:param calls: how many call can you make by period of time
:param period: in which period ot time can the call be made in seconds
:param max_tries: how many retries should we do
:param return_type:
:return:
"""
if is_dataframe(df):
df = df.rows.create_id(COL_ID)
# Load the dataframe data in the enricher
self.load(df)
collection_name = self.collection_name
collection = self.get_collection(collection_name)
# Get data that is not yet enriched
cursor = collection.find({COL_RESULTS: {"$exists": False}})
total_docs = cursor.count(True)
if func_request is None:
func_request = requests.get
collection = self.get_collection(collection_name)
@on_exception(expo, RateLimitException, max_tries=max_tries)
@limits(calls=calls, period=period)
def _func_request(v):
return func_request(v)
if total_docs > 0:
for c in tqdm_notebook(cursor,
total=total_docs,
desc='Processing...'):
# Send request to the API
response = _func_request(c)
mongo_id = c["_id"]
if response.status_code == 200:
if return_type == "json":
response = json.loads(response.text)
elif return_type == "text":
response = response.text
# Process the result with an external function
if is_function(func_response):
response = func_response(response)
# Update the mongo id with the result
collection.find_and_modify(
query={"_id": mongo_id},
update={"$set": {
COL_RESULTS: response
}},
upsert=False,
full_response=True)
else:
# The response key will remain blank so we can filter it to try in future request
logger.print(response.status_code)
# Append the data in enrichment to the dataframe
logger.print("Appending collection info into the dataframe")
# TODO: An elegant way to handle pickling?
# take care to the pickling
db_name = self.db_name
# collection_name = self.collection_name
url = self.url
@pandas_udf('string', PandasUDFType.SCALAR)
def func(value):
# More about pickling
from pymongo import MongoClient
_client = MongoClient(url)
_db = _client[db_name]
_collection = _db[collection_name]
def func_serie(serie):
_cursor = _collection.find_one({COL_ID: serie},
projection={
"_id": 0,
COL_RESULTS: 1
})
return _cursor[COL_RESULTS]
return value.apply(func_serie)
df = df.withColumn(COL_RESULTS,
func(df[COL_ID])).cols.drop(COL_ID).run()
# If the process is finished, flush the Mongo collection
self.flush()
return df
else:
print("No records available to process")
