# df3.show()
# df4 = df2.withColumn("new", my_udf2(df2.values))
# df4.show()
#
# print(temp1)
#Question 2
data = [Row(101,[[1,2],[3]]),
Row(102,[[1],[2],[1,2]]),
Row(103,[[1],[1],[1]]),
]
my_schema = StructType([
StructField("id", IntegerType()),
StructField("value", ArrayType(ArrayType(IntegerType())))
])
def my_def1(col2):
# x = dict(Counter(col2))
# y = [k for k,v in x.items() if v > 1 ]
print(col2)
print(type(col2))
temp = [item for items in col2 for item in items]
x = dict(Counter(temp))
print(x)
y = [k for k, v in x.items() if v > 1]
print(y)
return len(y)
my_udf1 = udf(lambda x: my_def1(x), IntegerType())
from pyspark.sql import SparkSession
from pyspark.sql.functions import from_json, to_json, col, unbase64, base64, split, expr
from pyspark.sql.types import StructField, StructType, StringType, FloatType, BooleanType, ArrayType, DateType
# TO-DO: create a StructType for the Kafka redis-server topic which has all changes made to Redis - before Spark 3.0.0, schema inference is not automatic
redisMessageSchema = StructType(
[
StructField("key", StringType()),
StructField("existType", StringType()),
StructField("Ch", BooleanType()),
StructField("Incr", BooleanType()),
StructField("zSetEntries", ArrayType(
StructType([
StructField("element", StringType()),
StructField("Score", StringType())
])
))
]
)
# TO-DO: create a StructType for the Customer JSON that comes from Redis- before Spark 3.0.0, schema inference is not automatic
redisCustomerSchema = StructType(
[
StructField("customerName", StringType()),
StructField("email", StringType()),
StructField("phone", StringType()),
StructField("birthDay", StringType()),
]
)
# TO-DO: create a StructType for the Kafka stedi-events topic which has the Customer Risk JSON that comes from Redis- before Spark 3.0.0, schema inference is not automatic
# Read input from 5 text sources using a spark data frame
documents = spark.read.text("dataset/*.txt")
documents = documents.withColumn("doc_id",
F.row_number().over(Window.orderBy('value')))
documents.printSchema()
# Toekns identified and extracted from input data
tokenizer = Tokenizer(inputCol="value", outputCol="words")
wordsData = tokenizer.transform(documents)
wordsData.show()
# Stem words derived using the tokens identified
stemmer = SnowballStemmer(language='english')
stemmer_udf = F.udf(lambda tokens: [stemmer.stem(token) for token in tokens],
ArrayType(StringType()))
wordsData = wordsData.withColumn("lemms", stemmer_udf("words"))
# Learn a mapping from words to Vectors.
word2Vec = Word2Vec(vectorSize=3,
minCount=0,
inputCol="lemms",
outputCol="result")
model = word2Vec.fit(wordsData)
result = model.transform(wordsData)
# Display synonyms and cosine similarity of words within input data
synonyms = model.findSynonyms(
"5g", 5) # its okay for certain words , real bad for others
synonyms.show(5)
#For databricks related packages
#./bin/pyspark --packages com.databricks:spark-csv_2.10:1.3.0
#Before Spark 1.4
train = sqlContext.load(source="com.databricks.spark.csv", path = 'PATH/train.csv', header = True,inferSchema = True)
test = sqlContext.load(source="com.databricks.spark.csv", path = 'PATH/test-comb.csv', header = True,inferSchema = True)
#Current Spark 2.1 and ...
from pyspark .sql import SparkSession
spark = SparkSession.builder.master("yarn").getOrCreate()
df = spark.read.csv('hdfs://hadoop-master:9000/index/train.csv',mode="DROPMALFORMED")
#Defining schema with ArrayType
schema = StructType([StructField('array_column', ArrayType(StructType([StructField('element_of_array', StringType(), True)])), True)])
#From local : The third parameter i.e. Boolean Type : True / False denote whether the corresponding filed can be nullable
from pyspark.sql.types import StructType,StructField,LongType,StringType,TimestampType
schema=StructType([StructField('col0', LongType(), True), StructField('col1', LongType(), True), StructField('col2', StringType(), True), StructField('col3', StringType(), True),StructField('col4',TimestampType(),True),StructField('col5',TimestampType(),True),StructField('col6',StringType(),True)])
df = spark.read.csv('file:///index/data_extract_restart2_without_cert/data_refined.csv',mode="DROPMALFORMED"),schema=schema)
#Creating UDF
def dict(sk):
new_sk=sk.replace(',','|')#replacing comma by pipe in column col2 and putting the result in column named new_column_name
return new_sk
udf_dict = udf(dict, StringType())
df.withColumn('new_column_name', udf_dict("col2")).write.csv(path="/index/skill_clean_v3")#col2 is the column to be changed
def __type_for(field_descriptor):
get_type = possible_types.get(field_descriptor.type, lambda t: StringType())
if field_descriptor.label == field_descriptor.LABEL_REPEATED:
return ArrayType(get_type(field_descriptor))
return get_type(field_descriptor)
# remove stop words
remover = StopWordsRemover(inputCol=column, outputCol='remove_stop_words')
return remover.transform(df).drop(column).withColumnRenamed(
'remove_stop_words', column)
def remove_not_alphabetic(tokens):
# isalpha remove words with numbers
# return [w for w in tokens if w.isalpha()]
# isalnum allow words and numbers
return [w for w in tokens if w.isalnum()]
# udf_remove_not_alphabetic = udf(remove_not_alphabetic, ArrayType(StringType()))
udf_remove_not_alphabetic = udf(
lambda tokens: [w for w in tokens if w.isalnum()], ArrayType(StringType()))
def remove_only_numbers(tokens):
return [w for w in tokens if not w.isnumeric()]
# udf_remove_only_numbers = udf(remove_only_numbers, ArrayType(StringType()))
udf_remove_only_numbers = udf(
lambda tokens: [w for w in tokens if not w.isnumeric()],
ArrayType(StringType()))
def remove_only_spaces(tokens):
return [w for w in tokens if not w.isspace()]
def to_array(col):
def to_array_(v):
return v.toArray().tolist()
return udf(to_array_, ArrayType(DoubleType()))(col)
def get_date_list(start_date):
if period == PERIOD_WEEKLY:
end_date = start_date + timedelta(6 - start_date.weekday())
else:
end_date = start_date + timedelta(1)
date_id_list = []
date_focus = start_date
while (date_focus < end_date):
date_id_list.append(int(date_focus.strftime("%Y%m%d")))
date_focus += timedelta(1)
return date_id_list
udf_get_date_list = f.udf(get_date_list, ArrayType(LongType()))
def get_df_boom_point_weekly(start_date, df_boom_point):
end_date = start_date + timedelta(7)
start_date_id = int(start_date.strftime("%Y%m%d"))
end_date_id = int(end_date.strftime("%Y%m%d"))
print('start_date_id: ' + str(start_date_id))
print('end_date_id: ' + str(end_date_id))
date_id_list = []
date_focus = start_date
while (date_focus < end_date):
date_id_list.append(int(date_focus.strftime("%Y%m%d")))
date_focus += timedelta(1)
def derive_directories_with_spark(input_dir: str,
output_dir: str,
workflows: dict,
mode='batch',
print_output=False,
spark=None):
if not spark:
spark = SparkSession \
.builder \
.appName("derive_directories") \
.getOrCreate()
if mode == 'stream':
# infer schema from existing file
tiny_loc = os.path.dirname(os.path.realpath(
__file__)) + "/../../../data/examples/panoptes_raw.txt"
assert os.path.exists(tiny_loc)
schema = spark.read.json(tiny_loc).schema
logging.warning(
'Attempting to stream derived files to {}'.format(input_dir))
df = spark.readStream.json(input_dir, schema=schema)
else:
df = spark.read.json(input_dir)
df = df.filter(df['links']['workflow'].isin(list(
workflows.keys()))) # include only allowed workflows
df = clarify_workflow_version(df)
df = rename_metadata_like_exports(df)
workflows_str = json.dumps(workflows)
def match_and_insert_workflow(annotations, workflow_id, major_version,
minor_version, workflows_str):
workflow_versions = pd.DataFrame(
data=json.loads(workflows_str)[workflow_id])
workflow = find_matching_version(major_version, minor_version,
workflow_versions)
annotations_dict = json.loads(annotations)
updated_annotations_dict = insert_workflow_contents(
annotations_dict, workflow)
return json.dumps(updated_annotations_dict)
match_and_insert_workflow_udf = udf(
lambda a, b, c, d: match_and_insert_workflow(a, b, c, d, workflows_str
), StringType())
# apparently can't pass struct as udf argument, need to use as string
df = df.withColumn('annotations', to_json(df['annotations']))
df = df.withColumn(
'annotations',
match_and_insert_workflow_udf(
df['annotations'],
df['workflow_id'], # added by rename_metadata_like_exports
df['workflow_major_version'], # added by clarify_workflow_version
df['workflow_minor_version'] # added by clarify_workflow_version
))
# parse annotations back out again (using the new schema, of course)
annotations_schema = ArrayType(
StructType([
StructField("task", StringType(), False),
StructField("value", StringType(), False),
StructField("task_label", StringType(), False),
StructField("task_id", StringType(), False),
StructField("value_index", StringType(), False),
StructField("multiple_choice", BooleanType(), False)
]))
df = df.withColumn('annotations',
from_json(df['annotations'], schema=annotations_schema))
if mode == 'stream':
query = df.writeStream \
.outputMode('append') \
.option('checkpointLocation', os.path.join(output_dir, 'checkpoints')) \
.trigger(processingTime='3 seconds') \
.start(path=output_dir, format='json')
print('Derived data ready to stream')
if print_output:
while True:
time.sleep(0.1)
if query.status['isDataAvailable']:
print(datetime.now(), query.status['message'])
else:
df.show()
df.write.save(output_dir, format='json', mode='overwrite')
def task4_generate_grams_udf(context):
# TASK 4
# Code for task 4...
context.udf.register("sanitize", sanitize, ArrayType(StringType()))
df.printSchema()
def extract_cat(text):
all_groups = re.findall(r'(\[\[Category:)(\w+)(\]\])', text)
words = [group[1] for group in all_groups]
return words
def extract_links(text):
all_groups = re.findall(r'(\[\[)(\w+)(\]\])', text)
words = [group[1] for group in all_groups]
return words
extract_cat_udf = udf(extract_cat, ArrayType(StringType()))
extract_links_udf = udf(extract_links, ArrayType(StringType()))
##Select sample for analyze
df = df.limit(1000)
df = df.withColumn('categories',extract_cat_udf('revision.text'))\
.withColumn('page_links',extract_links_udf('revision.text'))\
.withColumn('last_modify_date',to_timestamp('revision.timestamp',TIME_FORMAT))
df_pages = df.select('id', 'title', 'last_modify_date', 'categories',
'page_links')
df_pages.show()
df_categories = df.selectExpr('id', 'title', 'explode(categories) as category',
'last_modify_date')
def task5_combine_trigrams_subreddits(labeled_comments):
# TASK 5
# Code for task 5...
sanitize_udf = udf(sanitize, ArrayType(StringType()))
return labeled_comments.select('*', sanitize_udf('body').alias('ngrams'))
def test_supported_types(self):
values = [
1, 2, 3, 4, 5, 1.1, 2.2,
Decimal(1.123), [1, 2, 2], True, 'hello',
bytearray([0x01, 0x02])
]
output_fields = [('id', IntegerType()), ('byte', ByteType()),
('short', ShortType()), ('int', IntegerType()),
('long', LongType()), ('float', FloatType()),
('double', DoubleType()),
('decim', DecimalType(10, 3)),
('array', ArrayType(IntegerType())),
('bool', BooleanType()), ('str', StringType()),
('bin', BinaryType())]
output_schema = StructType([StructField(*x) for x in output_fields])
df = self.spark.createDataFrame([values], schema=output_schema)
# Different forms of group map pandas UDF, results of these are the same
udf1 = pandas_udf(
lambda pdf: pdf.assign(byte=pdf.byte * 2,
short=pdf.short * 2,
int=pdf.int * 2,
long=pdf.long * 2,
float=pdf.float * 2,
double=pdf.double * 2,
decim=pdf.decim * 2,
bool=False if pdf.bool else True,
str=pdf.str + 'there',
array=pdf.array,
bin=pdf.bin), output_schema,
PandasUDFType.GROUPED_MAP)
udf2 = pandas_udf(
lambda _, pdf: pdf.assign(byte=pdf.byte * 2,
short=pdf.short * 2,
int=pdf.int * 2,
long=pdf.long * 2,
float=pdf.float * 2,
double=pdf.double * 2,
decim=pdf.decim * 2,
bool=False if pdf.bool else True,
str=pdf.str + 'there',
array=pdf.array,
bin=pdf.bin), output_schema,
PandasUDFType.GROUPED_MAP)
udf3 = pandas_udf(
lambda key, pdf: pdf.assign(id=key[0],
byte=pdf.byte * 2,
short=pdf.short * 2,
int=pdf.int * 2,
long=pdf.long * 2,
float=pdf.float * 2,
double=pdf.double * 2,
decim=pdf.decim * 2,
bool=False if pdf.bool else True,
str=pdf.str + 'there',
array=pdf.array,
bin=pdf.bin), output_schema,
PandasUDFType.GROUPED_MAP)
result1 = df.groupby('id').apply(udf1).sort('id').toPandas()
expected1 = df.toPandas().groupby('id').apply(
udf1.func).reset_index(drop=True)
result2 = df.groupby('id').apply(udf2).sort('id').toPandas()
expected2 = expected1
result3 = df.groupby('id').apply(udf3).sort('id').toPandas()
expected3 = expected1
assert_frame_equal(expected1, result1)
assert_frame_equal(expected2, result2)
assert_frame_equal(expected3, result3)
interested_frame = sql_context.sql(
"SELECT sa2_name16 AS suburb_name , sa2_main16 AS suburb_code, geometry.coordinates AS coordinates FROM raw"
)
# group and aggregate table by suburb
interested_frame.registerTempTable('interested_table')
gr = sql_context.sql(
"SELECT suburb_name, FIRST(suburb_code) AS suburb_code, collect_list(coordinates) AS multipolygon "
"FROM interested_table GROUP BY suburb_name")
# calculate suburb area
user_define_function = func.udf(calculate_multipolygon_area, FloatType())
df1 = gr.withColumn('suburb_area', user_define_function('multipolygon'))
multipolygons_bounds_udf = func.udf(calculate_multipolygon_bounds,
ArrayType(ArrayType(FloatType())))
# Dataframe of melbourne sa2 suburb
df1 = df1.withColumn('suburb_bound', multipolygons_bounds_udf('multipolygon'))
# load forest data
schema = StructType([
StructField('area', FloatType(), nullable=False),
StructField('polygon', StringType(), nullable=False)
])
df2 = sql_context.read.format("com.databricks.spark.csv").option("header", "false").option('delimiter', ' '). \
load(os.path.join(root, "melb_urban_forest_2016.txt/part-*"), schema=schema)
# transform geometry string to formatted string
df2 = df2.withColumn(
'polygon_formatted',
classifiers_dict.get("ORGANIZATION", {}))
ordered_dict["person_ss"] = list(classifiers_dict.get("PERSON", {}))
ordered_dict["id"] = row["id"]
return ordered_dict
read_opts = {
"collection": "cnn",
"fields": "id, body_t",
"max_rows": "10",
"query": "body_t:[* TO *]"
}
df = spark.read.format("solr").options(**read_opts).load()
cnn_rdd = df.rdd.map(lambda r: Row(**classify_text(r.asDict())))
schema = StructType([
StructField("location_ss", ArrayType(StringType())),
StructField("organization_ss", ArrayType(StringType())),
StructField("person_ss", ArrayType(StringType())),
StructField("id", StringType())
])
newdf = spark.createDataFrame(cnn_rdd, schema)
# newdf.cache
spark.sparkContext._jvm.com.lucidworks.spark.util.DatasetLoader.sendAtomicAddsToSolr(
newdf._jdf, "cnn", "id", "localhost:9983/lwfusion/4.0.0-SNAPSHOT/solr")
spark.stop()
query = "select count_array,day,uckey from factdata where day in ('2020-05-15','2020-05-14','2020-05-13','2020-05-12','2020-05-11','2020-05-10','2020-05-09')"
sc = SparkContext()
hive_context = HiveContext(sc)
df = hive_context.sql(query)
df = add_count_map(df)
df = df.select('uckey', 'day',
explode(df.count_map)).withColumnRenamed("value", "impr_count")
df = df.withColumn('impr_count',
udf(lambda x: int(x), IntegerType())(df.impr_count))
df = df.groupBy('uckey', 'day').sum('impr_count').withColumnRenamed(
"sum(impr_count)", 'impr_count')
split_uckey_udf = udf(lambda x: x.split(","), ArrayType(StringType()))
df = df.withColumn('col', split_uckey_udf(df.uckey))
df = df.select('uckey', 'impr_count', 'day',
df.col[1]).withColumnRenamed("col[1]", 'slot_id')
df_slot = df.select('slot_id', 'impr_count', 'day')
df_slot = df_slot.groupBy('slot_id',
'day').sum('impr_count').withColumnRenamed(
"sum(impr_count)", "impr_total")
bc_df_slot = broadcast(df_slot)
df_new = df.join(bc_df_slot, on=["slot_id", 'day'], how="inner")
df_new = df_new.withColumn(
'percent',
udf(lambda x, y: (x * 100) / y, FloatType())(df_new.impr_count,
'py': self.py,
'pz': self.pz,
'ke': to_electron_volt(self.ke)
}
Model = NewType('Model', Callable[[Hit], Optional[AnalyzedHit]])
Analyzer = NewType('Analyzer', Callable[[Hit], Mapping[str,
Optional[AnalyzedHit]]])
if pyspark_exists:
from pyspark.sql.types import StructType, StructField, DoubleType, IntegerType, ArrayType, MapType, StringType
SpkAnalyzedHit = StructType([
StructField('px', DoubleType(), nullable=False),
StructField('py', DoubleType(), nullable=False),
StructField('pz', DoubleType(), nullable=False),
StructField('ke', DoubleType(), nullable=False),
])
SpkHit = StructType([
StructField('t', DoubleType(), nullable=False),
StructField('x', DoubleType(), nullable=False),
StructField('y', DoubleType(), nullable=False),
StructField('flag', IntegerType(), nullable=True),
StructField('as', MapType(StringType(), SpkAnalyzedHit),
nullable=True),
])
SpkHits = ArrayType(SpkHit)
else:
SpkAnalyzedHit, SpkHit, SpkHits = None, None, None
def convert_extract_to_parquet(extract_loc, save_dir, spark=None):
if not spark:
spark = SparkSession \
.builder \
.appName("shared") \
.getOrCreate()
# https://spark.apache.org/docs/latest/api/python/pyspark.sql.html?highlight=read%20csv
# ignore (escape) " if already within quotes to avoid splitting by , within the columnwise jsons
# NullPointerException if you try to access something you promised would never be null
# cannot read nested structs straight from CSV, sadly, so will parse columns individually
metadata_struct = StructType([
StructField('source', StringType(), False),
StructField('session', StringType(), False),
StructField(
'viewport',
StructType([
StructField('width', StringType(), False),
StructField('height', StringType(), False)
]), False),
StructField('started_at', TimestampType(), False),
StructField('user_agent', StringType(), False),
StructField('utc_offset', StringType(), False),
StructField('finished_at', TimestampType(), False),
StructField('live_project', BooleanType(), False),
StructField('interventions', StringType(), False), # actually struct
StructField('user_language', StringType(), False),
StructField('source', StringType(), False),
StructField('subject_dimensions', StringType(),
False), # actually struct
StructField('subject_selection_state', StringType(),
False), # actually struct
StructField('workflow_translation_id', StringType(),
True), # actually struct, sometimes null
])
# TODO answer (at the very least) is very occasionally null, and this causes either EOF/Null pointer (if not nullable) or raise error like
# ValueError: Answer None of type <class 'NoneType'> not found in schema for question T0
# should filter out tasks with missing keys for these
annotations_struct = ArrayType(
StructType([
StructField('task', StringType(), True),
StructField('task_id', StringType(), True),
StructField('task_label', StringType(), True),
StructField('value', StringType(), True),
StructField('multiple_choice', BooleanType(), True),
]))
# subject_data_internal_struct = StructType(
# # StructField('!iauname', StringType(), True),
# # StructField('iauname', StringType(), True)
# )
# subject_data_struct = ArrayType(MapType(StringType(), subject_data_internal_struct))
schema = StructType([
StructField('classification_id', StringType(), False),
StructField('user_name', StringType(), True),
StructField('user_id', StringType(), True),
StructField('user_ip', StringType(), True),
StructField('workflow_id', StringType(), False),
StructField('workflow_name', StringType(), False),
StructField('workflow_version', FloatType(), False),
StructField('created_at', StringType(), False),
StructField('gold_standard', StringType(), False),
StructField('expert', StringType(), False),
StructField('metadata', StringType(), False),
StructField('annotations', StringType(), False),
StructField('subject_data', StringType(), False),
StructField('subject_ids', StringType(), False)
])
# schema = StructType([
# StructField('name', StructType([
# StructField('firstname', StringType(), True),
# StructField('middlename', StringType(), True),
# StructField('lastname', StringType(), True)
# ])),
# StructField('id', StringType(), True),
# StructField('gender', StringType(), True),
# StructField('salary', IntegerType(), True)
# ])
ds = spark.read.csv(extract_loc,
header=True,
quote='"',
escape='"',
schema=schema,
mode='FAILFAST')
# for debugging
# ds = ds.sample(withReplacement=False, fraction=.1, seed=42)
# print(ds.head())
# need to unpack metadata and subject data
# print(ds.head()['metadata'])
# print(ds.head()['annotations'])
metadata_str_to_struct_udf = udf(metadata_str_to_struct,
returnType=metadata_struct)
annotations_str_to_struct_udf = udf(annotation_to_struct,
returnType=annotations_struct)
subject_data_str_to_iauname_udf = udf(subject_data_str_to_iauname,
returnType=StringType())
get_person_id_udf = udf(get_person_id, returnType=StringType())
ds = ds.withColumn('metadata', metadata_str_to_struct_udf(ds['metadata']))
ds = ds.withColumn('annotations',
annotations_str_to_struct_udf(ds['annotations']))
ds = ds.withColumn('iauname',
subject_data_str_to_iauname_udf(ds['subject_data']))
ds = ds.withColumn('person_id',
get_person_id_udf(ds['user_id'], ds['user_ip']))
ds = ds.withColumnRenamed('subject_ids', 'subject_id')
ds = ds.withColumn(
'project_id', lit('5733')
) # TODO hardcoded for now as not in export. lit to make it a column, as Spark requires.
flattened = flatten.api_df_to_responses(ds)
flattened.write.parquet(save_dir, mode='overwrite')
sess.run(tf.compat.v1.global_variables_initializer())
sess.run(iterator.initializer, feed_dict={image_input: image_batch})
softmax_tensor = sess.graph.get_tensor_by_name('softmax_tensor:0')
result = []
try:
while True:
batch = sess.run(image)
preds = sess.run(softmax_tensor, {'input_tensor:0': batch})
result = result + list(preds)
except tf.errors.OutOfRangeError:
pass
return pd.Series(result)
df = spark.read.format("tfrecords").schema(schema).load(input_local_dir+'/flowers_train*.tfrecord')
df = df.limit(3200)
# In[ ]:
# image_batch = df.limit(128).toPandas().loc[: , "image/encoded"].apply(lambda x: bytes(x))
# images = predict_batch(image_batch)
# print(images.shape)
predict_batch_udf = pandas_udf(ArrayType(FloatType()), PandasUDFType.SCALAR)(predict_batch)
predictions = df.select(predict_batch_udf(col("image/encoded")).alias("prediction"))
predictions.write.mode("overwrite").save("/tmp/predictions")
result_df = spark.read.load("/tmp/predictions")
display(result_df)
def indices_to_terms(vocabulary):
def indices_to_terms(xs):
return [vocabulary[int(x)] for x in xs]
return udf(indices_to_terms, ArrayType(StringType()))
outputMode="vector")
transformer
fs = !ls content/train/*.jpg
uri_df = spark.createDataFrame(fs, StringType()).toDF("filename")
keras_pred_df = transformer.transform(uri_df)
from keras.models import Sequential
from keras.layers import Dense
import numpy as np
from pyspark.sql.types import StructType, StructField, ArrayType, FloatType
num_features = 10
num_examples = 100
input_data = [{"features" : np.random.randn(num_features).astype(float).tolist()} for i in range(num_examples)]
schema = StructType([ StructField("features", ArrayType(FloatType()), True)])
input_df = spark.createDataFrame(input_data, schema)
model = Sequential()
model.add(Dense(units=20, input_shape=[num_features], activation='relu'))
model.add(Dense(units=1, activation='sigmoid'))
model_path = "simple-binary-classification"
model.save('model_path')
transformer = KerasImageFileTransformer(inputCol="features", outputCol="category",
modelFile='model_path',
imageLoader=loadAndPreprocessKerasInceptionV3,
outputMode="vector")
final_df = transformer.transform(input_df)
def create_test_scalar_dataset(tmp_url, num_rows, num_files=4, spark=None):
shutdown = False
if not spark:
spark_session = SparkSession \
.builder \
.appName('petastorm_end_to_end_test') \
.master('local[*]')
spark = spark_session.getOrCreate()
shutdown = True
expected_data = [{
'id':
np.int32(i),
'int_fixed_size_list':
np.arange(1 + i, 10 + i).astype(np.int32),
'datetime':
np.datetime64('2019-01-02'),
'timestamp':
np.datetime64('2005-02-25T03:30'),
'string':
np.unicode_('hello_{}'.format(i)),
'string2':
np.unicode_('world_{}'.format(i)),
'float64':
np.float64(i) * .66
} for i in range(num_rows)]
expected_data_as_scalars = [{
k: np.asscalar(v) if isinstance(v, np.generic) else v
for k, v in row.items()
} for row in expected_data]
# np.datetime64 is converted to a timezone unaware datetime instances. Working explicitly in UTC so we don't need
# to think about local timezone in the tests
for row in expected_data_as_scalars:
row['timestamp'] = row['timestamp'].replace(tzinfo=pytz.UTC)
row['int_fixed_size_list'] = row['int_fixed_size_list'].tolist()
rows = [Row(**row) for row in expected_data_as_scalars]
# WARNING: surprisingly, schema fields and row fields are matched only by order and not name.
# We must maintain alphabetical order of the struct fields for the code to work!!!
schema = StructType([
StructField('datetime', DateType(), False),
StructField('float64', DoubleType(), False),
StructField('id', IntegerType(), False),
StructField('int_fixed_size_list', ArrayType(IntegerType(), False),
False),
StructField('string', StringType(), False),
StructField('string2', StringType(), False),
StructField('timestamp', TimestampType(), False),
])
dataframe = spark.createDataFrame(rows, schema)
dataframe. \
coalesce(num_files). \
write.option('compression', 'none'). \
mode('overwrite'). \
parquet(tmp_url)
if shutdown:
spark.stop()
return expected_data
day = cfg['today']
allocated = {'b1': 100, 'b2': 50}
result = optimizer.dao.query_builder.index_bb(day, ands, minus, allocated,
es_client_bb)
print(result)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='optimizer')
parser.add_argument('config_file')
parser.add_argument('today', help='today %Y%m%d e.g. 20181230')
args = parser.parse_args()
schema = StructType([
StructField('day', StringType(), True),
StructField('ands', ArrayType(StringType()), True),
StructField('minus', ArrayType(StringType()), True),
StructField('allocated', MapType(StringType(), IntegerType()), True),
StructField('amount', IntegerType(), True)
])
# Load config file
try:
with open(args.config_file, 'r') as ymlfile:
cfg = yaml.load(ymlfile)
cfg['today'] = args.today
except Exception as e:
print(e)
test_9(cfg)
StructField("locale", StringType(), True),
StructField(
"active_addons",
# active_addons is a list of dictionaries holding all
# metadata related to an addon
ArrayType(
StructType([
StructField("addon_id", StringType(), True),
StructField("app_disabled", BooleanType(), True),
StructField("blocklisted", BooleanType(), True),
StructField("foreign_install", BooleanType(), True),
StructField("has_binary_components", BooleanType(), True),
StructField("install_day", LongType(), True),
StructField("is_system", BooleanType(), True),
StructField("is_web_extension", BooleanType(), True),
StructField("multiprocess_compatible", BooleanType(), True),
StructField("name", StringType(), True),
StructField("scope", LongType(), True),
StructField("signed_state", LongType(), True),
StructField("type", StringType(), True),
StructField("update_day", LongType(), True),
StructField("user_disabled", BooleanType(), True),
StructField("version", StringType(), True),
]),
True,
),
),
StructField("places_bookmarks_count_mean", LongType(), True),
StructField(
"scalar_parent_browser_engagement_tab_open_event_count_sum",
LongType(),
def sqlType(cls):
return StructType([
StructField("type", ByteType(), False),
StructField("size", IntegerType(), True),
StructField("indices", ArrayType(IntegerType(), False), True),
StructField("values", ArrayType(DoubleType(), False), True)])
import nltk
from functools import partial
from pyspark.sql.types import StructType, StructField,DoubleType,ArrayType,StringType
print "Start preprocessing all data"
t0 = time()
def preProcess(doc):
clean = doc.review.replace("<br /><br />"," ")
tok = nltk.tokenize.wordpunct_tokenize(clean)
tags = nltk.pos_tag(tok,tagset='universal')
low = [word.lower() for word in tok]
return low,zip(*tags)[1],doc.label
schema = StructType([StructField('words',ArrayType(StringType()),True), StructField('tags',ArrayType(StringType()),True), StructField('label',DoubleType())])
dfPre=df.map(preProcess).toDF(schema).cache()
trigram = NGram(n=3,inputCol="tags", outputCol="tagTrigrams")
dfTriAux = trigram.transform(dfPre).cache()
trigram.setInputCol("words")
trigram.setOutputCol("wordTrigrams")
dfTri = trigram.transform(dfTriAux).cache()
dfTrain, dfValid = dfTri.randomSplit([0.8,0.2])
lists=dfTrain.map(lambda r : r.words).collect()
dictUnigrams=list(set(itertools.chain(*lists)))
dictionaryUni={}
for i,word in enumerate(dictUnigrams):
"double": DoubleType,
"boolean": BooleanType,
"struct": StructType,
"array": ArrayType,
"bigint": LongType,
"date": DateType,
"byte": ByteType,
"short": ShortType,
"datetime": TimestampType,
"binary": BinaryType,
"null": NullType,
"vector": VectorUDT
}
SPARK_DTYPES_DICT_OBJECTS = \
{"string": StringType(), "int": IntegerType(), "float": FloatType(),
"double": DoubleType(), "boolean": BooleanType(), "struct": StructType(), "array": ArrayType(StringType()),
"bigint": LongType(), "date": DateType(), "byte": ByteType(), "short": ShortType(),
"datetime": TimestampType(), "binary": BinaryType(), "null": NullType()
}
PROFILER_COLUMN_TYPES = {
"categorical", "numeric", "date", "null", "array", "binary"
}
PYTHON_TO_PROFILER = {
"string": "categorical",
"boolean": "categorical",
"int": "numeric",
"decimal": "numeric",
"date": "date",
"array": "array",
"binaty": "binary",
"null": "null"
# MAGIC <img alt="Opens in new tab" src="https://files.training.databricks.com/static/images/external-link-icon-16x16.png"/> Watch full-screen.</a>
# MAGIC </div>
# COMMAND ----------
# MAGIC %md
# MAGIC The ZIP Code dataset contains an array with the latitude and longitude of the cities. Use an `ArrayType`, which takes the primitive type of its elements as an argument.
# COMMAND ----------
from pyspark.sql.types import StructType, StructField, IntegerType, StringType, ArrayType, FloatType
zipsSchema3 = StructType([
StructField("city", StringType(), True),
StructField("loc",
ArrayType(FloatType(), True), True),
StructField("pop", IntegerType(), True)
])
# COMMAND ----------
# MAGIC %md
# MAGIC Apply the schema using the `.schema()` method and observe the results. Expand the array values in the column `loc` to explore further.
# COMMAND ----------
zipsDF3 = (spark.read
.schema(zipsSchema3)
.json("/mnt/training/zips.json")
)
display(zipsDF3)
def get_hrv_features(rr_data, acceptable_percentage=50, window_length=60):
"""
Args:
rr_data (DataStream):
acceptable_percentage (int):
window_length (int):
Returns:
"""
stream_name = 'org.md2k.autosense.ecg.features'
def get_metadata():
stream_metadata = Metadata()
stream_metadata.set_name(stream_name).set_description("HRV Features from ECG RR interval") \
.add_input_stream(rr_data.metadata.get_name()) \
.add_dataDescriptor(
DataDescriptor()
.set_name("var")
.set_type("double")
.set_attribute("description","variance")) \
.add_dataDescriptor(
DataDescriptor()
.set_name("iqr")
.set_type("double")
.set_attribute("description","Inter Quartile Range")) \
.add_dataDescriptor(
DataDescriptor()
.set_name("mean")
.set_type("double")
.set_attribute("description","Mean RR Interval")) \
.add_dataDescriptor(
DataDescriptor()
.set_name("median")
.set_type("double")
.set_attribute("description","Median RR Interval")) \
.add_dataDescriptor(
DataDescriptor()
.set_name("80th")
.set_type("double")
.set_attribute("description","80th percentile RR Interval")) \
.add_dataDescriptor(
DataDescriptor()
.set_name("20th")
.set_type("double")
.set_attribute("description","20th percentile RR Interval")) \
.add_dataDescriptor(
DataDescriptor()
.set_name("heartrate")
.set_type("double")
.set_attribute("description","Heart Rate in BPM")) \
.add_dataDescriptor(
DataDescriptor()
.set_name("vlf")
.set_type("double")
.set_attribute("description","Very Low Frequency Energy")) \
.add_dataDescriptor(
DataDescriptor()
.set_name("lf")
.set_type("double")
.set_attribute("description","Low Frequency Energy")) \
.add_dataDescriptor(
DataDescriptor()
.set_name("hf")
.set_type("double")
.set_attribute("description","High Frequency Energy")) \
.add_dataDescriptor(
DataDescriptor()
.set_name("lfhf")
.set_type("double")
.set_attribute("description","Low frequency to High Frequency energy ratio")) \
.add_dataDescriptor(
DataDescriptor()
.set_name("window")
.set_type("struct")
.set_attribute("description","window start and end time in UTC")
.set_attribute('start','start of window')
.set_attribute('end','end of window')) \
.add_module(
ModuleMetadata().set_name("HRV Features from ECG RR Interval")
.set_attribute("url", "http://md2k.org/")
.set_attribute('algorithm','ecg feature computation')
.set_attribute('unit','ms')
.set_author("Md Azim Ullah", "*****@*****.**"))
return stream_metadata
def get_rr_features(a):
return np.array([
np.var(a),
iqr(a),
np.mean(a),
np.median(a),
np.percentile(a, 80),
np.percentile(a, 20), 60000 / np.median(a)
])
def frequencyDomain(RRints,
tmStamps,
band_type=None,
lf_bw=0.11,
hf_bw=0.1,
vlf=(0.003, 0.04),
lf=(0.04, 0.15),
hf=(0.15, 0.4)):
"""
Args:
RRints:
tmStamps:
band_type:
lf_bw:
hf_bw:
vlf:
lf:
hf:
Returns:
"""
NNs = RRints
tss = tmStamps
frequency_range = np.linspace(0.001, 1, 10000)
NNs = np.array(NNs)
NNs = NNs - np.mean(NNs)
result = signal.lombscargle(tss, NNs, frequency_range)
#Pwelch w/ zero pad
fxx = frequency_range
pxx = result
if band_type == 'adapted':
vlf_peak = fxx[np.where(pxx == np.max(pxx[np.logical_and(
fxx >= vlf[0], fxx < vlf[1])]))[0][0]]
lf_peak = fxx[np.where(pxx == np.max(pxx[np.logical_and(
fxx >= lf[0], fxx < lf[1])]))[0][0]]
hf_peak = fxx[np.where(pxx == np.max(pxx[np.logical_and(
fxx >= hf[0], fxx < hf[1])]))[0][0]]
peak_freqs = (vlf_peak, lf_peak, hf_peak)
hf = (peak_freqs[2] - hf_bw / 2, peak_freqs[2] + hf_bw / 2)
lf = (peak_freqs[1] - lf_bw / 2, peak_freqs[1] + lf_bw / 2)
vlf = (0.003, lf[0])
if lf[0] < 0:
print(
'***Warning***: Adapted LF band lower bound spills into negative frequency range'
)
print('Lower thresold of LF band has been set to zero')
print('Adjust LF and HF bandwidths accordingly')
lf = (0, lf[1])
vlf = (0, 0)
elif hf[0] < 0:
print(
'***Warning***: Adapted HF band lower bound spills into negative frequency range'
)
print('Lower thresold of HF band has been set to zero')
print('Adjust LF and HF bandwidths accordingly')
hf = (0, hf[1])
lf = (0, 0)
vlf = (0, 0)
df = fxx[1] - fxx[0]
vlf_power = np.trapz(pxx[np.logical_and(fxx >= vlf[0], fxx < vlf[1])],
dx=df)
lf_power = np.trapz(pxx[np.logical_and(fxx >= lf[0], fxx < lf[1])],
dx=df)
hf_power = np.trapz(pxx[np.logical_and(fxx >= hf[0], fxx < hf[1])],
dx=df)
totalPower = vlf_power + lf_power + hf_power
#Normalize and take log
vlf_NU_log = np.log((vlf_power / (totalPower - vlf_power)) + 1)
lf_NU_log = np.log((lf_power / (totalPower - vlf_power)) + 1)
hf_NU_log = np.log((hf_power / (totalPower - vlf_power)) + 1)
lfhfRation_log = np.log((lf_power / hf_power) + 1)
freqDomainFeats = {
'VLF_Power': vlf_NU_log,
'LF_Power': lf_NU_log,
'HF_Power': hf_NU_log,
'LF/HF': lfhfRation_log
}
return freqDomainFeats
schema = StructType([
StructField("timestamp", TimestampType()),
StructField("start", TimestampType()),
StructField("end", TimestampType()),
StructField("localtime", TimestampType()),
StructField("version", IntegerType()),
StructField("user", StringType()),
StructField("features", ArrayType(DoubleType()))
])
@pandas_udf(schema, PandasUDFType.GROUPED_MAP)
@CC_MProvAgg('org.md2k.autosense.ecg.rr', 'get_hrv_features', stream_name,
['user', 'timestamp'], ['user', 'timestamp'])
def ecg_r_peak(key, data):
"""
Args:
key:
data:
Returns:
"""
if data.shape[0] >= acceptable_percentage * window_length / 100:
data = data.sort_values('time')
data['time'] = 1000 * data['time']
a = data['rr'].values
features = [
np.double(
np.array(
list(get_rr_features(a)) + list(
frequencyDomain(
np.array(a) / 1000,
np.cumsum(a) / 1000).values())))
]
data = data[:1]
data['features'] = features
data['start'] = [key[2]['start']]
data['end'] = [key[2]['end']]
data = data[[
'timestamp', 'localtime', 'version', 'user', 'start', 'end',
'features'
]]
return data
else:
return pd.DataFrame([],
columns=[
'timestamp', 'localtime', 'version',
'user', 'features', 'start', 'end'
])
rr_data = rr_data.withColumn('time', F.col('timestamp').cast('double'))
ecg_features = rr_data.compute(ecg_r_peak,
windowDuration=window_length,
startTime='0 seconds')
df = ecg_features.select('timestamp',
F.struct('start', 'end').alias('window'),
'localtime', 'features', 'user', 'version')
df = df.withColumn('var', F.col('features').getItem(0))
df = df.withColumn('iqr', F.col('features').getItem(1))
df = df.withColumn('vlf', F.col('features').getItem(7))
df = df.withColumn('lf', F.col('features').getItem(8))
df = df.withColumn('hf', F.col('features').getItem(9))
df = df.withColumn('lfhf', F.col('features').getItem(10))
df = df.withColumn('mean', F.col('features').getItem(2))
df = df.withColumn('median', F.col('features').getItem(3))
df = df.withColumn('80th', F.col('features').getItem(4))
df = df.withColumn('20th', F.col('features').getItem(5))
ecg_features_final = df.withColumn('heartrate',
F.col('features').getItem(6))
ecg_features_final = ecg_features_final.drop('features')
feature_names = [
'var', 'iqr', 'mean', 'median', '80th', '20th', 'heartrate', 'vlf',
'lf', 'hf', 'lfhf'
]
stress_features = ecg_features_final.withColumn(
'features', F.array([F.col(i) for i in feature_names]))
stress_features.metadata = get_metadata()
return stress_features
resultMap_FilterUlr = resultMap.map(lambda (a,b,c,d,e,f,g,h,i,j,l): (a,b,c,d,e,f,g,h,i,j,regularExpression(l.split(",")))). \
filter(lambda (a,b,c,d,e,f,g,h,i,j,l): len(l) >1)
#put on Json
fields = StructType( \
[StructField("GSN", StringType(), False), \
StructField("ChargingID", IntegerType(), False), \
StructField("RecordSequence", IntegerType(), False), \
StructField("RecordOpeningDate", TimestampType(), False), \
StructField("rATType", IntegerType(), False), \
StructField("UserLocation", StringType(), False), \
StructField("Accuracy", IntegerType(), False), \
StructField("BrowsingSession", IntegerType(), False), \
StructField("Uplink", IntegerType(), False), \
StructField("Downlink", IntegerType(), False), \
StructField("Urls", ArrayType(StringType(),False))])
#The new Json Format
newStructure = StructType( \
[StructField("GSN", StringType(), False), \
StructField("ChargingID", IntegerType(), False), \
StructField("RecordSequence", IntegerType(), False), \
StructField("RecordOpeningDate", TimestampType(), False), \
StructField("rATType", IntegerType(), False), \
StructField("UserLocation", StringType(), False), \
StructField("Accuracy", IntegerType(), False), \
StructField("BrowsingSession", IntegerType(), False), \
StructField("Uplink", IntegerType(), False), \
StructField("Downlink", IntegerType(), False), \
StructField("Urls",ArrayType( \
StructType([StructField("name", StringType(), False),StructField("domain", StringType(), True), \
