def db_connection(config, db):
src = connector.connect(host=rd(op.getitem, [db, "host"], config),
user=rd(op.getitem, [db, "user"], config),
passwd=op.getitem(config, "password"),
db=rd(op.getitem, [db, "db_name"], config))
cur = src.cursor(buffered=True)
return cur, src
def reduceByKey(self, _function):
inter = lambda x, y: (x[0], _function(x[1:], y[1:])
if len(x) > 2 and len(y) > 2 else _function(
x[1], y[1]))
return Dataset([
rd(inter, group)
for _, group in groupby(sorted(self.arr, key=lambda x: x[0]),
key=itemgetter(0))
])
def reduce(iterable, reducer):
"""
Get a merged value using an associative reduce function,
so as to reduce the iterable to a single value from left to right.
:param iterable:
:param reducer:
"""
value = rd(reducer, iterable)
return value
def reduce(iterable, reducer):
"""
Get a merged value using an associative reduce function,
so as to reduce the iterable to a single value from left to right.
:param iterable:
:param reducer:
"""
value = rd(reducer, iterable)
return value
def ReadData(dirPath, city_config, schema, spark):
dfs = []
for fn, city in city_config.items():
fp = dirPath + fn
df_temp = spark.read\
.option('encoding', 'UTF-8')\
.schema(schema)\
.csv(fp)
df_temp = df_temp.withColumn("index",
monotonically_increasing_id()).filter(
col('index') > 1).drop('index')
df_temp = df_temp.withColumn('city', lit(city))
dfs.append(df_temp)
df = rd(DataFrame.unionAll, dfs)
return df
def reduceNumbers(myList: list) -> list:
"""
Simple reduce function for numbers in the list using reduce and lambda.
"""
allMultiplied = rd(lambda a, b: a * b, myList)
return allMultiplied
c = [result.append(num) if num not in result else num for num in a]
print(result)
'''
5. Реализовать формирование списка, используя функцию range() и возможности генератора.
В список должны войти четные числа от 100 до 1000 (включая границы).
Необходимо получить результат вычисления произведения всех элементов списка.
Подсказка: использовать функцию reduce().
'''
from functools import reduce as rd
from random import randint
a = [randint(100, 1000 + 1) for i in range(15)]
print(a)
a_reduce = rd(lambda x, y: x * y, a)
print(a_reduce)
'''
6. Реализовать два небольших скрипта:
а) бесконечный итератор, генерирующий целые числа, начиная с указанного,
б) бесконечный итератор, повторяющий элементы некоторого списка, определенного заранее.
Подсказка: использовать функцию count() и cycle() модуля itertools.
'''
from itertools import cycle, count
for el in count(3):
print(el)
for el in cycle('abc'):
print(el)
def reduce(self, _function):
return rd(_function, self.arr)
print(list(map(lambda x: str(x) if x % 3 == 0 else x, a)))
a = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
print(list(map(lambda x: str(x) if x == 1 else float(x) if x == 2 else x + 10, a)))
a = [1, 2, 3, 4, 5]
b = [2, 4, 6, 8, 10]
print(list(map(lambda x, y: x * y, a, b)))
a = [8, 3, 2, 10, 15, 7, 1, 9, 0, 11]
print(list(filter(f, a)))
print(list(filter(lambda x: x > 5 and x < 10, a)))
a = [1, 2, 3, 4, 5]
print(rd(f2, a))
print(rd(lambda x, y: x + y, a))
files = input().split()
print(list(map(lambda x: x.zfill if len(x.split('.')[0]) < 3 else x, files)))
x = 10
print(locals())
def calc():
a = 3
b = 5
def mul_add(x):
return a * x + b
return mul_add