Exemplos de sparkInitialize em Python

Exemplo n.º 1

#coding=utf-8

from perform.SparkInit import sparkInitialize
from utils.logging import logging
from perform.MongoExecute import mongoExecute
import time

# 声明spark session
sparkSession = sparkInitialize().setAppName(
    "firstlogin_yaodou_user").onHive().onMongo()

# 开始
spark = sparkSession.getOrCreate()

# 声明日志
logger = logging(spark, "warn")

# banner
logger.warn(sparkSession.showConf(), 'config')
logger.warn("Author:huangsai", "banner")


def logic():
    '''
    处理逻辑
    大致步骤  全表扫描
    '''

    execute_sql_full_table = '''
  select t4.uid,t4.date,t7.group,t7.area_province,t7.area_city,t7.area_district
from

Exemplo n.º 2

from perform.MongoExecute import mongoExecute
from perform.SparkInit import sparkInitialize
from utils.HiveAlter import partition
from utils.SubmitArguments import arguments
from utils.logging import logging

DEFAULT_START_DATE = str(
    datetime.datetime.strptime(time.strftime('%Y%m%d', time.localtime()),
                               '%Y%m%d') +
    datetime.timedelta(days=-1))[0:10].replace("-", "")

DEFAULT_END_DATE = time.strftime('%Y%m%d', time.localtime())

# spark session
sparkSession = sparkInitialize().setAppName(
    "firstlogin_dcplaytogetheractive").onHive().onMongo()

spark = sparkSession.getOrCreate()

logger = logging(spark, "WARN")

# banner
logger.warn(sparkSession.showConf(), 'config')
logger.warn("liugf", "author")


def logic(start_date=DEFAULT_START_DATE, end_date=DEFAULT_END_DATE):
    # Queries are expressed in HiveQL  currencytype(3) & currencytype(100)
    gspropsmalldb_mobileprops_sql = "select t1.game as game_id, " \
                                    "t1.gamecode as game_code, " \
                                    "t1.date as date, " \

Exemplo n.º 3

Arquivo: firstlogin_collection_package_user.py Projeto: heavylgf/liugf.python

#coding=utf-8

from perform.SparkInit import sparkInitialize
from utils.logging import logging
from perform.MongoExecute import mongoExecute
import time

# 声明spark session
sparkSession = sparkInitialize().setAppName(
    "uid_firstlogin_collection_package").onHive().onMongo()

# 开始
spark = sparkSession.getOrCreate()

# 声明日志
logger = logging(spark, "warn")

# banner
logger.warn(sparkSession.showConf(), 'config')
logger.warn("Author:huangsai", "banner")


def logic():
    '''
    处理逻辑
    大致步骤  全表扫描
    '''

    execute_sql_full_table = '''
                  select a.uid,a.`date`,a.time,b.fromappCode,b.channel,b.group,b.app
                from

Exemplo n.º 4

from pyspark import SparkConf
import time, datetime, sys
from utils.SubmitArguments import arguments
from utils.HiveAlter import partition
from perform.MongoExecute import mongoExecute

# 默认开始时间
DEFAULT_START_DATE = str(
    datetime.datetime.strptime(time.strftime('%Y%m%d', time.localtime()),
                               '%Y%m%d') +
    datetime.timedelta(days=-1))[0:10].replace("-", "")
# 默认结束时间
DEFAULT_END_DATE = time.strftime('%Y%m%d', time.localtime())

# 声明spark session
sparkSession = sparkInitialize().setAppName(
    "revenue_spend_coupon_cash_daily_agg").onHive().onMongo()

# 开始
spark = sparkSession.getOrCreate()

# 声明日志
logger = logging(spark, "warn")

# banner
logger.warn(sparkSession.showConf(), 'config')
logger.warn("huangsai", "author")


def logic(start_date=DEFAULT_START_DATE, end_date=DEFAULT_END_DATE):
    '''
    处理逻辑

Exemplo n.º 5

Arquivo: Demo.py Projeto: heavylgf/liugf.python

#coding=utf-8

from perform.SparkInit import sparkInitialize
from utils.logging import logging
from pyspark import SparkConf
import time,datetime,sys
from utils.SubmitArguments import arguments

# 默认开始时间
DEFAULT_START_DATE = str(datetime.datetime.strptime(time.strftime('%Y%m%d',time.localtime()), '%Y%m%d') + datetime.timedelta(days=-1))[0:10].replace("-","")
# 默认结束时间
DEFAULT_END_DATE = time.strftime('%Y%m%d',time.localtime())

# 声明spark session
sparkSession = sparkInitialize().setAppName("demo").onHive().onMongo()

# 开始
spark = sparkSession.getOrCreate()

# 声明日志
logger = logging(spark,"warn")

# banner
logger.warn(sparkSession.showConf(),'config')
logger.warn("zhangwei","author")


def logic(start_date=DEFAULT_START_DATE,end_date=DEFAULT_END_DATE):
    '''
    处理逻辑
    大致步骤  读取转换数据 -》 写入hive  -》 写入mongo

Exemplo n.º 6

Arquivo: firstlogin_whole.py Projeto: heavylgf/liugf.python

from pyspark import SparkConf
import time, datetime, sys
from utils.SubmitArguments import arguments
from utils.HiveAlter import partition
from perform.MongoExecute import mongoExecute

# 默认开始时间
DEFAULT_START_DATE = str(
    datetime.datetime.strptime(time.strftime('%Y%m%d', time.localtime()),
                               '%Y%m%d') +
    datetime.timedelta(days=-1))[0:10].replace("-", "")
# 默认结束时间
DEFAULT_END_DATE = time.strftime('%Y%m%d', time.localtime())

# 声明spark session
sparkSession = sparkInitialize().setAppName("firstlogin_whole").onHive()

# 开始
spark = sparkSession.getOrCreate()

# 声明日志
logger = logging(spark, "warn")

# banner
logger.warn(sparkSession.showConf(), 'config')
logger.warn("zhangwei", "author")


def logic(start_date=DEFAULT_START_DATE, end_date=DEFAULT_END_DATE):
    '''
    处理逻辑

Exemplo n.º 7

Arquivo: merge_logsdklog_mobile_detail.py Projeto: heavylgf/liugf.python

import time

from perform.MongoExecute import mongoExecute
from perform.SparkInit import sparkInitialize
from utils.HiveAlter import partition
from utils.SubmitArguments import arguments
from utils.logging import logging


DEFAULT_START_DATE = str(datetime.datetime.strptime(time.strftime('%Y%m%d', time.localtime()), '%Y%m%d')
                         + datetime.timedelta(days=-1))[0:10].replace("-", "")

DEFAULT_END_DATE = time.strftime('%Y%m%d', time.localtime())

# spark session
sparkSession = sparkInitialize().setAppName("merge_logsdklog_mobile_detail").onHive()

spark = sparkSession.getOrCreate()

logger = logging(spark, "WARN")

# banner
logger.warn(sparkSession.showConf(), 'config')
logger.warn("liugf", "author")

def logic(start_date=DEFAULT_START_DATE, end_date=DEFAULT_END_DATE):
    # logsdklog_mobile_detail_sql_1 = "select uid, " \
    #                               "date, " \
    #                               "time, " \
    #                               "app_id, " \
    #                               "app_code, " \

Exemplo n.º 8

Arquivo: revenue_income_propsmall_daily_agg.py Projeto: heavylgf/liugf.python

from perform.MongoExecute import mongoExecute
from perform.SparkInit import sparkInitialize
from utils.HiveAlter import partition
from utils.SubmitArguments import arguments
from utils.logging import logging

DEFAULT_START_DATE = str(
    datetime.datetime.strptime(time.strftime('%Y%m%d', time.localtime()),
                               '%Y%m%d') +
    datetime.timedelta(days=-1))[0:10].replace("-", "")

DEFAULT_END_DATE = time.strftime('%Y%m%d', time.localtime())

# spark session
sparkSession = sparkInitialize().setAppName(
    "revenue_income_propsmall_daily_agg").onHive().onMongo()

spark = sparkSession.getOrCreate()

logger = logging(spark, "WARN")

# banner
logger.warn(sparkSession.showConf(), 'config')
logger.warn("liugf", "author")


def logic(start_date=DEFAULT_START_DATE, end_date=DEFAULT_END_DATE):
    # Queries are expressed in HiveQL  currencytype(3) & currencytype(100)
    gspropsmalldb_mobileprops_sql = "select t1.game as game_id, " \
                                    "t1.gamecode as game_code, " \
                                    "t1.date as date, " \

Exemplo n.º 9

Arquivo: firstlogin_playtogether_room_account1st.py Projeto: heavylgf/liugf.python

from utils.logging import logging
from perform.MongoExecute import mongoExecute
import time, datetime, sys
from utils.SubmitArguments import arguments
from utils.HiveAlter import partition

# 默认开始时间
DEFAULT_START_DATE = str(
    datetime.datetime.strptime(time.strftime('%Y%m%d', time.localtime()),
                               '%Y%m%d') +
    datetime.timedelta(days=-1))[0:10].replace("-", "")

DEFAULT_END_DATE = time.strftime('%Y%m%d', time.localtime())

# 声明spark session
sparkSession = sparkInitialize().setAppName(
    "firstlogin_playtogether_room_account1st").onHive().onMongo()

# 开始
spark = sparkSession.getOrCreate()

# 声明日志
logger = logging(spark, "warn")

# banner
logger.warn(sparkSession.showConf(), 'config')
logger.warn("Author:liugf", "banner")


def logic(start_date=DEFAULT_START_DATE, end_date=DEFAULT_END_DATE):
    '''
    处理逻辑

Exemplo n.º 10

from pyspark import SparkConf
import time, datetime, sys
from utils.SubmitArguments import arguments
from utils.HiveAlter import partition
from perform.MongoExecute import mongoExecute

# 默认开始时间
DEFAULT_START_DATE = str(
    datetime.datetime.strptime(time.strftime('%Y%m%d', time.localtime()),
                               '%Y%m%d') +
    datetime.timedelta(days=-1))[0:10].replace("-", "")
# 默认结束时间
DEFAULT_END_DATE = time.strftime('%Y%m%d', time.localtime())

# 声明spark session
sparkSession = sparkInitialize().setAppName(
    "revenue_silver_daily_agg").onHive().onMongo()

# 开始
spark = sparkSession.getOrCreate()

# 声明日志
logger = logging(spark, "warn")

# banner
logger.warn(sparkSession.showConf(), 'config')
logger.warn("huangsai", "author")


def logic(start_date=DEFAULT_START_DATE, end_date=DEFAULT_END_DATE):
    '''
    处理逻辑

Exemplo n.º 11

Arquivo: StreamingListenerDemo.py Projeto: heavylgf/liugf.python

import datetime
import sys
import time

from perform.MongoExecute import mongoExecute
from perform.SparkInit import sparkInitialize
from utils.HiveAlter import partition
from utils.SubmitArguments import arguments
from utils.logging import logging


sparkSession = sparkInitialize()\
    .setAppName("StreamingListenerDemo")

spark = sparkSession.getOrCreate()

# val sc = new SparkContext(conf)
#     val ssc = new StreamingContext(sc, Seconds(10))

if __name__ == "__main__":
    argv = arguments(sys.argv)
    # if argv["start_date"] is None or argv["end_date"] is None:
    #     logic()
    # else:
    #     logic(argv["start_date"], argv["end_date"])

Exemplo n.º 12

from perform.MongoExecute import mongoExecute
from perform.SparkInit import sparkInitialize
from utils.HiveAlter import partition
from utils.SubmitArguments import arguments
from utils.logging import logging

DEFAULT_START_DATE = str(
    datetime.datetime.strptime(time.strftime('%Y%m%d', time.localtime()),
                               '%Y%m%d') +
    datetime.timedelta(days=-1))[0:10].replace("-", "")

DEFAULT_END_DATE = time.strftime('%Y%m%d', time.localtime())

# spark session
sparkSession = sparkInitialize().setAppName(
    "revenue_spend_exchange_daily_agg").onHive().onMongo()

spark = sparkSession.getOrCreate()

logger = logging(spark, "WARN")

# banner
logger.warn(sparkSession.showConf(), 'config')
logger.warn("liugf", "author")


def logic(start_date=DEFAULT_START_DATE, end_date=DEFAULT_END_DATE):
    gsgiftcoupon_acquiregc_sql = "select t1.game as game_id, " \
                                 "t1.gamecode as game_code, " \
                                 "t1.date as date, " \
                                 "t1.pkgtype as package_type, " \

Exemplo n.º 13

from utils.logging import logging
from perform.MongoExecute import mongoExecute
import time, datetime, sys
from utils.SubmitArguments import arguments
from utils.HiveAlter import partition

# 默认开始时间
DEFAULT_START_DATE = str(
    datetime.datetime.strptime(time.strftime('%Y%m%d', time.localtime()),
                               '%Y%m%d') +
    datetime.timedelta(days=-1))[0:10].replace("-", "")

DEFAULT_END_DATE = time.strftime('%Y%m%d', time.localtime())

# 声明spark session
sparkSession = sparkInitialize().setAppName(
    "firstlogin_mobile_lianyun_user").onHive().onMongo()

# 开始
spark = sparkSession.getOrCreate()

# 声明日志
logger = logging(spark, "warn")

# banner
logger.warn(sparkSession.showConf(), 'config')
logger.warn("Author:liugf", "banner")


def logic(start_date=DEFAULT_START_DATE, end_date=DEFAULT_END_DATE):
    '''
    处理逻辑

Exemplo n.º 14

Arquivo: revenue_spend_common_props_daily_agg.py Projeto: heavylgf/liugf.python

from perform.SparkInit import sparkInitialize
from utils.logging import logging
from pyspark import SparkConf
import time,datetime,sys
from utils.SubmitArguments import arguments
from utils.HiveAlter import partition
from perform.MongoExecute import mongoExecute

# 默认开始时间
DEFAULT_START_DATE = str(datetime.datetime.strptime(time.strftime('%Y%m%d',time.localtime()), '%Y%m%d') + datetime.timedelta(days=-1))[0:10].replace("-","")
# 默认结束时间
DEFAULT_END_DATE = time.strftime('%Y%m%d',time.localtime())

# 声明spark session
sparkSession = sparkInitialize().setAppName("revenue_spend_common_props_daily_agg").onHive().onMongo()

# 开始
spark = sparkSession.getOrCreate()

# 声明日志
logger = logging(spark,"warn")

# banner
logger.warn(sparkSession.showConf(),'config')
logger.warn("zhangwei","author")


def logic(start_date=DEFAULT_START_DATE,end_date=DEFAULT_END_DATE):

    '''