Python exec_odps_commandsの例

コード例 #1

def downloaddatafromodpsbypartten(tablename, filepathname, partitionpertten,  prefixfilepath='default'):
    if prefixfilepath == 'default':
        filepathname = os.path.dirname(__file__) + filepathname
    else:
        filepathname = prefixfilepath + filepathname
    downloaddata = 'tunnel d %s/%s %s' % (tablename, partitionpertten, filepathname)
    commandlines = ';'.join((downloaddata,))
    exec_odps_commands('\"' + commandlines + '\"')

コード例 #2

def downloaddatafromodps(tablename, filepathname, offsetbasedate=0, prefixfilepath='default'):
    if prefixfilepath == 'default':
        filepathname = os.path.dirname(__file__) + filepathname
    else:
        filepathname = prefixfilepath + filepathname
 #   downloaddata = 'tunnel d %s/recorddate=%s %s' % (tablename, str(int(constvars.recorddate) + int(offsetbasedate)), filepathname)
    downloaddata = 'tunnel d %s/recorddate=%s %s' % (tablename, (datetime.date(int(constvars.recorddate[0:4]),int(constvars.recorddate[4:6]),int(constvars.recorddate[6:])) + datetime.timedelta(days=int(offsetbasedate))).strftime('%Y%m%d'), filepathname)
    print downloaddata
    commandlines = ';'.join((downloaddata,))
    exec_odps_commands('\"' + commandlines + '\"')

コード例 #3

ファイル: pipeofodps.py プロジェクト: ParkPan/ATCasePackage

def uploaddatatoodps(tablename, filepathname, offsetbasedate=0):
    droppartition = 'alter table %s drop if exists partition(recorddate=%s)' % (
        tablename, str(int(constvars.recorddate) + int(offsetbasedate)))
    addpartition = 'alter table %s add partition(recorddate=%s)' % (
        tablename, str(int(constvars.recorddate) + int(offsetbasedate)))
    uploaddata = 'tunnel u %s %s/recorddate=%s' % (
        filepathname, tablename,
        str(int(constvars.recorddate) + int(offsetbasedate)))
    commandlines = ';'.join((droppartition, addpartition, uploaddata))
    exec_odps_commands('\"' + commandlines + '\"')

コード例 #4

ファイル: pipeofodps.py プロジェクト: ParkPan/ATCasePackage

def downloaddatafromodps(tablename,
                         filepathname,
                         offsetbasedate=0,
                         prefixfilepath='default'):
    if prefixfilepath == 'default':
        filepathname = os.path.dirname(__file__) + filepathname
    else:
        filepathname = prefixfilepath + filepathname
    downloaddata = 'tunnel d %s/recorddate=%s %s' % (
        tablename, str(int(constvars.recorddate) + int(offsetbasedate)),
        filepathname)
    commandlines = ';'.join((downloaddata, ))
    exec_odps_commands('\"' + commandlines + '\"')

コード例 #5

def uploaddatatoodps(tablename, filepathname, offsetbasedate=0, isnull= True):
#    droppartition = 'alter table %s drop if exists partition(recorddate=%s)' % (
#        tablename, str(int(constvars.recorddate) + int(offsetbasedate)))
    droppartition = 'alter table %s drop if exists partition(recorddate=%s)' % (
         tablename, (datetime.date(int(constvars.recorddate[0:4]),int(constvars.recorddate[4:6]),int(constvars.recorddate[6:])) + datetime.timedelta(days=int(offsetbasedate))).strftime('%Y%m%d'))
    print droppartition
    addpartition = 'alter table %s add partition(recorddate=%s)' % (
        tablename, (datetime.date(int(constvars.recorddate[0:4]),int(constvars.recorddate[4:6]),int(constvars.recorddate[6:])) + datetime.timedelta(days=int(offsetbasedate))).strftime('%Y%m%d'))
    print addpartition
    uploaddata = 'tunnel u %s %s/recorddate=%s' % (
        filepathname, tablename, (datetime.date(int(constvars.recorddate[0:4]),int(constvars.recorddate[4:6]),int(constvars.recorddate[6:])) + datetime.timedelta(days=int(offsetbasedate))).strftime('%Y%m%d'))
    commandlines = ';'.join((droppartition, addpartition, uploaddata))
    print uploaddata
    if isnull:
        exec_odps_commands('\"' + commandlines + '\"')
    else:
        exec_odps_commands_notnull('\"' + commandlines + '\"')

コード例 #6

ファイル: startodpsjob.py プロジェクト: ParkPan/ATCasePackage

def dropodpsdata(tablename, offsetbasedate=0):
    droppartition = 'alter table %s drop if exists partition(recorddate=%s)' % (
        tablename, str(int(constvars.recorddate) + int(offsetbasedate)))
    commandlines = droppartition
    exec_odps_commands('\"' + commandlines + '\"')