def downloaddatafromodpsbypartten(tablename, filepathname, partitionpertten, prefixfilepath='default'):
if prefixfilepath == 'default':
filepathname = os.path.dirname(__file__) + filepathname
else:
filepathname = prefixfilepath + filepathname
downloaddata = 'tunnel d %s/%s %s' % (tablename, partitionpertten, filepathname)
commandlines = ';'.join((downloaddata,))
exec_odps_commands('\"' + commandlines + '\"')
def downloaddatafromodps(tablename, filepathname, offsetbasedate=0, prefixfilepath='default'):
if prefixfilepath == 'default':
filepathname = os.path.dirname(__file__) + filepathname
else:
filepathname = prefixfilepath + filepathname
# downloaddata = 'tunnel d %s/recorddate=%s %s' % (tablename, str(int(constvars.recorddate) + int(offsetbasedate)), filepathname)
downloaddata = 'tunnel d %s/recorddate=%s %s' % (tablename, (datetime.date(int(constvars.recorddate[0:4]),int(constvars.recorddate[4:6]),int(constvars.recorddate[6:])) + datetime.timedelta(days=int(offsetbasedate))).strftime('%Y%m%d'), filepathname)
print downloaddata
commandlines = ';'.join((downloaddata,))
exec_odps_commands('\"' + commandlines + '\"')
def uploaddatatoodps(tablename, filepathname, offsetbasedate=0):
droppartition = 'alter table %s drop if exists partition(recorddate=%s)' % (
tablename, str(int(constvars.recorddate) + int(offsetbasedate)))
addpartition = 'alter table %s add partition(recorddate=%s)' % (
tablename, str(int(constvars.recorddate) + int(offsetbasedate)))
uploaddata = 'tunnel u %s %s/recorddate=%s' % (
filepathname, tablename,
str(int(constvars.recorddate) + int(offsetbasedate)))
commandlines = ';'.join((droppartition, addpartition, uploaddata))
exec_odps_commands('\"' + commandlines + '\"')
def downloaddatafromodps(tablename,
filepathname,
offsetbasedate=0,
prefixfilepath='default'):
if prefixfilepath == 'default':
filepathname = os.path.dirname(__file__) + filepathname
else:
filepathname = prefixfilepath + filepathname
downloaddata = 'tunnel d %s/recorddate=%s %s' % (
tablename, str(int(constvars.recorddate) + int(offsetbasedate)),
filepathname)
commandlines = ';'.join((downloaddata, ))
exec_odps_commands('\"' + commandlines + '\"')
def uploaddatatoodps(tablename, filepathname, offsetbasedate=0, isnull= True):
# droppartition = 'alter table %s drop if exists partition(recorddate=%s)' % (
# tablename, str(int(constvars.recorddate) + int(offsetbasedate)))
droppartition = 'alter table %s drop if exists partition(recorddate=%s)' % (
tablename, (datetime.date(int(constvars.recorddate[0:4]),int(constvars.recorddate[4:6]),int(constvars.recorddate[6:])) + datetime.timedelta(days=int(offsetbasedate))).strftime('%Y%m%d'))
print droppartition
addpartition = 'alter table %s add partition(recorddate=%s)' % (
tablename, (datetime.date(int(constvars.recorddate[0:4]),int(constvars.recorddate[4:6]),int(constvars.recorddate[6:])) + datetime.timedelta(days=int(offsetbasedate))).strftime('%Y%m%d'))
print addpartition
uploaddata = 'tunnel u %s %s/recorddate=%s' % (
filepathname, tablename, (datetime.date(int(constvars.recorddate[0:4]),int(constvars.recorddate[4:6]),int(constvars.recorddate[6:])) + datetime.timedelta(days=int(offsetbasedate))).strftime('%Y%m%d'))
commandlines = ';'.join((droppartition, addpartition, uploaddata))
print uploaddata
if isnull:
exec_odps_commands('\"' + commandlines + '\"')
else:
exec_odps_commands_notnull('\"' + commandlines + '\"')
def dropodpsdata(tablename, offsetbasedate=0):
droppartition = 'alter table %s drop if exists partition(recorddate=%s)' % (
tablename, str(int(constvars.recorddate) + int(offsetbasedate)))
commandlines = droppartition
exec_odps_commands('\"' + commandlines + '\"')