def syncFile_mp3(mp3file): ''' 将文件转换成base64 ''' try: print 'do syncFile:',mp3file digest = utils.getfiledigest(mp3file) # return prepare(digest) f = open(mp3file,'rb') c = f.read() f.close() s = base64.encodestring(c).strip() params = urllib.urlencode({'token':getApp().getToken(),'digest':digest,'content':s}) print 'file len:',len(s) server = getApp().getSettings().get('webserver') if server.find('http')==-1: server = 'http://'+server f = urllib.urlopen('%s/WebApi/Terminal/syncFile'%(server),params) # POST d = f.read() print 'sync file returns:',d d = json.loads(d) if d['status'] == 0 : return True except: traceback.print_exc() return False
def scan(): ''' 扫描文件,写入db,再进行文件传输 ''' import dbsql path = getApp().getAudioStorePath() db = getApp().getDB() #扫描文件目录,找出差异文件,并写入db print 'scan :',path for root, dirs, files in os.walk(path): for file in files: time.sleep(0.001) if not getApp().running: return file = file.strip().lower() if file.find('.spx') == -1: continue spxfile = root+'/'+file try: # print 'ready for ',spxfile if db.getRecCountBySpxFile(spxfile): # print 'file has registered in db!' continue #已处理 spx_digest = utils.getfiledigest(spxfile) if db.getRecCountByDigest(spx_digest): # print 'has upload by digest!' continue #已处理 db.appendSpxFile(spxfile,spx_digest,0) #未上传 except: traceback.print_exc()
def scan(): ''' 扫描文件,写入db,再进行文件传输 ''' import dbsql path = getApp().getAudioStorePath() db = getApp().getDB() #扫描文件目录,找出差异文件,并写入db print 'scan :', path for root, dirs, files in os.walk(path): for file in files: time.sleep(0.001) if not getApp().running: return file = file.strip().lower() if file.find('.spx') == -1: continue spxfile = root + '/' + file try: # print 'ready for ',spxfile if db.getRecCountBySpxFile(spxfile): # print 'file has registered in db!' continue #已处理 spx_digest = utils.getfiledigest(spxfile) if db.getRecCountByDigest(spx_digest): # print 'has upload by digest!' continue #已处理 db.appendSpxFile(spxfile, spx_digest, 0) #未上传 except: traceback.print_exc()
def syncFile_mp3(mp3file): ''' 将文件转换成base64 ''' try: print 'do syncFile:', mp3file digest = utils.getfiledigest(mp3file) # return prepare(digest) f = open(mp3file, 'rb') c = f.read() f.close() s = base64.encodestring(c).strip() params = urllib.urlencode({ 'token': getApp().getToken(), 'digest': digest, 'content': s }) print 'file len:', len(s) server = getApp().getSettings().get('webserver') if server.find('http') == -1: server = 'http://' + server f = urllib.urlopen('%s/WebApi/Terminal/syncFile' % (server), params) # POST d = f.read() print 'sync file returns:', d d = json.loads(d) if d['status'] == 0: return True except: traceback.print_exc() return False
def sync_mp3(): db = getApp().getDB() #从db中取出未上传部分,执行文件同步 sql = 'select * from core_audiofile where status= 0 ' cr = db.handle().cursor() cr.execute(sql) rs = fetchallDict(cr) cr = None for r in rs: spxfile,digest = r['spxfile'],r['digest'] try: if not getApp().running: return spxfile = spxfile.decode('utf-8') if not spxfile or not os.path.exists(spxfile): sql = 'delete from core_audiofile where digest=?' cr = db.handle().cursor() cr.execute(sql,(digest,)) db.handle().commit() continue print __file__,inspect.currentframe().f_lineno, spxfile rc = prepare(digest) print 'prepare result:',rc if rc not in (0,1): # 2 getApp().token = '' return #网络访问异常,返回,等待下次再次执行sync if rc ==1: #开始同步文件到服务器 #读文件 失败 可能文件未写完,要求话机上传程序写入文件时,独占写入 mp3file ='' try: mp3file = convert.spx_convert_mp3(spxfile,'') except: traceback.print_exc() if not mp3file: #文件转换失败,更名 处理失败,删除数据库记录 #print spxfile,spxfile.replace('.spx','._spx') if os.path.exists(spxfile): os.rename(spxfile,spxfile.replace('.spx','._spx')) sql = 'delete from core_audiofile where digest=?' cr = db.handle().cursor() cr.execute(sql,(digest,)) db.handle().commit() continue digest = utils.getfiledigest(mp3file) if not digest: continue if not syncFile_mp3(mp3file): print 'syncFile failed!' getApp().token = '' return #上传okay ,标识文件已经被上传了 sql = 'update core_audiofile set status=1 ,uptime=? where digest=?' cr = db.handle().cursor() cr.execute(sql,(int(time.time()),digest)) db.handle().commit() except: traceback.print_exc()
def sync_mp3(): db = getApp().getDB() #从db中取出未上传部分,执行文件同步 sql = 'select * from core_audiofile where status= 0 ' cr = db.handle().cursor() cr.execute(sql) rs = fetchallDict(cr) cr = None for r in rs: spxfile, digest = r['spxfile'], r['digest'] try: if not getApp().running: return spxfile = spxfile.decode('utf-8') if not spxfile or not os.path.exists(spxfile): sql = 'delete from core_audiofile where digest=?' cr = db.handle().cursor() cr.execute(sql, (digest, )) db.handle().commit() continue print __file__, inspect.currentframe().f_lineno, spxfile rc = prepare(digest) print 'prepare result:', rc if rc not in (0, 1): # 2 getApp().token = '' return #网络访问异常,返回,等待下次再次执行sync if rc == 1: #开始同步文件到服务器 #读文件 失败 可能文件未写完,要求话机上传程序写入文件时,独占写入 mp3file = '' try: mp3file = convert.spx_convert_mp3(spxfile, '') except: traceback.print_exc() if not mp3file: #文件转换失败,更名 处理失败,删除数据库记录 #print spxfile,spxfile.replace('.spx','._spx') if os.path.exists(spxfile): os.rename(spxfile, spxfile.replace('.spx', '._spx')) sql = 'delete from core_audiofile where digest=?' cr = db.handle().cursor() cr.execute(sql, (digest, )) db.handle().commit() continue digest = utils.getfiledigest(mp3file) if not digest: continue if not syncFile_mp3(mp3file): print 'syncFile failed!' getApp().token = '' return #上传okay ,标识文件已经被上传了 sql = 'update core_audiofile set status=1 ,uptime=? where digest=?' cr = db.handle().cursor() cr.execute(sql, (int(time.time()), digest)) db.handle().commit() except: traceback.print_exc()
def isExisted(mov,db): name,ext = os.path.splitext(mov) ff = name +'.dat' #从.dat生成摘要 digest = utils.getfiledigest(ff) #检测image是否存在 ,根据digest dbconn = db sql = "select count(*) from core_imagefile where degist=%s" cr = dbconn.cursor() cr.execute(sql,(digest,)) r = cr.fetchone()[0] if r==0: return False return True
def spx_convert_mp3(spxfile,mp3file,subfix=''): TEMPDIR = getApp().getTempPath() riffhdr = [0x4952,0x4646,0x3EA4,0x0,0x4157,0x4556,0x6D66,0x2074, 0x0010,0x0,0x1,0x1,0x1F40,0x0,0x3E80,0x0, 0x2,0x10,0x6164,0x6174,0x0,0x0] print spxfile f = open(spxfile,'rb') d = f.read() f.close() hdrsize = 32 magic,= struct.unpack('I',d[28:28+4]) if magic != 0xffffffff: hdrsize = 19 # hdrsize = 32 print 'hdrsize:',hdrsize npos = hdrsize wavlen = int((len(d) - hdrsize)/500) wavlen*=8000 riffhdr[21] = (wavlen >> 16) & 0xffff riffhdr[20] = wavlen & 0xffff riffhdr[3] = ((wavlen+36)>>16)& 0xffff riffhdr[2] = (wavlen+36) & 0xffff if not os.path.exists(TEMPDIR): os.mkdir(TEMPDIR) segfile = TEMPDIR +'/tempseg.dat'+subfix pcmfile = TEMPDIR +'/temppcm.dat'+subfix pcmbytes='' while True: block = d[npos:npos+CODEC_BLOCK_SIZE] if not block: break npos+=CODEC_BLOCK_SIZE f = open(segfile,'wb') f.write(block) f.close() # print '--'*20 # print segfile,pcmfile segfile = os.path.normpath(segfile) pcmfile = os.path.normpath(pcmfile) cmd = u'speex.exe "%s" "%s"'%(segfile,pcmfile) # print cmd.encode('gbk') # r = os.system(cmd.encode('gbk')) r = subprocess.call(cmd.encode('gbk'),shell=True) if r : print 'speex fail!' return '' f = open(pcmfile,'rb') pcmbytes+=f.read() print len(pcmbytes) f.close() pcmfile = spxfile.replace('.spx','.pcm') print 'spx to pcm completed! >> ',pcmfile f = open(pcmfile,'wb') f.write(pcmbytes) f.close() print 'preparing pcm to wav ..' wavfile = spxfile.replace('.spx','.wav') f = open(wavfile,'wb') for n in riffhdr: f.write( struct.pack('H',n)) f.write(pcmbytes) f.close() #-- copy wav to $app/temp/spx-digest.wav if getApp().getSystemConfig('retain_wav_to_temp','0') == '1': try:#检测是否已经存在对应的 wav文件了 digest = utils.getfiledigest(spxfile) destpath = os.path.join(getApp().getTempPath(),digest+'.wav') if not os.path.exists(destpath): shutil.copyfile(wavfile,destpath) except: traceback.print_exc() #-- end ---- mp3file = TEMPDIR +'/audiofinal.mp3' if not wav_to_mp3(spxfile,wavfile,mp3file): return '' os.remove(pcmfile) os.remove(wavfile) return mp3file
def wav_to_mp3(spxfile,wavfile,mp3file): try: spxinfo = format.parseSpxFileInfo(spxfile) if not spxinfo: getApp().getLogger().info(u'corrupted file %s, be deleted'%(spxinfo)) #os.remove(spxfile) #删除非法的spx文件 return False if os.path.exists(mp3file): os.remove(mp3file) wavfile = os.path.normpath(wavfile) mp3file = os.path.normpath(mp3file) cmd = u'ffmpeg.exe -y -aq 128 -i "%s" "%s"'%(wavfile,mp3file) # os.system(cmd.encode('gbk')) r = subprocess.call(cmd.encode('gbk'),shell=True) if r: print 'ffmpeg convert failed!' return False #添加附属信息 # (phone,filename,createtime,duration) encode json array #2013.10.7 spx 头部32字节写入mp3尾部,spx的degist也写入尾部, # mp3-content + magic(4) + spx-degist(32) + spx-hdr(32) spx_digest = utils.getfiledigest(spxfile) if not spx_digest: print 'calc file digest failed!' return False f = open(spxfile,'rb') hdrinfo = f.read(32) f.close() filename= os.path.basename(spxfile).lower() filename = filename.replace('.spx','.mp3') # phone = spxinfo.phone #getApp().getSettings()['phone'] # createtime = time.mktime( # (spxinfo.year, # spxinfo.month, # spxinfo.day, # spxinfo.hour, # spxinfo.minute, # 0, # 0,0,0)) # int(time.time()) # createtime = int(createtime) duration = utils.readImageDuration(mp3file) if not duration: print 'calc duration failed: 0 size' return False f = open(mp3file,'ab') magic = struct.pack('!I',0xEEAAEEBB) f.write(magic) f.write(spx_digest) f.write(hdrinfo) # params=(phone,filename,createtime,duration,spxinfo.index,spxinfo.attr,spxinfo.serial,spx_digest) # print params # d = json.dumps(params) # padding = 'leadtel '+ d + ' '*200 # padding = padding[:200] # f.write(padding) f.close() except: traceback.print_exc() return False return True
def beforePut_Init(self,file): ''' file - {filesize,filename,...} 找寻trp文件并写入 file trp在.或者../INFO/.trp|.dat ''' import string self.lastpercent = 0 #文件是否可被解码 times = utils.readImageTimes(file['filename']) if not times: # decode asscess denied print 'decode file failed!' return False st,mt = times if st <= 1325350861:#文件创建时间是否合法 print 'file md time so early!' return False #文件时长是否过小 if mt - st < 5*60: # less than 5 minutes, skipped print 'file duration too short!' return False filename = os.path.normpath(file['filename']) file['fullname'] = filename file['filename'] = os.path.basename(filename) #重载文件大小为MB file['filesize'] = int( file['filesize']/ (1024*1024)) d,b = os.path.split(filename) c,ext = os.path.splitext(filename) if os.path.exists( c+'.pass'): #已处理过了 print '.pass skipped..' return False trp = c+'.trp' dat= c+'.dat' #同级目录是否存在trp和dat if os.path.exists( trp): # and os.path.exists( dat): pass else:#去上级的INFO下检索 dat,trp d = d.split(os.path.sep) name,ext = os.path.splitext(b) trp = string.join(d[:-1],'/')+'/INFO/'+name+'.trp' #dat = string.join(d[:-1],'/')+'/INFO/'+name+'.dat' if os.path.exists( trp):# and os.path.exists( dat): pass else: print trp #print dat trp=dat='' print '.trp or .dat missed' return False #无法找到dat,trp文件 try: #将trp,dat文件内容上传,并将dat内容digest作为资源唯一标示 fd = open(trp,'rb') cont = fd.read() fd.close() file['trp'] = base64.encodestring(cont) print 'trp size:',len(file['trp']) # fd = open(dat,'rb') # cont = fd.read() # fd.close() # file['dat'] = base64.encodestring(cont) file['digest'] = utils.getfiledigest(trp) #将trp的digest上传 2012.7.17 except: traceback.print_exc() print 'create digest failed!' return False return True
# -*- coding:utf-8 -*-
def regdb_ImageFile(stodir,base,zone,app): ff = os.path.join(stodir,base)+'.dat' #从.dat生成摘要 digest = utils.getfiledigest(ff) if not digest: print 'digest make failed!' return False# 生成摘要失败 print 'digest:',digest #检测image是否存在 ,根据digest dbconn = app.getDbConn() sql = "select count(*) from core_imagefile where degist=%s" cr = dbconn.cursor() cr.execute(sql,(digest,)) r = cr.fetchone()[0] if r!=0: print 'same file is existed!(with digest)' return False #读取image视频信息 ff = os.path.join(stodir,base)+'.wmv' print ff csec,msec = readImageTimes(ff) if csec ==0: print 'read image time failed!' return False profile={'starttime':utils.mk_datetime(csec),'duration':msec-csec, 'uri':app.id+'.'+digest, 'filesize':os.path.getsize(ff), 'digest':digest, 'gpsdata':''} #读取trp生成gpsdata (json格式) trp2 = os.path.join(stodir,base)+'.trp2' trp = os.path.join(stodir,base)+'.trp' cwd = os.getcwd() os.chdir('c:/dvr_bin') cmd = "%s fixpoint %s %s"%('c:/dvr_bin/trpMapFix.exe',trp,trp2) cmd = os.path.normpath(cmd) #print cmd os.system(cmd) os.chdir(cwd) fp = open(trp2) lines = fp.readlines() fp.close() gpsdata=[] wkt = 'LINESTRING(%s)' pts=[] for line in lines: line = line.strip() pp = map(string.strip,line.split(',')) if len(pp)!=6: continue lon,lat,timestr,tick,speed,angle = pp lon = float(lon)/3600. lat = float(lat)/3600. lon = round(lon,6) lat = round(lat,6) tick = int(tick) speed = float(speed) angle = float(angle) gps={ 'lon':lon,'lat':lat,'time':tick,'speed':speed,'angle':angle } gpsdata.append(gps) pts.append( "%s %s"%(lon,lat)) wkt = wkt%(string.join(pts,',')) #LINESTRING() profile['gpsdata']= json.dumps(gpsdata) #写入数据库 录像文件 dbconn = app.getDbConn() seq = utils.getdbsequence_pg(dbconn,'core_imagefile_id_seq') sql = "INSERT INTO core_imagefile(id, uri, starttime, \ addtime, duration, node, degist, filesize, removed, \ gpsdata,wkt) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);" dbconn = app.getDbConn() cur = dbconn.cursor() cur.execute(sql,(seq,profile['uri'], profile['starttime'], datetime.datetime.now(), # add time profile['duration'], app.id, # node id profile['digest'], profile['filesize'], 0, # removed profile['gpsdata'], wkt )) #print profile['gpsdata'] #写入nodeimageindex sql = "insert into core_StoNodeImageIndex (node,zone,imgdigest,imgpath) values \ (%s,%s,%s,%s)" cr = dbconn.cursor() #movpath = os.path.join(stodir,base)+'.mov' cr.execute(sql,(app.id,zone.id,digest,ff)) return True
def importImageFile(stodir,base,zone,app): trp = os.path.join(stodir,base)+'.trp' digest = utils.getfiledigest(trp) if not digest: getLog().debug('digest make failed!') return False# 生成摘要失败 getLog().debug('digest:',digest) #检测image是否存在 ,根据digest dbconn = app.getDbConn() #读取image视频信息 mov = os.path.join(stodir,base)+'.mov' csec,msec = readImageTimes(mov) if csec ==0: getLog().debug( 'read image time failed!') return False profile={'starttime':utils.mk_datetime(csec),'duration':msec-csec, 'uri':app.id+'.'+digest, 'filesize':os.path.getsize(mov), 'digest':digest, 'gpsdata':''} fp = open(trp) lines = fp.readlines() fp.close() gpsdata=[] wkt = 'LINESTRING(%s)' pts=[] for line in lines: line = line.strip() pp = map(string.strip,line.split(',')) if len(pp)!=6: continue lon,lat,timestr,tick,speed,angle = pp lon = float(lon)/3600. lat = float(lat)/3600. lon = round(lon,6) lat = round(lat,6) tick = int(tick) speed = float(speed) angle = float(angle) gps={ 'lon':lon,'lat':lat,'time':tick,'speed':speed,'angle':angle } gpsdata.append(gps) pts.append( "%s %s"%(lon,lat)) wkt = wkt%(string.join(pts,',')) #LINESTRING() profile['gpsdata']= json.dumps(gpsdata) #写入数据库 录像文件 dbconn = app.getDbConn() seq = utils.getdbsequence_pg(dbconn,'core_imagefile_id_seq') sql = "INSERT INTO core_imagefile(id, uri, starttime, \ addtime, duration, node, degist, filesize, removed, \ gpsdata,wkt,visible,check_passed) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);" dbconn = app.getDbConn() cur = dbconn.cursor() cur.execute(sql,(seq,profile['uri'], profile['starttime'], datetime.datetime.now(), # add time profile['duration'], app.id, # node id profile['digest'], profile['filesize'], 0, # removed profile['gpsdata'], wkt, True, True )) sql = "insert into core_StoNodeImageIndex (node,zone,imgdigest,imgpath) values \ (%s,%s,%s,%s)" cr = dbconn.cursor() cr.execute(sql,(app.id,zone.id,digest,mov)) dbconn.commit() return True
def spx_convert_mp3(spxfile, mp3file, subfix=''): TEMPDIR = getApp().getTempPath() riffhdr = [ 0x4952, 0x4646, 0x3EA4, 0x0, 0x4157, 0x4556, 0x6D66, 0x2074, 0x0010, 0x0, 0x1, 0x1, 0x1F40, 0x0, 0x3E80, 0x0, 0x2, 0x10, 0x6164, 0x6174, 0x0, 0x0 ] print spxfile f = open(spxfile, 'rb') d = f.read() f.close() hdrsize = 32 magic, = struct.unpack('I', d[28:28 + 4]) if magic != 0xffffffff: hdrsize = 19 # hdrsize = 32 print 'hdrsize:', hdrsize npos = hdrsize wavlen = int((len(d) - hdrsize) / 500) wavlen *= 8000 riffhdr[21] = (wavlen >> 16) & 0xffff riffhdr[20] = wavlen & 0xffff riffhdr[3] = ((wavlen + 36) >> 16) & 0xffff riffhdr[2] = (wavlen + 36) & 0xffff if not os.path.exists(TEMPDIR): os.mkdir(TEMPDIR) segfile = TEMPDIR + '/tempseg.dat' + subfix pcmfile = TEMPDIR + '/temppcm.dat' + subfix pcmbytes = '' while True: block = d[npos:npos + CODEC_BLOCK_SIZE] if not block: break npos += CODEC_BLOCK_SIZE f = open(segfile, 'wb') f.write(block) f.close() # print '--'*20 # print segfile,pcmfile segfile = os.path.normpath(segfile) pcmfile = os.path.normpath(pcmfile) cmd = u'speex.exe "%s" "%s"' % (segfile, pcmfile) # print cmd.encode('gbk') # r = os.system(cmd.encode('gbk')) r = subprocess.call(cmd.encode('gbk'), shell=True) if r: print 'speex fail!' return '' f = open(pcmfile, 'rb') pcmbytes += f.read() print len(pcmbytes) f.close() pcmfile = spxfile.replace('.spx', '.pcm') print 'spx to pcm completed! >> ', pcmfile f = open(pcmfile, 'wb') f.write(pcmbytes) f.close() print 'preparing pcm to wav ..' wavfile = spxfile.replace('.spx', '.wav') f = open(wavfile, 'wb') for n in riffhdr: f.write(struct.pack('H', n)) f.write(pcmbytes) f.close() #-- copy wav to $app/temp/spx-digest.wav if getApp().getSystemConfig('retain_wav_to_temp', '0') == '1': try: #检测是否已经存在对应的 wav文件了 digest = utils.getfiledigest(spxfile) destpath = os.path.join(getApp().getTempPath(), digest + '.wav') if not os.path.exists(destpath): shutil.copyfile(wavfile, destpath) except: traceback.print_exc() #-- end ---- mp3file = TEMPDIR + '/audiofinal.mp3' if not wav_to_mp3(spxfile, wavfile, mp3file): return '' os.remove(pcmfile) os.remove(wavfile) return mp3file
def wav_to_mp3(spxfile, wavfile, mp3file): try: spxinfo = format.parseSpxFileInfo(spxfile) if not spxinfo: getApp().getLogger().info(u'corrupted file %s, be deleted' % (spxinfo)) #os.remove(spxfile) #删除非法的spx文件 return False if os.path.exists(mp3file): os.remove(mp3file) wavfile = os.path.normpath(wavfile) mp3file = os.path.normpath(mp3file) cmd = u'ffmpeg.exe -y -aq 128 -i "%s" "%s"' % (wavfile, mp3file) # os.system(cmd.encode('gbk')) r = subprocess.call(cmd.encode('gbk'), shell=True) if r: print 'ffmpeg convert failed!' return False #添加附属信息 # (phone,filename,createtime,duration) encode json array #2013.10.7 spx 头部32字节写入mp3尾部,spx的degist也写入尾部, # mp3-content + magic(4) + spx-degist(32) + spx-hdr(32) spx_digest = utils.getfiledigest(spxfile) if not spx_digest: print 'calc file digest failed!' return False f = open(spxfile, 'rb') hdrinfo = f.read(32) f.close() filename = os.path.basename(spxfile).lower() filename = filename.replace('.spx', '.mp3') # phone = spxinfo.phone #getApp().getSettings()['phone'] # createtime = time.mktime( # (spxinfo.year, # spxinfo.month, # spxinfo.day, # spxinfo.hour, # spxinfo.minute, # 0, # 0,0,0)) # int(time.time()) # createtime = int(createtime) duration = utils.readImageDuration(mp3file) if not duration: print 'calc duration failed: 0 size' return False f = open(mp3file, 'ab') magic = struct.pack('!I', 0xEEAAEEBB) f.write(magic) f.write(spx_digest) f.write(hdrinfo) # params=(phone,filename,createtime,duration,spxinfo.index,spxinfo.attr,spxinfo.serial,spx_digest) # print params # d = json.dumps(params) # padding = 'leadtel '+ d + ' '*200 # padding = padding[:200] # f.write(padding) f.close() except: traceback.print_exc() return False return True