def run(self):
logger.info('Start SaltMan.')
lc = util.LoopChecker('Main loop')
while True:
try:
targets = self.collect_delegate.collect(self.config)
self.compress_delegate.compress(self.config, targets)
time.sleep(1)
lc.check()
if len(targets) > 0:
lc.reset()
except:
e = sys.exc_info()[0]
logger.error(e)
time.sleep(5)
def file_import(request): # 文件导入处理
filename = ""
filetype = ""
tablename = ""
sparatorvalue = ","
fields = {} # 已选择字段列表
target = [] # 目标文件列号
errmode = 1 # 出错处理方式 1 跳过错误 继续处理 2 退出，并删除已导入记录
headln = 1
recordln = 2
unicode_ = "utf-8"
relatefield = []
relrecs = []
addrelate = 1 # 关联记录处理 1 自动增加 2 关键记录不存在，跳过当次记录,不增加关联记录
data = dict(request.POST)
# print data
errmode = int(data["errmode"][0])
filename = data["txtfilename"][0]
filetype = data["txtfiletype"][0]
sparatorvalue = data["sparatorvalue"][0]
headln = int(data["headln"][0])
recordln = int(data["recordln"][0])
tablename = data["txttablename"][0]
unicode_ = data["unicode"][0].decode()
addrelate = int(data["addrelate"][0])
# 查找字段列表，目标列号
for n, v in data.items():
if n.startswith("_chk_"):
field = str(data["_select_" + n[5:]][0]).decode(unicode_)
fields[field] = int(n[5:])
# 查找模型
model, flds, rlfield = findAllFieldsAndModel(tablename)
objlist = []
Employee = GetModel("personnel", "Employee")
Department = GetModel("personnel", "Department")
error_list = []
try:
if filetype == "txt" or filetype == "csv":
fs = file(uploadpath + filename, "r")
rec = fs.readline()
ln = 1
while rec != "":
try:
# logger.info(ln)
rec = rec.decode(unicode_)
if rec.endswith("\r\n"):
rec = rec[: len(rec) - 2]
linedata = rec.split(sparatorvalue)
ltmp = []
# logger.info("linedata:",linedata)
for l in linedata:
if l.startswith('"') and l.endswith('"'):
ltmp.append(u"%s" % l[1 : len(l) - 1])
else:
ltmp.append(u"%s" % l)
linedata = ltmp
currelobj = [] # 当前行外键实例列表
if len(linedata) >= len(fields.keys()): # 当前行记录不满足已选择字段
if ln >= recordln:
strwhere = {}
upobj = ""
isSave = True
try:
for tmpfld, tmpfldvalue in fields.items():
rf = ""
if tmpfld.find(".") > 0:
for f in model._meta.fields:
if isinstance(f, ForeignKey):
if f.rel.to.__name__ == tmpfld.split(".")[0]:
strwhere[
str(f.name + "__" + tmpfld.split(".")[1] + "__exact")
] = linedata[tmpfldvalue]
else:
strwhere[str(tmpfld + "__exact")] = linedata[tmpfldvalue]
upobj = model.objects.filter(Q(**strwhere))
except Exception, e:
logger.error("%s" % e)
if upobj:
obj = upobj[0]
else:
obj = model()
for fld, value in fields.items():
relObj = ""
isForeignKey = False
if fld.find(".") > 0: # 查找到需要保存关联字段
print "find relate"
for nkey, val in rlfield.items(): # 查找关联记录,更新或创建,并保存
if nkey.__name__ == fld.split(".")[0]:
strfilter = {}
for f in fields: # 查找关联表多个相应的字段，并生成表达式
if f.find(".") > 0:
if f.split(".")[0] == nkey.__name__:
tmpvalue = linedata[fields[f]]
# tmpvalue=tmpvalue.encode('utf-8')
strfilter[str(f.split(".")[1] + "__exact")] = tmpvalue
if strfilter:
relObj = nkey.objects.filter(Q(**strfilter))
if len(relObj) <= 0: # 查找不到记录，生成新记录
if addrelate != 1:
isSave = False
break
else:
relObj = nkey()
for tfld in fields.keys():
if tfld.find(".") > 0:
if tfld.split(".")[0] == nkey.__name__:
relObj.__setattr__(
tfld.split(".")[1], linedata[fields[tfld]]
)
isForeignKey = True
relObj.save()
relrecs.append(relObj)
else:
isForeignKey = True
relObj = relObj[0]
currelobj.append(relObj)
break
if not isSave:
break # 跳出当前行
tobj = ""
fieldname = ""
if isForeignKey:
for f in obj._meta.fields: # 查找字段是否是外键
if isinstance(f, ForeignKey) and f.rel.to.__name__ == fld.split(".")[0]:
for tobj in currelobj:
if tobj == f.rel.to:
break
fieldname = f.name
break
obj.__setattr__(fieldname, tobj)
else:
if fld == "PIN":
model_emp = sys.modules["mysite.personnel.models.model_emp"]
settings = sys.modules["mysite.settings"]
if len(str(linedata[value]).strip()) > getattr(settings, "PIN_WIDTH"):
raise Exception(u"%s" % _(u"人员编号长度过长"))
else:
linedata[value] = getattr(model_emp, "format_pin")(str(linedata[value]))
if fld == "code":
dept = Department.objects.filter(code=linedata[value])
if dept:
raise Exception(u"%s" % _(u"部门编号已存在"))
obj.__setattr__(fld, linedata[value])
# logger.info("file Line:%s save is %s "%(ln,isSave))
if isSave:
obj.save()
if type(obj) == Employee:
obj.__setattr__("attarea", (1,))
obj.save()
if type(obj) == Department:
if obj.parent == None or obj.parent == "":
obj.parent_id = 1
obj.save()
objlist.append(obj)
ln += 1
rec = fs.readline()
except Exception, e:
# logger.error("%s"%e)
try:
error_list.append(str(ln) + u" 行 " + u"%s" % e)
except:
error_list.append(str(ln))
pass
if errmode == 1:
ln += 1
rec = fs.readline()
continue
else:
raise
if datash[row][tmpfldvalue][0] == 2:
dv = int(datash[row][tmpfldvalue][1])
else:
dv = datash[row][tmpfldvalue][1]
if tmpfld.find(".") > 0:
for f in model._meta.fields:
if isinstance(f, ForeignKey):
if f.rel.to.__name__ == tmpfld.split(".")[0]:
strwhere[str(f.name + "__" + tmpfld.split(".")[1] + "__exact")] = dv
else:
strwhere[str(tmpfld + "__exact")] = datash[row][tmpfldvalue][1] = dv
# print strwhere
# print(Q(**strwhere))
upobj = model.objects.filter(Q(**strwhere))
except Exception, e:
logger.error("%s" % e)
if upobj:
obj = upobj[0]
else:
obj = model()
currelobj = []
for fld, value in fields.items():
relObj = ""
isForeignKey = False
if fld.find(".") > 0: # 查找到需要保存关联字段
# print "find relate"
for nkey, val in rlfield.items(): # 查找关联记录,更新或创建,并保存
if nkey.__name__ == fld.split(".")[0]:
def file_import(request): #文件导入处理
filename=""
filetype=""
tablename=""
sparatorvalue=","
fields={} #已选择字段列表
target=[] #目标文件列号
errmode=1 #出错处理方式 1 跳过错误 继续处理 2 退出，并删除已导入记录
headln=1
recordln=2
unicode_="utf-8"
relatefield=[]
relrecs=[]
addrelate=1 #关联记录处理 1 自动增加 2 关键记录不存在，跳过当次记录,不增加关联记录
data=dict(request.POST)
#print data
errmode=int(data["errmode"][0])
filename=data["txtfilename"][0]
filetype=data["txtfiletype"][0]
sparatorvalue=data["sparatorvalue"][0]
headln=int(data["headln"][0])
recordln=int(data["recordln"][0])
tablename=data["txttablename"][0]
unicode_=data["unicode"][0].decode()
addrelate=int(data["addrelate"][0])
#查找字段列表，目标列号
for n,v in data.items():
if n.startswith("_chk_"):
field=str(data["_select_"+ n[5:]][0]).decode(unicode_)
fields[field]=int(n[5:])
#查找模型
model,flds,rlfield=findAllFieldsAndModel(tablename)
objlist=[]
Employee=GetModel("personnel","Employee")
Department=GetModel("personnel","Department")
error_list=[]
try:
if filetype=="txt" or filetype=="csv":
fs=file(uploadpath+filename,"r")
rec=fs.readline()
ln=1
while rec!="":
try:
#logger.info(ln)
rec=rec.decode(unicode_)
if rec.endswith("\r\n"):
rec=rec[:len(rec)-2]
linedata=rec.split(sparatorvalue)
ltmp=[]
#logger.info("linedata:",linedata)
for l in linedata:
if l.startswith('"') and l.endswith('"'):
ltmp.append(u"%s"%l[1:len(l)-1])
else:
ltmp.append(u"%s"%l)
linedata=ltmp
currelobj=[] #当前行外键实例列表
if len(linedata)>=len(fields.keys()): #当前行记录不满足已选择字段
if ln>=recordln:
strwhere={}
upobj=""
isSave=True
try:
for tmpfld,tmpfldvalue in fields.items():
rf=""
if tmpfld.find(".")>0:
for f in model._meta.fields:
if isinstance(f,ForeignKey):
if f.rel.to.__name__==tmpfld.split(".")[0]:
strwhere[str(f.name+"__"+tmpfld.split(".")[1]+"__exact")]=linedata[tmpfldvalue]
else:
strwhere[str(tmpfld+"__exact")]=linedata[tmpfldvalue]
upobj=model.objects.filter(Q(**strwhere))
except Exception, e:
logger.error("%s"%e)
if upobj:
obj=upobj[0]
else:
obj=model()
for fld,value in fields.items():
relObj=""
isForeignKey=False
if fld.find(".")>0: #查找到需要保存关联字段
print "find relate"
for nkey,val in rlfield.items(): #查找关联记录,更新或创建,并保存
if nkey.__name__==fld.split(".")[0]:
strfilter={}
for f in fields: #查找关联表多个相应的字段，并生成表达式
if f.find(".")>0 :
if f.split(".")[0]==nkey.__name__:
tmpvalue=linedata[fields[f]]
#tmpvalue=tmpvalue.encode('utf-8')
strfilter[str(f.split(".")[1]+"__exact")]=tmpvalue
if strfilter:
relObj=nkey.objects.filter(Q(**strfilter))
if len(relObj)<=0: #查找不到记录，生成新记录
if addrelate!=1:
isSave=False
break
else:
relObj=nkey()
for tfld in fields.keys():
if tfld.find(".")>0:
if tfld.split(".")[0]==nkey.__name__:
relObj.__setattr__(tfld.split(".")[1],linedata[fields[tfld]])
isForeignKey=True
relObj.save()
relrecs.append(relObj)
else:
isForeignKey=True
relObj=relObj[0]
currelobj.append(relObj)
break
if not isSave:
break #跳出当前行
tobj=""
fieldname=""
if isForeignKey:
for f in obj._meta.fields: #查找字段是否是外键
if isinstance(f,ForeignKey) and f.rel.to.__name__==fld.split(".")[0]:
for tobj in currelobj:
if tobj==f.rel.to:
break
fieldname=f.name
break
obj.__setattr__(fieldname,tobj)
else:
if fld=="PIN":
model_emp=sys.modules['mysite.personnel.models.model_emp']
settings=sys.modules['mysite.settings']
if len(str(linedata[value]).strip())>getattr(settings,"PIN_WIDTH"):
raise Exception(u"%s"%_(u"人员编号长度过长"))
else:
linedata[value]=getattr(model_emp,"format_pin")(str(linedata[value]))
if fld=="code":
dept=Department.objects.filter(code=linedata[value])
if dept:
raise Exception(u"%s"%_(u'部门编号已存在'))
obj.__setattr__(fld,linedata[value])
#logger.info("file Line:%s save is %s "%(ln,isSave))
if isSave:
obj.save()
if type(obj)==Employee:
obj.__setattr__('attarea',(1,))
obj.save()
if type(obj)==Department:
if obj.parent==None or obj.parent=="":
obj.parent_id=1
obj.save()
objlist.append(obj)
ln+=1
rec=fs.readline()
except Exception,e:
#logger.error("%s"%e)
try:
error_list.append(str(ln)+u" 行 "+u"%s"%e)
except:
error_list.append(str(ln))
pass
if errmode==1:
ln+=1
rec=fs.readline()
continue
else:
raise
if datash[row][tmpfldvalue][0]==2:
dv=int(datash[row][tmpfldvalue][1])
else:
dv=datash[row][tmpfldvalue][1]
if tmpfld.find(".")>0:
for f in model._meta.fields:
if isinstance(f,ForeignKey):
if f.rel.to.__name__==tmpfld.split(".")[0]:
strwhere[str(f.name+"__"+tmpfld.split(".")[1]+"__exact")]=dv
else:
strwhere[str(tmpfld+"__exact")]=datash[row][tmpfldvalue][1]=dv
#print strwhere
#print(Q(**strwhere))
upobj=model.objects.filter(Q(**strwhere))
except Exception, e:
logger.error("%s"%e)
if upobj:
obj=upobj[0]
else:
obj=model()
currelobj=[]
for fld,value in fields.items():
relObj=""
isForeignKey=False
if fld.find(".")>0: #查找到需要保存关联字段
#print "find relate"
for nkey,val in rlfield.items(): #查找关联记录,更新或创建,并保存
if nkey.__name__==fld.split(".")[0]:
def detailthread(data,session_key):
filename=""
filetype=""
tablename=""
sparatorvalue=","
fields={} #已选择字段列表
target=[] #目标文件列号
errmode=1 #出错处理方式 1 跳过错误 继续处理 2 退出，并删除已导入记录
headln=1
recordln=2
unicode_="utf-8"
relatefield=[]
relrecs=[]
addrelate=1 #关联记录处理 1 自动增加 2 关键记录不存在，跳过当次记录,不增加关联记录
# data=dict(request.POST)
#print data
errmode=int(data["errmode"][0])
filename=data["txtfilename"][0]
filetype=data["txtfiletype"][0]
sparatorvalue=data["sparatorvalue"][0]
headln=int(data["headln"][0])
recordln=int(data["recordln"][0])
tablename=data["txttablename"][0]
unicode_=data["unicode"][0].decode()
addrelate=int(data["addrelate"][0])
# session_key=request.session.session_key
session_key = str(session_key)
#查找字段列表，目标列号
for n,v in data.items():
if n.startswith("_chk_"):
field=str(data["_select_"+ n[5:]][0]).decode(unicode_)
fields[field]=int(n[5:])
#查找模型
model,flds,rlfield=findAllFieldsAndModel(tablename)
objlist=[]
Employee=GetModel("personnel","Employee")
Department=GetModel("personnel","Department")
error_list=[]
# global user_process
# #加入全局变量
# processdata_global={}
#
# user_process[session_key]=processdata_global
# user_process[session_key]["process"]=0
# user_process[session_key]["index"]=1
# user_process[session_key]["return_value"]=""
# user_process[session_key]["total"]=1
processdata={}
# processdata["process"]=1
# processdata["index"]=1
# processdata["return_value"]=""
# processdata["total"]=1
# cache.set(session_key,processdata,3600)
try:
if filetype=="txt" or filetype=="csv":
fs=file(uploadpath+filename,"r")
rec=fs.readline()#读取每行
sheet_data = []
ln=1
while rec!="":
if ln>=recordln:
rec=rec.decode(unicode_)
if rec.endswith("\r\n"):
rec=rec[:len(rec)-2]
linedata=rec.split(sparatorvalue)
ltmp=[]
for l in linedata:
if l.startswith('"') and l.endswith('"'):
ltmp.append(u"%s"%l[1:len(l)-1])
else:
ltmp.append(u"%s"%l)
sheet_data.append(ltmp)
ln+=1
rec=fs.readline()
fs.close()
return sheet_data
elif filetype == "xls" : # word 2007 xls文件后缀应该添加
sheetdata=ParseXlsUxlrd(uploadpath+filename)
# global UPLOAD_FILEDATA
#datash = UPLOAD_FILEDATA
datash=sheetdata[0][1]
#在运行开始让进度条 初始化为 0
processdata['total'] = len(datash)#文件总数
cache.set(session_key,processdata,3600)
# user_process[session_key]['total'] = len(datash)#文件总数
# deptfkey={}#部门外键存放的地方
# hasdeptf=False
# perfkey={}# 模型的数据存放的地方
# for fld,value in fields.items():#遍历提交 对应关系
# if fld.find(".")>0:
# fr=fld.split(".")
# if fr[0]=="Department":
# deptfkey[fr[1]]=value
# else:
# perfkey[fld]=value
#
# if deptfkey.has_key("code"):
# hasdeptf=True
for row in range(len(datash)):#每行每行开始读取
if row==len(datash):
processdata["process"]=processdata["total"]
#变量
# user_process[session_key]["process"]=user_process[session_key]["total"]
else:
processdata["process"] = processdata["process"]+1
#全局变量
# user_process[session_key]["process"] = user_process[session_key]["process"]+1
cache.set(session_key,processdata,3600)
try:
if row>=recordln-1:#从第二行开始读取数据
isSave=True #这里定义几个标识符
has_property=False
strwhere={}
upobj=""
try:
for tmpfld,tmpfldvalue in fields.items():#fields.item() [(u'Department.code', 2), (u'EName', 1), (u'PIN', 0), (u'Department.name', 3)]
rf=""
#datash[1]=[[2, 1128.0], [1, u'\u5b8b\u7231\u73b2'], [1, u'2'], [1, u'\u90e8\u95e8\u540d\u79f0']],
#datash[1][tmpfldvalue] 选取的是 数据对应的列 datash[row][tmpfldvalue][0]=1
if datash[row][tmpfldvalue][0]==2:#datsh[row]读取的每行数据 然后部选择对应的部门 编号所在的列
dv=str(int(datash[row][tmpfldvalue][1]))
else:
dv=datash[row][tmpfldvalue][1]# 为什么 判断，感觉没什么必要？？ dv=2
if tmpfld.find(".")>0:#查找是否有外键的开始， 以.号 判断
for f in model._meta.fields:#遍历循环（估计这个也最 消耗时间）
if isinstance(f,ForeignKey):#匹配 外键 属性 也就是模型中的 每个数据对象
if f.rel.to.__name__==tmpfld.split(".")[0]:#其中 f.rel.to.__name__ 是 模型下面的对象名称 如果找到了 tmpfld.split(".")[0] = Department
if str(dv).strip():#先strwhere = {'DeptID__code__exact':'1'}
strwhere[str(f.name+"__"+tmpfld.split(".")[1]+"__exact")]=dv.strip()#
else:
if tmpfld=="code" and model==Department:
strwhere={}
strwhere["code__exact"]=datash[row][tmpfldvalue][1].strip()
break
if tmpfld=="PIN" and model==Employee:
strwhere={}
strwhere["PIN__exact"]=str(int(datash[row][tmpfldvalue][1])).strip()
break
#strwhere[str(tmpfld+"__exact")]=datash[row][tmpfldvalue][1]=dv.strip()
#print strwhere
#print(Q(**strwhere))
upobj=model.objects.filter(Q(**strwhere))
except Exception, e:
logger.error("%s"%e)
if upobj:
obj=upobj[0]
else:
obj=model()
currelobj=[]
for fld,value in fields.items():
relObj=""
isForeignKey=False
if fld.find(".")>0: #查找到需要保存关联字段
for nkey,val in rlfield.items(): #查找关联记录,更新或创建,并保存
if nkey.__name__==fld.split(".")[0]:
strfilter={}
for f in fields: #查找关联表多个相应的字段，并生成表达式
if f.find(".")>0 :
if f.split(".")[0]==nkey.__name__:
if datash[row][value][0]==2:
tmpvalue=str(int(datash[row][fields[f]][1]))
else:
tmpvalue=datash[row][fields[f]][1]
if tmpvalue.strip():
if f.split(".")[1]=="code" and model==Employee:
strfilter={} #人员表，查找部门表时，当有选择部门名称时，只需要查找部门编号
strfilter[str(f.split(".")[1]+"__exact")]=tmpvalue.strip()
break
else:
strfilter[str(f.split(".")[1]+"__exact")]=tmpvalue.strip()
if strfilter:
#print "strfilter:",strfilter
dir(nkey)
relObj=nkey.objects.filter(Q(**strfilter))
if len(relObj)<=0: #查找不到记录，生成新记录
# print "not found"
if addrelate!=1:
isSave=False
break #跳出当前行
else:
relObj=nkey()
is_save_rel=False
for tfld in fields.keys():
if tfld.find(".")>0:
if tfld.split(".")[0]==nkey.__name__:
if datash[row][fields[tfld]][0]==2:
t_value=str(int(datash[row][fields[tfld]][1]))
else:
t_value=datash[row][fields[tfld]][1]
#print "t_value,",t_value
if t_value.strip():
relObj.__setattr__(tfld.split(".")[1],t_value.strip())
is_save_rel=True
if is_save_rel:
relObj.save()
relrecs.append(relObj)
isForeignKey=True
else:
relObj=None
else:
isForeignKey=True
relObj=relObj[0]
#print "find: %s "%relObj.__doc__
currelobj.append(relObj)
break
if not isSave:
break #跳过当前行
tobj=""
fieldname=""
#print"%s:%s"%(fld,datash[row][value][1])
if isForeignKey:
for f in obj._meta.fields: #查找字段是否是外键
if isinstance(f,ForeignKey) and f.rel.to.__name__==fld.split(".")[0]:
for tobj in currelobj:
if tobj==f.rel.to:
break
fieldname=f.name
break
#print "%s :%s"%(fieldname,tobj.pk)
obj.__setattr__(fieldname,tobj)
else:
if datash[row][value][0]==2:
cellvalue=str(int(datash[row][value][1]))
else:
cellvalue=datash[row][value][1]
#print "field :%s value:%s"%(fld,cellvalue)
if cellvalue.strip():
if fld=="PIN":
model_emp=sys.modules['mysite.personnel.models.model_emp']
settings=sys.modules['erp.settings']
if len(str(cellvalue).strip())>getattr(settings,"PIN_WIDTH"):
raise Exception(u"%s"%_(u"人员编号长度过长"))
else:
cellvalue=getattr(model_emp,"format_pin")(str(cellvalue.strip()))
if fld=="code":
dept=Department.objects.filter(code=cellvalue.strip())
if dept:
raise Exception(u"%s"%_(u'部门编号已存在'))
obj.__setattr__(fld,cellvalue.strip())
has_property=True
if isSave and has_property:
obj.save()
if(type(obj)==Employee):
if not obj.DeptID:#导入人员的时候 部门为空的时候 给默认的
obj.__setattr__('DeptID',Department.objects.get(pk=1))
obj.__setattr__('attarea',(1,))
obj.save()
if(type(obj)==Department):
if obj.parent==None or obj.parent=="":
obj.parent_id=1
obj.save()
objlist.append(obj)
except Exception,e:
#logger.error("%s"%e)
try:
error_list.append(str(row+1)+u"%s"%_(u" 行 ")+u"%s"%e)
except:
error_list.append(str(row+1))
pass
if errmode==1: # 按错误处理方式处理数据
continue
else:
raise
else:
