def mapLargeProcess(ags):
mapLog = logging.getLogger(str(os.getpid()))
logOutputPath = ags[12]
if len(mapLog.handlers) == 0:
mapLog = Logger(logname=os.path.join(
logOutputPath, "largeGDB_process_%s.log" % os.getpid()),
loglevel=3,
callfile=str(os.getpid())).get_logger()
try:
sourceFilePath = ags[0]
outputGDBPath = ags[1]
serverName = ags[2]
templatePath = ags[3]
x = ags[4]
y = ags[5]
sourceFileFields = ags[6]
fileEncode = ags[7]
convertSr = ags[8]
serverInfo = ags[9]
pythonPath = ags[10]
toolsPyPath = ags[11]
mapLog.info('process pid(%s) running...' % str(os.getpid()))
sshConn = SSHControl(None, None, mapLog, serverInfo=serverInfo)
status = sshConn.ssh_exec_cmd(
'%s %s %s %s %s %s %s %s %s %s %s %s' %
(pythonPath, toolsPyPath, sourceFilePath, outputGDBPath,
serverName, templatePath, x, y, sourceFileFields, str(
os.getpid()), fileEncode, convertSr))
sshConn.ssh_close()
mapLog.info('proces pid(%s) complate ')
return status
except:
mapLog.warning(traceback.format_exc())
raise
def mapProcess(ags):
fileName = ags[0]
sourcePath = ags[1]
x = ags[2]
y = ags[3]
sourceFileFields = ags[4]
gdbPathList = ags[5]
extentList = ags[6]
outputPath = gdbPathList.pop()
mapLog = logging.getLogger(__file__)
if len(mapLog.handlers) == 0:
mapLog = Logger(logname=os.path.join(
logOutputPath, "largeGDB_process_%s.log" % os.getpid()),
loglevel=3,
callfile=__file__).get_logger() # 感谢匿名网友指正
try:
mapLog.info('PID(%s) convert GDB:%s' % (os.getpid(), fileName))
layerPath = os.path.join(outputPath, fileName)
ac.copyLayers(os.path.join(outputPath, "tempLayer"), layerPath)
layerFields = ac.getLayerFields(layerPath)
# 插入记录到gdb
extent = ac.insertRow_Point(layerPath, sourcePath, x, y,
sourceFileFields, layerFields, mapLog)
extentList.append([
fileName,
os.path.split(outputPath)[1] + os.path.sep + fileName, extent
])
# os.remove(sourcePath) # 把源文件删除
gdbPathList.append(outputPath)
mapLog.info('PID(%s) convert GDB:%s Done' % (os.getpid(), fileName))
if len(extent) > 0:
mapLog.info('PID(%s) GDB extent:%s' %
(os.getpid(), "%f,%f,%f,%f" %
(extent[0], extent[1], extent[2], extent[3])))
except:
gdbPathList.append(outputPath)
mapLog.warning(traceback.format_exc())
raise
shutil.rmtree(gdbPath_temp)
logger.info(
'Step 8: insertUpdate ** update GDB soucefile:%s complate！' %
sourceFilePath)
return True
except Exception as e:
logger.warning(e.message)
logger.warning(traceback.format_exc())
return False
if __name__ == '__main__':
logger = Logger(logname=os.path.join(logOutputPath,
"updateGDBMulti_main.log"),
loglevel=3,
callfile=__file__).get_logger()
try:
sourceFilePath = r'C:\Users\esri\Desktop\customerupdate' # 带经纬度坐标信息文件
outputGDBPath = r'D:\dataHandle\GISData\gdb\crm\clientdata' # 转换GDB输出路径
serverGDBPath = r'D:\dataHandle\GISData\arcgisserver\gdb\crm\clientdata'
serverName = r'clientdata' # 数据服务名
x = 10 # 经度序号
y = 11 # 纬度序号
uniqueId = 0
updateModel = 'update' # 更新模式：add\update
sourceFileFields = "CID,CITY,CNAME,CDATE,CADDRESS,CCAPITAL,CARTIFICIA,CTELEPHONE,CPHONE,CBUSITYPE,X,Y,BZ1,BZ2".split(
',') # 数据源文件表头
fileEncode = 'utf-8'
convertSr = '3857,3857'
# fileEncode = 'utf-8'
# convertSr = 'wgs84|webmercator'
sourceFilePath = sys.argv[1] # 带经纬度坐标信息文件
outputGDBPath = sys.argv[2] # 转换GDB输出路径
serverName = sys.argv[3] # 数据服务名
templatePath = sys.argv[4] # 图层模板路径
x = int(sys.argv[5]) # 经度序号
y = int(sys.argv[6]) # 纬度序号
sourceFileFields = sys.argv[7].split(',') # 数据源文件表头
pid = sys.argv[8]
fileEncode = sys.argv[9]
convertSr = sys.argv[10]
logger = Logger(
logname=os.path.join(logOutputPath, "largeGDB_%s.log" % pid), loglevel=3,
callfile=__file__).get_logger()
logger.info('''start convert GDB...
source File Path:%s
output GDBPath:%s
log Path:%s''' % (
sourceFilePath, outputGDBPath, os.path.join(runWorkspace, 'workspace', "logs")))
status = buildGDB(sourceFilePath, outputGDBPath, serverName, x, y, sourceFileFields, fileEncode, convertSr)
if status:
logger.info('insertLarge_GDB ** convert success pid(%s)!' % pid)
print "sshsuccess"
else:
logger.info('insertLarge_GDB ** convert error pid(%s)!' % pid)
print "ssherror"
def encodeProcess(ags):
# filePath:数据源路径
# addrField:地址列标题名
# threadNum：线程数量
# geocodeOutput：地理编码结果输出路径
fileIndex = str(ags[0])
filePath = ags[1]
fields = ags[2]
addrField = ags[3]
fileEncode = ags[4]
threadNum = ags[5]
geocodeOutput = ags[6]
geocodeUrl = ags[7]
logOutputPath = ags[8]
# encodeLog = Logger("geocodeLog_" + str(os.getpid()), os.path.join(geocodeOutput, "processLog"))
logId = 'geocode_%s' % str(os.getpid())
geocodeLog = logging.getLogger(logId)
if len(geocodeLog.handlers) == 0:
geocodeLog = Logger(logname=os.path.join(
logOutputPath, "geocode_process_%s.log" % os.getpid()),
loglevel=1,
callfile=logId).get_logger() # 感谢匿名网友指正
encodeResultPath = os.path.join(geocodeOutput, "geocodeResult")
if os.path.exists(encodeResultPath) is not True:
os.mkdir(encodeResultPath)
try:
setdefaulttimeout(5) # 设置超时时间
try:
sourceFileObj = file(filePath, 'rb')
encodeSuccess = []
encodeError = []
threads = []
geocodeLog.info("analysis file: %s" % fileIndex)
for i in xrange(threadNum):
th = threading.Thread(target=encodeThread,
args=(fileIndex, sourceFileObj, fields,
addrField, fileEncode, geocodeUrl,
encodeSuccess, encodeError,
geocodeLog))
th.start()
threads.append(th)
for process in threads:
process.join()
sourceFileObj.close()
fileName = os.path.basename(filePath).split('.')[0]
errorFile = file(
os.path.join(encodeResultPath, fileName + '_error'), 'wb')
for i in encodeError:
errorFile.write(i)
errorFile.close()
xyFile = os.path.join(encodeResultPath, fileName + '_success')
successFile = file(xyFile, 'wb')
for i in encodeSuccess:
successFile.write(FIELD_DEL.join(i) + FIELD_DEL + '\n')
geocodeLog.info("file done: %s" % filePath)
successFile.close()
except Exception as e:
geocodeLog.warning(traceback.format_exc())
geocodeLog.info("file analysis error: %s--%s" % (filePath, str(e)))
return ""
except Exception as e:
geocodeLog.warning(e.message)
geocodeLog.warning(traceback.format_exc())
return ""
sourceFilePath = sys.argv[1] # 带经纬度坐标信息文件
outputGDBPath = sys.argv[2] # 转换GDB输出路径
serverName = sys.argv[3] # 数据服务名
templatePath = sys.argv[4] # 图层模板路径
x = int(sys.argv[5]) # 经度序号
y = int(sys.argv[6]) # 纬度序号
sourceFileFields = sys.argv[7].split(',') # 数据源文件表头
processCount = 3
if len(sys.argv) > 8:
processCount = int(sys.argv[8]) # 进程数量
maxGDBCount = 1000000
if len(sys.argv) > 9:
maxGDBCount = sys.argv[9]
logger = Logger(logname=os.path.join(logOutputPath, "largeGDB_main.log"),
loglevel=3,
callfile=__file__).get_logger()
logger.info('''start convert GDB...
source File Path:%s
output Path:%s
process Count:%s
log Path:%s''' %
(sourceFilePath, outputGDBPath, processCount,
os.path.join(runWorkspace, 'workspace', "logs")))
status = buildGDB(sourceFilePath, outputGDBPath, serverName, templatePath,
x, y, sourceFileFields, processCount, maxGDBCount)
if status:
logger.info('inserGDB_large success!')
print "success"
return False
if __name__ == '__main__':
type = sys.argv[1] # 业务类型
serverName = sys.argv[2] # 数据服务名
gdbPath = sys.argv[3] # 转换GDB输出路径
serverGDBPath = sys.argv[4]
ip = sys.argv[5]
port = sys.argv[6]
user = sys.argv[7]
password = sys.argv[8]
logger = Logger(logname=os.path.join(logOutputPath, "publishMs_main.log"),
loglevel=3,
callfile=__file__).get_logger()
logger.info('''Publish MapServer...
MapServer Name:%s/%s
sourceGDB Path:%s
serverGDB Path:%s''' %
(type, serverName, gdbPath, serverGDBPath))
status = publishServer(type, serverName, gdbPath, serverGDBPath, ip, port,
user, password)
if status:
logger.info('publish ** success!')
print "sshsuccess"
else:
logger.info('publish ** error!')
if sys.argv[1].upper() == 'NONE':
type = ""
else:
type = sys.argv[1]
serverName = sys.argv[2] # 数据服务名
gdbPath = sys.argv[3] # 转换GDB输出路径
serverGDBPath = sys.argv[4] # 图层模板路径
ip = sys.argv[5]
port = sys.argv[6]
user = sys.argv[7]
password = sys.argv[8]
logger = Logger(logname=os.path.join(logOutputPath, "updateMs_main.log"),
loglevel=3,
callfile=__file__).get_logger()
logger.info('''start update Data & restart MapServer...
MapServer Name:%s/%s
sourceGDB Path:%s
serverGDB Path:%s''' %
(type, serverName, gdbPath, serverGDBPath))
status = UpdateMS(type, serverName, gdbPath, serverGDBPath, ip, port, user,
password)
if status:
logger.info('update ** update Data success!')
print "sshsuccess"
else:
logger.info('update ** update Data error!')
sys.path.append(runWorkspace)
from common.logModel import Logger
os.environ['NLS_LANG'] = 'SIMPLIFIED CHINESE_CHINA.UTF8'
timeStr = time.strftime("%Y%m%d", time.localtime())
logger = None
logOutputPath = os.path.join(runWorkspace, 'workspace', 'logs', timeStr)
if os.path.exists(os.path.join(runWorkspace, 'workspace', 'logs',
timeStr)) is not True:
os.mkdir(os.path.join(runWorkspace, 'workspace', 'logs', timeStr))
if __name__ == '__main__':
logger = Logger(logname=os.path.join(logOutputPath, "updateGDB_main.log"),
loglevel=3,
callfile=__file__).get_logger()
try:
strPath = sys.argv[1]
# strPath = r'D:\5000W\customer1.gdb,D:\5000W\customer2.gdb'
extentList = strPath.split(',')
list = []
outputGDB = ""
for extent in extentList:
if extent.endswith('1.gdb'):
outputGDB = os.path.join(extent, 'extent')
else:
list.append(os.path.join(extent, 'extent'))
ac.appendLayers(list, outputGDB)
for extent in list: