def getInnerTextNvl(node, xpath=''):
u'''
TITLE:[STATIC]ノードの指定XPATHのInnerText文字列を取得
PARAMETER:
node:対象ノード
xpath:対象XPATH
RETURN:文字列（取得できない場合は空文字を【返す）
'''
if node == None:
return ''
if xpath == None or xpath == '':
return UcfUtil.nvl(node.getInnerText())
# hash_node = node.exchangeToHash(isAttr=True, isChild=True)
# return UcfUtil.nvl(UcfUtil.getHashStr(hash_node, xpath))
# XPATH 解析（属性とノードに分ける）
xpath_parts = xpath.split('@')
xpath_node = xpath_parts[0].rstrip('/')
xpath_atr = xpath_parts[1] if len(xpath_parts) >= 2 else ''
node_target = node.selectSingleNode(xpath_node)
if node_target == None:
return ''
if xpath_atr != '':
return UcfUtil.nvl(node_target.getAttribute(xpath_atr))
else:
return UcfUtil.nvl(node_target.getInnerText())
def processOfRequest(self):
self._approot_path = os.path.dirname(__file__)
last_modified_language = '' # Wed, 21 Jun 2006 07:00:25 GMT
last_modified_default = '' # Wed, 21 Jun 2006 07:00:25 GMT
language = UcfUtil.nvl(self.getRequest('ln'));
if language != '':
last_modified_language = UcfMessage.getLangFileLastModified(self._approot_path, language=language)
if language.lower() != UcfConfig.MESSAGE_DEFAULT_FILE.lower():
last_modified_default = UcfMessage.getLangFileLastModified(self._approot_path, language=UcfConfig.MESSAGE_DEFAULT_FILE)
else:
last_modified_default = last_modified_language
last_modified = str(last_modified_language) + '|' + str(last_modified_default)
# if self.request.headers.has_key('If-Modified-Since'):
# logging.info('If-Modified-Since=' + self.request.headers['If-Modified-Since'])
# logging.info('last_modified=' + last_modified)
if self.request.headers.has_key('If-Modified-Since') and last_modified is not None and self.request.headers['If-Modified-Since'] == last_modified:
self.response.set_status(304)
return
msgs = UcfMessage.getMessageList(self._approot_path, language=language,is_only_js=True)
template_vals = {
'lang_json':JSONEncoder().encode(msgs)
}
#self.appendBasicInfoToTemplateVals(template_vals)
self.response.headers['Last-Modified'] = last_modified
self.render('jslang.html', self._design_type, template_vals, content_type='text/javascript')
def getInnerText(self): u''' TITLE:InnerText文字列を取得 ABSTRACT:直下のTEXTのみを取得.サブノードがあっても無視. PARAMETER: RETURN:文字列 ''' return UcfUtil.nvl(self._element.text)
def setAttribute(self, name, value): u''' TITLE:属性に値をセット ABSTRACT:属性がなければ作成してセット PARAMETER: name:属性名 value:属性値 ''' self._element.attrib[name] = UcfUtil.nvl(value)
def editVoForRegist(cls, helper, vo, entry_vo, edit_type):
if edit_type == UcfConfig.EDIT_TYPE_NEW:
vo['dept_id'] = UcfUtil.getHashStr(helper.getDeptInfo(), 'dept_id')
vo['operator_id_lower'] = UcfUtil.getHashStr(vo, 'operator_id').lower()
# アクセス期限の日付と時分からアクセス期限を作成
access_expire = ''
if UcfUtil.getHashStr(vo, 'access_expire_date') != '':
access_expire = UcfUtil.getHashStr(vo, 'access_expire_date')
if UcfUtil.getHashStr(vo, 'access_expire_time') != '':
time_ary = UcfUtil.getHashStr(vo, 'access_expire_time').split(':')
if len(time_ary) >= 2:
access_expire = access_expire + ' ' + time_ary[0] + ':' + time_ary[1] + ':00'
vo['access_expire'] = access_expire
# 承認ステータスが変更された場合はステータス日付を更新
if UcfUtil.getHashStr(vo, 'approval_status') != UcfUtil.getHashStr(entry_vo, 'approval_status'):
vo['approval_status_date'] = UcfUtil.nvl(UcfUtil.getNowLocalTime(helper._timezone))
vo['approval_operator_id'] = UcfUtil.nvl(helper.getLoginID())
def updateTaskStatus(self,
file_vo,
file_entry,
log_msg,
is_error,
login_operator_unique_id,
login_operator_id,
is_after_process=False):
# ステータス後処理
datNow = UcfUtil.getLocalTime(UcfUtil.getNow(), self._timezone)
if is_error:
file_vo['status'] = 'FAILED'
elif is_after_process:
file_vo['status'] = 'SUCCESS'
log_text = file_vo.get('log_text', '')
for msg in log_msg:
log_text += msg + '\n'
file_vo['log_text'] = log_text
if is_after_process:
file_vo['deal_status'] = 'FIN'
file_vo['expire_date'] = UcfUtil.add_months(datNow, 1) # 一ヶ月有効とする
file_vo['upload_count'] = '1'
file_vo['last_upload_date'] = UcfUtil.nvl(datNow)
file_vo['upload_operator_id'] = login_operator_id
file_vo['upload_operator_unique_id'] = login_operator_unique_id
file_vo['last_upload_operator_id'] = login_operator_id
file_vo['last_upload_operator_unique_id'] = login_operator_unique_id
FileUtils.editVoForRegist(self, file_vo, UcfConfig.EDIT_TYPE_RENEW)
# Voからモデルにマージ
file_entry.margeFromVo(file_vo, self._timezone)
# 更新
file_entry.updater_name = login_operator_id
file_entry.date_changed = UcfUtil.getNow()
file_entry.put()
def processOfRequest(self, tenant, token):
self._approot_path = os.path.dirname(__file__)
# エラーが1回おきたら処理を終了する
if (int(self.request.headers.environ['HTTP_X_APPENGINE_TASKRETRYCOUNT']
) > 1):
logging.error('error over_1_times')
return
data_key = UcfUtil.nvl(self.getRequest('data_key'))
data_kind = UcfUtil.nvl(self.getRequest('data_kind'))
login_operator_id = UcfUtil.nvl(self.getRequest('login_operator_id'))
login_operator_unique_id = UcfUtil.nvl(
self.getRequest('login_operator_unique_id'))
login_operator_mail_address = UcfUtil.nvl(
self.getRequest('login_operator_mail_address'))
login_operator_client_ip = UcfUtil.nvl(
self.getRequest('login_operator_client_ip'))
# オペレータ情報を取得
login_operator_entry = None
if login_operator_unique_id != '':
login_operator_entry = OperatorUtils.getData(
self, login_operator_unique_id)
if login_operator_entry is None:
raise Exception('Not found login operator information.')
return
# preparing blob reader
# blobstore 保存しているバイナリデータを取得
blob_key = str(urllib.unquote(self.request.get('key')))
blob_reader = blobstore.BlobReader(blob_key)
# ファイルデータを取得（ステータス=CREATINGで作成済）
file_entry = FileUtils.getDataEntryByDataKey(self, data_key)
if file_entry is None:
raise Exception(self.getMsg('MSG_NOTFOUND_TARGET_FILE',
(data_key)))
return
# タスクトークンの取得と更新
last_task_token = file_entry.task_token if file_entry.task_token is not None else ''
file_entry.task_token = token
file_entry.put()
file_vo = file_entry.exchangeVo(self._timezone)
FileUtils.editVoForSelect(self, file_vo)
file_encoding = UcfUtil.getHashStr(self.getDeptInfo(True),
'file_encoding')
if file_encoding == '' or file_encoding == 'SJIS':
data_encoding = 'cp932'
elif file_encoding == 'JIS':
data_encoding = 'jis'
elif file_encoding == 'EUC':
data_encoding = 'euc-jp'
elif file_encoding == 'UTF7':
data_encoding = 'utf-7'
elif file_encoding == 'UTF8':
data_encoding = 'utf-8'
elif file_encoding == 'UNICODE':
data_encoding = 'utf-16'
else:
data_encoding = 'cp932'
log_msg = []
#is_error = False
record_cnt = 0
#insert_cnt = 0
#update_cnt = 0
#delete_cnt = 0
#skip_cnt = 0
#error_cnt = 0
shutdown_record_cnt_str = self.request.get('shutdown_record_cnt')
if shutdown_record_cnt_str is not None and shutdown_record_cnt_str != '':
shutdown_record_cnt = int(shutdown_record_cnt_str)
else:
shutdown_record_cnt = 0
logging.info('shutdown_record_cnt=' + str(shutdown_record_cnt))
is_error_str = self.request.get('is_error')
if is_error_str is not None and is_error_str.lower() == 'true':
is_error = True
else:
is_error = False
logging.info('is_error=' + str(is_error))
insert_cnt_str = self.request.get('insert_cnt')
if insert_cnt_str is not None and insert_cnt_str != '':
insert_cnt = int(insert_cnt_str)
else:
insert_cnt = 0
update_cnt_str = self.request.get('update_cnt')
if update_cnt_str is not None and update_cnt_str != '':
update_cnt = int(update_cnt_str)
else:
update_cnt = 0
delete_cnt_str = self.request.get('delete_cnt')
if delete_cnt_str is not None and delete_cnt_str != '':
delete_cnt = int(delete_cnt_str)
else:
delete_cnt = 0
skip_cnt_str = self.request.get('skip_cnt')
if skip_cnt_str is not None and skip_cnt_str != '':
skip_cnt = int(skip_cnt_str)
else:
skip_cnt = 0
error_cnt_str = self.request.get('error_cnt')
if error_cnt_str is not None and error_cnt_str != '':
error_cnt = int(error_cnt_str)
else:
error_cnt = 0
try:
# 同じトークンで既に処理済みの場合、ＧＡＥのタスクが強制終了した後のリトライなのでログを出しておく
if last_task_token == token:
is_error = True
log_msg.append(
self._formatLogRecord(
UcfMessage.getMessage(
self.getMsg('MSG_TASK_FORCE_RETRY'))))
self.updateTaskStatus(file_vo, file_entry, log_msg, is_error,
login_operator_unique_id,
login_operator_id)
del log_msg[:]
logging.info('csv_analysis start...')
new_lines = []
str_record = ''
quote_num = 0
old_lines = blob_reader.read().splitlines()
for line in old_lines:
#str_record += lineline + '\n'
#if str_record.count('"') % 2 == 0:
# new_lines.append(str_record.rstrip('\n'))
# str_record = ''
quote_num += line.count('"')
if quote_num % 2 == 0:
new_lines.append(str_record + line)
str_record = ''
quote_num = 0
else:
str_record += line + '\n'
logging.info('csv_analysis end. the record count is ' +
str(len(new_lines)) + ' with title line.')
# 巨大なCSVファイルを扱えるように対応 2015.03.27
csv.field_size_limit(1000000000)
# process uploaded csv file
# universal-newline mode に対応
#csvfile = csv.reader(blob_reader, dialect=csv.excel)
#csvfile = csv.reader(blob_reader.read().splitlines(), dialect=csv.excel)
csvfile = csv.reader(new_lines, dialect=csv.excel)
col_names = []
for row in csvfile:
# タイトル行の処理
if record_cnt == 0:
# first row: column list
col_index = 0
for col in row:
# BOM付CSVに対応 2016.10.13
if data_encoding == 'utf-8' and col_index == 0:
col = col.decode('utf-8-sig').encode('utf-8')
col_name = col.strip().strip('"')
# # 条件を削除し、一列目の情報は全て列を作成
col_names.append(col_name)
col_index += 1
# データ行の処理
elif shutdown_record_cnt <= record_cnt - 1:
is_runtime_shutdown = False
is_force_runtime_shutdown = False
# 5レコードに一回チェックしてみる
# シャットダウンを検知した場合
if record_cnt % 5 == 0:
is_runtime_shutdown = runtime.is_shutting_down()
# 強制対応はとりあえずコメントアウト
## シャットダウン検知しない場合も多いので、500レコードに一回ずつ別タスクにする ⇒ 100 に変更 2014.06.12
#if (shutdown_record_cnt < record_cnt - 1) and (record_cnt - 1) % 100 == 0:
# is_force_runtime_shutdown = True
if is_runtime_shutdown or is_force_runtime_shutdown:
is_shutting_down = True
current_memory_usage = runtime.memory_usage().current()
logging.info('is_shutting_down=' +
str(is_runtime_shutdown) +
' current_memory_usage=' +
str(current_memory_usage))
# instance will be shut down soon!
# exit here and kick same batch to start next record
logging.info(
'***** kicking same batch and stopping: shutdown_record_cnt='
+ str(record_cnt - 1))
# サマリをログ出力
log_msg.append(
self._formatLogRecord(
'development process [record:' +
UcfUtil.nvl(record_cnt - 1) + ' skip:' +
UcfUtil.nvl(skip_cnt) + ' insert:' +
UcfUtil.nvl(insert_cnt) + ' update:' +
UcfUtil.nvl(update_cnt) + ' delete:' +
UcfUtil.nvl(delete_cnt) + ' error:' +
UcfUtil.nvl(error_cnt) + ' ]'))
log_msg.append(
self._formatLogRecord(
'kicking same batch and stopping: shutdown_record_cnt='
+ str(record_cnt - 1)))
self.updateTaskStatus(file_vo, file_entry, log_msg,
is_error,
login_operator_unique_id,
login_operator_id)
del log_msg[:]
# kick start import
import_q = taskqueue.Queue('csv-export-import')
params = {
'shutdown_record_cnt': record_cnt - 1,
'insert_cnt': insert_cnt,
'update_cnt': update_cnt,
'delete_cnt': delete_cnt,
'skip_cnt': skip_cnt,
'error_cnt': error_cnt,
'is_error': is_error,
'key': blob_key,
'data_key': data_key,
'data_kind': data_kind,
'login_operator_id': login_operator_id,
'login_operator_unique_id':
login_operator_unique_id,
'login_operator_mail_address':
login_operator_mail_address,
'login_operator_client_ip':
login_operator_client_ip
}
import_t = taskqueue.Task(
url='/a/' + tenant + '/' + token +
'/queue_csv_import',
params=params,
target=sateraito_func.getBackEndsModuleName(
tenant),
countdown='1')
import_q.add(import_t)
return
col_index = 0
# params に配列を作成する。
csv_record = {}
for col_value in row:
if col_index < len(col_names):
# cut off too much csv data columns
# csv_record[col_names[col_index]] = unicode(col_value, UcfConfig.DL_ENCODING).strip().strip('"')
# csv_record[col_names[col_index]] = unicode(col_value, data_encoding).strip().strip('"')
csv_record[col_names[col_index]] = unicode(
col_value, data_encoding)
col_index += 1
# 1行処理
deal_type = ''
row_log_msg = None
code = ''
if data_kind == 'importgroupcsv':
deal_type, code, row_log_msg = self.importOneRecordGroup(
csv_record, record_cnt, blob_key, data_key,
data_kind, login_operator_unique_id,
login_operator_id, login_operator_mail_address,
login_operator_client_ip, login_operator_entry)
# elif data_kind == 'importusercsv':
# deal_type, code, row_log_msg = self.importOneRecordUser(csv_record, record_cnt, blob_key, data_key, data_kind, login_operator_unique_id, login_operator_id, login_operator_mail_address, login_operator_client_ip, login_operator_entry)
# elif data_kind == 'importchangeuseridcsv':
# deal_type, code, row_log_msg = self.importOneRecordChangeUserID(csv_record, record_cnt, blob_key, data_key, data_kind, login_operator_unique_id, login_operator_id, login_operator_mail_address, login_operator_client_ip, login_operator_entry)
# 件数やエラーメッセージを集計
if row_log_msg is not None:
log_msg.extend(row_log_msg)
if code != '':
error_cnt += 1
is_error = True
if deal_type == UcfConfig.EDIT_TYPE_NEW:
insert_cnt += 1
elif deal_type == UcfConfig.EDIT_TYPE_RENEW:
update_cnt += 1
elif deal_type == UcfConfig.EDIT_TYPE_DELETE:
delete_cnt += 1
elif deal_type == UcfConfig.EDIT_TYPE_SKIP:
skip_cnt += 1
# ユーザーID変更処理はデリケートなので毎回ログを出す
if data_kind == 'importchangeuseridcsv' and log_msg is not None and len(
log_msg) > 0:
self.updateTaskStatus(file_vo, file_entry, log_msg,
is_error,
login_operator_unique_id,
login_operator_id)
del log_msg[:]
# ときどきメモリ開放
if record_cnt % 100 == 0:
current_memory_usage = runtime.memory_usage().current()
gc.collect()
current_memory_usage2 = runtime.memory_usage().current()
logging.info('[memory_usage]record=' + str(record_cnt) +
' before:' + str(current_memory_usage) +
' after:' + str(current_memory_usage2))
record_cnt += 1
except BaseException, e:
self.outputErrorLog(e)
log_msg.append(self._formatLogRecord('system error.'))
is_error = True
def _formatLogRecord(self, log):
return '[' + UcfUtil.nvl(UcfUtil.getNowLocalTime(
self._timezone)) + ']' + log
for key, value in vc.msg.iteritems():
#row_log_msg.extend(value)
row_log_msg.extend(
['[' + key + ']' + msg for msg in value])
except BaseException, e:
self.outputErrorLog(e)
code = '500'
row_log_msg.append(self._formatLogRecord('system error.'))
# エラーメッセージ処理
if code != '':
#row_log_msg.append(self._formatLogRecord('[row:' + UcfUtil.nvl(record_cnt) + ',code=' + code + ']'))
row_log_msg.insert(
0,
self._formatLogRecord('[row:' + UcfUtil.nvl(record_cnt) +
',code=' + code + ']'))
return deal_type, code, row_log_msg
############################################################
# 一行インポート：ユーザー
############################################################
# def importOneRecordUser(self, csv_record, record_cnt, blob_key, data_key, data_kind, login_operator_unique_id, login_operator_id, login_operator_mail_address, login_operator_client_ip, login_operator_entry):
#
# titles = UserUtils.getCsvTitles(self)
#
# code = ''
# row_log_msg = []
# entry_vo = None
# edit_type = ''
# deal_type = ''
def getKey(cls, helper, vo):
# 最新ものを上に出したいので. ※TODO BigTableのInsertパフォーマンス大丈夫かなー？
return UcfUtil.nvl(int(''.ljust(10, '9')) - int(UcfUtil.nvl(int(time.time())).ljust(10, '0'))) + UcfConfig.KEY_PREFIX + UcfUtil.getHashStr(vo, 'unique_id')