def getInnerTextNvl(node, xpath=''):
u'''
TITLE:[STATIC]ノードの指定XPATHのInnerText文字列を取得
PARAMETER:
node:対象ノード
xpath:対象XPATH
RETURN:文字列（取得できない場合は空文字を【返す）
'''
if node == None:
return ''
if xpath == None or xpath == '':
return UcfUtil.nvl(node.getInnerText())
# hash_node = node.exchangeToHash(isAttr=True, isChild=True)
# return UcfUtil.nvl(UcfUtil.getHashStr(hash_node, xpath))
# XPATH 解析（属性とノードに分ける）
xpath_parts = xpath.split('@')
xpath_node = xpath_parts[0].rstrip('/')
xpath_atr = xpath_parts[1] if len(xpath_parts) >= 2 else ''
node_target = node.selectSingleNode(xpath_node)
if node_target == None:
return ''
if xpath_atr != '':
return UcfUtil.nvl(node_target.getAttribute(xpath_atr))
else:
return UcfUtil.nvl(node_target.getInnerText())
def removeAttribute(self, name): u''' TITLE:属性を削除 ABSTRACT:属性がない場合は無視 PARAMETER: name:属性名 ''' UcfUtil.removeHash(self._element.attrib, name)
def _sendMail(to,
message_subject,
message_body,
cc='',
reply_to='',
is_html=False):
# check to
if to is None or str(to).strip() == '':
logging.error('to is None')
return False, 'invalid_to'
if not mail.IsEmailValid(to):
logging.error('invalid to=' + str(to))
return False, 'invalid_to'
# check cc
if cc is not None and str(cc).strip() != '':
if not mail.IsEmailValid(cc):
logging.error('invalid cc=' + str(cc))
return False, 'invalid_cc'
# check reply_to
if reply_to is not None and str(reply_to) != '':
if not mail.IsEmailValid(reply_to):
logging.error('invalid reply_to=' + str(reply_to))
return False, 'invalid_reply_to'
# calc check_key
now = UcfUtil.getNow() # 標準時
check_key = UcfUtil.md5(
now.strftime('%Y%m%d%H%M') + MD5_SUFFIX_KEY_MAIL_SERVER +
SENDER_ADDON_PROJECT_ID)
# post data
url = SATERAITO_MAIL_SERVER_URL + '/api/sendmail'
values = {
'addon_project_id': SENDER_ADDON_PROJECT_ID,
'sender_email': MESSAGE_SENDER_EMAIL,
'to': to,
'cc': cc,
'reply_to': reply_to,
'message_subject': message_subject,
'message_body': message_body,
'is_html': str(is_html),
'check_key': check_key,
}
headers = {}
response = HttpPostAccess(url, values, headers)
logging.info('response=' + str(response))
try:
response_dict = json.JSONDecoder().decode(response)
if response_dict.get('status') != 'ok':
return False, response_dict.get('error_code')
except BaseException, e:
logging.error('error: class name:' + e.__class__.__name__ +
' message=' + str(e))
return False, 'unexpected_error'
def create_api_key(cls, creator_email):
unique_id = UcfUtil.guid()
entry = cls(id=unique_id)
entry.unique_id = unique_id
api_key = UcfUtil.guid()
entry.api_key = api_key
entry.creator_email = creator_email
entry_key = entry.put()
entry_dict = entry_key.get().to_dict()
return entry_dict
def processOfRequest(self):
self._approot_path = os.path.dirname(__file__)
last_modified_language = '' # Wed, 21 Jun 2006 07:00:25 GMT
last_modified_default = '' # Wed, 21 Jun 2006 07:00:25 GMT
language = UcfUtil.nvl(self.getRequest('ln'));
if language != '':
last_modified_language = UcfMessage.getLangFileLastModified(self._approot_path, language=language)
if language.lower() != UcfConfig.MESSAGE_DEFAULT_FILE.lower():
last_modified_default = UcfMessage.getLangFileLastModified(self._approot_path, language=UcfConfig.MESSAGE_DEFAULT_FILE)
else:
last_modified_default = last_modified_language
last_modified = str(last_modified_language) + '|' + str(last_modified_default)
# if self.request.headers.has_key('If-Modified-Since'):
# logging.info('If-Modified-Since=' + self.request.headers['If-Modified-Since'])
# logging.info('last_modified=' + last_modified)
if self.request.headers.has_key('If-Modified-Since') and last_modified is not None and self.request.headers['If-Modified-Since'] == last_modified:
self.response.set_status(304)
return
msgs = UcfMessage.getMessageList(self._approot_path, language=language,is_only_js=True)
template_vals = {
'lang_json':JSONEncoder().encode(msgs)
}
#self.appendBasicInfoToTemplateVals(template_vals)
self.response.headers['Last-Modified'] = last_modified
self.render('jslang.html', self._design_type, template_vals, content_type='text/javascript')
def addMailSendQueue(cls, helper, apply_vo):
# token作成
token = UcfUtil.guid()
params = {
'apply_unique_id': UcfUtil.getHashStr(apply_vo, 'unique_id')
}
# taskに追加
import_q = taskqueue.Queue('send-mail')
import_t = taskqueue.Task(
url='/a/' + helper._tenant + '/' + token + '/sendmail_update_approval_status',
params=params,
#target='mail',
target='default',
countdown='1'
)
import_q.add(import_t)
logging.info('add taskqueue:sendmail_update_approval_status')
def getInnerText(self): u''' TITLE:InnerText文字列を取得 ABSTRACT:直下のTEXTのみを取得.サブノードがあっても無視. PARAMETER: RETURN:文字列 ''' return UcfUtil.nvl(self._element.text)
def expandApplyInfo(cls, helper, vo):
try:
apply_info = JSONDecoder().decode(UcfUtil.getHashStr(vo, 'apply_info'))
# TODO セキュリティブラウザの端末情報を展開 2015.09.04
if apply_info.has_key('DeviceInfo'):
pass
except BaseException, e:
logging.exception(e)
apply_info = {}
def setAttribute(self, name, value): u''' TITLE:属性に値をセット ABSTRACT:属性がなければ作成してセット PARAMETER: name:属性名 value:属性値 ''' self._element.attrib[name] = UcfUtil.nvl(value)
def getApprovalApplyVoListByUser(cls, helper, operator_unique_id):
apply_vos = []
q = UCFMDLAccessApply.all()
q.filter('operator_unique_id =', operator_unique_id)
q.filter('approval_status IN', ['APPROVAL', 'DENY']) # 申請済みはカウントしないべきなので修正 2016.11.24
active_apply_count = 0
for entry in q:
vo = entry.exchangeVo(helper._timezone)
AccessApplyUtils.editVoForSelect(helper, vo)
is_approval = True
# ステータス.承認済みかどうか
if is_approval and UcfUtil.getHashStr(vo, 'approval_status') != 'APPROVAL':
is_approval = False
# アクセス期限.切れていないかどうか
if is_approval:
access_expire = UcfUtil.getHashStr(vo, 'access_expire')
if access_expire != '' and UcfUtil.getNowLocalTime(helper._timezone) > UcfUtil.getDateTime(access_expire):
is_approval = False
if is_approval:
apply_vos.append(vo)
return apply_vos
def validate(self, helper, vo):
# 初期化
self.init()
check_name = ''
check_key = ''
check_value = ''
########################
check_name = helper.getMsg('FLD_APPROVAL_COMMENT')
check_key = 'approval_comment'
check_value = UcfUtil.getHashStr(vo, check_key)
# 最大長チェック：500文字
if not self.maxLengthValidator(check_value, 500):
self.appendValidate(check_key, UcfMessage.getMessage(helper.getMsg('MSG_VC_MAXLENGTH'), (check_name, 500)))
check_name = helper.getMsg('FLD_COMMENT')
check_key = 'comment'
check_value = UcfUtil.getHashStr(vo, check_key)
# 最大長チェック：500文字
if not self.maxLengthValidator(check_value, 500):
self.appendValidate(check_key, UcfMessage.getMessage(helper.getMsg('MSG_VC_MAXLENGTH'), (check_name, 500)))
def editVoForSelect(cls, helper, vo):
# 申請日時を日付と時間（時分）に分ける
access_expire_date = ''
access_expire_time = ''
access_expire = UcfUtil.getHashStr(vo, 'access_expire')
if access_expire != '':
date_time_ary = access_expire.split(' ')
access_expire_date = date_time_ary[0]
if len(date_time_ary) >= 2:
access_expire_time = date_time_ary[1]
time_ary = access_expire_time.split(':')
if len(time_ary) >= 2:
access_expire_time = time_ary[0] + ':' + time_ary[1]
vo['access_expire_date'] = access_expire_date
vo['access_expire_time'] = access_expire_time
def updateTaskStatus(self,
file_vo,
file_entry,
log_msg,
is_error,
login_operator_unique_id,
login_operator_id,
is_after_process=False):
# ステータス後処理
datNow = UcfUtil.getLocalTime(UcfUtil.getNow(), self._timezone)
if is_error:
file_vo['status'] = 'FAILED'
elif is_after_process:
file_vo['status'] = 'SUCCESS'
log_text = file_vo.get('log_text', '')
for msg in log_msg:
log_text += msg + '\n'
file_vo['log_text'] = log_text
if is_after_process:
file_vo['deal_status'] = 'FIN'
file_vo['expire_date'] = UcfUtil.add_months(datNow, 1) # 一ヶ月有効とする
file_vo['upload_count'] = '1'
file_vo['last_upload_date'] = UcfUtil.nvl(datNow)
file_vo['upload_operator_id'] = login_operator_id
file_vo['upload_operator_unique_id'] = login_operator_unique_id
file_vo['last_upload_operator_id'] = login_operator_id
file_vo['last_upload_operator_unique_id'] = login_operator_unique_id
FileUtils.editVoForRegist(self, file_vo, UcfConfig.EDIT_TYPE_RENEW)
# Voからモデルにマージ
file_entry.margeFromVo(file_vo, self._timezone)
# 更新
file_entry.updater_name = login_operator_id
file_entry.date_changed = UcfUtil.getNow()
file_entry.put()
if result_json['success']:
access_token = result_json.get('access_token', '')
expires_in = result_json.get('expire_timestamp', 0)
break # 成功なのでwhile抜ける
else:
return 1
except BaseException, e:
if process_cnt <= MAX_RETRY_CNT:
logging.warning('[process_cnt=' + str(process_cnt) + ']' + ' '.join(e.args))
else:
return 2
process_cnt += 1
if expires_in != 0:
memcache_expire_secs = expires_in - time.mktime(UcfUtil.getNow().timetuple()) - sateraito_inc.session_timeout
else:
memcache_expire_secs = sateraito_inc.session_timeout - 600
logging.info('memcache_expire_secs=' + str(memcache_expire_secs))
if memcache_expire_secs > 0 and directcloudbox_token_memcache_key != '':
if not memcache.set(key=directcloudbox_token_memcache_key, value=access_token, time=memcache_expire_secs):
logging.warning("Memcache set failed.")
else:
logging.info("Memcache set success.key=" + directcloudbox_token_memcache_key)
return access_token
#####################################################
# DIRECT CLOUD BOX コール
#####################################################
def getData(cls, helper, unique_id):
query = UCFMDLAccessApply.gql("where unique_id = :1", UcfUtil.escapeGql(unique_id))
entry = query.get()
return entry
def getKey(cls, helper, vo):
# 最新ものを上に出したいので. ※TODO BigTableのInsertパフォーマンス大丈夫かなー？
return UcfUtil.nvl(int(''.ljust(10, '9')) - int(UcfUtil.nvl(int(time.time())).ljust(10, '0'))) + UcfConfig.KEY_PREFIX + UcfUtil.getHashStr(vo, 'unique_id')
def createCsv(cls, helper, login_operator_entry=None, sk_operator_unique_id='', optional_scond=None):
logging.info('start create csv...')
with_cursor = True
csv_records = []
# タイトル
titles = AccessApplyUtils.getCsvTitles(helper)
csv_records.append(UcfUtil.createCsvRecordEx(titles))
# データ一覧取得
q = UCFMDLAccessApply.all()
if sk_operator_unique_id != '':
q.filter('operator_unique_id =', sk_operator_unique_id)
## ユーザーごとのレコードは従来通り1000件固定
#max_export_cnt = 1000 # 最大出力件数
else:
sk_search_type = UcfUtil.getHashStr(optional_scond, 'sk_search_type') if optional_scond is not None else ''
sk_operator_id = UcfUtil.getHashStr(optional_scond, 'sk_operator_id').lower() if optional_scond is not None else ''
#sk_operator_unique_id = UcfUtil.getHashStr(optional_scond, 'sk_operator_unique_id') if optional_scond is not None else ''
sk_apply_date_date_from = UcfUtil.getHashStr(optional_scond, 'sk_apply_date_date_from') if optional_scond is not None else ''
sk_apply_date_time_from = UcfUtil.getHashStr(optional_scond, 'sk_apply_date_time_from') if optional_scond is not None else ''
sk_apply_date_date_to = UcfUtil.getHashStr(optional_scond, 'sk_apply_date_date_to') if optional_scond is not None else ''
sk_apply_date_time_to = UcfUtil.getHashStr(optional_scond, 'sk_apply_date_time_to') if optional_scond is not None else ''
if sk_search_type == '':
sk_search_type = 'operator_id'
# 委託管理者なら自分が触れるデータのみ対象
if ucffunc.isDelegateOperator(login_operator_entry) and login_operator_entry.delegate_management_groups is not None and len(login_operator_entry.delegate_management_groups) > 0:
q.filter('management_group IN', login_operator_entry.delegate_management_groups)
# 管理グループが複数ある場合はカーソル使えないので
if len(login_operator_entry.delegate_management_groups) >= 2:
with_cursor = False
# ログインIDで検索
if sk_search_type == 'operator_id' and sk_operator_id != '':
q.filter('operator_id_lower >=', sk_operator_id)
q.filter('operator_id_lower <', sk_operator_id + u'\uFFFD')
# 申請日時で検索
elif sk_search_type == 'apply_date' and (sk_apply_date_date_from != '' or sk_apply_date_date_to != ''):
if sk_apply_date_date_from != '':
if sk_apply_date_time_from != '':
time_ary = sk_apply_date_time_from.split(':')
sk_apply_date_from = sk_apply_date_date_from + ' ' + time_ary[0] + ':' + time_ary[1] + ':00'
sk_apply_date_from_utc = UcfUtil.getUTCTime(UcfUtil.getDateTime(sk_apply_date_from), helper._timezone)
else:
sk_apply_date_from = sk_apply_date_date_from + ' 00:00:00'
sk_apply_date_from_utc = UcfUtil.getUTCTime(UcfUtil.getDateTime(sk_apply_date_from), helper._timezone)
#wheres.append("apply_date >= '" + UcfUtil.escapeGql(sk_apply_date_from_utc) + "'")
q.filter('apply_date >=', sk_apply_date_from_utc)
if sk_apply_date_date_to != '':
if sk_apply_date_time_to != '':
time_ary = sk_apply_date_time_to.split(':')
sk_apply_date_to = sk_apply_date_date_to + ' ' + time_ary[0] + ':' + time_ary[1] + ':00'
sk_apply_date_to_utc = UcfUtil.getUTCTime(UcfUtil.getDateTime(sk_apply_date_to), helper._timezone)
else:
sk_apply_date_to = sk_apply_date_date_to + ' 00:00:00'
sk_apply_date_to_utc = UcfUtil.getUTCTime(UcfUtil.add_days(UcfUtil.getDateTime(sk_apply_date_to), 1), helper._timezone)
#wheres.append("apply_date < '" + UcfUtil.escapeGql(sk_apply_date_to_utc) + "'")
q.filter('apply_date <', sk_apply_date_to_utc)
q.order('-apply_date')
# ユーザーごとも全体も上限を統一 2017.02.14
# 全件取得の場合は、fetchのメモリ使用量が大きいため、過去何ヶ月分の制約を設けてみる（ログイン履歴の設定を流用）
login_history_max_export_cnt = helper.getDeptInfo().get('login_history_max_export_cnt')
max_export_cnt = UcfUtil.toInt(login_history_max_export_cnt) # 最大出力件数
if max_export_cnt <= 0:
max_export_cnt = 1000
logging.info('max_export_cnt=' + str(max_export_cnt))
cnt = 0
limit = 500
#limit = 1000 # 通常の、max_export_cnt == 1000 のドメインは1発で取れたほうがいいはずなので 1000 とする
start_cursor = None
while True:
if with_cursor and start_cursor is not None:
fetch_data = q.with_cursor(start_cursor=start_cursor).fetch(limit)
else:
fetch_data = q.fetch(limit, cnt)
each_cnt = 0
for entry in fetch_data:
vo = entry.exchangeVo(helper._timezone)
AccessApplyUtils.editVoForCsv(helper, vo)
data = []
data.append('IU') # command
data.append(UcfUtil.getHashStr(vo, 'apply_date')) # apply_date
data.append(UcfUtil.getHashStr(vo, 'operator_id')) # email
data.append(UcfUtil.getHashStr(vo, 'device_distinguish_id')) # device_distinguish_id
data.append(UcfUtil.getHashStr(vo, 'device_mac_address')) # mac_address
data.append(UcfUtil.getHashStr(vo, 'identifier_for_vendor')) # identifier_for_vendor
data.append(UcfUtil.getHashStr(vo, 'target_career')) # target_career
data.append(UcfUtil.getHashStr(vo, 'target_env')) # target_env
data.append(UcfUtil.getHashStr(vo, 'use_profile_id')) # use_profile_id
data.append(UcfUtil.getHashStr(vo, 'useragent_id')) # user_agent
data.append(UcfUtil.getHashStr(vo, 'approval_status')) # status
data.append(UcfUtil.getHashStr(vo, 'approval_status_date')) # status_date
data.append(UcfUtil.getHashStr(vo, 'access_expire')) # access_expire
data.append(UcfUtil.getHashStr(vo, 'last_login_date')) # last_login_date
data.append(UcfUtil.getHashStr(vo, 'apply_comment')) # apply_comment
data.append(UcfUtil.getHashStr(vo, 'approval_comment')) # approval_comment
csv_records.append(UcfUtil.createCsvRecordEx(data))
each_cnt += 1
vo = None
entry = None
if each_cnt % 100 == 0:
gc.collect()
cnt += each_cnt
if with_cursor:
start_cursor = q.cursor()
logging.info(start_cursor)
logging.info(cnt)
# 件数上限
if cnt >= max_export_cnt or each_cnt < limit:
break
csv_text = '\r\n'.join(csv_records)
return csv_text
def processOfRequest(self, tenant, token):
self._approot_path = os.path.dirname(__file__)
# エラーが1回おきたら処理を終了する
if (int(self.request.headers.environ['HTTP_X_APPENGINE_TASKRETRYCOUNT']
) > 1):
logging.error('error over_1_times')
return
data_key = UcfUtil.nvl(self.getRequest('data_key'))
data_kind = UcfUtil.nvl(self.getRequest('data_kind'))
login_operator_id = UcfUtil.nvl(self.getRequest('login_operator_id'))
login_operator_unique_id = UcfUtil.nvl(
self.getRequest('login_operator_unique_id'))
login_operator_mail_address = UcfUtil.nvl(
self.getRequest('login_operator_mail_address'))
login_operator_client_ip = UcfUtil.nvl(
self.getRequest('login_operator_client_ip'))
# オペレータ情報を取得
login_operator_entry = None
if login_operator_unique_id != '':
login_operator_entry = OperatorUtils.getData(
self, login_operator_unique_id)
if login_operator_entry is None:
raise Exception('Not found login operator information.')
return
# preparing blob reader
# blobstore 保存しているバイナリデータを取得
blob_key = str(urllib.unquote(self.request.get('key')))
blob_reader = blobstore.BlobReader(blob_key)
# ファイルデータを取得（ステータス=CREATINGで作成済）
file_entry = FileUtils.getDataEntryByDataKey(self, data_key)
if file_entry is None:
raise Exception(self.getMsg('MSG_NOTFOUND_TARGET_FILE',
(data_key)))
return
# タスクトークンの取得と更新
last_task_token = file_entry.task_token if file_entry.task_token is not None else ''
file_entry.task_token = token
file_entry.put()
file_vo = file_entry.exchangeVo(self._timezone)
FileUtils.editVoForSelect(self, file_vo)
file_encoding = UcfUtil.getHashStr(self.getDeptInfo(True),
'file_encoding')
if file_encoding == '' or file_encoding == 'SJIS':
data_encoding = 'cp932'
elif file_encoding == 'JIS':
data_encoding = 'jis'
elif file_encoding == 'EUC':
data_encoding = 'euc-jp'
elif file_encoding == 'UTF7':
data_encoding = 'utf-7'
elif file_encoding == 'UTF8':
data_encoding = 'utf-8'
elif file_encoding == 'UNICODE':
data_encoding = 'utf-16'
else:
data_encoding = 'cp932'
log_msg = []
#is_error = False
record_cnt = 0
#insert_cnt = 0
#update_cnt = 0
#delete_cnt = 0
#skip_cnt = 0
#error_cnt = 0
shutdown_record_cnt_str = self.request.get('shutdown_record_cnt')
if shutdown_record_cnt_str is not None and shutdown_record_cnt_str != '':
shutdown_record_cnt = int(shutdown_record_cnt_str)
else:
shutdown_record_cnt = 0
logging.info('shutdown_record_cnt=' + str(shutdown_record_cnt))
is_error_str = self.request.get('is_error')
if is_error_str is not None and is_error_str.lower() == 'true':
is_error = True
else:
is_error = False
logging.info('is_error=' + str(is_error))
insert_cnt_str = self.request.get('insert_cnt')
if insert_cnt_str is not None and insert_cnt_str != '':
insert_cnt = int(insert_cnt_str)
else:
insert_cnt = 0
update_cnt_str = self.request.get('update_cnt')
if update_cnt_str is not None and update_cnt_str != '':
update_cnt = int(update_cnt_str)
else:
update_cnt = 0
delete_cnt_str = self.request.get('delete_cnt')
if delete_cnt_str is not None and delete_cnt_str != '':
delete_cnt = int(delete_cnt_str)
else:
delete_cnt = 0
skip_cnt_str = self.request.get('skip_cnt')
if skip_cnt_str is not None and skip_cnt_str != '':
skip_cnt = int(skip_cnt_str)
else:
skip_cnt = 0
error_cnt_str = self.request.get('error_cnt')
if error_cnt_str is not None and error_cnt_str != '':
error_cnt = int(error_cnt_str)
else:
error_cnt = 0
try:
# 同じトークンで既に処理済みの場合、ＧＡＥのタスクが強制終了した後のリトライなのでログを出しておく
if last_task_token == token:
is_error = True
log_msg.append(
self._formatLogRecord(
UcfMessage.getMessage(
self.getMsg('MSG_TASK_FORCE_RETRY'))))
self.updateTaskStatus(file_vo, file_entry, log_msg, is_error,
login_operator_unique_id,
login_operator_id)
del log_msg[:]
logging.info('csv_analysis start...')
new_lines = []
str_record = ''
quote_num = 0
old_lines = blob_reader.read().splitlines()
for line in old_lines:
#str_record += lineline + '\n'
#if str_record.count('"') % 2 == 0:
# new_lines.append(str_record.rstrip('\n'))
# str_record = ''
quote_num += line.count('"')
if quote_num % 2 == 0:
new_lines.append(str_record + line)
str_record = ''
quote_num = 0
else:
str_record += line + '\n'
logging.info('csv_analysis end. the record count is ' +
str(len(new_lines)) + ' with title line.')
# 巨大なCSVファイルを扱えるように対応 2015.03.27
csv.field_size_limit(1000000000)
# process uploaded csv file
# universal-newline mode に対応
#csvfile = csv.reader(blob_reader, dialect=csv.excel)
#csvfile = csv.reader(blob_reader.read().splitlines(), dialect=csv.excel)
csvfile = csv.reader(new_lines, dialect=csv.excel)
col_names = []
for row in csvfile:
# タイトル行の処理
if record_cnt == 0:
# first row: column list
col_index = 0
for col in row:
# BOM付CSVに対応 2016.10.13
if data_encoding == 'utf-8' and col_index == 0:
col = col.decode('utf-8-sig').encode('utf-8')
col_name = col.strip().strip('"')
# # 条件を削除し、一列目の情報は全て列を作成
col_names.append(col_name)
col_index += 1
# データ行の処理
elif shutdown_record_cnt <= record_cnt - 1:
is_runtime_shutdown = False
is_force_runtime_shutdown = False
# 5レコードに一回チェックしてみる
# シャットダウンを検知した場合
if record_cnt % 5 == 0:
is_runtime_shutdown = runtime.is_shutting_down()
# 強制対応はとりあえずコメントアウト
## シャットダウン検知しない場合も多いので、500レコードに一回ずつ別タスクにする ⇒ 100 に変更 2014.06.12
#if (shutdown_record_cnt < record_cnt - 1) and (record_cnt - 1) % 100 == 0:
# is_force_runtime_shutdown = True
if is_runtime_shutdown or is_force_runtime_shutdown:
is_shutting_down = True
current_memory_usage = runtime.memory_usage().current()
logging.info('is_shutting_down=' +
str(is_runtime_shutdown) +
' current_memory_usage=' +
str(current_memory_usage))
# instance will be shut down soon!
# exit here and kick same batch to start next record
logging.info(
'***** kicking same batch and stopping: shutdown_record_cnt='
+ str(record_cnt - 1))
# サマリをログ出力
log_msg.append(
self._formatLogRecord(
'development process [record:' +
UcfUtil.nvl(record_cnt - 1) + ' skip:' +
UcfUtil.nvl(skip_cnt) + ' insert:' +
UcfUtil.nvl(insert_cnt) + ' update:' +
UcfUtil.nvl(update_cnt) + ' delete:' +
UcfUtil.nvl(delete_cnt) + ' error:' +
UcfUtil.nvl(error_cnt) + ' ]'))
log_msg.append(
self._formatLogRecord(
'kicking same batch and stopping: shutdown_record_cnt='
+ str(record_cnt - 1)))
self.updateTaskStatus(file_vo, file_entry, log_msg,
is_error,
login_operator_unique_id,
login_operator_id)
del log_msg[:]
# kick start import
import_q = taskqueue.Queue('csv-export-import')
params = {
'shutdown_record_cnt': record_cnt - 1,
'insert_cnt': insert_cnt,
'update_cnt': update_cnt,
'delete_cnt': delete_cnt,
'skip_cnt': skip_cnt,
'error_cnt': error_cnt,
'is_error': is_error,
'key': blob_key,
'data_key': data_key,
'data_kind': data_kind,
'login_operator_id': login_operator_id,
'login_operator_unique_id':
login_operator_unique_id,
'login_operator_mail_address':
login_operator_mail_address,
'login_operator_client_ip':
login_operator_client_ip
}
import_t = taskqueue.Task(
url='/a/' + tenant + '/' + token +
'/queue_csv_import',
params=params,
target=sateraito_func.getBackEndsModuleName(
tenant),
countdown='1')
import_q.add(import_t)
return
col_index = 0
# params に配列を作成する。
csv_record = {}
for col_value in row:
if col_index < len(col_names):
# cut off too much csv data columns
# csv_record[col_names[col_index]] = unicode(col_value, UcfConfig.DL_ENCODING).strip().strip('"')
# csv_record[col_names[col_index]] = unicode(col_value, data_encoding).strip().strip('"')
csv_record[col_names[col_index]] = unicode(
col_value, data_encoding)
col_index += 1
# 1行処理
deal_type = ''
row_log_msg = None
code = ''
if data_kind == 'importgroupcsv':
deal_type, code, row_log_msg = self.importOneRecordGroup(
csv_record, record_cnt, blob_key, data_key,
data_kind, login_operator_unique_id,
login_operator_id, login_operator_mail_address,
login_operator_client_ip, login_operator_entry)
# elif data_kind == 'importusercsv':
# deal_type, code, row_log_msg = self.importOneRecordUser(csv_record, record_cnt, blob_key, data_key, data_kind, login_operator_unique_id, login_operator_id, login_operator_mail_address, login_operator_client_ip, login_operator_entry)
# elif data_kind == 'importchangeuseridcsv':
# deal_type, code, row_log_msg = self.importOneRecordChangeUserID(csv_record, record_cnt, blob_key, data_key, data_kind, login_operator_unique_id, login_operator_id, login_operator_mail_address, login_operator_client_ip, login_operator_entry)
# 件数やエラーメッセージを集計
if row_log_msg is not None:
log_msg.extend(row_log_msg)
if code != '':
error_cnt += 1
is_error = True
if deal_type == UcfConfig.EDIT_TYPE_NEW:
insert_cnt += 1
elif deal_type == UcfConfig.EDIT_TYPE_RENEW:
update_cnt += 1
elif deal_type == UcfConfig.EDIT_TYPE_DELETE:
delete_cnt += 1
elif deal_type == UcfConfig.EDIT_TYPE_SKIP:
skip_cnt += 1
# ユーザーID変更処理はデリケートなので毎回ログを出す
if data_kind == 'importchangeuseridcsv' and log_msg is not None and len(
log_msg) > 0:
self.updateTaskStatus(file_vo, file_entry, log_msg,
is_error,
login_operator_unique_id,
login_operator_id)
del log_msg[:]
# ときどきメモリ開放
if record_cnt % 100 == 0:
current_memory_usage = runtime.memory_usage().current()
gc.collect()
current_memory_usage2 = runtime.memory_usage().current()
logging.info('[memory_usage]record=' + str(record_cnt) +
' before:' + str(current_memory_usage) +
' after:' + str(current_memory_usage2))
record_cnt += 1
except BaseException, e:
self.outputErrorLog(e)
log_msg.append(self._formatLogRecord('system error.'))
is_error = True
def _formatLogRecord(self, log):
return '[' + UcfUtil.nvl(UcfUtil.getNowLocalTime(
self._timezone)) + ']' + log
def importOneRecordGroup(self, csv_record, record_cnt, blob_key, data_key,
data_kind, login_operator_unique_id,
login_operator_id, login_operator_mail_address,
login_operator_client_ip, login_operator_entry):
titles = GroupUtils.getCsvTitles(self)
code = ''
row_log_msg = []
entry_vo = None
current_belong_members = [] # 編集時に使用。編集前の所属メンバー（今回解除されたメンバーの判定に使用する）
deal_type = ''
edit_type = ''
try:
# CSVチェック
vc = GroupCsvValidator()
vc.validate(self, csv_record)
# 入力エラーがあれば
if vc.total_count > 0:
code = '100'
for title in titles:
if vc.msg.has_key(title):
#row_log_msg.extend(vc.msg[title])
row_log_msg.extend(
['[' + title + ']' + msg for msg in vc.msg[title]])
if code == '':
command = UcfUtil.getHashStr(csv_record, 'command')
# セールスフォースではグループIDとメールアドレスは別 2015.12.18
#email = UcfUtil.getHashStr(csv_record, 'email')
group_id = UcfUtil.getHashStr(csv_record, 'group_id')
q = sateraito_db.Group.query()
q = q.filter(
sateraito_db.Group.group_id_lower == group_id.lower())
key = q.get(keys_only=True)
entry = key.get() if key is not None else None
# 委託管理者の場合は自分がアクセスできるカテゴリかをチェック
if entry is not None and ucffunc.isDelegateOperator(
login_operator_entry
) and not ucffunc.isDelegateTargetManagementGroup(
entry.management_group,
login_operator_entry.delegate_management_groups
if login_operator_entry is not None else None):
code = '400'
row_log_msg.append(
UcfMessage.getMessage(
self.getMsg(
'MSG_INVALID_ACCESS_BY_DELEGATE_MANAGEMENT_GROUPS'
)))
else:
vo = {}
# 削除処理の場合
if command == 'D':
if entry is None:
code = '400'
row_log_msg.append(
self._formatLogRecord(
UcfMessage.getMessage(
self.getMsg('MSG_NOT_EXIST_DATA'))))
else:
edit_type = UcfConfig.EDIT_TYPE_DELETE
entry_vo = entry.exchangeVo(
self._timezone) # 既存データをVoに変換
# 新規登録の場合
elif command == 'I':
if entry is not None:
code = '400'
row_log_msg.append(
self._formatLogRecord(
UcfMessage.getMessage(
self.getMsg('MSG_VC_ALREADY_EXIST'))))
else:
edit_type = UcfConfig.EDIT_TYPE_NEW
GroupUtils.editVoForDefault(self, vo)
# csv_dataからマージ
GroupUtils.margeVoFromCsvRecord(
self, vo, csv_record, login_operator_entry)
GroupUtils.editVoForSelectCsvImport(
self, vo) # データ加工（取得用）
# 編集の場合
elif command == 'U':
if entry is None:
code = '400'
row_log_msg.append(
self._formatLogRecord(
UcfMessage.getMessage(
self.getMsg('MSG_NOT_EXIST_DATA'))))
else:
edit_type = UcfConfig.EDIT_TYPE_RENEW
entry_vo = entry.exchangeVo(
self._timezone) # 既存データをVoに変換
GroupUtils.editVoForSelect(
self,
entry_vo,
is_with_parent_group_info=False,
is_with_belong_member_info=False) # データ加工（取得用）
UcfUtil.margeHash(vo, entry_vo) # 既存データをVoにコピー
# csv_dataからマージ
GroupUtils.margeVoFromCsvRecord(
self, vo, csv_record, login_operator_entry)
GroupUtils.editVoForSelectCsvImport(
self, vo) # データ加工（取得用）
current_belong_members = UcfUtil.csvToList(
UcfUtil.getHashStr(entry_vo, 'belong_members'))
# 新規 or 編集の場合
elif command == 'IU':
if entry is not None:
edit_type = UcfConfig.EDIT_TYPE_RENEW
entry_vo = entry.exchangeVo(
self._timezone) # 既存データをVoに変換
GroupUtils.editVoForSelect(
self,
entry_vo,
is_with_parent_group_info=False,
is_with_belong_member_info=False) # データ加工（取得用）
UcfUtil.margeHash(vo, entry_vo) # 既存データをVoにコピー
else:
edit_type = UcfConfig.EDIT_TYPE_NEW
# csv_dataからマージ
GroupUtils.margeVoFromCsvRecord(
self, vo, csv_record, login_operator_entry)
GroupUtils.editVoForSelectCsvImport(self,
vo) # データ加工（取得用）
########################
# 削除
if edit_type == UcfConfig.EDIT_TYPE_DELETE:
# トップグループなら子グループをトップに昇格
if UcfUtil.getHashStr(entry_vo,
'top_group_flag') == 'TOP':
GroupUtils.updateBelongMembersTopGroupFlag(
self,
UcfUtil.getHashStr(entry_vo, 'group_id_lower'),
UcfUtil.csvToList(entry_vo['belong_members']),
'SET',
operator_id=login_operator_id)
# このグループをメイン組織に設定しているユーザのメイン組織をクリア
OperatorUtils.removeUsersTargetMainGroup(
self,
UcfUtil.getHashStr(entry_vo, 'group_id_lower'),
operator_id=login_operator_id)
# このグループをメイン組織に設定しているグループのメイン組織をクリア
GroupUtils.removeGroupsTargetMainGroup(
self,
UcfUtil.getHashStr(entry_vo, 'group_id_lower'),
operator_id=login_operator_id)
# このグループを所属メンバーとして持っている親グループからメンバーとして解除 2016.10.24
GroupUtils.removeOneMemberFromBelongGroups(
self,
UcfUtil.getHashStr(entry_vo, 'group_id_lower'),
operator_id=login_operator_id)
# このグループ自体を削除
entry.key.delete()
# オペレーションログ出力
UCFMDLOperationLog.addLog(
login_operator_mail_address,
login_operator_unique_id,
UcfConfig.SCREEN_GROUP,
UcfConfig.OPERATION_TYPE_REMOVE,
entry_vo.get('group_id', ''),
entry_vo.get('unique_id', ''),
login_operator_client_ip,
'',
is_async=True)
deal_type = edit_type
# 更新、新規
elif edit_type == UcfConfig.EDIT_TYPE_NEW or edit_type == UcfConfig.EDIT_TYPE_RENEW:
# 入力チェック
vc = GroupValidator(
edit_type,
ucffunc.isDelegateOperator(login_operator_entry),
login_operator_entry.delegate_management_groups
if login_operator_entry is not None else None)
vc.validate(self, vo)
# 入力エラーがなければ登録処理
if vc.total_count <= 0:
# オペレーションログ詳細用に更新フィールドを取得（加工前に比較しておく）
if edit_type == UcfConfig.EDIT_TYPE_NEW:
is_diff = True
diff_for_operation_log = []
else:
is_diff, diff_for_operation_log = GroupUtils.isDiff(
self, vo, entry_vo)
# 差分があるかを判定
is_skip = not is_diff
# 加工データ
GroupUtils.editVoForRegist(self, vo, entry_vo,
edit_type)
# 新規登録場合モデルを新規作成
if edit_type == UcfConfig.EDIT_TYPE_NEW:
unique_id = UcfUtil.guid()
vo['unique_id'] = unique_id
entry = sateraito_db.Group(
unique_id=unique_id,
id=GroupUtils.getKey(self, vo))
#logging.info('vo=' + str(vo))
# Voからモデルにマージ
#logging.info(vo)
entry.margeFromVo(vo, self._timezone)
# 更新日時、更新者の更新
entry.updater_name = login_operator_id
entry.date_changed = UcfUtil.getNow()
# 新規登録場合ユニークＩＤを生成
if edit_type == UcfConfig.EDIT_TYPE_NEW:
# 作成日時、作成者の更新
entry.creator_name = login_operator_id
entry.date_created = UcfUtil.getNow()
belong_members = UcfUtil.csvToList(
vo['belong_members'])
release_members = [] # 今回の変更で所属メンバーから解除されたメンバー
for current_belong_member in current_belong_members:
if current_belong_member not in belong_members:
release_members.append(
current_belong_member)
# 子グループのトップフラグを更新
GroupUtils.updateBelongMembersTopGroupFlag(
self,
UcfUtil.getHashStr(vo, 'group_id'),
belong_members,
'REMOVE',
operator_id=login_operator_id)
# 今回このグループからメンバー解除されたグループのトップフラグ更新
GroupUtils.updateBelongMembersTopGroupFlag(
self,
UcfUtil.getHashStr(vo, 'group_id'),
release_members,
'SET',
operator_id=login_operator_id)
# 登録、更新処理（※トランザクションは制約やデメリットが多いので使用しない）
if not is_skip:
entry.put()
# オペレーションログ出力
operation_log_detail = {}
if edit_type == UcfConfig.EDIT_TYPE_RENEW:
operation_log_detail[
'fields'] = diff_for_operation_log
UCFMDLOperationLog.addLog(
login_operator_mail_address,
login_operator_unique_id,
UcfConfig.SCREEN_GROUP,
UcfConfig.OPERATION_TYPE_ADD
if edit_type == UcfConfig.EDIT_TYPE_NEW
else UcfConfig.OPERATION_TYPE_MODIFY,
vo.get('group_id', ''),
vo.get('unique_id', ''),
login_operator_client_ip,
JSONEncoder().encode(operation_log_detail),
is_async=True)
deal_type = edit_type
else:
deal_type = UcfConfig.EDIT_TYPE_SKIP
# 入力エラーがあれば
else:
code = '100'
for key, value in vc.msg.iteritems():
#row_log_msg.extend(value)
row_log_msg.extend(
['[' + key + ']' + msg for msg in value])
except BaseException, e:
self.outputErrorLog(e)
code = '500'
row_log_msg.append(self._formatLogRecord('system error.'))
for key, value in vc.msg.iteritems():
#row_log_msg.extend(value)
row_log_msg.extend(
['[' + key + ']' + msg for msg in value])
except BaseException, e:
self.outputErrorLog(e)
code = '500'
row_log_msg.append(self._formatLogRecord('system error.'))
# エラーメッセージ処理
if code != '':
#row_log_msg.append(self._formatLogRecord('[row:' + UcfUtil.nvl(record_cnt) + ',code=' + code + ']'))
row_log_msg.insert(
0,
self._formatLogRecord('[row:' + UcfUtil.nvl(record_cnt) +
',code=' + code + ']'))
return deal_type, code, row_log_msg
############################################################
# 一行インポート：ユーザー
############################################################
# def importOneRecordUser(self, csv_record, record_cnt, blob_key, data_key, data_kind, login_operator_unique_id, login_operator_id, login_operator_mail_address, login_operator_client_ip, login_operator_entry):
#
# titles = UserUtils.getCsvTitles(self)
#
# code = ''
# row_log_msg = []
# entry_vo = None
# edit_type = ''
# deal_type = ''
def createCookieKeyForAccessKey(cls, helper, operator_unique_id): return 'ACSDID' + UcfUtil.md5(helper.getDeptInfo()['unique_id'] + '|' + operator_unique_id)
def getMsg(self, msgid, ls_param=()):
msgid = oem_func.exchangeMessageID(msgid, self._oem_company_code)
return UcfMessage.getMessage(UcfUtil.getHashStr(self.getMsgs(), msgid), ls_param)
def createAccessKey(cls, helper, useragent_id, operator_unique_id): return UcfUtil.md5(useragent_id + '|' + operator_unique_id)
def editVoForRegist(cls, helper, vo, entry_vo, edit_type):
if edit_type == UcfConfig.EDIT_TYPE_NEW:
vo['dept_id'] = UcfUtil.getHashStr(helper.getDeptInfo(), 'dept_id')
vo['operator_id_lower'] = UcfUtil.getHashStr(vo, 'operator_id').lower()
# アクセス期限の日付と時分からアクセス期限を作成
access_expire = ''
if UcfUtil.getHashStr(vo, 'access_expire_date') != '':
access_expire = UcfUtil.getHashStr(vo, 'access_expire_date')
if UcfUtil.getHashStr(vo, 'access_expire_time') != '':
time_ary = UcfUtil.getHashStr(vo, 'access_expire_time').split(':')
if len(time_ary) >= 2:
access_expire = access_expire + ' ' + time_ary[0] + ':' + time_ary[1] + ':00'
vo['access_expire'] = access_expire
# 承認ステータスが変更された場合はステータス日付を更新
if UcfUtil.getHashStr(vo, 'approval_status') != UcfUtil.getHashStr(entry_vo, 'approval_status'):
vo['approval_status_date'] = UcfUtil.nvl(UcfUtil.getNowLocalTime(helper._timezone))
vo['approval_operator_id'] = UcfUtil.nvl(helper.getLoginID())
def getServerToken(open_api_id, server_id, priv_key, no_memcache_use=False):
lineworksapi_server_token_memcache_key = ''
if open_api_id != '' and server_id != '':
lineworksapi_server_token_memcache_key = 'script=lineworksapi_server_token&open_api_id=' + open_api_id + '&server_id=' + server_id
logging.info(lineworksapi_server_token_memcache_key)
if not no_memcache_use and lineworksapi_server_token_memcache_key != '':
server_token = memcache.get(lineworksapi_server_token_memcache_key)
if server_token is not None:
logging.info('retrieve server_token from memcache.')
return server_token
server_token = ''
expires_in = 0
process_cnt = 0
MAX_RETRY_CNT = 0
while True:
try:
# １．JWT 生成 - RFC-7519
jwt_header = {
'typ': 'JWT',
'alg': 'RS256',
}
jwt_payload = {
'iss': server_id,
'iat': sateraito_func.datetimeToEpoch(UcfUtil.add_minutes(datetime.datetime.now(), -5)),
'exp': sateraito_func.datetimeToEpoch(UcfUtil.add_minutes(datetime.datetime.now(), 50)),
# MAX3600秒の範囲であること
}
jwt_header_str = json.JSONEncoder().encode(jwt_header)
jwt_header_enc = base64.urlsafe_b64encode(jwt_header_str)
jwt_payload_str = json.JSONEncoder().encode(jwt_payload)
jwt_payload_enc = base64.urlsafe_b64encode(jwt_payload_str)
# ２．JWT 電子署名(signature) - RFC-7515
signature_source = str(jwt_header_enc + '.' + jwt_payload_enc)
rsa_private_key = RSA.importKey(priv_key, passphrase='')
logging.info(process_cnt)
signer = PKCS1_v1_5.new(rsa_private_key)
digest = SHA256.new()
digest.update(signature_source)
signature = signer.sign(digest)
jwt_sig_enc = base64.urlsafe_b64encode(signature)
logging.info(jwt_sig_enc)
assertion_jwt = jwt_header_enc + '.' + jwt_payload_enc + '.' + jwt_sig_enc
logging.info(assertion_jwt)
# ３．LINE WORKS 認証サーバーへの Token リクエスト - RFC-7523
form_fields = {}
form_fields['grant_type'] = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
form_fields['assertion'] = assertion_jwt
form_data = urllib.urlencode(form_fields)
request_url = 'https://authapi.worksmobile.com/b/%s/server/token' % (open_api_id)
deadline = 10
headers = {
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
}
logging.info('urlfetch:' + request_url)
result = urlfetch.fetch(url=request_url, method=urlfetch.POST, payload=form_data, follow_redirects=True,
deadline=deadline, headers=headers)
if result.status_code != 200:
raise Exception(
'failed retrieve access_token of LINE WORKS API. status code=' + str(result.status_code))
# 結果サンプル（成功）
# {"access_token":"AAAA/arPRL4dqjpKAsg5Vt4r2P/zoP+c2VydmVySWQ+otLu9qTV5IqO0jbXfzaJQ5UZ0EYA22s5sMRySYIKpF0Va8NyW5Vu0okP9dlVqOKiZQczlzVTD0Ull8RI/rlBJN8gWrslOIA1WdOzLyM7byVQSqYL1AY43Ltrc9xqTDDJVMsS/Z8x1drlqeSCBXohIfb+ddk4h/gFPghzRmyG2qPueC4YxWArmCBOpbXH0zVdw/z2bXtgnHr33/XnRnhwx2j9JKx1nNW7JsAK4SIbgqZMxzr9YJ7iWEMJKS0ACsyMBa6Ba7izijUvRG43QZZ4BlSFtaykA7dDcAezaclmCNxRj9G0JszzrKfMK0PEI=","token_type":"Bearer","expires_in":86400}
# 結果サンプル（失敗）
# {"message":"invalid param","detail":"jwt exp - iat term should be max 3600 seconds","code":"11"}
response_json = json.JSONDecoder().decode(result.content)
# 結果チェック
if response_json.get('access_token', '') == '':
# エラー
raise Exception('failed retrieve access token of line works api.code=%s message=%s detail=%s' % (
response_json.get('code', ''), response_json.get('message', ''), response_json.get('detail', '')))
server_token = response_json.get('access_token', '')
token_type = response_json.get('token_type', '')
expires_in = response_json.get('expires_in', 0) # 例：86400 （24時間）
break # 成功なのでwhile抜ける
except BaseException, e:
logging.info(e)
if process_cnt <= MAX_RETRY_CNT:
logging.warning('[process_cnt=' + str(process_cnt) + ']' + ' '.join(e.args))
else:
# logging.exception(e)
# pass
raise e
process_cnt += 1
