def getInnerTextNvl(node, xpath=''): u''' TITLE:[STATIC]ノードの指定XPATHのInnerText文字列を取得 PARAMETER: node:対象ノード xpath:対象XPATH RETURN:文字列(取得できない場合は空文字を【返す) ''' if node == None: return '' if xpath == None or xpath == '': return UcfUtil.nvl(node.getInnerText()) # hash_node = node.exchangeToHash(isAttr=True, isChild=True) # return UcfUtil.nvl(UcfUtil.getHashStr(hash_node, xpath)) # XPATH 解析(属性とノードに分ける) xpath_parts = xpath.split('@') xpath_node = xpath_parts[0].rstrip('/') xpath_atr = xpath_parts[1] if len(xpath_parts) >= 2 else '' node_target = node.selectSingleNode(xpath_node) if node_target == None: return '' if xpath_atr != '': return UcfUtil.nvl(node_target.getAttribute(xpath_atr)) else: return UcfUtil.nvl(node_target.getInnerText())
def removeAttribute(self, name): u''' TITLE:属性を削除 ABSTRACT:属性がない場合は無視 PARAMETER: name:属性名 ''' UcfUtil.removeHash(self._element.attrib, name)
def _sendMail(to, message_subject, message_body, cc='', reply_to='', is_html=False): # check to if to is None or str(to).strip() == '': logging.error('to is None') return False, 'invalid_to' if not mail.IsEmailValid(to): logging.error('invalid to=' + str(to)) return False, 'invalid_to' # check cc if cc is not None and str(cc).strip() != '': if not mail.IsEmailValid(cc): logging.error('invalid cc=' + str(cc)) return False, 'invalid_cc' # check reply_to if reply_to is not None and str(reply_to) != '': if not mail.IsEmailValid(reply_to): logging.error('invalid reply_to=' + str(reply_to)) return False, 'invalid_reply_to' # calc check_key now = UcfUtil.getNow() # 標準時 check_key = UcfUtil.md5( now.strftime('%Y%m%d%H%M') + MD5_SUFFIX_KEY_MAIL_SERVER + SENDER_ADDON_PROJECT_ID) # post data url = SATERAITO_MAIL_SERVER_URL + '/api/sendmail' values = { 'addon_project_id': SENDER_ADDON_PROJECT_ID, 'sender_email': MESSAGE_SENDER_EMAIL, 'to': to, 'cc': cc, 'reply_to': reply_to, 'message_subject': message_subject, 'message_body': message_body, 'is_html': str(is_html), 'check_key': check_key, } headers = {} response = HttpPostAccess(url, values, headers) logging.info('response=' + str(response)) try: response_dict = json.JSONDecoder().decode(response) if response_dict.get('status') != 'ok': return False, response_dict.get('error_code') except BaseException, e: logging.error('error: class name:' + e.__class__.__name__ + ' message=' + str(e)) return False, 'unexpected_error'
def create_api_key(cls, creator_email): unique_id = UcfUtil.guid() entry = cls(id=unique_id) entry.unique_id = unique_id api_key = UcfUtil.guid() entry.api_key = api_key entry.creator_email = creator_email entry_key = entry.put() entry_dict = entry_key.get().to_dict() return entry_dict
def processOfRequest(self): self._approot_path = os.path.dirname(__file__) last_modified_language = '' # Wed, 21 Jun 2006 07:00:25 GMT last_modified_default = '' # Wed, 21 Jun 2006 07:00:25 GMT language = UcfUtil.nvl(self.getRequest('ln')); if language != '': last_modified_language = UcfMessage.getLangFileLastModified(self._approot_path, language=language) if language.lower() != UcfConfig.MESSAGE_DEFAULT_FILE.lower(): last_modified_default = UcfMessage.getLangFileLastModified(self._approot_path, language=UcfConfig.MESSAGE_DEFAULT_FILE) else: last_modified_default = last_modified_language last_modified = str(last_modified_language) + '|' + str(last_modified_default) # if self.request.headers.has_key('If-Modified-Since'): # logging.info('If-Modified-Since=' + self.request.headers['If-Modified-Since']) # logging.info('last_modified=' + last_modified) if self.request.headers.has_key('If-Modified-Since') and last_modified is not None and self.request.headers['If-Modified-Since'] == last_modified: self.response.set_status(304) return msgs = UcfMessage.getMessageList(self._approot_path, language=language,is_only_js=True) template_vals = { 'lang_json':JSONEncoder().encode(msgs) } #self.appendBasicInfoToTemplateVals(template_vals) self.response.headers['Last-Modified'] = last_modified self.render('jslang.html', self._design_type, template_vals, content_type='text/javascript')
def addMailSendQueue(cls, helper, apply_vo): # token作成 token = UcfUtil.guid() params = { 'apply_unique_id': UcfUtil.getHashStr(apply_vo, 'unique_id') } # taskに追加 import_q = taskqueue.Queue('send-mail') import_t = taskqueue.Task( url='/a/' + helper._tenant + '/' + token + '/sendmail_update_approval_status', params=params, #target='mail', target='default', countdown='1' ) import_q.add(import_t) logging.info('add taskqueue:sendmail_update_approval_status')
def getInnerText(self): u''' TITLE:InnerText文字列を取得 ABSTRACT:直下のTEXTのみを取得.サブノードがあっても無視. PARAMETER: RETURN:文字列 ''' return UcfUtil.nvl(self._element.text)
def expandApplyInfo(cls, helper, vo): try: apply_info = JSONDecoder().decode(UcfUtil.getHashStr(vo, 'apply_info')) # TODO セキュリティブラウザの端末情報を展開 2015.09.04 if apply_info.has_key('DeviceInfo'): pass except BaseException, e: logging.exception(e) apply_info = {}
def setAttribute(self, name, value): u''' TITLE:属性に値をセット ABSTRACT:属性がなければ作成してセット PARAMETER: name:属性名 value:属性値 ''' self._element.attrib[name] = UcfUtil.nvl(value)
def getApprovalApplyVoListByUser(cls, helper, operator_unique_id): apply_vos = [] q = UCFMDLAccessApply.all() q.filter('operator_unique_id =', operator_unique_id) q.filter('approval_status IN', ['APPROVAL', 'DENY']) # 申請済みはカウントしないべきなので修正 2016.11.24 active_apply_count = 0 for entry in q: vo = entry.exchangeVo(helper._timezone) AccessApplyUtils.editVoForSelect(helper, vo) is_approval = True # ステータス.承認済みかどうか if is_approval and UcfUtil.getHashStr(vo, 'approval_status') != 'APPROVAL': is_approval = False # アクセス期限.切れていないかどうか if is_approval: access_expire = UcfUtil.getHashStr(vo, 'access_expire') if access_expire != '' and UcfUtil.getNowLocalTime(helper._timezone) > UcfUtil.getDateTime(access_expire): is_approval = False if is_approval: apply_vos.append(vo) return apply_vos
def validate(self, helper, vo): # 初期化 self.init() check_name = '' check_key = '' check_value = '' ######################## check_name = helper.getMsg('FLD_APPROVAL_COMMENT') check_key = 'approval_comment' check_value = UcfUtil.getHashStr(vo, check_key) # 最大長チェック:500文字 if not self.maxLengthValidator(check_value, 500): self.appendValidate(check_key, UcfMessage.getMessage(helper.getMsg('MSG_VC_MAXLENGTH'), (check_name, 500))) check_name = helper.getMsg('FLD_COMMENT') check_key = 'comment' check_value = UcfUtil.getHashStr(vo, check_key) # 最大長チェック:500文字 if not self.maxLengthValidator(check_value, 500): self.appendValidate(check_key, UcfMessage.getMessage(helper.getMsg('MSG_VC_MAXLENGTH'), (check_name, 500)))
def editVoForSelect(cls, helper, vo): # 申請日時を日付と時間(時分)に分ける access_expire_date = '' access_expire_time = '' access_expire = UcfUtil.getHashStr(vo, 'access_expire') if access_expire != '': date_time_ary = access_expire.split(' ') access_expire_date = date_time_ary[0] if len(date_time_ary) >= 2: access_expire_time = date_time_ary[1] time_ary = access_expire_time.split(':') if len(time_ary) >= 2: access_expire_time = time_ary[0] + ':' + time_ary[1] vo['access_expire_date'] = access_expire_date vo['access_expire_time'] = access_expire_time
def updateTaskStatus(self, file_vo, file_entry, log_msg, is_error, login_operator_unique_id, login_operator_id, is_after_process=False): # ステータス後処理 datNow = UcfUtil.getLocalTime(UcfUtil.getNow(), self._timezone) if is_error: file_vo['status'] = 'FAILED' elif is_after_process: file_vo['status'] = 'SUCCESS' log_text = file_vo.get('log_text', '') for msg in log_msg: log_text += msg + '\n' file_vo['log_text'] = log_text if is_after_process: file_vo['deal_status'] = 'FIN' file_vo['expire_date'] = UcfUtil.add_months(datNow, 1) # 一ヶ月有効とする file_vo['upload_count'] = '1' file_vo['last_upload_date'] = UcfUtil.nvl(datNow) file_vo['upload_operator_id'] = login_operator_id file_vo['upload_operator_unique_id'] = login_operator_unique_id file_vo['last_upload_operator_id'] = login_operator_id file_vo['last_upload_operator_unique_id'] = login_operator_unique_id FileUtils.editVoForRegist(self, file_vo, UcfConfig.EDIT_TYPE_RENEW) # Voからモデルにマージ file_entry.margeFromVo(file_vo, self._timezone) # 更新 file_entry.updater_name = login_operator_id file_entry.date_changed = UcfUtil.getNow() file_entry.put()
if result_json['success']: access_token = result_json.get('access_token', '') expires_in = result_json.get('expire_timestamp', 0) break # 成功なのでwhile抜ける else: return 1 except BaseException, e: if process_cnt <= MAX_RETRY_CNT: logging.warning('[process_cnt=' + str(process_cnt) + ']' + ' '.join(e.args)) else: return 2 process_cnt += 1 if expires_in != 0: memcache_expire_secs = expires_in - time.mktime(UcfUtil.getNow().timetuple()) - sateraito_inc.session_timeout else: memcache_expire_secs = sateraito_inc.session_timeout - 600 logging.info('memcache_expire_secs=' + str(memcache_expire_secs)) if memcache_expire_secs > 0 and directcloudbox_token_memcache_key != '': if not memcache.set(key=directcloudbox_token_memcache_key, value=access_token, time=memcache_expire_secs): logging.warning("Memcache set failed.") else: logging.info("Memcache set success.key=" + directcloudbox_token_memcache_key) return access_token ##################################################### # DIRECT CLOUD BOX コール #####################################################
def getData(cls, helper, unique_id): query = UCFMDLAccessApply.gql("where unique_id = :1", UcfUtil.escapeGql(unique_id)) entry = query.get() return entry
def getKey(cls, helper, vo): # 最新ものを上に出したいので. ※TODO BigTableのInsertパフォーマンス大丈夫かなー? return UcfUtil.nvl(int(''.ljust(10, '9')) - int(UcfUtil.nvl(int(time.time())).ljust(10, '0'))) + UcfConfig.KEY_PREFIX + UcfUtil.getHashStr(vo, 'unique_id')
def createCsv(cls, helper, login_operator_entry=None, sk_operator_unique_id='', optional_scond=None): logging.info('start create csv...') with_cursor = True csv_records = [] # タイトル titles = AccessApplyUtils.getCsvTitles(helper) csv_records.append(UcfUtil.createCsvRecordEx(titles)) # データ一覧取得 q = UCFMDLAccessApply.all() if sk_operator_unique_id != '': q.filter('operator_unique_id =', sk_operator_unique_id) ## ユーザーごとのレコードは従来通り1000件固定 #max_export_cnt = 1000 # 最大出力件数 else: sk_search_type = UcfUtil.getHashStr(optional_scond, 'sk_search_type') if optional_scond is not None else '' sk_operator_id = UcfUtil.getHashStr(optional_scond, 'sk_operator_id').lower() if optional_scond is not None else '' #sk_operator_unique_id = UcfUtil.getHashStr(optional_scond, 'sk_operator_unique_id') if optional_scond is not None else '' sk_apply_date_date_from = UcfUtil.getHashStr(optional_scond, 'sk_apply_date_date_from') if optional_scond is not None else '' sk_apply_date_time_from = UcfUtil.getHashStr(optional_scond, 'sk_apply_date_time_from') if optional_scond is not None else '' sk_apply_date_date_to = UcfUtil.getHashStr(optional_scond, 'sk_apply_date_date_to') if optional_scond is not None else '' sk_apply_date_time_to = UcfUtil.getHashStr(optional_scond, 'sk_apply_date_time_to') if optional_scond is not None else '' if sk_search_type == '': sk_search_type = 'operator_id' # 委託管理者なら自分が触れるデータのみ対象 if ucffunc.isDelegateOperator(login_operator_entry) and login_operator_entry.delegate_management_groups is not None and len(login_operator_entry.delegate_management_groups) > 0: q.filter('management_group IN', login_operator_entry.delegate_management_groups) # 管理グループが複数ある場合はカーソル使えないので if len(login_operator_entry.delegate_management_groups) >= 2: with_cursor = False # ログインIDで検索 if sk_search_type == 'operator_id' and sk_operator_id != '': q.filter('operator_id_lower >=', sk_operator_id) q.filter('operator_id_lower <', sk_operator_id + u'\uFFFD') # 申請日時で検索 elif sk_search_type == 'apply_date' and (sk_apply_date_date_from != '' or sk_apply_date_date_to != ''): if sk_apply_date_date_from != '': if sk_apply_date_time_from != '': time_ary = sk_apply_date_time_from.split(':') sk_apply_date_from = sk_apply_date_date_from + ' ' + time_ary[0] + ':' + time_ary[1] + ':00' sk_apply_date_from_utc = UcfUtil.getUTCTime(UcfUtil.getDateTime(sk_apply_date_from), helper._timezone) else: sk_apply_date_from = sk_apply_date_date_from + ' 00:00:00' sk_apply_date_from_utc = UcfUtil.getUTCTime(UcfUtil.getDateTime(sk_apply_date_from), helper._timezone) #wheres.append("apply_date >= '" + UcfUtil.escapeGql(sk_apply_date_from_utc) + "'") q.filter('apply_date >=', sk_apply_date_from_utc) if sk_apply_date_date_to != '': if sk_apply_date_time_to != '': time_ary = sk_apply_date_time_to.split(':') sk_apply_date_to = sk_apply_date_date_to + ' ' + time_ary[0] + ':' + time_ary[1] + ':00' sk_apply_date_to_utc = UcfUtil.getUTCTime(UcfUtil.getDateTime(sk_apply_date_to), helper._timezone) else: sk_apply_date_to = sk_apply_date_date_to + ' 00:00:00' sk_apply_date_to_utc = UcfUtil.getUTCTime(UcfUtil.add_days(UcfUtil.getDateTime(sk_apply_date_to), 1), helper._timezone) #wheres.append("apply_date < '" + UcfUtil.escapeGql(sk_apply_date_to_utc) + "'") q.filter('apply_date <', sk_apply_date_to_utc) q.order('-apply_date') # ユーザーごとも全体も上限を統一 2017.02.14 # 全件取得の場合は、fetchのメモリ使用量が大きいため、過去何ヶ月分の制約を設けてみる(ログイン履歴の設定を流用) login_history_max_export_cnt = helper.getDeptInfo().get('login_history_max_export_cnt') max_export_cnt = UcfUtil.toInt(login_history_max_export_cnt) # 最大出力件数 if max_export_cnt <= 0: max_export_cnt = 1000 logging.info('max_export_cnt=' + str(max_export_cnt)) cnt = 0 limit = 500 #limit = 1000 # 通常の、max_export_cnt == 1000 のドメインは1発で取れたほうがいいはずなので 1000 とする start_cursor = None while True: if with_cursor and start_cursor is not None: fetch_data = q.with_cursor(start_cursor=start_cursor).fetch(limit) else: fetch_data = q.fetch(limit, cnt) each_cnt = 0 for entry in fetch_data: vo = entry.exchangeVo(helper._timezone) AccessApplyUtils.editVoForCsv(helper, vo) data = [] data.append('IU') # command data.append(UcfUtil.getHashStr(vo, 'apply_date')) # apply_date data.append(UcfUtil.getHashStr(vo, 'operator_id')) # email data.append(UcfUtil.getHashStr(vo, 'device_distinguish_id')) # device_distinguish_id data.append(UcfUtil.getHashStr(vo, 'device_mac_address')) # mac_address data.append(UcfUtil.getHashStr(vo, 'identifier_for_vendor')) # identifier_for_vendor data.append(UcfUtil.getHashStr(vo, 'target_career')) # target_career data.append(UcfUtil.getHashStr(vo, 'target_env')) # target_env data.append(UcfUtil.getHashStr(vo, 'use_profile_id')) # use_profile_id data.append(UcfUtil.getHashStr(vo, 'useragent_id')) # user_agent data.append(UcfUtil.getHashStr(vo, 'approval_status')) # status data.append(UcfUtil.getHashStr(vo, 'approval_status_date')) # status_date data.append(UcfUtil.getHashStr(vo, 'access_expire')) # access_expire data.append(UcfUtil.getHashStr(vo, 'last_login_date')) # last_login_date data.append(UcfUtil.getHashStr(vo, 'apply_comment')) # apply_comment data.append(UcfUtil.getHashStr(vo, 'approval_comment')) # approval_comment csv_records.append(UcfUtil.createCsvRecordEx(data)) each_cnt += 1 vo = None entry = None if each_cnt % 100 == 0: gc.collect() cnt += each_cnt if with_cursor: start_cursor = q.cursor() logging.info(start_cursor) logging.info(cnt) # 件数上限 if cnt >= max_export_cnt or each_cnt < limit: break csv_text = '\r\n'.join(csv_records) return csv_text
def processOfRequest(self, tenant, token): self._approot_path = os.path.dirname(__file__) # エラーが1回おきたら処理を終了する if (int(self.request.headers.environ['HTTP_X_APPENGINE_TASKRETRYCOUNT'] ) > 1): logging.error('error over_1_times') return data_key = UcfUtil.nvl(self.getRequest('data_key')) data_kind = UcfUtil.nvl(self.getRequest('data_kind')) login_operator_id = UcfUtil.nvl(self.getRequest('login_operator_id')) login_operator_unique_id = UcfUtil.nvl( self.getRequest('login_operator_unique_id')) login_operator_mail_address = UcfUtil.nvl( self.getRequest('login_operator_mail_address')) login_operator_client_ip = UcfUtil.nvl( self.getRequest('login_operator_client_ip')) # オペレータ情報を取得 login_operator_entry = None if login_operator_unique_id != '': login_operator_entry = OperatorUtils.getData( self, login_operator_unique_id) if login_operator_entry is None: raise Exception('Not found login operator information.') return # preparing blob reader # blobstore 保存しているバイナリデータを取得 blob_key = str(urllib.unquote(self.request.get('key'))) blob_reader = blobstore.BlobReader(blob_key) # ファイルデータを取得(ステータス=CREATINGで作成済) file_entry = FileUtils.getDataEntryByDataKey(self, data_key) if file_entry is None: raise Exception(self.getMsg('MSG_NOTFOUND_TARGET_FILE', (data_key))) return # タスクトークンの取得と更新 last_task_token = file_entry.task_token if file_entry.task_token is not None else '' file_entry.task_token = token file_entry.put() file_vo = file_entry.exchangeVo(self._timezone) FileUtils.editVoForSelect(self, file_vo) file_encoding = UcfUtil.getHashStr(self.getDeptInfo(True), 'file_encoding') if file_encoding == '' or file_encoding == 'SJIS': data_encoding = 'cp932' elif file_encoding == 'JIS': data_encoding = 'jis' elif file_encoding == 'EUC': data_encoding = 'euc-jp' elif file_encoding == 'UTF7': data_encoding = 'utf-7' elif file_encoding == 'UTF8': data_encoding = 'utf-8' elif file_encoding == 'UNICODE': data_encoding = 'utf-16' else: data_encoding = 'cp932' log_msg = [] #is_error = False record_cnt = 0 #insert_cnt = 0 #update_cnt = 0 #delete_cnt = 0 #skip_cnt = 0 #error_cnt = 0 shutdown_record_cnt_str = self.request.get('shutdown_record_cnt') if shutdown_record_cnt_str is not None and shutdown_record_cnt_str != '': shutdown_record_cnt = int(shutdown_record_cnt_str) else: shutdown_record_cnt = 0 logging.info('shutdown_record_cnt=' + str(shutdown_record_cnt)) is_error_str = self.request.get('is_error') if is_error_str is not None and is_error_str.lower() == 'true': is_error = True else: is_error = False logging.info('is_error=' + str(is_error)) insert_cnt_str = self.request.get('insert_cnt') if insert_cnt_str is not None and insert_cnt_str != '': insert_cnt = int(insert_cnt_str) else: insert_cnt = 0 update_cnt_str = self.request.get('update_cnt') if update_cnt_str is not None and update_cnt_str != '': update_cnt = int(update_cnt_str) else: update_cnt = 0 delete_cnt_str = self.request.get('delete_cnt') if delete_cnt_str is not None and delete_cnt_str != '': delete_cnt = int(delete_cnt_str) else: delete_cnt = 0 skip_cnt_str = self.request.get('skip_cnt') if skip_cnt_str is not None and skip_cnt_str != '': skip_cnt = int(skip_cnt_str) else: skip_cnt = 0 error_cnt_str = self.request.get('error_cnt') if error_cnt_str is not None and error_cnt_str != '': error_cnt = int(error_cnt_str) else: error_cnt = 0 try: # 同じトークンで既に処理済みの場合、GAEのタスクが強制終了した後のリトライなのでログを出しておく if last_task_token == token: is_error = True log_msg.append( self._formatLogRecord( UcfMessage.getMessage( self.getMsg('MSG_TASK_FORCE_RETRY')))) self.updateTaskStatus(file_vo, file_entry, log_msg, is_error, login_operator_unique_id, login_operator_id) del log_msg[:] logging.info('csv_analysis start...') new_lines = [] str_record = '' quote_num = 0 old_lines = blob_reader.read().splitlines() for line in old_lines: #str_record += lineline + '\n' #if str_record.count('"') % 2 == 0: # new_lines.append(str_record.rstrip('\n')) # str_record = '' quote_num += line.count('"') if quote_num % 2 == 0: new_lines.append(str_record + line) str_record = '' quote_num = 0 else: str_record += line + '\n' logging.info('csv_analysis end. the record count is ' + str(len(new_lines)) + ' with title line.') # 巨大なCSVファイルを扱えるように対応 2015.03.27 csv.field_size_limit(1000000000) # process uploaded csv file # universal-newline mode に対応 #csvfile = csv.reader(blob_reader, dialect=csv.excel) #csvfile = csv.reader(blob_reader.read().splitlines(), dialect=csv.excel) csvfile = csv.reader(new_lines, dialect=csv.excel) col_names = [] for row in csvfile: # タイトル行の処理 if record_cnt == 0: # first row: column list col_index = 0 for col in row: # BOM付CSVに対応 2016.10.13 if data_encoding == 'utf-8' and col_index == 0: col = col.decode('utf-8-sig').encode('utf-8') col_name = col.strip().strip('"') # # 条件を削除し、一列目の情報は全て列を作成 col_names.append(col_name) col_index += 1 # データ行の処理 elif shutdown_record_cnt <= record_cnt - 1: is_runtime_shutdown = False is_force_runtime_shutdown = False # 5レコードに一回チェックしてみる # シャットダウンを検知した場合 if record_cnt % 5 == 0: is_runtime_shutdown = runtime.is_shutting_down() # 強制対応はとりあえずコメントアウト ## シャットダウン検知しない場合も多いので、500レコードに一回ずつ別タスクにする ⇒ 100 に変更 2014.06.12 #if (shutdown_record_cnt < record_cnt - 1) and (record_cnt - 1) % 100 == 0: # is_force_runtime_shutdown = True if is_runtime_shutdown or is_force_runtime_shutdown: is_shutting_down = True current_memory_usage = runtime.memory_usage().current() logging.info('is_shutting_down=' + str(is_runtime_shutdown) + ' current_memory_usage=' + str(current_memory_usage)) # instance will be shut down soon! # exit here and kick same batch to start next record logging.info( '***** kicking same batch and stopping: shutdown_record_cnt=' + str(record_cnt - 1)) # サマリをログ出力 log_msg.append( self._formatLogRecord( 'development process [record:' + UcfUtil.nvl(record_cnt - 1) + ' skip:' + UcfUtil.nvl(skip_cnt) + ' insert:' + UcfUtil.nvl(insert_cnt) + ' update:' + UcfUtil.nvl(update_cnt) + ' delete:' + UcfUtil.nvl(delete_cnt) + ' error:' + UcfUtil.nvl(error_cnt) + ' ]')) log_msg.append( self._formatLogRecord( 'kicking same batch and stopping: shutdown_record_cnt=' + str(record_cnt - 1))) self.updateTaskStatus(file_vo, file_entry, log_msg, is_error, login_operator_unique_id, login_operator_id) del log_msg[:] # kick start import import_q = taskqueue.Queue('csv-export-import') params = { 'shutdown_record_cnt': record_cnt - 1, 'insert_cnt': insert_cnt, 'update_cnt': update_cnt, 'delete_cnt': delete_cnt, 'skip_cnt': skip_cnt, 'error_cnt': error_cnt, 'is_error': is_error, 'key': blob_key, 'data_key': data_key, 'data_kind': data_kind, 'login_operator_id': login_operator_id, 'login_operator_unique_id': login_operator_unique_id, 'login_operator_mail_address': login_operator_mail_address, 'login_operator_client_ip': login_operator_client_ip } import_t = taskqueue.Task( url='/a/' + tenant + '/' + token + '/queue_csv_import', params=params, target=sateraito_func.getBackEndsModuleName( tenant), countdown='1') import_q.add(import_t) return col_index = 0 # params に配列を作成する。 csv_record = {} for col_value in row: if col_index < len(col_names): # cut off too much csv data columns # csv_record[col_names[col_index]] = unicode(col_value, UcfConfig.DL_ENCODING).strip().strip('"') # csv_record[col_names[col_index]] = unicode(col_value, data_encoding).strip().strip('"') csv_record[col_names[col_index]] = unicode( col_value, data_encoding) col_index += 1 # 1行処理 deal_type = '' row_log_msg = None code = '' if data_kind == 'importgroupcsv': deal_type, code, row_log_msg = self.importOneRecordGroup( csv_record, record_cnt, blob_key, data_key, data_kind, login_operator_unique_id, login_operator_id, login_operator_mail_address, login_operator_client_ip, login_operator_entry) # elif data_kind == 'importusercsv': # deal_type, code, row_log_msg = self.importOneRecordUser(csv_record, record_cnt, blob_key, data_key, data_kind, login_operator_unique_id, login_operator_id, login_operator_mail_address, login_operator_client_ip, login_operator_entry) # elif data_kind == 'importchangeuseridcsv': # deal_type, code, row_log_msg = self.importOneRecordChangeUserID(csv_record, record_cnt, blob_key, data_key, data_kind, login_operator_unique_id, login_operator_id, login_operator_mail_address, login_operator_client_ip, login_operator_entry) # 件数やエラーメッセージを集計 if row_log_msg is not None: log_msg.extend(row_log_msg) if code != '': error_cnt += 1 is_error = True if deal_type == UcfConfig.EDIT_TYPE_NEW: insert_cnt += 1 elif deal_type == UcfConfig.EDIT_TYPE_RENEW: update_cnt += 1 elif deal_type == UcfConfig.EDIT_TYPE_DELETE: delete_cnt += 1 elif deal_type == UcfConfig.EDIT_TYPE_SKIP: skip_cnt += 1 # ユーザーID変更処理はデリケートなので毎回ログを出す if data_kind == 'importchangeuseridcsv' and log_msg is not None and len( log_msg) > 0: self.updateTaskStatus(file_vo, file_entry, log_msg, is_error, login_operator_unique_id, login_operator_id) del log_msg[:] # ときどきメモリ開放 if record_cnt % 100 == 0: current_memory_usage = runtime.memory_usage().current() gc.collect() current_memory_usage2 = runtime.memory_usage().current() logging.info('[memory_usage]record=' + str(record_cnt) + ' before:' + str(current_memory_usage) + ' after:' + str(current_memory_usage2)) record_cnt += 1 except BaseException, e: self.outputErrorLog(e) log_msg.append(self._formatLogRecord('system error.')) is_error = True
def _formatLogRecord(self, log): return '[' + UcfUtil.nvl(UcfUtil.getNowLocalTime( self._timezone)) + ']' + log
def importOneRecordGroup(self, csv_record, record_cnt, blob_key, data_key, data_kind, login_operator_unique_id, login_operator_id, login_operator_mail_address, login_operator_client_ip, login_operator_entry): titles = GroupUtils.getCsvTitles(self) code = '' row_log_msg = [] entry_vo = None current_belong_members = [] # 編集時に使用。編集前の所属メンバー(今回解除されたメンバーの判定に使用する) deal_type = '' edit_type = '' try: # CSVチェック vc = GroupCsvValidator() vc.validate(self, csv_record) # 入力エラーがあれば if vc.total_count > 0: code = '100' for title in titles: if vc.msg.has_key(title): #row_log_msg.extend(vc.msg[title]) row_log_msg.extend( ['[' + title + ']' + msg for msg in vc.msg[title]]) if code == '': command = UcfUtil.getHashStr(csv_record, 'command') # セールスフォースではグループIDとメールアドレスは別 2015.12.18 #email = UcfUtil.getHashStr(csv_record, 'email') group_id = UcfUtil.getHashStr(csv_record, 'group_id') q = sateraito_db.Group.query() q = q.filter( sateraito_db.Group.group_id_lower == group_id.lower()) key = q.get(keys_only=True) entry = key.get() if key is not None else None # 委託管理者の場合は自分がアクセスできるカテゴリかをチェック if entry is not None and ucffunc.isDelegateOperator( login_operator_entry ) and not ucffunc.isDelegateTargetManagementGroup( entry.management_group, login_operator_entry.delegate_management_groups if login_operator_entry is not None else None): code = '400' row_log_msg.append( UcfMessage.getMessage( self.getMsg( 'MSG_INVALID_ACCESS_BY_DELEGATE_MANAGEMENT_GROUPS' ))) else: vo = {} # 削除処理の場合 if command == 'D': if entry is None: code = '400' row_log_msg.append( self._formatLogRecord( UcfMessage.getMessage( self.getMsg('MSG_NOT_EXIST_DATA')))) else: edit_type = UcfConfig.EDIT_TYPE_DELETE entry_vo = entry.exchangeVo( self._timezone) # 既存データをVoに変換 # 新規登録の場合 elif command == 'I': if entry is not None: code = '400' row_log_msg.append( self._formatLogRecord( UcfMessage.getMessage( self.getMsg('MSG_VC_ALREADY_EXIST')))) else: edit_type = UcfConfig.EDIT_TYPE_NEW GroupUtils.editVoForDefault(self, vo) # csv_dataからマージ GroupUtils.margeVoFromCsvRecord( self, vo, csv_record, login_operator_entry) GroupUtils.editVoForSelectCsvImport( self, vo) # データ加工(取得用) # 編集の場合 elif command == 'U': if entry is None: code = '400' row_log_msg.append( self._formatLogRecord( UcfMessage.getMessage( self.getMsg('MSG_NOT_EXIST_DATA')))) else: edit_type = UcfConfig.EDIT_TYPE_RENEW entry_vo = entry.exchangeVo( self._timezone) # 既存データをVoに変換 GroupUtils.editVoForSelect( self, entry_vo, is_with_parent_group_info=False, is_with_belong_member_info=False) # データ加工(取得用) UcfUtil.margeHash(vo, entry_vo) # 既存データをVoにコピー # csv_dataからマージ GroupUtils.margeVoFromCsvRecord( self, vo, csv_record, login_operator_entry) GroupUtils.editVoForSelectCsvImport( self, vo) # データ加工(取得用) current_belong_members = UcfUtil.csvToList( UcfUtil.getHashStr(entry_vo, 'belong_members')) # 新規 or 編集の場合 elif command == 'IU': if entry is not None: edit_type = UcfConfig.EDIT_TYPE_RENEW entry_vo = entry.exchangeVo( self._timezone) # 既存データをVoに変換 GroupUtils.editVoForSelect( self, entry_vo, is_with_parent_group_info=False, is_with_belong_member_info=False) # データ加工(取得用) UcfUtil.margeHash(vo, entry_vo) # 既存データをVoにコピー else: edit_type = UcfConfig.EDIT_TYPE_NEW # csv_dataからマージ GroupUtils.margeVoFromCsvRecord( self, vo, csv_record, login_operator_entry) GroupUtils.editVoForSelectCsvImport(self, vo) # データ加工(取得用) ######################## # 削除 if edit_type == UcfConfig.EDIT_TYPE_DELETE: # トップグループなら子グループをトップに昇格 if UcfUtil.getHashStr(entry_vo, 'top_group_flag') == 'TOP': GroupUtils.updateBelongMembersTopGroupFlag( self, UcfUtil.getHashStr(entry_vo, 'group_id_lower'), UcfUtil.csvToList(entry_vo['belong_members']), 'SET', operator_id=login_operator_id) # このグループをメイン組織に設定しているユーザのメイン組織をクリア OperatorUtils.removeUsersTargetMainGroup( self, UcfUtil.getHashStr(entry_vo, 'group_id_lower'), operator_id=login_operator_id) # このグループをメイン組織に設定しているグループのメイン組織をクリア GroupUtils.removeGroupsTargetMainGroup( self, UcfUtil.getHashStr(entry_vo, 'group_id_lower'), operator_id=login_operator_id) # このグループを所属メンバーとして持っている親グループからメンバーとして解除 2016.10.24 GroupUtils.removeOneMemberFromBelongGroups( self, UcfUtil.getHashStr(entry_vo, 'group_id_lower'), operator_id=login_operator_id) # このグループ自体を削除 entry.key.delete() # オペレーションログ出力 UCFMDLOperationLog.addLog( login_operator_mail_address, login_operator_unique_id, UcfConfig.SCREEN_GROUP, UcfConfig.OPERATION_TYPE_REMOVE, entry_vo.get('group_id', ''), entry_vo.get('unique_id', ''), login_operator_client_ip, '', is_async=True) deal_type = edit_type # 更新、新規 elif edit_type == UcfConfig.EDIT_TYPE_NEW or edit_type == UcfConfig.EDIT_TYPE_RENEW: # 入力チェック vc = GroupValidator( edit_type, ucffunc.isDelegateOperator(login_operator_entry), login_operator_entry.delegate_management_groups if login_operator_entry is not None else None) vc.validate(self, vo) # 入力エラーがなければ登録処理 if vc.total_count <= 0: # オペレーションログ詳細用に更新フィールドを取得(加工前に比較しておく) if edit_type == UcfConfig.EDIT_TYPE_NEW: is_diff = True diff_for_operation_log = [] else: is_diff, diff_for_operation_log = GroupUtils.isDiff( self, vo, entry_vo) # 差分があるかを判定 is_skip = not is_diff # 加工データ GroupUtils.editVoForRegist(self, vo, entry_vo, edit_type) # 新規登録場合モデルを新規作成 if edit_type == UcfConfig.EDIT_TYPE_NEW: unique_id = UcfUtil.guid() vo['unique_id'] = unique_id entry = sateraito_db.Group( unique_id=unique_id, id=GroupUtils.getKey(self, vo)) #logging.info('vo=' + str(vo)) # Voからモデルにマージ #logging.info(vo) entry.margeFromVo(vo, self._timezone) # 更新日時、更新者の更新 entry.updater_name = login_operator_id entry.date_changed = UcfUtil.getNow() # 新規登録場合ユニークIDを生成 if edit_type == UcfConfig.EDIT_TYPE_NEW: # 作成日時、作成者の更新 entry.creator_name = login_operator_id entry.date_created = UcfUtil.getNow() belong_members = UcfUtil.csvToList( vo['belong_members']) release_members = [] # 今回の変更で所属メンバーから解除されたメンバー for current_belong_member in current_belong_members: if current_belong_member not in belong_members: release_members.append( current_belong_member) # 子グループのトップフラグを更新 GroupUtils.updateBelongMembersTopGroupFlag( self, UcfUtil.getHashStr(vo, 'group_id'), belong_members, 'REMOVE', operator_id=login_operator_id) # 今回このグループからメンバー解除されたグループのトップフラグ更新 GroupUtils.updateBelongMembersTopGroupFlag( self, UcfUtil.getHashStr(vo, 'group_id'), release_members, 'SET', operator_id=login_operator_id) # 登録、更新処理(※トランザクションは制約やデメリットが多いので使用しない) if not is_skip: entry.put() # オペレーションログ出力 operation_log_detail = {} if edit_type == UcfConfig.EDIT_TYPE_RENEW: operation_log_detail[ 'fields'] = diff_for_operation_log UCFMDLOperationLog.addLog( login_operator_mail_address, login_operator_unique_id, UcfConfig.SCREEN_GROUP, UcfConfig.OPERATION_TYPE_ADD if edit_type == UcfConfig.EDIT_TYPE_NEW else UcfConfig.OPERATION_TYPE_MODIFY, vo.get('group_id', ''), vo.get('unique_id', ''), login_operator_client_ip, JSONEncoder().encode(operation_log_detail), is_async=True) deal_type = edit_type else: deal_type = UcfConfig.EDIT_TYPE_SKIP # 入力エラーがあれば else: code = '100' for key, value in vc.msg.iteritems(): #row_log_msg.extend(value) row_log_msg.extend( ['[' + key + ']' + msg for msg in value]) except BaseException, e: self.outputErrorLog(e) code = '500' row_log_msg.append(self._formatLogRecord('system error.'))
for key, value in vc.msg.iteritems(): #row_log_msg.extend(value) row_log_msg.extend( ['[' + key + ']' + msg for msg in value]) except BaseException, e: self.outputErrorLog(e) code = '500' row_log_msg.append(self._formatLogRecord('system error.')) # エラーメッセージ処理 if code != '': #row_log_msg.append(self._formatLogRecord('[row:' + UcfUtil.nvl(record_cnt) + ',code=' + code + ']')) row_log_msg.insert( 0, self._formatLogRecord('[row:' + UcfUtil.nvl(record_cnt) + ',code=' + code + ']')) return deal_type, code, row_log_msg ############################################################ # 一行インポート:ユーザー ############################################################ # def importOneRecordUser(self, csv_record, record_cnt, blob_key, data_key, data_kind, login_operator_unique_id, login_operator_id, login_operator_mail_address, login_operator_client_ip, login_operator_entry): # # titles = UserUtils.getCsvTitles(self) # # code = '' # row_log_msg = [] # entry_vo = None # edit_type = '' # deal_type = ''
def createCookieKeyForAccessKey(cls, helper, operator_unique_id): return 'ACSDID' + UcfUtil.md5(helper.getDeptInfo()['unique_id'] + '|' + operator_unique_id)
def getMsg(self, msgid, ls_param=()): msgid = oem_func.exchangeMessageID(msgid, self._oem_company_code) return UcfMessage.getMessage(UcfUtil.getHashStr(self.getMsgs(), msgid), ls_param)
def createAccessKey(cls, helper, useragent_id, operator_unique_id): return UcfUtil.md5(useragent_id + '|' + operator_unique_id)
def editVoForRegist(cls, helper, vo, entry_vo, edit_type): if edit_type == UcfConfig.EDIT_TYPE_NEW: vo['dept_id'] = UcfUtil.getHashStr(helper.getDeptInfo(), 'dept_id') vo['operator_id_lower'] = UcfUtil.getHashStr(vo, 'operator_id').lower() # アクセス期限の日付と時分からアクセス期限を作成 access_expire = '' if UcfUtil.getHashStr(vo, 'access_expire_date') != '': access_expire = UcfUtil.getHashStr(vo, 'access_expire_date') if UcfUtil.getHashStr(vo, 'access_expire_time') != '': time_ary = UcfUtil.getHashStr(vo, 'access_expire_time').split(':') if len(time_ary) >= 2: access_expire = access_expire + ' ' + time_ary[0] + ':' + time_ary[1] + ':00' vo['access_expire'] = access_expire # 承認ステータスが変更された場合はステータス日付を更新 if UcfUtil.getHashStr(vo, 'approval_status') != UcfUtil.getHashStr(entry_vo, 'approval_status'): vo['approval_status_date'] = UcfUtil.nvl(UcfUtil.getNowLocalTime(helper._timezone)) vo['approval_operator_id'] = UcfUtil.nvl(helper.getLoginID())
def getServerToken(open_api_id, server_id, priv_key, no_memcache_use=False): lineworksapi_server_token_memcache_key = '' if open_api_id != '' and server_id != '': lineworksapi_server_token_memcache_key = 'script=lineworksapi_server_token&open_api_id=' + open_api_id + '&server_id=' + server_id logging.info(lineworksapi_server_token_memcache_key) if not no_memcache_use and lineworksapi_server_token_memcache_key != '': server_token = memcache.get(lineworksapi_server_token_memcache_key) if server_token is not None: logging.info('retrieve server_token from memcache.') return server_token server_token = '' expires_in = 0 process_cnt = 0 MAX_RETRY_CNT = 0 while True: try: # 1.JWT 生成 - RFC-7519 jwt_header = { 'typ': 'JWT', 'alg': 'RS256', } jwt_payload = { 'iss': server_id, 'iat': sateraito_func.datetimeToEpoch(UcfUtil.add_minutes(datetime.datetime.now(), -5)), 'exp': sateraito_func.datetimeToEpoch(UcfUtil.add_minutes(datetime.datetime.now(), 50)), # MAX3600秒の範囲であること } jwt_header_str = json.JSONEncoder().encode(jwt_header) jwt_header_enc = base64.urlsafe_b64encode(jwt_header_str) jwt_payload_str = json.JSONEncoder().encode(jwt_payload) jwt_payload_enc = base64.urlsafe_b64encode(jwt_payload_str) # 2.JWT 電子署名(signature) - RFC-7515 signature_source = str(jwt_header_enc + '.' + jwt_payload_enc) rsa_private_key = RSA.importKey(priv_key, passphrase='') logging.info(process_cnt) signer = PKCS1_v1_5.new(rsa_private_key) digest = SHA256.new() digest.update(signature_source) signature = signer.sign(digest) jwt_sig_enc = base64.urlsafe_b64encode(signature) logging.info(jwt_sig_enc) assertion_jwt = jwt_header_enc + '.' + jwt_payload_enc + '.' + jwt_sig_enc logging.info(assertion_jwt) # 3.LINE WORKS 認証サーバーへの Token リクエスト - RFC-7523 form_fields = {} form_fields['grant_type'] = 'urn:ietf:params:oauth:grant-type:jwt-bearer' form_fields['assertion'] = assertion_jwt form_data = urllib.urlencode(form_fields) request_url = 'https://authapi.worksmobile.com/b/%s/server/token' % (open_api_id) deadline = 10 headers = { 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', } logging.info('urlfetch:' + request_url) result = urlfetch.fetch(url=request_url, method=urlfetch.POST, payload=form_data, follow_redirects=True, deadline=deadline, headers=headers) if result.status_code != 200: raise Exception( 'failed retrieve access_token of LINE WORKS API. status code=' + str(result.status_code)) # 結果サンプル(成功) # {"access_token":"AAAA/arPRL4dqjpKAsg5Vt4r2P/zoP+c2VydmVySWQ+otLu9qTV5IqO0jbXfzaJQ5UZ0EYA22s5sMRySYIKpF0Va8NyW5Vu0okP9dlVqOKiZQczlzVTD0Ull8RI/rlBJN8gWrslOIA1WdOzLyM7byVQSqYL1AY43Ltrc9xqTDDJVMsS/Z8x1drlqeSCBXohIfb+ddk4h/gFPghzRmyG2qPueC4YxWArmCBOpbXH0zVdw/z2bXtgnHr33/XnRnhwx2j9JKx1nNW7JsAK4SIbgqZMxzr9YJ7iWEMJKS0ACsyMBa6Ba7izijUvRG43QZZ4BlSFtaykA7dDcAezaclmCNxRj9G0JszzrKfMK0PEI=","token_type":"Bearer","expires_in":86400} # 結果サンプル(失敗) # {"message":"invalid param","detail":"jwt exp - iat term should be max 3600 seconds","code":"11"} response_json = json.JSONDecoder().decode(result.content) # 結果チェック if response_json.get('access_token', '') == '': # エラー raise Exception('failed retrieve access token of line works api.code=%s message=%s detail=%s' % ( response_json.get('code', ''), response_json.get('message', ''), response_json.get('detail', ''))) server_token = response_json.get('access_token', '') token_type = response_json.get('token_type', '') expires_in = response_json.get('expires_in', 0) # 例:86400 (24時間) break # 成功なのでwhile抜ける except BaseException, e: logging.info(e) if process_cnt <= MAX_RETRY_CNT: logging.warning('[process_cnt=' + str(process_cnt) + ']' + ' '.join(e.args)) else: # logging.exception(e) # pass raise e process_cnt += 1