def _recycle_records(self, batch_commits=False): self.env.flush_all() records_to_clean = [] is_test = bool(config['test_enable'] or config['test_file']) existing_recycle_records = self.env[ 'data_recycle.record'].with_context(active_test=False).search([ ('recycle_model_id', 'in', self.ids) ]) mapped_existing_records = defaultdict(list) for recycle_record in existing_recycle_records: mapped_existing_records[recycle_record.recycle_model_id].append( recycle_record.res_id) for recycle_model in self: rule_domain = ast.literal_eval( recycle_model.domain ) if recycle_model.domain and recycle_model.domain != '[]' else [] if recycle_model.time_field_id and recycle_model.time_field_delta and recycle_model.time_field_delta_unit: if recycle_model.time_field_id.ttype == 'date': now = fields.Date.today() else: now = fields.Datetime.now() delta = relativedelta( **{ recycle_model.time_field_delta_unit: recycle_model.time_field_delta }) rule_domain = expression.AND([ rule_domain, [(recycle_model.time_field_id.name, '<=', now - delta)] ]) model = self.env[recycle_model.res_model_name] if recycle_model.include_archived: model = model.with_context(active_test=False) records_to_recycle = model.search(rule_domain) records_to_create = [ { 'res_id': record.id, 'recycle_model_id': recycle_model.id, } for record in records_to_recycle if record.id not in mapped_existing_records[recycle_model] ] if recycle_model.recycle_mode == 'automatic': for records_to_create_batch in split_every( DR_CREATE_STEP_AUTO, records_to_create): self.env['data_recycle.record'].create( records_to_create_batch).action_validate() if batch_commits and not is_test: # Commit after each batch iteration to avoid complete rollback on timeout as # this can create lots of new records. self.env.cr.commit() else: records_to_clean = records_to_clean + records_to_create for records_to_clean_batch in split_every(DR_CREATE_STEP_MANUAL, records_to_clean): self.env['data_recycle.record'].create(records_to_clean_batch) if batch_commits and not is_test: self.env.cr.commit()
def _refresh_statistics(self): super(SocialLivePostLinkedin, self)._refresh_statistics() accounts = self.env['social.account'].search([('media_type', '=', 'linkedin')]) for account in accounts: linkedin_post_ids = self.env['social.live.post'].sudo().search( [('account_id', '=', account.id)], order='create_date DESC', limit=1000) if not linkedin_post_ids: continue linkedin_post_ids = { post.linkedin_post_id: post for post in linkedin_post_ids } session = requests.Session() # The LinkedIn API limit the query parameters to 4KB # An LinkedIn URN is approximatively 40 characters # So we keep a big margin and we split over 50 LinkedIn posts for batch_linkedin_post_ids in tools.split_every( 50, linkedin_post_ids): endpoint = url_join( self.env['social.media']._LINKEDIN_ENDPOINT, 'organizationalEntityShareStatistics?shares=List(%s)' % ','.join(map(quote, batch_linkedin_post_ids))) response = session.get( endpoint, params={ 'q': 'organizationalEntity', 'organizationalEntity': account.linkedin_account_urn, 'count': 50 }, headers=account._linkedin_bearer_headers(), timeout=10) if response.status_code != 200 or 'elements' not in response.json( ): account.sudo().is_media_disconnected = True break for stats in response.json()['elements']: urn = stats.get('share') stats = stats.get('totalShareStatistics') if not urn or not stats or urn not in batch_linkedin_post_ids: continue linkedin_post_ids[urn].update({ 'engagement': stats.get('likeCount', 0) + stats.get('commentCount', 0) + stats.get('shareCount', 0) })
def _split_messages(self): """ 拆分消息 """ groups = defaultdict(list) for record_ids in groups.items(): for message_batch in tools.split_every(record_ids): yield message_batch
def _notify_members(self, message): """Send the given message to all members of the mail group (except the author).""" self.ensure_one() if message.mail_group_id != self: raise UserError(_('The group of the message do not match.')) if not message.mail_message_id.reply_to: _logger.error('The alias or the catchall domain is missing, group might not work properly.') base_url = self.get_base_url() body = self.env['mail.render.mixin']._replace_local_links(message.body) access_token = self._generate_group_access_token() mail_values = [] for batch_members in tools.split_every(GROUP_SEND_BATCH_SIZE, self.member_ids): for member in batch_members: if member.email_normalized == message.email_from_normalized: # Do not send the email to his author continue # SMTP headers related to the subscription email_url_encoded = urls.url_quote(member.email_normalized) headers = { 'List-Archive': f'<{base_url}/groups/{slug(self)}>', 'List-Subscribe': f'<{base_url}/groups?email={email_url_encoded}>', 'List-Unsubscribe': f'<{base_url}/groups?unsubscribe&email={email_url_encoded}>', } # Add the footer (member specific) in the body template_values = { 'mailto': f'{self.alias_name}@{self.alias_domain}', 'group_url': f'{base_url}/groups/{slug(self)}', 'unsub_url': f'{base_url}/groups?unsubscribe&group_id={self.id}&token={access_token}' } template = self.env.ref('mail_group.mail_group_footer') footer = template._render(template_values, engine='ir.qweb', minimal_qcontext=True) member_body = tools.append_content_to_html(body, footer) mail_values.append({ 'auto_delete': True, 'attachment_ids': message.attachment_ids.ids, 'body_html': member_body, 'email_from': message.email_from, 'email_to': member.email, 'headers': json.dumps(headers), 'mail_message_id': message.mail_message_id.id, 'message_id': message.mail_message_id.message_id, 'model': 'mail.group', 'reply_to': message.mail_message_id.reply_to, 'res_id': self.id, 'subject': message.subject, }) if mail_values: self.env['mail.mail'].sudo().create(mail_values)
def _split_by_mail_configuration(self): """Group the <mail.mail> based on their "email_from" and their "mail_server_id". The <mail.mail> will have the "same sending configuration" if they have the same mail server or the same mail from. For performance purpose, we can use an SMTP session in batch and therefore we need to group them by the parameter that will influence the mail server used. The same "sending configuration" may repeat in order to limit batch size according to the `mail.session.batch.size` system parameter. Return iterators over mail_server_id, email_from, Records<mail.mail>.ids """ mail_values = self.read(['id', 'email_from', 'mail_server_id']) # First group the <mail.mail> per mail_server_id and per email_from group_per_email_from = defaultdict(list) for values in mail_values: mail_server_id = values['mail_server_id'][0] if values[ 'mail_server_id'] else False group_per_email_from[(mail_server_id, values['email_from'])].append(values['id']) # Then find the mail server for each email_from and group the <mail.mail> # per mail_server_id and smtp_from mail_servers = self.env['ir.mail_server'].sudo().search( [], order='sequence') group_per_smtp_from = defaultdict(list) for (mail_server_id, email_from), mail_ids in group_per_email_from.items(): if not mail_server_id: mail_server, smtp_from = self.env[ 'ir.mail_server']._find_mail_server( email_from, mail_servers) mail_server_id = mail_server.id if mail_server else False else: smtp_from = email_from group_per_smtp_from[(mail_server_id, smtp_from)].extend(mail_ids) sys_params = self.env['ir.config_parameter'].sudo() batch_size = int(sys_params.get_param('mail.session.batch.size', 1000)) for (mail_server_id, smtp_from), record_ids in group_per_smtp_from.items(): for batch_ids in tools.split_every(batch_size, record_ids): yield mail_server_id, smtp_from, batch_ids
def _split_by_server(self): """Returns an iterator of pairs `(mail_server_id, record_ids)` for current recordset. The same `mail_server_id` may repeat in order to limit batch size according to the `mail.session.batch.size` system parameter. """ groups = defaultdict(list) # Turn prefetch OFF to avoid MemoryError on very large mail queues, we only care # about the mail server ids in this case. for mail in self.with_context(prefetch_fields=False): groups[mail.mail_server_id.id].append(mail.id) sys_params = self.env['ir.config_parameter'].sudo() batch_size = int(sys_params.get_param('mail.session.batch.size', 1000)) for server_id, record_ids in groups.items(): for mail_batch in tools.split_every(batch_size, record_ids): yield server_id, mail_batch
def _compute_allowed_picking_ids(self): # Backport of f329de26: allowed_picking_ids is useless, view_stock_landed_cost_form no longer uses it, # the field and its compute are kept since this is a stable version. Still, this compute has been made # more resilient to MemoryErrors. valued_picking_ids_per_company = defaultdict(list) if self.company_id: self.env.cr.execute( """SELECT sm.picking_id, sm.company_id FROM stock_move AS sm INNER JOIN stock_valuation_layer AS svl ON svl.stock_move_id = sm.id WHERE sm.picking_id IS NOT NULL AND sm.company_id IN %s GROUP BY sm.picking_id, sm.company_id""", [tuple(self.company_id.ids)]) for res in self.env.cr.fetchall(): valued_picking_ids_per_company[res[1]].append(res[0]) for cost in self: n = 5000 cost.allowed_picking_ids = valued_picking_ids_per_company[ cost.company_id.id][:n] for ids_chunk in tools.split_every( n, valued_picking_ids_per_company[cost.company_id.id][n:]): cost.allowed_picking_ids = [(4, id_) for id_ in ids_chunk]
def _firebase_send_message_from_iap(self, data, visitors): social_iap_endpoint = self.env['ir.config_parameter'].sudo().get_param( 'social.social_iap_endpoint', self.env['social.media']._DEFAULT_SOCIAL_IAP_ENDPOINT) batch_size = 100 tokens = visitors.mapped('push_token') data.update({ 'db_uuid': self.env['ir.config_parameter'].sudo().get_param('database.uuid') }) for tokens_batch in tools.split_every(batch_size, tokens, piece_maker=list): batch_data = dict(data) batch_data['tokens'] = tokens_batch iap_tools.iap_jsonrpc(url_join( social_iap_endpoint, '/iap/social_push_notifications/firebase_send_message'), params=batch_data) return []
def _procure_orderpoint_confirm(self, use_new_cursor=False, company_id=None, raise_user_error=True): """ Create procurements based on orderpoints. :param bool use_new_cursor: if set, use a dedicated cursor and auto-commit after processing 1000 orderpoints. This is appropriate for batch jobs only. """ self = self.with_company(company_id) orderpoints_noprefetch = self.read(['id']) orderpoints_noprefetch = [orderpoint['id'] for orderpoint in orderpoints_noprefetch] for orderpoints_batch in split_every(1000, orderpoints_noprefetch): if use_new_cursor: cr = registry(self._cr.dbname).cursor() self = self.with_env(self.env(cr=cr)) orderpoints_batch = self.env['stock.warehouse.orderpoint'].browse(orderpoints_batch) # ensure that qty_* which depends on datetime.now() are correctly # recomputed orderpoints_batch._compute_qty_to_order() orderpoints_exceptions = [] while orderpoints_batch: procurements = [] for orderpoint in orderpoints_batch: if float_compare(orderpoint.qty_to_order, 0.0, precision_rounding=orderpoint.product_uom.rounding) == 1: date = datetime.combine(orderpoint.lead_days_date, time.min) values = orderpoint._prepare_procurement_values(date=date) procurements.append(self.env['procurement.group'].Procurement( orderpoint.product_id, orderpoint.qty_to_order, orderpoint.product_uom, orderpoint.location_id, orderpoint.name, orderpoint.name, orderpoint.company_id, values)) try: with self.env.cr.savepoint(): self.env['procurement.group'].with_context(from_orderpoint=True).run(procurements, raise_user_error=raise_user_error) except ProcurementException as errors: for procurement, error_msg in errors.procurement_exceptions: orderpoints_exceptions += [(procurement.values.get('orderpoint_id'), error_msg)] failed_orderpoints = self.env['stock.warehouse.orderpoint'].concat(*[o[0] for o in orderpoints_exceptions]) if not failed_orderpoints: _logger.error('Unable to process orderpoints') break orderpoints_batch -= failed_orderpoints except OperationalError: if use_new_cursor: cr.rollback() continue else: raise else: orderpoints_batch._post_process_scheduler() break # Log an activity on product template for failed orderpoints. for orderpoint, error_msg in orderpoints_exceptions: existing_activity = self.env['mail.activity'].search([ ('res_id', '=', orderpoint.product_id.product_tmpl_id.id), ('res_model_id', '=', self.env.ref('product.model_product_template').id), ('note', '=', error_msg)]) if not existing_activity: orderpoint.product_id.product_tmpl_id.activity_schedule( 'mail.mail_activity_data_warning', note=error_msg, user_id=orderpoint.product_id.responsible_id.id or SUPERUSER_ID, ) if use_new_cursor: cr.commit() cr.close() return {}
def _split_batch(self): batch_size = int(self.env['ir.config_parameter'].sudo().get_param( 'sms.session.batch.size', 500)) for sms_batch in tools.split_every(batch_size, self.ids): yield sms_batch
def _notify_members(self, message): """Send the given message to all members of the mail group (except the author).""" self.ensure_one() if message.mail_group_id != self: raise UserError(_('The group of the message do not match.')) if not message.mail_message_id.reply_to: _logger.error( 'The alias or the catchall domain is missing, group might not work properly.' ) base_url = self.get_base_url() body = self.env['mail.render.mixin']._replace_local_links(message.body) access_token = self._generate_group_access_token() mail_values = [] # Email added in a dict to be sure to send only once the email to each address member_emails = { email_normalize(member.email): member.email for member in self.member_ids } for batch_email_member in tools.split_every(GROUP_SEND_BATCH_SIZE, member_emails.items()): for email_member_normalized, email_member in batch_email_member: if email_member_normalized == message.email_from_normalized: # Do not send the email to his author continue # SMTP headers related to the subscription email_url_encoded = urls.url_quote(email_member) headers = { **self._notify_email_header_dict(), 'List-Archive': f'<{base_url}/groups/{slug(self)}>', 'List-Subscribe': f'<{base_url}/groups?email={email_url_encoded}>', 'List-Unsubscribe': f'<{base_url}/groups?unsubscribe&email={email_url_encoded}>', 'Precedence': 'list', 'X-Auto-Response-Suppress': 'OOF', # avoid out-of-office replies from MS Exchange } if self.alias_name and self.alias_domain: headers.update({ 'List-Id': f'<{self.alias_name}.{self.alias_domain}>', 'List-Post': f'<mailto:{self.alias_name}@{self.alias_domain}>', 'X-Forge-To': f'"{self.name}" <{self.alias_name}@{self.alias_domain}>', }) if message.mail_message_id.parent_id: headers[ 'In-Reply-To'] = message.mail_message_id.parent_id.message_id # Add the footer (member specific) in the body template_values = { 'mailto': f'{self.alias_name}@{self.alias_domain}', 'group_url': f'{base_url}/groups/{slug(self)}', 'unsub_label': f'{base_url}/groups?unsubscribe', 'unsub_url': f'{base_url}/groups?unsubscribe&group_id={self.id}&token={access_token}&email={email_url_encoded}', } template = self.env.ref('mail_group.mail_group_footer') footer = template._render(template_values, engine='ir.qweb', minimal_qcontext=True) member_body = tools.append_content_to_html(body, footer, plaintext=False) mail_values.append({ 'auto_delete': True, 'attachment_ids': message.attachment_ids.ids, 'body_html': member_body, 'email_from': message.email_from, 'email_to': email_member, 'headers': json.dumps(headers), 'mail_message_id': message.mail_message_id.id, 'message_id': message.mail_message_id.message_id, 'model': 'mail.group', 'reply_to': message.mail_message_id.reply_to, 'res_id': self.id, 'subject': message.subject, }) if mail_values: self.env['mail.mail'].sudo().create(mail_values)
def _firebase_send_message_from_configuration(self, data, visitors): """ This method now has a dual implementation to handle cases when the firebase_admin python library is not installed / not in the correct version. 1. When firebase_admin is available: Sends messages by batch of 100 (max limit from firebase). Returns a tuple containing: - The matched website.visitors (search_read records). - A list of firebase_admin.messaging.BatchResponse to be handled by the caller. 2. When firebase_admin is NOT available. Sends messages one by one using the firebase REST API. (Which is what firebase_admin does under the hood anyway) It requires a bearer token for authentication that we obtain using the google_auth library. Returns a tuple containing: - The matched website.visitors (search_read records). - An empty list. """ if not visitors: return [], [] if not self.firebase_admin_key_file: raise UserError( _("Firebase Admin Key File is missing from the configuration.") ) results = [] tokens = visitors.mapped('push_token') if firebase_admin and self._check_firebase_version(): self._init_firebase_app() batch_size = 100 for tokens_batch in tools.split_every(batch_size, tokens, piece_maker=list): firebase_message = messaging.MulticastMessage( data=data, tokens=tokens_batch) results.append(messaging.send_multicast(firebase_message)) elif service_account: firebase_data = json.loads( base64.b64decode(self.firebase_admin_key_file).decode()) firebase_credentials = service_account.Credentials.from_service_account_info( firebase_data, scopes=['https://www.googleapis.com/auth/firebase.messaging']) firebase_credentials.refresh(google_requests.Request()) auth_token = firebase_credentials.token for token in tokens: requests.post( f'https://fcm.googleapis.com/v1/projects/{firebase_data["project_id"]}/messages:send', json={'message': { 'data': data, 'token': token }}, headers={'authorization': f'Bearer {auth_token}'}, timeout=10) else: raise UserError( _('You have to either install "firebase_admin>=2.17.0" or ' '"google_auth>=1.18.0" to be able to send push ' 'notifications.')) return tokens, results
def _procure_orderpoint_confirm(self, use_new_cursor=False, company_id=None, raise_user_error=True): """ Create procurements based on orderpoints. :param bool use_new_cursor: if set, use a dedicated cursor and auto-commit after processing 1000 orderpoints. This is appropriate for batch jobs only. """ self = self.with_company(company_id) for orderpoints_batch_ids in split_every(1000, self.ids): if use_new_cursor: cr = registry(self._cr.dbname).cursor() self = self.with_env(self.env(cr=cr)) orderpoints_batch = self.env['stock.warehouse.orderpoint'].browse( orderpoints_batch_ids) all_orderpoints_exceptions = [] while orderpoints_batch: procurements = [] for orderpoint in orderpoints_batch: origins = orderpoint.env.context.get('origins', {}).get( orderpoint.id, False) if origins: origin = '%s - %s' % (orderpoint.display_name, ','.join(origins)) else: origin = orderpoint.name if float_compare(orderpoint.qty_to_order, 0.0, precision_rounding=orderpoint.product_uom. rounding) == 1: date = orderpoint._get_orderpoint_procurement_date() values = orderpoint._prepare_procurement_values( date=date) procurements.append( self.env['procurement.group'].Procurement( orderpoint.product_id, orderpoint.qty_to_order, orderpoint.product_uom, orderpoint.location_id, orderpoint.name, origin, orderpoint.company_id, values)) try: with self.env.cr.savepoint(): self.env['procurement.group'].with_context( from_orderpoint=True).run( procurements, raise_user_error=raise_user_error) except ProcurementException as errors: orderpoints_exceptions = [] for procurement, error_msg in errors.procurement_exceptions: orderpoints_exceptions += [ (procurement.values.get('orderpoint_id'), error_msg) ] all_orderpoints_exceptions += orderpoints_exceptions failed_orderpoints = self.env[ 'stock.warehouse.orderpoint'].concat( *[o[0] for o in orderpoints_exceptions]) if not failed_orderpoints: _logger.error('Unable to process orderpoints') break orderpoints_batch -= failed_orderpoints except OperationalError: if use_new_cursor: cr.rollback() continue else: raise else: orderpoints_batch._post_process_scheduler() break # Log an activity on product template for failed orderpoints. for orderpoint, error_msg in all_orderpoints_exceptions: existing_activity = self.env['mail.activity'].search([ ('res_id', '=', orderpoint.product_id.product_tmpl_id.id), ('res_model_id', '=', self.env.ref('product.model_product_template').id), ('note', '=', error_msg) ]) if not existing_activity: orderpoint.product_id.product_tmpl_id.activity_schedule( 'mail.mail_activity_data_warning', note=error_msg, user_id=orderpoint.product_id.responsible_id.id or SUPERUSER_ID, ) if use_new_cursor: cr.commit() cr.close() _logger.info( "A batch of %d orderpoints is processed and committed", len(orderpoints_batch_ids)) return {}