class BindingDataMixin(models.AbstractModel): _name = 'binding.data.mixin' updated = fields.Boolean() data = fields.Serialized() def _synchronize_magento_record(self, backend_id): backend = self.env['magento.backend'].browse(backend_id) with backend.work_on(self._name) as work: adapter = work.component(usage='backend.adapter') records = self.search([ ('backend_id', '=', backend_id), ('updated', '=', False), ]) total = len(records) missing_ids = [] for idx, record in enumerate(records): if idx % 10 == 0: _logger.info('progress {} / {}'.format(idx, total)) try: data = {} for storeview in backend.mapped( 'website_ids.store_ids.storeview_ids'): data[storeview.code] = adapter.read( record.external_id, storeview_id=storeview.external_id) except Exception as e: missing_ids.append(record.id) record.write({'updated': True, 'data': data}) self._cr.commit() return missing_ids
class ShopinvaderVariant(models.Model): _inherit = ['shopinvader.variant', 'se.binding'] _name = 'shopinvader.variant' _description = 'Shopinvader Variant' index_id = fields.Many2one(compute="_compute_index", store=True, required=False) price = fields.Serialized(compute='_compute_price', string='Shopinvader Price') def _compute_price(self): for record in self: record.price = {} for role in record.backend_id.role_ids: fposition = role.fiscal_position_ids if len(fposition) > 0: fposition = fposition[0] record.price[role.code] = record._get_price( role.pricelist_id, fposition) @api.depends('lang_id', 'backend_id.se_backend_id') def _compute_index(self): for record in self: se_backend = record.backend_id.se_backend_id if se_backend: record.index_id = self.env['se.index'].search([ ('backend_id', '=', se_backend.id), ('lang_id', '=', record.lang_id.id), ('model_id.model', '=', record._name), ])
class BindingDataMixin(models.AbstractModel): _name = 'binding.data.mixin' data = fields.Serialized() @api.model def fields_get(self, allfields=None, attributes=None): res = super(BindingDataMixin, self).fields_get(allfields=allfields, attributes=attributes) if 'data' in res: res['data']['translate'] = True return res def write(self, vals): if 'data' in vals: self.ensure_one() lang = self._context.get('lang', 'en_US') data = self.data data[lang] = vals['data'] vals['data'] = data return super(BindingDataMixin, self).write(vals) def create(self, vals): if 'data' in vals: lang = self._context.get('lang', 'en_US') vals['data'] = {lang: vals['data']} return super(BindingDataMixin, self).write(vals)
class tms_host_group(models.Model): _name = "tms.host.group" _description = "Host Group" # Columns name = fields.Char('Name', size=256, required=True) config = fields.Serialized('Config', help="Config variable used by Ansible")
class Data(models.Model): _name = 'pubsub.data' ref = fields.Id(string='Ref', required=True) type = fields.Char(string='Type', required=True) data = fields.Serialized(string='Data') def _get_headers(self, ids): record = self.env['pubsub.header'].search_read([('id', 'in', ids)]) headers = {} # Iterate over the headers and add them to the object for header in record: headers[header.get('field_id')[1]] = header.get('value') # Set `Content-Type` to `application/json` regarding of what is set in the settings headers['Content-Type'] = 'application/json' return headers def _in_type_list(self, type, ids): # Check if the type is associated with the subscription length = self.env['pubsub.type'].search_count([('id', 'in', ids), ('type', '=', type)]) return length > 0 def _publish(self, vals): # Copy the `vals` object and change `ref` to `id` data = copy.copy(vals) data['id'] = data['ref'] del data['ref'] result = self.env['pubsub.subscription'].search_read([]) for r in result: types = r.get('type_ids') if len(types) == 0 or self._in_type_list(data['type'], types): _logger.info('invoke') url = r.get('url') headers = self._get_headers(r.get('header_ids')) requests.post(url, headers=headers, data=json.dumps(data)) @api.model def create(self, vals): rec = super(Data, self).create(vals) # Publish to all the subscriptions self._publish(vals) return rec
class Sparse(models.Model): _name = 'test_new_api.sparse' data = fields.Serialized() boolean = fields.Boolean(sparse='data') integer = fields.Integer(sparse='data') float = fields.Float(sparse='data') char = fields.Char(sparse='data') selection = fields.Selection([('one', 'One'), ('two', 'Two')], sparse='data') partner = fields.Many2one('res.partner', sparse='data')
class TestSparse(models.TransientModel): _name = 'sparse_fields.test' data = fields.Serialized() boolean = fields.Boolean(sparse='data') integer = fields.Integer(sparse='data') float = fields.Float(sparse='data') char = fields.Char(sparse='data') selection = fields.Selection([('one', 'One'), ('two', 'Two')], sparse='data') partner = fields.Many2one('res.partner', sparse='data')
class ProductTemplate(models.Model): _inherit = 'product.template' price_quantity_tiers = fields.Serialized( compute='_compute_price_quantity_tiers', help='Unit prices by pricelist and minimum quantity', ) @api.multi def _compute_price_quantity_tiers(self): pricelists = self.env['product.pricelist'].search([('id', '>', 0)]) for record in self: if not record.product_variant_ids: continue results = {} for pricelist in pricelists: pricelist_items = self.env['product.pricelist.item'].search([ ('pricelist_id', '=', pricelist.id), ('product_tmpl_id', '=', record.id), '|', ('date_start', '<=', fields.Date.today()), ('date_start', '=', False), '|', ('date_end', '>=', fields.Date.today()), ('date_end', '=', False), ]) min_quantities = set([]) for price in pricelist_items: min_quantities.add( 1 if price.min_quantity < 1 else price.min_quantity ) min_quantities.add(1) list_results = set([]) for min_qty in min_quantities: res = pricelist.price_rule_get( record.product_variant_ids[0].id, min_qty ) if res[pricelist.id][1] in pricelist_items.ids \ or min_qty == 1: list_results.add((min_qty, res[pricelist.id][0])) list_results = sorted(list(list_results)) # A tier with min qty of 1 should not exist on its own if len(list_results) == 1 and list_results[0][0] == 1: list_results = [] results[pricelist.id] = list_results record.price_quantity_tiers = results
class TmsDockerRepoTags(models.Model): _name = "tms.docker.repo.tags" _description = "Tms docker repository tags" _order = "create_date desc" name = fields.Char('Name') create_date = fields.Datetime('Creation Date') size = fields.Float('Size (MB)') active = fields.Boolean('Active', default=True) tms_docker_repo_id = fields.Many2one('tms.docker.repo', string='Tms docker repository') db_file = fields.Char('Database file') meta_data = fields.Serialized('Meta Data', readonly=True)
class ShopinvaderVariant(models.Model): _inherit = 'shopinvader.variant' hierarchical_categories = fields.Serialized( compute='_compute_shopinvader_category', string='Hierarchical Categories') def _compute_shopinvader_category(self): super(ShopinvaderVariant, self)._compute_shopinvader_category() def get_full_name(categ): result = [] while categ: result.insert(0, categ.name) categ = categ.parent_id return ' > '.join(result) for record in self: record.hierarchical_categories = {} for categ in record.shopinvader_categ_ids: record.hierarchical_categories['lvl%s' % categ.level] =\ get_full_name(categ.record_id)
class ShopinvaderVariant(models.Model): _inherit = 'shopinvader.variant' rating = fields.Serialized(compute='_compute_rating', string='Rating') def _compute_rating(self): for record in self: reviews = [] distribution = {1: 0, 2: 0, 3: 0, 4: 0, 5: 0} for rating in record.rating_ids: if rating.state == 'approved': reviews.append({ 'nickname': rating.nickname, 'name': rating.name, 'comment': rating.comment, 'rating': rating.rating, 'product_code': rating.product_id.default_code, }) distribution[rating.rating] += 1 if reviews: count = len(reviews) average = sum([c['rating'] for c in reviews]) / count record.rating = { 'reviews': reviews, 'summary': { 'average': average, 'count': count, 'distribution': distribution, } } else: record.rating = {}
class KeychainBackend(models.AbstractModel): _name = 'keychain.backend' _backend_name = None password = fields.Char(compute="_compute_password", inverse="_inverse_password", required=True) data = fields.Serialized(compute="_compute_keychain", inverse="_inverse_keychain") def _get_technical_name(self): return '%s,%s' % (self._name, self.id) def _get_existing_keychain(self): self.ensure_one() return self.env['keychain.account'].retrieve([ ('namespace', '=', self._backend_name), ('technical_name', '=', self._get_technical_name()) ]) def _prepare_keychain(self): env = config.get('running_env') return { 'name': "%s %s" % (self.name, env), 'technical_name': self._get_technical_name(), 'namespace': self._backend_name, 'environment': env, } def _get_keychain_account(self): self.ensure_one() account = self._get_existing_keychain() if not account: vals = self._prepare_keychain() account = self.env['keychain.account'].create(vals) return account def _inverse_password(self): for record in self: account = self._get_keychain_account() if record.password and record.password != '******': account.clear_password = record.password def _compute_password(self): for record in self: account = record._get_existing_keychain() if account and account.password: record.password = "******" else: record.password = "" def _inverse_keychain(self): for record in self: account = record._get_keychain_account() account.data = account._serialize_data(record.data) def _compute_keychain(self): for record in self: account = record._get_existing_keychain() if account: record.data = account.get_data() else: record.data = {}
class BaseFieldSerializedTestModel(models.Model): _name = 'base.field.serialized.test.model' serialized = fields.Serialized('Serialized')
class ShopinvaderVariant(models.Model): _name = 'shopinvader.variant' _description = 'Shopinvader Variant' _inherits = { 'shopinvader.product': 'shopinvader_product_id', 'product.product': 'record_id'} default_code = fields.Char( related='record_id.default_code', readonly=True) shopinvader_product_id = fields.Many2one( 'shopinvader.product', required=True, ondelete='cascade') record_id = fields.Many2one( 'product.product', required=True, ondelete='cascade') object_id = fields.Integer( compute='_compute_object_id', store=True) shopinvader_categ_ids = fields.Many2many( comodel_name='shopinvader.category', compute='_compute_shopinvader_category', string='Shopinvader Categories') images = fields.Serialized( compute='_compute_image', string='Shopinvader Image') variant_count = fields.Integer( related='product_variant_count') attributes = fields.Serialized( compute='_compute_attributes', string='Shopinvader Attributes') main = fields.Boolean( compute='_compute_main_product') redirect_url_key = fields.Serialized( compute='_compute_redirect_url_key', string='Redirect Url Keys') @api.depends('record_id') def _compute_object_id(self): for record in self: record.object_id = record.record_id.id def _compute_redirect_url_key(self): for record in self: res = [] for url in record.redirect_url_key_ids: res.append(url.url_key) record.redirect_url_key = res def _get_categories(self): self.ensure_one() return self.categ_id def _compute_shopinvader_category(self): for record in self: ids = [] categs = record._get_categories() for categ in categs: parents = self.env['shopinvader.category'].search([ ('parent_left', '<=', categ.parent_left), ('parent_right', '>=', categ.parent_right), ('backend_id', '=', record.backend_id.id), ]) ids += parents.ids record.shopinvader_categ_ids = ids def _compute_image(self): for record in self: images = [] for image in record.record_id.image_ids: res = {'original': image.url} for resize in record.backend_id.product_image_resize_ids: res[resize.key] = \ image.get_thumbnail_from_resize(resize).url images.append(res) record.images = images def _compute_attributes(self): for record in self: attributes = dict() for att_value in record.attribute_value_ids: sanitized_key = sanitize_attr_name(att_value.attribute_id) attributes[sanitized_key] = att_value.name record.attributes = attributes def _get_price(self, pricelist, fposition): self.ensure_one() return self._get_price_per_qty(1, pricelist, fposition) def _extract_price_from_onchange(self, pricelist, onchange_vals): tax_ids = onchange_vals['value']['tax_id'] tax_included = False if tax_ids: # Becarefull we only take in account the first tax for # determinating if the price is tax included or tax excluded # This may not work in multi-tax case tax = self.env['account.tax'].browse(tax_ids[0]) tax_included = tax.price_include prec = self.env['decimal.precision'].precision_get('Product') return { 'value': round(onchange_vals['value']['price_unit'], prec), 'tax_included': tax_included, } def _get_price_per_qty(self, qty, pricelist, fposition): # Get an partner in order to avoid the raise in the onchange # the partner selected will not impacted the result partner = self.env['res.partner'].search([], limit=1) result = self.env['sale.order.line'].product_id_change( pricelist.id, self.record_id.id, qty=qty, fiscal_position=fposition.id, partner_id=partner.id) return self._extract_price_from_onchange(pricelist, result) def _compute_main_product(self): for record in self: if record.record_id \ == record.product_tmpl_id.product_variant_ids[0]: record.main = True else: record.main = False
class tms_host(models.Model): _name = "tms.host" _description = "Host" _order = 'operating_system_id' _inherit = ['mail.thread'] @api.one @api.onchange('physical_host_id', 'container_id') def on_change_physical_host_container(self): """ F#13948 - auto compute ssh port for host (virtual machine) """ if self.virtual_machine: self.ip = self.env['ir.config_parameter'].get_param( 'default_ip_host', False) + str(self.container_id) temp = self.config.copy() node_add = self.physical_host_id and \ self.physical_host_id.host_address or False if node_add: if 'eu' in node_add: temp['timezone'] = 'Europe/Berlin' self.config = temp else: temp['timezone'] = 'Asia/Ho_Chi_Minh' self.config = temp if self.virtual_machine and \ self.virtualization_id and \ self.virtualization_id.name == 'Proxmox': suggest_ssh_port = '{AREA_CODE}{NODE_NUMBER}{CONTAINER_ID}' if self.physical_host_id and \ self.physical_host_id.name: host_name_prefix = self.physical_host_id.name[:2] host_name_suffix = self.physical_host_id.name[-2:] if host_name_prefix == 'eu': suggest_ssh_port = suggest_ssh_port.replace( '{AREA_CODE}', '1') elif host_name_prefix == 'vn': suggest_ssh_port = suggest_ssh_port.replace( '{AREA_CODE}', '2') if host_name_suffix.isnumeric(): suggest_ssh_port = suggest_ssh_port.replace( '{NODE_NUMBER}', str(int(host_name_suffix))) if self.container_id >= 0: suggest_ssh_port = suggest_ssh_port.replace( '{CONTAINER_ID}', str(self.container_id)) self.port = '{' not in suggest_ssh_port and \ suggest_ssh_port or '22' @api.multi def _get_num_of_vm(self): for host in self: host.num_VM = len(host.virtual_host_ids) @api.multi @api.depends('name', 'virtual_machine', 'physical_host_id') def _get_physical_host(self): """ get physical host for virtual machine if host is virtual machine then get Physical Host if host is not virtual machine then get name of this Host """ # looking for all virtual host belong to this physical_host and # including it hosts = self.search(["|", ('physical_host_id', 'in', self._ids), ('id', 'in', self._ids)]) for host in hosts: if host.virtual_machine: host.physical_host = host.physical_host_id.name else: host.physical_host = host.name @api.model def get_grouped_host(self): """ Retrieve host grouped by physical host """ self._cr.execute(''' SELECT ph.name, p.name, p.host_address, p.port, p.config FROM tms_host AS h JOIN tms_host AS ph ON h.physical_host_id = ph.id GROUP BY ph.name ''') @api.model def get_authorized_keys(self): """ Retrieve all authorized keys for hosts Note: used by configuration manager tools (Ansible) Format: results = { 'host-name': { 'admin': [list of admin users' keys], 'user': [list of host users' keys] } } """ # Read private token from cofig file # Request to git lab with number users is 400 results = {} private_token_key_config = config.get('private_token') or '' if private_token_key_config: # Get all user from gitlab payload = { 'private_token': private_token_key_config, 'per_page': 400 } r = requests.get( "https://gitlab.trobz.com/api/v3/users", data=payload, verify=False ) gitlab_users = json.loads(r.text) or [] # First, get all ssh keys from Admin users # and Sysadmin users who have full access. self._cr.execute(''' SELECT login FROM res_users WHERE is_sysadmin = TRUE AND has_full_sysadmin_access = TRUE AND active = TRUE AND id != 1 ORDER BY id; ''') admin_usernames = map(lambda r: r[0], self._cr.fetchall()) host_users = [] admin_keys = [] for gitlab_user in gitlab_users: if gitlab_user['state'] == 'active': req = requests.get( "https://gitlab.trobz.com/api/v3/users/" + str(gitlab_user['id']) + "/keys", data=payload, verify=False ) ssh_vals = json.loads(req.text) or [] ssh_keys = [] for val in ssh_vals: if 'key' in val: ssh_keys.append(val['key']) if gitlab_user['username'] in admin_usernames: admin_keys.append(val['key']) host_users.append({ 'login': gitlab_user['username'], 'ssh_key': ssh_keys }) self._cr.execute(''' SELECT h.name, u.login, u.is_sysadmin FROM tms_host AS h LEFT JOIN host_users_rel AS hu ON hu.host_id = h.id LEFT JOIN res_users AS u ON u.id = hu.user_id WHERE (u.is_trobz_member = TRUE AND u.active = TRUE) OR u.login IS NULL AND h.state != 'deleted' ORDER BY h.name, u.id; ''') # result[0] => host name # result[1] => user login name # result[2] => is_sysadmin for result in self._cr.fetchall(): if result[0] not in results: # All sysadmin users # who have full access are admins on this host. results[result[0]] = {'admin': admin_keys, 'user': []} if result[1]: for host_user in host_users: # All users who are responsible to maintain # odoo instances are users on this host. if result[1] == host_user['login']: results[result[0]]['user'] = list( set(results[result[0]]['user'] + host_user['ssh_key'])) # All sysadmin users who don't have full access, # but are responsible to maintain this host are # admins on this host. if result[2]: results[result[0]]['admin'] = list( set(results[result[0]]['admin'] + host_user['ssh_key'])) else: logging.info("You don't have a token key in the config file!") return results @api.constrains('container_id', 'virtual_machine', 'physical_host_id') def check_container_id(self): ''' For Host (Virtual Machine) inside a Physical Host: - Container ID between 1 and 255 - F#13552 - Unique container_id (only for Hosts that belong to the same Physical Host) ''' for host in self: if host.virtual_machine and host.physical_host_id: if host.container_id < 1 or host.container_id > 255: raise Warning( _('You have to input Container ID between 1 and 255 !') ) # find existing Hosts in Physical Host # using the Container ID existed_hosts = self.search([ ('virtual_machine', '=', True), ('physical_host_id', '=', host.physical_host_id.id), ('container_id', '=', host.container_id), ('id', '!=', host.id), ('state', '!=', 'deleted') ]) if existed_hosts: raise Warning( _('The Container ID should be unique for ' 'Hosts that belong to the same Physical Host! ' 'Container ID %s has been used for Host %s!' % ( host.container_id, existed_hosts[0].name)) ) @api.multi def _get_groups(self): for item in self: if item.group_ids: item.groups = map(lambda group: { 'name': group.name, 'config': group.config}, item.group_ids) else: item.groups = [] # Columns name = fields.Char('Host Name', size=256, required=True, track_visibility='onchange') host_address = fields.Char('Host Address', size=256, required=True, track_visibility='onchange') port = fields.Char('SSH Port', size=256, required=True, track_visibility='onchange') ip = fields.Char( 'IP', size=64, help='For nodes it is the IP address of the node, ' 'for VM it is the Virtual IP of the VM', track_visibility='onchange') internal_ip = fields.Char('Internal IP', size=64) using_pgbouncer = fields.Boolean( string='Using PgBouncer', default=False) pgbouncer_port = fields.Char('PgBouncer Port') operating_system_id = fields.Many2one( 'tms.operating.system', string='Operating System', track_visibility='onchange') # Adjustments of the Hosts service = fields.Char('Services', size=64) group_ids = fields.Many2many( "tms.host.group", "host_host_group_rel", "host_id", "group_id", string='Groups', help="Host group, used by ansible", track_visibility='onchange') groups = fields.Serialized( compute='_get_groups', string='Group list', readonly=True) user_ids = fields.Many2many( "res.users", "host_users_rel", "host_id", "user_id", string='Users', track_visibility='onchange', help="User with permissions to deploy on the host.", domain=[('is_trobz_member', '=', True)]) instance_ids = fields.One2many( 'tms.instance', 'host_id', string='Instances') config = fields.Serialized( string='Config', default={"timezone": "Asia/Ho_Chi_Minh"}) type = fields.Selection([ ('production', 'Production'), ('staging', 'Staging'), ('integration', 'Integration'), ('test', 'Test'), ('demo', 'Demo'), ('unknown', 'Unknown'), ('node', 'Node'), ('utils', 'Utils') ], string='Type', required=True, track_visibility='onchange') action_required = fields.Boolean('Action Required', track_visibility='onchange') backup_checking = fields.Date('Audit Date', track_visibility='onchange') update = fields.Datetime('Update', readonly=True) virtual_machine = fields.Boolean('Virtual Machine', track_visibility='onchange') virtualization_id = fields.Many2one('tms.host.virtualization', string='Virtualization', track_visibility='onchange') physical_host_id = fields.Many2one('tms.host', string='Physical Host', track_visibility='onchange') virtual_host_ids = fields.One2many('tms.host', 'physical_host_id', string='Virtual Machines') num_VM = fields.Integer( compute='_get_num_of_vm', string='Number of VM') # the temp for group by physical host physical_host = fields.Char( compute='_get_physical_host', string='Physical Host', store=True) state = fields.Selection( [ ('active', 'Active'), ('exception', 'Exception'), ('asleep', 'Asleep'), ('deleted', 'Deleted') ], string='Status', track_visibility='onchange', help='For the field status:\n' '- Active: The host is active and in a normal state.\n' '- Exception: Something is wrong (ex: We lost the connection ' 'to host). This is set manually.\n' '- Asleep: This host is temporarily disabled.' ' -Deleted: We do not handle this host anymore, keeping record.') is_managed_by_trobz = fields.Boolean( string='Managed by Trobz', help='Instances managed by Trobz.', track_visibility='onchange') container_id = fields.Integer( string='Container ID', help='required for Virtual Machines') # Resources processors = fields.Integer(string='Processors', track_visibility='onchange') ram = fields.Integer(string='Memory (RAM in MB)', track_visibility='onchange') disk_size = fields.Integer(string='Disk Size (GB)', track_visibility='onchange') swap = fields.Integer(string='Swap (MB)', track_visibility='onchange') allow_edit_users = fields.Boolean( 'Allow Edit Users', compute='_compute_allow_edit_users') awx_job_history_ids = fields.One2many( comodel_name='tms.awx.job.history', inverse_name='host_id', string='AWX Job History') _sql_constraints = [ ('host_unique', 'unique(name)', 'This host already exists!'), ('host_add_unique', 'unique(host_address)', 'This host address already exists!'), ] _defaults = { 'port': '22', 'state': 'active', 'is_managed_by_trobz': True, } @api.multi def _compute_allow_edit_users(self): ''' Users can only add users to hosts related to their projects ''' user = self.env.user for host in self: allow = False if user.id == SUPERUSER_ID or user.has_full_sysadmin_access: allow = True else: # Apply for PM without has_full_sysadmin_access project_owners = host.instance_ids.\ mapped('project_id').\ mapped('owner_id') if user in project_owners: allow = True host.allow_edit_users = allow return True @api.multi def write(self, vals): """ when change virtual machine, clear value of field physical host or virtual host """ if 'virtual_machine' in vals: if vals['virtual_machine']: virtual_machines = self.search( [('physical_host_id', '=', self._ids)]) virtual_machines.write({'physical_host_id': False}) else: vals['physical_host_id'] = False # User of group TMS Add User To Host can edit users of host only current_user = self.env['res.users'].browse(self._uid) if not current_user.has_full_sysadmin_access and \ current_user.has_groups('tms_modules.group_add_user_to_host'): if not 'bypass_security' in self._context and \ not ('user_ids' in vals and len(vals) == 1): raise Warning( 'Warning', 'You only allow modify users of host!') vals['update'] = time.strftime("%Y-%m-%d") return super(tms_host, self).write(vals) @api.multi def name_get(self): return [(host.id, u"{0} ({1})".format( host.name, host.physical_host_id.name)) if host.physical_host_id else (host.id, u"{0}".format(host.name)) for host in self]
class ShopinvaderCategory(models.Model): _name = 'shopinvader.category' _description = 'Shopinvader Category' _inherit = ['locomotive.binding', 'abstract.url'] _inherits = {'product.category': 'record_id'} record_id = fields.Many2one('product.category', required=True, ondelete='cascade') object_id = fields.Integer(compute='_compute_object_id', store=True) lang_id = fields.Many2one('res.lang', 'Lang', required=True) seo_title = fields.Char() meta_description = fields.Char() meta_keywords = fields.Char() subtitle = fields.Char() short_description = fields.Html() description = fields.Html() images = fields.Serialized(compute='_compute_image', string='Shopinvader Image') shopinvader_parent_id = fields.Many2one('shopinvader.category', 'Shopinvader Parent', compute='_compute_parent_category', store=True) shopinvader_child_ids = fields.Many2many('shopinvader.category', 'Shopinvader Childs', compute='_compute_child_category') level = fields.Integer(compute='_compute_level') redirect_url_key = fields.Serialized(compute='_compute_redirect_url_key', string='Redirect Url Keys') _sql_constraints = [ ('record_uniq', 'unique(backend_id, record_id, lang_id)', 'A category can only have one binding by backend.'), ] def _compute_redirect_url_key(self): for record in self: res = [] for url in record.redirect_url_key_ids: res.append(url.url_key) record.redirect_url_key = res def _compute_image(self): for record in self: images = [] for image in record.record_id.image_ids: res = {'original': image.url} for resize in record.backend_id.categ_image_resize_ids: res[resize.key] = \ image.get_thumbnail_from_resize(resize).url images.append(res) record.images = images @api.depends('record_id') def _compute_object_id(self): for record in self: record.object_id = record.record_id.id @api.depends('parent_id.shopinvader_bind_ids') def _compute_parent_category(self): for record in self: for binding in record.parent_id.shopinvader_bind_ids: if binding.backend_id == record.backend_id: record.shopinvader_parent_id = binding break def _compute_child_category(self): for record in self: record.shopinvader_child_ids = self.search([ ('record_id.parent_id', '=', record.record_id.id), ('backend_id', '=', record.backend_id.id), ]) def _build_url_key(self): key = super(ShopinvaderCategory, self)._build_url_key() if self.parent_id and self.shopinvader_parent_id: # TODO using self.shopinvader_parent_id.url_key fail... if self.shopinvader_parent_id.url_builder == 'manual': parent_url = self.shopinvader_parent_id.manual_url_key else: parent_url = self.shopinvader_parent_id._build_url_key() key = '/'.join([parent_url, key]) return key @api.depends('url_builder', 'record_id.name', 'shopinvader_parent_id.url_key') def _compute_url(self): return super(ShopinvaderCategory, self)._compute_url() @api.depends('shopinvader_parent_id.level') def _compute_level(self): for record in self: record.level = 0 parent = record.shopinvader_parent_id while parent: record.level += 1 parent = parent.shopinvader_parent_id
class ImportOdooDatabase(models.Model): _name = 'import.odoo.database' _description = 'An Odoo database to import' url = fields.Char(required=True) database = fields.Char(required=True) user = fields.Char(default='admin', required=True) password = fields.Char(default='admin') import_line_ids = fields.One2many( 'import.odoo.database.model', 'database_id', string='Import models', ) import_field_mappings = fields.One2many( 'import.odoo.database.field', 'database_id', string='Field mappings', ) cronjob_id = fields.Many2one( 'ir.cron', string='Import job', readonly=True, copy=False, ) cronjob_running = fields.Boolean(compute='_compute_cronjob_running') status_data = fields.Serialized('Status', readonly=True, copy=False) status_html = fields.Html( compute='_compute_status_html', readonly=True, sanitize=False, ) duplicates = fields.Selection( [ ('skip', 'Skip existing'), ('overwrite', 'Overwrite existing'), ('overwrite_empty', 'Overwrite empty fields'), ], 'Duplicate handling', default='skip', required=True, ) @api.multi def action_import(self): """Create a cronjob to run the actual import""" self.ensure_one() if self.cronjob_id: return self.cronjob_id.write({ 'numbercall': 1, 'doall': True, 'active': True, }) return self.write({ 'cronjob_id': self._create_cronjob().id, }) @api.multi def _run_import(self, commit=True, commit_threshold=100): """Run the import as cronjob, commit often""" self.ensure_one() if not self.password: return # model name: [ids] remote_ids = {} # model name: count remote_counts = {} # model name: count done = {} # mapping_key: local_id idmap = {} # mapping_key: local_id # this are records created or linked when we need to fill a required # field, but the local record is not yet created dummies = {} # model name: [local_id] # this happens when we create a dummy we can throw away again to_delete = {} # dummy_instance dummy_instances = [] remote = self._get_connection() self.write({'password': False}) if commit and not tools.config['test_enable']: # pylint: disable=invalid-commit self.env.cr.commit() for model_line in self.import_line_ids: model = model_line.model_id remote_ids[model.model] = remote.execute( model.model, 'search', tools.safe_eval(model_line.domain) if model_line.domain else []) remote_counts[model.model] = len(remote_ids[model.model]) self.write({ 'status_data': { 'counts': remote_counts, 'ids': remote_ids, 'error': None, 'dummies': None, 'done': {}, } }) if commit and not tools.config['test_enable']: # pylint: disable=invalid-commit self.env.cr.commit() for model_line in self.import_line_ids: model = self.env[model_line.model_id.model] done[model._name] = 0 chunk_len = commit and (commit_threshold or 1) or len( remote_ids[model._name]) for start_index in range( len(remote_ids[model._name]) / chunk_len + 1): index = start_index * chunk_len ids = remote_ids[model._name][index:index + chunk_len] context = ImportContext( remote, model_line, ids, idmap, dummies, dummy_instances, to_delete, field_context(None, None, None), ) try: self._run_import_model(context) except: # noqa: E722 # pragma: no cover error = traceback.format_exc() self.env.cr.rollback() self.write({ 'status_data': dict(self.status_data, error=error), }) # pylint: disable=invalid-commit self.env.cr.commit() raise done[model._name] += len(ids) self.write({'status_data': dict(self.status_data, done=done)}) if commit and not tools.config['test_enable']: # pylint: disable=invalid-commit self.env.cr.commit() missing = {} for dummy_model, remote_id in dummies.keys(): if remote_id: missing.setdefault(dummy_model, []).append(remote_id) self.write({ 'status_data': dict(self.status_data, dummies=dict(missing)), }) @api.multi def _run_import_model(self, context): """Import records of a configured model""" model = self.env[context.model_line.model_id.model] fields = self._run_import_model_get_fields(context) for data in context.remote.execute(model._name, 'read', context.ids, fields.keys()): self._run_import_get_record( context, model, data, create_dummy=False, ) if (model._name, data['id']) in context.idmap: # one of our mappings hit, create an xmlid to persist # this knowledge self._create_record_xmlid( model, context.idmap[(model._name, data['id'])], data['id']) if self.duplicates == 'skip': # there's a mapping for this record, nothing to do continue data = self._run_import_map_values(context, data) _id = data['id'] record = self._create_record(context, model, data) self._run_import_model_cleanup_dummies( context, model, _id, record.id, ) @api.multi def _create_record(self, context, model, record): """Create a record, add an xmlid""" _id = record.pop('id') xmlid = '%d-%s-%d' % ( self.id, model._name.replace('.', '_'), _id or 0, ) record = self._create_record_filter_fields(model, record) new = self.env.ref('base_import_odoo.%s' % xmlid, False) if new and new.exists(): if self.duplicates == 'overwrite_empty': record = { key: value for key, value in record.items() if not new[key] } new.with_context( **self._create_record_context(model, record)).write(record) _logger.debug('Updated record %s', xmlid) else: new = model.with_context( **self._create_record_context(model, record)).create(record) self._create_record_xmlid(model, new.id, _id) _logger.debug('Created record %s', xmlid) context.idmap[mapping_key(model._name, _id)] = new.id return new def _create_record_xmlid(self, model, local_id, remote_id): xmlid = '%d-%s-%d' % ( self.id, model._name.replace('.', '_'), remote_id or 0, ) if self.env.ref('base_import_odoo.%s' % xmlid, False): return return self.env['ir.model.data'].create({ 'name': xmlid, 'model': model._name, 'module': 'base_import_odoo', 'res_id': local_id, 'noupdate': True, 'import_database_id': self.id, 'import_database_record_id': remote_id, }) def _create_record_filter_fields(self, model, record): """Return a version of record with unknown fields for model removed""" return { key: value for key, value in record.items() if key in model._fields } def _create_record_context(self, model, record): """Return a context that is used when creating a record""" context = { 'tracking_disable': True, } if model._name == 'res.users': context['no_reset_password'] = True return context @api.multi def _run_import_get_record( self, context, model, record, create_dummy=True, ): """Find the local id of some remote record. Create a dummy if not available""" _id = context.idmap.get((model._name, record['id'])) logged = False if not _id: _id = context.dummies.get((model._name, record['id'])) if _id: context.dummy_instances.append( dummy_instance(*(context.field_context + (_id, )))) else: logged = True _logger.debug( 'Got %s(%d[%d]) from idmap', model._model, _id, record['id'] or 0, ) if not _id: _id = self._run_import_get_record_mapping( context, model, record, create_dummy=create_dummy, ) elif not logged: logged = True _logger.debug( 'Got %s(%d[%d]) from dummies', model._model, _id, record['id'], ) if not _id: xmlid = self.env['ir.model.data'].search([ ('import_database_id', '=', self.id), ('import_database_record_id', '=', record['id']), ('model', '=', model._name), ], limit=1) if xmlid: _id = xmlid.res_id context.idmap[(model._name, record['id'])] = _id elif not logged: logged = True _logger.debug( 'Got %s(%d[%d]) from mappings', model._model, _id, record['id'], ) if not _id and create_dummy: _id = self._run_import_create_dummy( context, model, record, forcecreate=record['id'] not in self.status_data['ids'].get(model._name, [])) elif _id and not logged: _logger.debug( 'Got %s(%d[%d]) from xmlid', model._model, _id, record['id'], ) return _id @api.multi def _run_import_get_record_mapping( self, context, model, record, create_dummy=True, ): current_field = self.env['ir.model.fields'].search([ ('name', '=', context.field_context.field_name), ('model_id.model', '=', context.field_context.record_model), ]) mappings = self.import_field_mappings.filtered( lambda x: x.mapping_type == 'fixed' and x.model_id.model == model. _name and (not x.field_ids or current_field in x.field_ids) and x. local_id and (x.remote_id == record['id'] or not x.remote_id) or x. mapping_type == 'by_field' and x.model_id.model == model._name) _id = None for mapping in mappings: if mapping.mapping_type == 'fixed': assert mapping.local_id _id = mapping.local_id context.idmap[(model._name, record['id'])] = _id break elif mapping.mapping_type == 'by_field': assert mapping.field_ids if len(record) == 1: # just the id of a record we haven't seen yet. # read the whole record from remote to check if # this can be mapped to an existing record record = context.remote.execute( model._name, 'read', record['id'], mapping.field_ids.mapped('name'), ) or None if not record: continue if isinstance(record, list): record = record[0] domain = [(field.name, '=', record.get(field.name)) for field in mapping.field_ids if record.get(field.name)] if len(domain) < len(mapping.field_ids): # play it save, only use mapping if we really select # something specific continue records = model.with_context(active_test=False).search( domain, limit=1, ) if records: _id = records.id context.idmap[(model._name, record['id'])] = _id break else: raise exceptions.UserError(_('Unknown mapping')) return _id @api.multi def _run_import_create_dummy( self, context, model, record, forcecreate=False, ): """Either misuse some existing record or create an empty one to satisfy required links""" dummy = model.search([ ('id', 'not in', [ v for (model_name, remote_id), v in context.dummies.items() if model_name == model._name ] + [ mapping.local_id for mapping in self.import_field_mappings if mapping.model_id.model == model._name and mapping.local_id ]), ], limit=1) if dummy and not forcecreate: context.dummies[mapping_key(model._name, record['id'])] = dummy.id context.dummy_instances.append( dummy_instance(*(context.field_context + (dummy.id, )))) _logger.debug( 'Using %d as dummy for %s(%d[%d]).%s[%d]', dummy.id, context.field_context.record_model, context.idmap.get(context.field_context.record_id, 0), context.field_context.record_id, context.field_context.field_name, record['id'], ) return dummy.id required = [ name for name, field in model._fields.items() if field.required ] defaults = model.default_get(required) values = {'id': record['id']} for name, field in model._fields.items(): if name not in required or defaults.get(name): continue value = None if field.type in ['char', 'text', 'html']: value = '/' elif field.type in ['boolean']: value = False elif field.type in ['integer', 'float']: value = 0 elif model._fields[name].type in ['date', 'datetime']: value = '2000-01-01' elif field.type in ['many2one']: if name in model._inherits.values(): continue new_context = context.with_field_context( model._name, name, record['id']) value = self._run_import_get_record( new_context, self.env[model._fields[name].comodel_name], {'id': record.get(name, [None])[0]}, ) elif field.type in ['selection'] and not callable(field.selection): value = field.selection[0][0] elif field.type in ['selection'] and callable(field.selection): value = field.selection(model)[0][0] values[name] = value dummy = self._create_record(context, model, values) del context.idmap[mapping_key(model._name, record['id'])] context.dummies[mapping_key(model._name, record['id'])] = dummy.id context.to_delete.setdefault(model._name, []) context.to_delete[model._name].append(dummy.id) context.dummy_instances.append( dummy_instance(*(context.field_context + (dummy.id, )))) _logger.debug( 'Created %d as dummy for %s(%d[%d]).%s[%d]', dummy.id, context.field_context.record_model, context.idmap.get(context.field_context.record_id, 0), context.field_context.record_id or 0, context.field_context.field_name, record['id'], ) return dummy.id @api.multi def _run_import_map_values(self, context, data): model = self.env[context.model_line.model_id.model] for field_name in data.keys(): if not isinstance(model._fields[field_name], fields._Relational) or not data[field_name]: continue if model._fields[field_name].type == 'one2many': # don't import one2many fields, use an own configuration # for this data.pop(field_name) continue ids = data[field_name] if (model._fields[field_name].type != 'many2one') else [data[field_name][0]] new_context = context.with_field_context(model._name, field_name, data['id']) comodel = self.env[model._fields[field_name].comodel_name] data[field_name] = [ self._run_import_get_record( new_context, comodel, {'id': _id}, create_dummy=model._fields[field_name].required or any(m.model_id.model == comodel._name for m in self.import_line_ids), ) for _id in ids ] data[field_name] = filter(None, data[field_name]) if model._fields[field_name].type == 'many2one': if data[field_name]: data[field_name] = data[field_name] and data[field_name][0] else: data[field_name] = None else: data[field_name] = [(6, 0, data[field_name])] for mapping in self.import_field_mappings: if mapping.model_id.model != model._name: continue if mapping.mapping_type == 'unique': for field in mapping.field_ids: value = data.get(field.name, '') counter = 1 while model.with_context(active_test=False).search([ (field.name, '=', data.get(field.name, value)), ]): data[field.name] = '%s (%d)' % (value, counter) counter += 1 elif mapping.mapping_type == 'by_reference': res_model = data.get(mapping.model_field_id.name) res_id = data.get(mapping.id_field_id.name) update = { mapping.model_field_id.name: None, mapping.id_field_id.name: None, } if res_model in self.env.registry and res_id: new_context = context.with_field_context( model._name, res_id, data['id']) record_id = self._run_import_get_record( new_context, self.env[res_model], {'id': res_id}, create_dummy=False) if record_id: update.update({ mapping.model_field_id.name: res_model, mapping.id_field_id.name: record_id, }) data.update(update) return data @api.multi def _run_import_model_get_fields(self, context): return { name: field for name, field in self.env[ context.model_line.model_id.model]._fields.items() if not field.compute or field.related } @api.multi def _run_import_model_cleanup_dummies(self, context, model, remote_id, local_id): if not (model._name, remote_id) in context.dummies: return for instance in context.dummy_instances: key = mapping_key(instance.model_name, instance.remote_id) if key not in context.idmap: continue dummy_id = context.dummies[(model._name, remote_id)] record_model = self.env[instance.model_name] comodel = record_model._fields[instance.field_name].comodel_name if comodel != model._name or instance.dummy_id != dummy_id: continue record = record_model.browse(context.idmap[key]) field_name = instance.field_name _logger.debug( 'Replacing dummy %d on %s(%d).%s with %d', dummy_id, record_model._name, record.id, field_name, local_id, ) if record._fields[field_name].type == 'many2one': record.write({field_name: local_id}) elif record._fields[field_name].type == 'many2many': record.write({field_name: [ (3, dummy_id), (4, local_id), ]}) else: raise exceptions.UserError( _('Unhandled field type %s') % record._fields[field_name].type) context.dummy_instances.remove(instance) if dummy_id in context.to_delete: model.browse(dummy_id).unlink() _logger.debug('Deleting dummy %d', dummy_id) if (model._name, remote_id) in context.dummies: del context.dummies[(model._name, remote_id)] def _get_connection(self): self.ensure_one() url = urlparse(self.url) hostport = url.netloc.split(':') if len(hostport) == 1: hostport.append('80') host, port = hostport if not odoorpc: # pragma: no cover raise exceptions.UserError( _('Please install the "odoorpc" libary in your environment')) remote = odoorpc.ODOO( host, protocol='jsonrpc+ssl' if url.scheme == 'https' else 'jsonrpc', port=int(port), ) remote.login(self.database, self.user, self.password) return remote @api.constrains('url', 'database', 'user', 'password') @api.multi def _constrain_url(self): for this in self: if this == self.env.ref('base_import_odoo.demodb', False): continue if tools.config['test_enable']: continue if not this.password: continue this._get_connection() @api.depends('status_data') @api.multi def _compute_status_html(self): for this in self: if not this.status_data: continue this.status_html = self.env.ref( 'base_import_odoo.view_import_odoo_database_qweb').render( {'object': this}) @api.depends('cronjob_id') @api.multi def _compute_cronjob_running(self): for this in self: if not this.cronjob_id: continue try: with self.env.cr.savepoint(): self.env.cr.execute( 'select id from "%s" where id=%%s for update nowait' % self.env['ir.cron']._table, (this.cronjob_id.id, ), log_exceptions=False, ) except psycopg2.OperationalError: this.cronjob_running = True @api.multi def _create_cronjob(self): self.ensure_one() return self.env['ir.cron'].create({ 'name': self.display_name, 'model': self._name, 'function': '_run_import', 'doall': True, 'args': str((self.ids, )), }) @api.multi def name_get(self): return [(this.id, '%s@%s, %s' % (this.user, this.url, this.database)) for this in self]
class TmsInstance(models.SecureModel): _name = "tms.instance" _inherit = ['mail.thread'] _order = "name" _sql_constraints = [ ('instance_unique', 'unique (name)', 'This instance already exists!'), ('host_xmlrpc_unique', 'unique (host_id,xmlrpc_port)', 'This xmlrpc port already exists on that host!') ] # ======================================================================== # FIELD DEFINITIONS # ======================================================================== https_password = fields.Secure( # @UndefinedVariable string="HTTP Authen Test Password", security="_security_https_password") name = fields.Char('Instance Name', size=256, required=True, track_visibility='onchange') mail_instance = fields.Char(compute='_get_mail', string='Mail') project_id = fields.Many2one('tms.project', 'Project name', required=True, track_visibility='onchange') milestone_id = fields.Many2one( 'tms.milestone', 'Milestone', track_visibility='onchange', required=True) host_id = fields.Many2one('tms.host', 'Host', required=True, track_visibility='onchange') physical_host_id = fields.Many2one('tms.host', related='host_id.physical_host_id', string="Node", store=True) parent_bzr_repository_suffix = fields.Char( 'Parent bzr Repository Suffix', size=64, help='A possible value is "-stable" ' 'if we want to use the repositories ' 'in {project_name}-stable instead of ' 'the one in {project_name}.') server_type = fields.Selection( server_type_array, 'Server Type', required=True, track_visibility='onchange') url = fields.Char('URL', size=256, required=True, track_visibility='onchange') xmlrpc_port = fields.Char('xmlrpc port', size=256, track_visibility='onchange') psql_host = fields.Char('PostgreSQL Host', size=120, required=True, track_visibility='onchange', default='localhost') psql_port = fields.Char('PostgreSQL Port', size=120, required=True, track_visibility='onchange', default='5432') psql_user = fields.Char('PostgreSQL User', size=120, required=True, track_visibility='onchange') psql_pass = fields.Char('PostgreSQL Pass', size=120, required=True, track_visibility='onchange') is_set_up_domain = fields.Boolean( 'Domain', help='Create the instance for: {projectname}.tms.com') is_set_up_http_authentication = fields.Boolean( 'Http Authentication', help='Set http authentication to ' '{projectname}.tms.com to (denis, jc, ' 'tam, TPM,{projectname})') is_set_up_https = fields.Boolean('https') is_set_up_xmlrpc = fields.Boolean('xmlrpc') is_set_up_ssh_access = fields.Boolean('ssh access', help='Set ssh access & ' 'passwordless to ' '{projectname}.tms.com to ' '(denis, jc, TPM,{projectname})') is_project_manager = fields.Boolean( compute='compute_is_project_manager', string='Is Project Manager') state = fields.Selection( instance_state_array, string='Status', required=True, default='active', help='Sleep: the instance is active but is facing temporary ' 'issues not under our control. Inactive: This instance will ' 'not be used anymore. Deleted: This instance has been removed ' 'from our servers.', track_visibility='onchange' ) datetime_test = fields.Datetime('Last Test', readonly=True) note = fields.Text('Note') last_error = fields.Text('Last Error') operating_system = fields.Char(compute='compute_operating_system', method=True, string="Operating System", store=True) ssh_port = fields.Char(compute='compute_ssh_port', string="SSH Port", store=True) custom_parameter = fields.Serialized('Custom parameter', track_visibility='onchange') instance_database_ids = fields.One2many('instance.database', 'tms_instance_id', string='Instance Database') active = fields.Boolean('Active', default=True, help="When unchecked, the instance will not " "be visible in the user interface unless the " "search filters specify that you want to " "display non-active records (this a native " "behavior of Odoo)") test_instance = fields.Selection( test_instance_array, default='access_login', required=True, string='Test instance', help="- Access only: Test if instance is up by connecting xmlrpc " "and test database names\n" "- Access and login: Test if instance is up by connecting xmlrpc " "and can be login into admin account and test database names\n" "- None: Don't test status of the instance, " "don't check the database names.", track_visibility='onchange' ) xmlrpc_url = fields.Char(compute='compute_xmlrpc_url', string="XML-RPC URL") https_login = fields.Char('HTTP Authen Test Login', size=64) proj_owner_id = fields.Many2one('res.users', related='project_id.owner_id', string="Project's Owner", store=True) team_id = fields.Many2one(string='Team', related='project_id.team_id', store=True) team_manager_id = fields.Many2one( string="Team Manager", related='project_id.team_id.team_manager', store=True) backend_ip = fields.Char('Backend IP') backend_port = fields.Char('Backend Port', default='8069') ssl = fields.Boolean( 'SSL', default=True, help="SSL termination is handled by nginx" ) http_auth = fields.Boolean('HTTP Auth', default=True) htpasswd_file = fields.Char('htpasswd file') instance_user_ids = fields.Many2many( comodel_name='res.users', relation='tms_instance_user_rel', column1='instance_id', column2='user_id', string="Users of Instance", help="list users who can access to this instance " "( to generate htpasswd file)") multi_host = fields.Boolean(string="is Multi-host?", track_visibility='onchange') haproxy_host_id = fields.Many2one('tms.host', string="HA Proxy Host", track_visibility='onchange') front_end_ids = fields.Many2many( comodel_name='tms.host', relation="tms_instance_host_front_end_rel", column1="instance_id", column2="host_id", string="Front End", track_visibility='onchange') back_end_ids = fields.Many2many( comodel_name='tms.host', relation="tms_instance_host_back_end_rel", column1="instance_id", column2="host_id", string="Back Office", track_visibility='onchange') database_ids = fields.One2many( 'multi.host.database', 'instance_id', string="Databases", track_visibility='onchange') nfs_host_id = fields.Many2one( 'tms.host', string="NFS Host", track_visibility='onchange') awx_job_history_ids = fields.One2many( comodel_name='tms.awx.job.history', inverse_name='instance_id', string='AWX Job History') @api.multi def write(self, vals): res = super(TmsInstance, self).write(vals) if vals.get('database_ids', False): for instance in self: databases = instance.database_ids master_no = len(databases.filtered('master')) if master_no > 1: raise Warning('ONLY ONE DATABASE CAN BE SET AS MASTER') return res # ======================================================================== # COMPUTE FUNCTION DEFINITIONS # ======================================================================== @api.multi def _get_mail(self): mail_content = '' numb = 1 # Only check on active instances with databases defined domain_build = [ ('state', 'in', ['active']), ('instance_database_ids', '!=', False), ('test_instance', '=', 'access_login') ] instances = self.search(domain_build) logging.info('{0} instance(s) have databases need ' 'to be compared'.format(len(instances))) for instance in instances: list_db_in_instance, list_db_in_tms_instances = \ self.get_lst_db_instance(instance) if not list_db_in_instance and not list_db_in_tms_instances: continue mail_content += '<div><b>' + str(numb) + '.' mail_content += instance.name + '</b>:</div>' if list_db_in_instance and list_db_in_tms_instances: mail_content += '<ul>' mail_content += '<li>Database(s) in instance:' + \ ','.join(list_db_in_instance) + '</li>' mail_content += '<li>Database(s) in TMS:' + \ ','.join(list_db_in_tms_instances) + '</li>' mail_content += '</ul>' else: mail_content += '<ul>' mail_content += "<li><font color=red>Cannot access" +\ " the instance!</font></li>" mail_content += '</ul>' numb += 1 for user in self: user.mail_instance = mail_content @api.model def get_lst_db_instance(self, instance): # Get instance address to test - including port # Get the https login and password https_login = instance.https_login or 'guest' https_password = instance.read_secure( fields=['https_password'])[0].get('https_password', 'n0-@pplY') inject_idx = instance.xmlrpc_url.find('://') uri_base = '%s://%s:%s@%s' % (instance.xmlrpc_url[:inject_idx], https_login, https_password, instance.xmlrpc_url[inject_idx + 3:]) # List to store database for each instances and in TMS instance_databases_list = [] # Databases configured for instance in TMS tms_instances_databases_list = [ db_info.name for db_info in instance.instance_database_ids] # log in uri message = '' error_stack = [] try: # Link to instance's db service, should be: # http://<user>:<pass>@<host-name>:<port>/xmlrpc/db # can be changed by later versions (odoo) conn = xmlrpclib.ServerProxy(uri_base + '/xmlrpc/db') instance_databases_list = conn.list() except Exception as e: logging.warning('Failed at sock common') message += 'Warning: Failed at sock common\n' error_stack.append(e) return [], [] # Compare 2 database list before calling check_database to improve the # performance tms_instances_databases_list = sorted(tms_instances_databases_list) instance_databases_list = sorted(instance_databases_list) compare_list = set(tms_instances_databases_list).symmetric_difference( instance_databases_list) if not compare_list: return [], [] list_db_in_tms_instances = self.check_database( tms_instances_databases_list, instance_databases_list ) list_db_in_instance = self.check_database( instance_databases_list, tms_instances_databases_list ) return list_db_in_instance, list_db_in_tms_instances @api.multi def compute_is_project_manager(self): user = self.env["res.users"].browse(self._uid) user_group_ids = [group.id for group in user.groups_id] pm_group_ids = self.env["res.groups"].search( [('name', '=', 'TMS Activity Viewer')]) if pm_group_ids and user_group_ids: for record in self: record.is_project_manager = pm_group_ids[0].id in \ user_group_ids @api.multi @api.depends( "host_id", "host_id.operating_system_id", "host_id.operating_system_id.name") def compute_operating_system(self): for instance in self: instance.operating_system = \ instance.host_id \ and instance.host_id.operating_system_id \ and instance.host_id.operating_system_id.name \ or False @api.multi @api.depends( "host_id", "host_id.port") def compute_ssh_port(self): for instance in self: ssh_port = instance.host_id and instance.host_id.port or False instance.ssh_port = ssh_port @api.multi def compute_xmlrpc_url(self): for instance in self: # Normal authentication, use custom xmlrpc port if instance.xmlrpc_port: uri_base = '%s:%s' % (instance.url, instance.xmlrpc_port) if uri_base[:4] != 'http': uri_base = 'http://%s' % uri_base # 8113: Special uri for the instances which require https # authentication else: uri_base = '%s:443' % instance.url if uri_base[:5] != 'https': uri_base = 'https://' + uri_base instance.xmlrpc_url = uri_base # ======================================================================== # ONCHANGE FUNCTION DEFINITIONS # ======================================================================== @api.onchange('server_type') def on_change_server_type(self): for instance in self: project = instance.project_id server_type = instance.server_type if not project or not server_type: return {} name = 'openerp-%s-%s' % (project.name, server_type) if server_type == 'production': url = '%s.trobz.com' % (project.name) else: url = '%s-%s.trobz.com' % (project.name, server_type) instance.name = name instance.url = url instance.psql_user = name instance.psql_pass = name # F#13799 - htpasswd_file field on instance should be editable if project and server_type: htpasswd_file = '/usr/local/var/auth/htpasswd_%s_%s' %\ (project.name, server_type) instance.htpasswd_file = htpasswd_file @api.onchange('state') def on_change_state(self): for instance in self: if instance.state in ('inactive', 'deleted'): instance.active = False else: instance.active = True @api.onchange('project_id') def on_change_project_id(self): for instance in self: project = instance.project_id server_type = instance.server_type # F#13799 - htpasswd_file field on instance should be editable if project and server_type: htpasswd_file = '/usr/local/var/auth/htpasswd_%s_%s' %\ (project.name, server_type) instance.htpasswd_file = htpasswd_file @api.onchange('host_id') def on_change_host(self): for instance in self: host = instance.host_id if host: instance.backend_ip = '10.26.%d.y' % host.container_id instance.ssh_port = host.port else: instance.backend_ip = False instance.ssh_port = False # ======================================================================== # FORM BUTTON FUNCTION DEFINITIONS # ======================================================================== @api.multi def button_request_in_ticket(self): # FIXME: should we consider to remove this function, this is not used ticket_pool = self.env['tms.ticket'] for instance in self: ticket_pool.create({ 'summary': u'Configure the instance {0}'.format(instance.name) }) # ======================================================================== # Daily Check List Instance Databases scheduler (email: Test List DB) # ======================================================================== @api.model def check_database(self, list_check, list_to_check): RENDER_COLOR = '<font color=red>%s</font>' list_new = [] for i in list_check: if i not in list_to_check: i = RENDER_COLOR % i list_new.append(i) return list_new @api.model def run_scheduler_compare_instance_in_tms_and_database(self): logging.info('[Scheduler] [Start] Compare list of databases in TMS ' 'and in instances') domain_build = [ ('state', 'in', ['active']), ('instance_database_ids', '!=', False), ('test_instance', '!=', 'none') ] instances = self.search(domain_build) for instance in instances: list_db_in_instance, list_db_in_tms_instances = \ self.get_lst_db_instance(instance) if not list_db_in_instance and not list_db_in_tms_instances: continue # send notification email template = self.env.ref( 'tms_modules.daily_instances_db_template' ) template._send_mail_asynchronous(instance.id) logging.info('[Scheduler] [End] Compare list of databases ' 'in TMS and in instances') return True # ======================================================================== # Test Instances Scheduler (email: Instance Down Mail) # ======================================================================== @api.model def run_test_instance_scheduler(self): logging.info("run_test_instance_scheduler: start") try: self.button_test_all_instances() except Exception as e: logging.error("Error in run_test_instance_scheduler: %s" % str(e)) logging.info("run_test_instance_scheduler: end") return True @api.multi def button_test(self): """ Test instance. """ logging.info('Entered button_test of tms_instance ' '(to test availability of instances)...') # Number of attempts and Sleep time between each attempt NB_ATTEMPS = 2 TIME_SLEEP = 3 error_stack = [] # message for test instance log message = '' for instance in self: message = '' vals = {} # get the db_login, db_password and db_name # from instance database field db_login = '******' if instance.test_instance == 'access_login': # check if a database is created for this instance if not (instance.instance_database_ids and instance.instance_database_ids.ids or False): logging.warning( 'No database defined for the instance %s.' % instance.name ) continue # only check the first database in the list instance_db = instance.instance_database_ids and \ instance.instance_database_ids[0] or False if instance_db: # extract db name and password from first line db_name = instance_db.name db_password = instance_db\ .read_secure(fields=['password'])[0]\ .get('password', 'n0-@pplY') else: db_name = 'unnamed' db_password = '******' # Get the https login and password (from instance) https_login = instance.https_login or 'guest' https_password = instance\ .read_secure(fields=['https_password'])[0]\ .get('https_password', 'n0-@pplY') inject_idx = instance.xmlrpc_url.find('://') uri_base = '%s://%s:%s@%s' % (instance.xmlrpc_url[:inject_idx], https_login, https_password, instance.xmlrpc_url[inject_idx + 3:]) # To remove the https_login_pass from the instance down message https_real_pass = '******' % https_password https_replace_pass = '******' logging.info('Checking ' + instance.xmlrpc_url) for i in range(NB_ATTEMPS): instance_exc = None state = 'active' # Try to reach the instance try: sock_common = xmlrpclib.ServerProxy( '%s/xmlrpc/common' % uri_base) except Exception as e: logging.warning('Failed at sock common') message += 'Warning: Failed at sock common\n' state = 'exception' instance_exc = str(e).replace( https_real_pass, https_replace_pass ) # Try to login at the instance try: if instance.test_instance == 'access_login': logging.info( 'Trying to login as Admin into instance %s...' % instance.name) message += 'Info: Trying to login as Admin into '\ 'instance %s...\n' % instance.name connection = sock_common.login( db_name, db_login, db_password) if not connection: raise Exception( 'Could not login to the instance %s' % instance.name) else: # reset state and message then break, no more try # after successful check state, message = 'active', '' break elif instance.test_instance == 'access': try: connection = sock_common.login( 'test_access_only', 'test', 'test') except Exception as e: if 'FATAL: database "test_access_only" ' +\ 'does not exist' in str(e): state, message = 'active', '' break else: raise Exception( 'Could not connect to the instance %s' % instance.name) except Exception as e: # TODO: to remove the password from the exception exc_str = str(e).replace(https_real_pass, https_replace_pass) logging.warning('ATTEMPT FAILED: Could not connect ' 'to the instance %s: %s' % (instance.name, exc_str)) message += 'Warning: ATTEMPT FAILED: Could not connect '\ 'to the instance %s: %s' % (instance.name, exc_str) state = 'exception' instance_exc = exc_str finally: sock_common = None if state == 'exception': logging.warning('ATTEMPT FAILED: Tried %s time(s), ' 'trying again ...' % (i + 1,)) if i < NB_ATTEMPS - 1: time.sleep(TIME_SLEEP) elif i == NB_ATTEMPS - 1: error_stack.append(instance_exc) # after N times to check on the instance ==> if instance down if state == 'exception': logging.error('The instance %s is down !!!' % instance.name) message += 'Error: The instance %s is down !!!\n'\ % instance.name logging.error('Could not connect to the instance %s after %s ' 'attempts!!' % (instance.name, NB_ATTEMPS)) message += 'Error: Could not connect to the instance %s '\ 'after %s attempts!!\n' % (instance.name, NB_ATTEMPS) logging.error('uri_base: %s, db_name: %s, db_login: %s, ' 'db_password: ****' % (instance.xmlrpc_url, db_name, db_login)) message += 'Error: uri_base: %s, db_name: %s, '\ 'db_login: %s, db_password: ****\n' % (instance.xmlrpc_url, db_name, db_login) logging.error(error_stack) vals['last_error'] = message state = 'exception' else: logging.info( 'Connect to the instance %s successful!' % instance.name) state = 'active' vals['last_error'] = False vals.update({ 'state': state, 'datetime_test': datetime.now() }) logging.info('Writing new data to current instance: %s' % vals) instance.write(vals) # write test data to instance @api.model def button_test_all_instances(self): """ Test instance. """ logging.info('Entered button_test of tms_instance ' '(to test availability of instances)...') instances = self.search( [('state', 'in', ('active', 'exception')), ('test_instance', '!=', 'none')] ) for instance in instances: instance.button_test() # commit the write transaction (before sending email) self.env.cr.commit() down_instances = self.search([ ('state', '=', 'exception'), ('test_instance', '!=', 'none') ]) if down_instances and down_instances.ids: # context should be passed in email process context = {'instances_down_ids': down_instances.ids} # send email to inform number of down instances email_template = self.env.ref( 'tms_modules.tms_instance_down_mail_template' ) email_template.with_context(context).send_mail( down_instances[0].id) logging.info('Leaving button_test of tms_instance ' '(to test availability of instances)...') @api.multi def get_mail_down_instances_subject(self): """ this function will be called by email template to prepare the subject of the email """ context = self._context and self._context.copy() or {} # build domain to get only down instances domain = [ ('state', '=', 'exception'), ('test_instance', '!=', 'none') ] if context.get('instances_down_ids'): domain.append(('id', 'in', context.get('instances_down_ids'))) # get down instances down_instances = self.search(domain) if not down_instances.ids: return 'No instance down' nb_production_instance_down = 0 nb_instance_down = 0 for down_instance in down_instances: nb_instance_down += 1 if down_instance.name[-11:] == '-production': nb_production_instance_down += 1 if nb_production_instance_down == 0: return '%s Instance(s) down, none from production' %\ str(nb_instance_down) return '%s Instance(s) down, including %s from production' %\ (str(nb_instance_down), str(nb_production_instance_down)) @api.multi def get_mail_down_instances(self): """ # Ticket #1075 Send list of down instances. """ context = self._context and self._context.copy() or {} domain = [('state', '=', 'exception'), ('test_instance', '!=', 'none')] if context.get('instances_down_ids'): domain.append(('id', 'in', context.get('instances_down_ids'))) down_instances = self.search(domain) if not down_instances: return 'No instance down' config_pool = self.env['ir.config_parameter'] base_url = config_pool.get_param( 'web.base.url', default='https://tms.trobz.com') base_url = u'{0}#model=tms.instance&id='.format(base_url) # 3051 list_instances_production, list_instances_down = [], [] # Get name of down instances - classified through instance type for instance in down_instances: if "production" in instance.name: list_instances_production.append(instance.name) else: list_instances_down.append(instance.name) # predefined mail templates and mail contents mail_content = u'' list_template = u"<ol>{0}</ol>" details_template = u"<li>{0}: {1}</li>" instance_down_contents, instance_down_details = u"", u"" # compose list of down production instance names for instance in list_instances_production: instance_down_contents += u'<li>{0} down</li>'.format(instance) # compose list of down staging/integration instances name for instance in list_instances_down: instance_down_contents += u'<li>{0} down</li>'.format(instance) # compose list of down instance names mail_content += list_template.format(instance_down_contents) # detailed information section for down instances mail_content += u'More details:<br />' # information to be displayed in details, read from database for instance in down_instances: # name of the down instance first - followed by the details instance_down_details += u'<li style="margin-bottom: 15px;">' instance_down_details += u'<span>{0} down</span>'.format( instance.name) # begin detailed information instance_down_details += u'<ul style="padding-left: 20px;">' instance_down_details += details_template.format( "URL", instance.url) instance_db_list = [ db_info.name for db_info in instance.instance_database_ids] instance_down_details += details_template.format( "Database info", str(instance_db_list)) instance_down_details += details_template.format( "XML-RPC port", instance.xmlrpc_port) instance_down_details += details_template.format( "Last error", escape(instance.last_error)) instance_down_details += details_template.format( "Link to TMS", base_url + str(instance.id)) instance_down_details += u'</ul>' # end detailed information instance_down_details += u'</li>' # compose list of down instance details mail_content += list_template.format(instance_down_details) return mail_content @api.multi def get_sysadmin_tpm_email_list(self): """ Get list of users who are active Trobz members and have jobs in (Manager, Technical Project Manager) and have departments in (Management, OpenErp). """ departments = self.env['hr.department'].search( [('name', 'in', ('Management', 'OpenErp', 'Sysadmin'))]) if not departments: return '[email protected],[email protected]' jobs = self.env['hr.job'].search([ ('name', 'in', ('Manager', 'Technical Project Manager', 'System Admin'))]) if not jobs: return '[email protected],[email protected]' employee_obj = self.env['hr.employee'] employees = employee_obj.search( [('job_id', 'in', jobs.ids), ('department_id', 'in', departments.ids)]) if not employees: return '[email protected],[email protected]' mail_list = ['*****@*****.**'] for employee in employees: if employee and employee.user_id\ and employee.user_id.active\ and employee.user_id.is_trobz_member\ and employee.user_id.email: mail_list.append(employee.user_id.email) if mail_list: all_mail = ",".join(mail_list) + ",[email protected]" else: all_mail = '[email protected],[email protected]' return all_mail @api.multi def get_admin_tpm_sysadmin_email(self): result = "" res_users_obj = self.env['res.users'] res_groups_obj = self.env['res.groups'] admin_tpm_profs = res_groups_obj.search( [('name', 'in', ('Admin Profile', 'Sysadmin Profile', 'Technical Project Manager Profile')), ('is_profile', '=', True)]) res_users = res_users_obj.search( [('group_profile_id', 'in', admin_tpm_profs.ids)]) mail_list = [] for user in res_users: if user.email: mail_list.append(user.email) if mail_list: result = ",".join(mail_list) else: result = '*****@*****.**' return result # ======================================================================== # OTHER FUNCTIONS # ======================================================================== @api.model def _check_password_security(self, instance): """ @param instance: recordset tms.instance @return: - Admin profile, return True - TPM/FC profiles and in the supporters, return True - The rest, return False """ if self._uid == SUPERUSER_ID: return True config_pool = self.env['ir.config_parameter'] user_profile = self.env.user.group_profile_id # If this user is a Sysadmin and has full access if user_profile.is_sysadmin and self.env.user.has_full_sysadmin_access: return True # If this user is a Sysadmin but don't have full access # check if he is in the list of users of the host of this instance if user_profile.is_sysadmin\ and self.env.user.id in instance.host_id.user_ids.ids: return True # If this user is not a sysadmin, he must be in the list of supporters db_instance_profiles = config_pool.get_param( 'db_instance_profiles') db_instance_profiles = safe_eval(db_instance_profiles) if user_profile.name not in db_instance_profiles: return False project_supporters = instance and\ instance.project_id.project_supporter_rel_ids.ids\ or [] if self.env.user.id not in project_supporters: return False return True @api.multi def _security_https_password(self): """ Only allow Admin/TPM or FC update read/update password field. """ is_allow = False for rec in self: if self._check_password_security(rec): is_allow = True else: is_allow = False break return is_allow @api.model def run_get_module_quality_check_scheduler(self): instance_ids = self.search([ ('state', '=', 'active')] ) self.button_module_quality_check(instance_ids) self.env['email.template']._send_mail_asynchronous( self._uid, 'Module Quality Check Result') return True def _build_result_detail(self, result, result_details): detail_dict = {} for detail in result_details: module_name = result[detail['quality_check_id'][0]]['name'] module_score = result[detail['quality_check_id'][0]]['final_score'] if module_name not in detail_dict: detail_dict[module_name] = [ u'<li> Module {0}: {1}</li>'.format( module_name, module_score) ] result_detail = u'Last update: {0}<br/>'.format(datetime.now())\ + u'<li>Modules: </li><ul>' for detail in detail_dict.values(): detail_str = u'<br /> '.join(tuple(detail)) result_detail = str(result_detail) + detail_str return result_detail @api.model def migrate_database_instance(self): logging.info('==== START migrate database instance ====') tms_instance_datas = self.search([]) instance_db_env = self.env['instance.database'] for data in tms_instance_datas: if data.databases: databases = data.databases.split(",") for database in databases: dict_data = database.split(":") vals = {'name': dict_data[0], 'tms_instance_id': data.id} new_data = instance_db_env.create(vals) if len(dict_data) > 1: vals.update({'password': dict_data[1]}) new_data.secure_write(vals) logging.info('==== END migrate database instance ====') return True @api.multi def get_databases_list(self): """ Get a list of databases for each given instance. """ databases_lst = {} # {instance: list of databases} for instance in self: db_names = [db_info.name for db_info in instance.instance_database_ids] databases_lst.update({ instance.name: db_names }) return databases_lst @api.multi def get_instance_info(self, context=None): """ Get cleartext password of a database. """ databases_lst = {} for instance in self: instance_db = instance.instance_database_ids and \ instance.instance_database_ids[0] or False if instance_db: db_name = instance_db.name db_password = instance_db.read_secure(fields=['password']) databases_lst.update({ instance.name: [ instance.state, instance.server_type, db_name, db_password[0]['password']] }) return databases_lst