class ExportTranslationsToBranch(LaunchpadCronScript): """Commit translations to translations_branches where requested.""" commit_message = "Launchpad automatic translations update." # Don't bother looking for a previous translations commit if it's # longer than this ago. previous_commit_cutoff_age = timedelta(days=7) # We can find out when the last translations commit to a branch # completed, and we can find out when the last transaction changing # a POFile started. This is exactly the wrong way around for # figuring out which POFiles need a fresh export, so assume a fudge # factor. fudge_factor = timedelta(hours=6) def add_my_options(self): """See `LaunchpadScript`.""" self.parser.add_option( '-n', '--no-fudge', action='store_true', dest='no_fudge', default=False, help="For testing: no fudge period for POFile changes.") def _checkForObjections(self, source): """Check for reasons why we can't commit to this branch. Raises `ConcurrentUpdateError` if there is such a reason. :param source: the series being exported to its translations_branch. """ if source.translations_branch is None: raise ConcurrentUpdateError( "Translations export for %s was just disabled." % ( source.title)) branch = source.translations_branch jobsource = getUtility(IRosettaUploadJobSource) unfinished_jobs = jobsource.findUnfinishedJobs( branch, since=datetime.now(pytz.UTC) - timedelta(days=1)) if unfinished_jobs.any(): raise ConcurrentUpdateError( "Translations branch for %s has pending translations " "changes. Not committing." % source.title) def _makeDirectBranchCommit(self, db_branch): """Create a `DirectBranchCommit`. :param db_branch: A `Branch` object as defined in Launchpad. :return: A `DirectBranchCommit` for `db_branch`. """ committer_id = 'Launchpad Translations on behalf of %s' % ( db_branch.owner.name) return DirectBranchCommit(db_branch, committer_id=committer_id) def _commit(self, source, committer): """Commit changes to branch. Check for race conditions.""" self._checkForObjections(source) committer.commit(self.commit_message, txn=self.txn) def _isTranslationsCommit(self, revision): """Is `revision` an automatic translations commit?""" return revision.message == self.commit_message def _getRevisionTime(self, revision): """Get timestamp of `revision`.""" # The bzr timestamp is a float representing UTC-based seconds # since the epoch. It stores the timezone as well, but we can # ignore it here. return datetime.fromtimestamp(revision.timestamp, pytz.UTC) def _getLatestTranslationsCommit(self, branch): """Get date of last translations commit to `branch`, if any.""" cutoff_date = datetime.now(pytz.UTC) - self.previous_commit_cutoff_age revno, current_rev = branch.last_revision_info() repository = branch.repository graph = repository.get_graph() for rev_id in graph.iter_lefthand_ancestry( current_rev, (NULL_REVISION, )): revision = repository.get_revision(rev_id) revision_date = self._getRevisionTime(revision) if self._isTranslationsCommit(revision): return revision_date if revision_date < cutoff_date: # Going too far back in history. Give up. return None return None def _findChangedPOFiles(self, source, changed_since): """Return an iterator of POFiles changed since `changed_since`. :param source: a `ProductSeries`. :param changed_since: a datetime object. """ subset = getUtility(IPOTemplateSet).getSubset( productseries=source, iscurrent=True) for template in subset: for pofile in template.pofiles: if (changed_since is None or pofile.date_changed > changed_since or template.date_last_updated > changed_since): yield pofile def _exportToBranch(self, source): """Export translations for source into source.translations_branch. :param source: a `ProductSeries`. """ self.logger.info("Exporting %s." % source.title) self._checkForObjections(source) branch = source.translations_branch branch = source.translations_branch try: committer = self._makeDirectBranchCommit(branch) except StaleLastMirrored as e: # Request a rescan of the branch. Do this on the master # store, or we won't be able to modify the branch object. # (The master copy may also be more recent, in which case # the rescan won't be necessary). master_branch = IMasterStore(branch).get(Branch, branch.id) master_branch.branchChanged(**get_db_branch_info(**e.info)) self.logger.warning( "Skipped %s due to stale DB info, and scheduled a new scan.", branch.bzr_identity) if self.txn: self.txn.commit() return self.logger.debug("Created DirectBranchCommit.") if self.txn: self.txn.commit() bzr_branch = committer.bzrbranch last_commit_date = self._getLatestTranslationsCommit(bzr_branch) if last_commit_date is None: self.logger.debug("No previous translations commit found.") changed_since = None else: # Export files that have been touched since the last export. # Subtract a fudge factor because the last-export date marks # the end of the previous export, and the POFiles' # last-touched timestamp marks the beginning of the last # transaction that changed them. self.logger.debug("Last commit was at %s." % last_commit_date) changed_since = last_commit_date - self.fudge_factor change_count = 0 try: for pofile in self._findChangedPOFiles(source, changed_since): base_path = os.path.dirname(pofile.potemplate.path) language_code = pofile.getFullLanguageCode() self.logger.debug("Exporting %s." % language_code) pofile_path = os.path.join( base_path, language_code + '.po') pofile_contents = pofile.export() committer.writeFile(pofile_path, pofile_contents) change_count += 1 # We're not actually writing any changes to the # database, but it's not polite to stay in one # transaction for too long. if self.txn: self.txn.commit() # We're done with this POFile. Don't bother caching # anything about it any longer. pofile.potemplate.clearPOFileCache() if change_count > 0: self.logger.debug("Writing to branch.") self._commit(source, committer) finally: committer.unlock() def _exportToBranches(self, productseries_iter): """Loop over `productseries_iter` and export their translations.""" items_done = 0 items_failed = 0 unpushed_branches = 0 productseries = shortlist(productseries_iter, longest_expected=2000) for source in productseries: try: self._exportToBranch(source) if self.txn: self.txn.commit() except (KeyboardInterrupt, SystemExit): raise except NotBranchError: unpushed_branches += 1 if self.txn: self.txn.abort() self._handleUnpushedBranch(source) if self.txn: self.txn.commit() except Exception as e: items_failed += 1 self.logger.error( "Failure in %s/%s: %s", source.product.name, source.name, repr(e)) if self.txn: self.txn.abort() items_done += 1 self.logger.info( "Processed %d item(s); %d failure(s), %d unpushed branch(es)." % ( items_done, items_failed, unpushed_branches)) def _sendMail(self, sender, recipients, subject, text): """Wrapper for `simple_sendmail`. Fakeable for easy testing.""" simple_sendmail(sender, recipients, subject, text) def _handleUnpushedBranch(self, productseries): """Branch has never been scanned. Notify owner. This means that as far as the Launchpad database knows, there is no actual bzr branch behind this `IBranch` yet. """ branch = productseries.translations_branch self.logger.info("Notifying %s of unpushed branch %s." % ( branch.owner.name, branch.bzr_identity)) template = get_email_template('unpushed-branch.txt', 'translations') text = template % { 'productseries': productseries.title, 'branch_url': branch.bzr_identity, } recipients = get_contact_email_addresses(branch.owner) sender = format_address( "Launchpad Translations", config.canonical.noreply_from_address) subject = "Launchpad: translations branch has not been set up." self._sendMail(sender, recipients, subject, text) def main(self): """See `LaunchpadScript`.""" # Avoid circular imports. from lp.registry.model.product import Product from lp.registry.model.productseries import ProductSeries errorlog.globalErrorUtility.configure(self.config_name) if self.options.no_fudge: self.fudge_factor = timedelta(0) self.logger.info("Exporting to translations branches.") self.store = ISlaveStore(Product) product_join = Join( ProductSeries, Product, ProductSeries.product == Product.id) productseries = self.store.using(product_join).find( ProductSeries, And( Product.translations_usage == ServiceUsage.LAUNCHPAD, ProductSeries.translations_branch != None)) # Anything deterministic will do, and even that is only for # testing. productseries = productseries.order_by(ProductSeries.id) bzrserver = get_rw_server() bzrserver.start_server() try: self._exportToBranches(productseries) finally: bzrserver.stop_server()