def start_sync(self, account_id): """ Starts a sync for the account with the given account_id. If that account doesn't exist, does nothing. """ with session_scope() as db_session: acc = db_session.query(Account).get(account_id) if acc is None: self.log.error('no such account', account_id=account_id) return fqdn = platform.node() self.log.info('starting sync', account_id=acc.id, email_address=acc.email_address) if acc.sync_host is not None and acc.sync_host != fqdn: self.log.error( 'Sync Host Mismatch', message='account is syncing on another host {}'.format( acc.sync_host), account_id=account_id) elif acc.id not in self.monitors: try: if acc.is_sync_locked and acc.is_killed: acc.sync_unlock() acc.sync_lock() monitor = self.monitor_cls_for[acc.provider](acc) self.monitors[acc.id] = monitor monitor.start() info = acc.provider_info if info.get('contacts', None) and acc.sync_contacts: contact_sync = ContactSync(acc.email_address, acc.provider, acc.id, acc.namespace.id) self.contact_sync_monitors[acc.id] = contact_sync contact_sync.start() if info.get('events', None) and acc.sync_events: event_sync = EventSync(acc.email_address, acc.provider, acc.id, acc.namespace.id) self.event_sync_monitors[acc.id] = event_sync event_sync.start() acc.sync_started() db_session.add(acc) db_session.commit() self.log.info('Sync started', account_id=account_id, sync_host=fqdn) except Exception as e: self.log.error('sync_error', message=str(e.message), account_id=account_id) else: self.log.info('sync already started', account_id=account_id)
def start_sync(self, account_id): """ Starts a sync for the account with the given account_id. If that account doesn't exist, does nothing. """ with session_scope() as db_session: acc = db_session.query(Account).get(account_id) if acc is None: self.log.error('no such account', account_id=account_id) return fqdn = platform.node() self.log.info('starting sync', account_id=acc.id, email_address=acc.email_address) if acc.sync_host is not None and acc.sync_host != fqdn: self.log.error('Sync Host Mismatch', message='account is syncing on another host {}' .format(acc.sync_host), account_id=account_id) elif acc.id not in self.monitors: try: if acc.is_sync_locked and acc.is_killed: acc.sync_unlock() acc.sync_lock() monitor = self.monitor_cls_for[acc.provider](acc) self.monitors[acc.id] = monitor monitor.start() info = acc.provider_info if info.get('contacts', None) and acc.sync_contacts: contact_sync = ContactSync(acc.email_address, acc.provider, acc.id, acc.namespace.id) self.contact_sync_monitors[acc.id] = contact_sync contact_sync.start() if info.get('events', None) and acc.sync_events: event_sync = EventSync(acc.email_address, acc.provider, acc.id, acc.namespace.id) self.event_sync_monitors[acc.id] = event_sync event_sync.start() acc.sync_started() db_session.add(acc) db_session.commit() self.log.info('Sync started', account_id=account_id, sync_host=fqdn) except Exception as e: self.log.error('sync_error', message=str(e.message), account_id=account_id) else: self.log.info('sync already started', account_id=account_id)
def start_sync(self, account_id): """ Starts a sync for the account with the given account_id. If that account doesn't exist, does nothing. """ with self.semaphore, session_scope(account_id) as db_session: acc = db_session.query(Account).get(account_id) if acc is None: self.log.error('no such account', account_id=account_id) return self.log.info('starting sync', account_id=acc.id, email_address=acc.email_address) if acc.id not in self.syncing_accounts: try: acc.sync_host = self.process_identifier if acc.sync_email: monitor = self.monitor_cls_for[acc.provider](acc) self.email_sync_monitors[acc.id] = monitor monitor.start() info = acc.provider_info if info.get('contacts', None) and acc.sync_contacts: contact_sync = ContactSync(acc.email_address, acc.verbose_provider, acc.id, acc.namespace.id) self.contact_sync_monitors[acc.id] = contact_sync contact_sync.start() if info.get('events', None) and acc.sync_events: if (USE_GOOGLE_PUSH_NOTIFICATIONS and acc.provider == 'gmail'): event_sync = GoogleEventSync( acc.email_address, acc.verbose_provider, acc.id, acc.namespace.id) else: event_sync = EventSync(acc.email_address, acc.verbose_provider, acc.id, acc.namespace.id) self.event_sync_monitors[acc.id] = event_sync event_sync.start() acc.sync_started() self.syncing_accounts.add(acc.id) db_session.commit() self.log.info('Sync started', account_id=account_id, sync_host=acc.sync_host) except Exception: self.log.error('Error starting sync', exc_info=True, account_id=account_id) else: self.log.info('sync already started', account_id=account_id)
def test_handle_changes(db, generic_account): namespace_id = generic_account.namespace.id event_sync = EventSync(generic_account.email_address, 'google', generic_account.id, namespace_id) # Sync calendars/events event_sync.provider.sync_calendars = calendar_response event_sync.provider.sync_events = event_response event_sync.sync() assert db.session.query(Calendar).filter( Calendar.namespace_id == namespace_id, Calendar.name != 'Emailed events').count() == 2 assert db.session.query(Event).join(Calendar).filter( Event.namespace_id == namespace_id, Calendar.uid == 'first_calendar_uid').count() == 3 assert db.session.query(Event).join(Calendar).filter( Event.namespace_id == namespace_id, Calendar.uid == 'second_calendar_uid').count() == 2 # Sync a calendar update with long name event_sync.provider.sync_calendars = calendar_long_name event_sync.sync() long_calendar = db.session.query(Calendar).filter( Calendar.namespace_id == namespace_id, Calendar.uid == 'long_calendar_uid').one() assert len(long_calendar.name) == MAX_INDEXABLE_LENGTH # Sync a calendar update event_sync.provider.sync_calendars = calendar_response_with_update event_sync.provider.sync_events = event_response event_sync.sync() # Check that we have the same number of calendars and events as before assert db.session.query(Calendar).filter( Calendar.namespace_id == namespace_id, Calendar.name != 'Emailed events').count() == 3 assert db.session.query(Event).join(Calendar).filter( Event.namespace_id == namespace_id, Calendar.uid == 'first_calendar_uid').count() == 3 assert db.session.query(Event).join(Calendar).filter( Event.namespace_id == namespace_id, Calendar.uid == 'second_calendar_uid').count() == 2 assert db.session.query(Event).join(Calendar).filter( Event.namespace_id == namespace_id, Calendar.uid == 'long_calendar_uid').count() == 2 # Check that calendar attribute was updated. first_calendar = db.session.query(Calendar).filter( Calendar.namespace_id == namespace_id, Calendar.uid == 'first_calendar_uid').one() assert first_calendar.name == 'Super Important Meetings' # Sync an event update event_sync.provider.sync_events = event_response_with_update event_sync.sync() # Make sure the update was persisted first_event = db.session.query(Event).filter( Event.namespace_id == namespace_id, Event.calendar_id == first_calendar.id, Event.uid == 'first_event_uid').one() assert first_event.title == 'Top Secret Plotting Meeting' # Sync an event delete event_sync.provider.sync_events = event_response_with_delete event_sync.sync() # Make sure the delete was persisted. first_event = db.session.query(Event).filter( Event.namespace_id == namespace_id, Event.calendar_id == first_calendar.id, Event.uid == 'first_event_uid').first() db.session.refresh(first_event) assert first_event.status == 'cancelled' # Sync a calendar delete event_public_ids = [ id_ for id_, in db.session.query(Event.public_id).filter( Event.namespace_id == namespace_id, Event.calendar_id == first_calendar.id) ] event_sync.provider.sync_calendars = calendar_response_with_delete event_sync.sync() assert db.session.query(Calendar).filter( Calendar.namespace_id == namespace_id, Calendar.uid == 'first_calendar_uid').first() is None # Check that delete transactions are created for events on the deleted # calendar. deleted_event_transactions = db.session.query(Transaction).filter( Transaction.object_type == 'event', Transaction.command == 'delete', Transaction.namespace_id == namespace_id, Transaction.object_public_id.in_(event_public_ids)).all() assert len(deleted_event_transactions) == 3 # Check that events with the same uid but associated to a different # calendar still survive. assert db.session.query(Event).filter( Event.namespace_id == namespace_id).count() == 4
def start_sync(self, account_id): """ Starts a sync for the account with the given account_id. If that account doesn't exist, does nothing. """ with self.semaphore, session_scope(account_id) as db_session: acc = db_session.query(Account).with_for_update().get(account_id) if acc is None: self.log.error("no such account", account_id=account_id) return False if not acc.sync_should_run: return False if (acc.desired_sync_host is not None and acc.desired_sync_host != self.process_identifier): return False if acc.sync_host is not None and acc.sync_host != self.process_identifier: return False self.log.info("starting sync", account_id=acc.id, email_address=acc.email_address) if acc.id in self.syncing_accounts: self.log.info("sync already started", account_id=account_id) return False try: acc.sync_host = self.process_identifier if acc.sync_email: monitor = self.monitor_cls_for[acc.provider](acc) self.email_sync_monitors[acc.id] = monitor monitor.start() info = acc.provider_info if info.get("contacts", None) and acc.sync_contacts: contact_sync = ContactSync( acc.email_address, acc.verbose_provider, acc.id, acc.namespace.id, ) self.contact_sync_monitors[acc.id] = contact_sync contact_sync.start() if info.get("events", None) and acc.sync_events: if USE_GOOGLE_PUSH_NOTIFICATIONS and acc.provider == "gmail": event_sync = GoogleEventSync( acc.email_address, acc.verbose_provider, acc.id, acc.namespace.id, ) else: event_sync = EventSync( acc.email_address, acc.verbose_provider, acc.id, acc.namespace.id, ) self.event_sync_monitors[acc.id] = event_sync event_sync.start() acc.sync_started() self.syncing_accounts.add(acc.id) # TODO (mark): Uncomment this after we've transitioned to from statsd to brubeck # statsd_client.gauge('mailsync.sync_hosts_counts.{}'.format(acc.id), 1, delta=True) db_session.commit() self.log.info("Sync started", account_id=account_id, sync_host=acc.sync_host) except Exception: self.log.error("Error starting sync", exc_info=True, account_id=account_id) return False return True
def alt_event_sync(config, db): return EventSync(2, 2)
def test_handle_changes(db, generic_account): namespace_id = generic_account.namespace.id event_sync = EventSync(generic_account.email_address, 'google', generic_account.id, namespace_id) # Sync calendars/events event_sync.provider.sync_calendars = calendar_response event_sync.provider.sync_events = event_response event_sync.sync() assert db.session.query(Calendar).filter( Calendar.namespace_id == namespace_id, Calendar.name != 'Emailed events').count() == 2 assert db.session.query(Event).join(Calendar).filter( Event.namespace_id == namespace_id, Calendar.uid == 'first_calendar_uid').count() == 3 assert db.session.query(Event).join(Calendar).filter( Event.namespace_id == namespace_id, Calendar.uid == 'second_calendar_uid').count() == 2 # Sync a calendar update event_sync.provider.sync_calendars = calendar_response_with_update event_sync.provider.sync_events = event_response event_sync.sync() # Check that we have the same number of calendars and events as before assert db.session.query(Calendar).filter( Calendar.namespace_id == namespace_id, Calendar.name != 'Emailed events').count() == 2 assert db.session.query(Event).join(Calendar).filter( Event.namespace_id == namespace_id, Calendar.uid == 'first_calendar_uid').count() == 3 assert db.session.query(Event).join(Calendar).filter( Event.namespace_id == namespace_id, Calendar.uid == 'second_calendar_uid').count() == 2 # Check that calendar attribute was updated. first_calendar = db.session.query(Calendar).filter( Calendar.namespace_id == namespace_id, Calendar.uid == 'first_calendar_uid').one() assert first_calendar.name == 'Super Important Meetings' # Sync an event update event_sync.provider.sync_events = event_response_with_update event_sync.sync() # Make sure the update was persisted first_event = db.session.query(Event).filter( Event.namespace_id == namespace_id, Event.calendar_id == first_calendar.id, Event.uid == 'first_event_uid').one() assert first_event.title == 'Top Secret Plotting Meeting' # Sync an event delete event_sync.provider.sync_events = event_response_with_delete event_sync.sync() # Make sure the delete was persisted. first_event = db.session.query(Event).filter( Event.namespace_id == namespace_id, Event.calendar_id == first_calendar.id, Event.uid == 'first_event_uid').first() db.session.refresh(first_event) assert first_event.status == 'cancelled' # Sync a calendar delete event_public_ids = [id_ for id_, in db.session.query(Event.public_id). filter(Event.namespace_id == namespace_id, Event.calendar_id == first_calendar.id)] event_sync.provider.sync_calendars = calendar_response_with_delete event_sync.sync() assert db.session.query(Calendar).filter( Calendar.namespace_id == namespace_id, Calendar.uid == 'first_calendar_uid').first() is None # Check that delete transactions are created for events on the deleted # calendar. deleted_event_transactions = db.session.query(Transaction).filter( Transaction.object_type == 'event', Transaction.command == 'delete', Transaction.namespace_id == namespace_id, Transaction.object_public_id.in_(event_public_ids)).all() assert len(deleted_event_transactions) == 3 # Check that events with the same uid but associated to a different # calendar still survive. assert db.session.query(Event).filter( Event.namespace_id == namespace_id).count() == 2
def start_sync(self, account_id): """ Starts a sync for the account with the given account_id. If that account doesn't exist, does nothing. """ with session_scope() as db_session: acc = db_session.query(Account).get(account_id) if acc is None: self.log.error('no such account', account_id=account_id) return fqdn = platform.node() self.log.info('starting sync', account_id=acc.id, email_address=acc.email_address) if acc.sync_host is not None and acc.sync_host != fqdn: self.log.warning('account is syncing on another host', account_id=account_id, email_address=acc.email_address, sync_host=acc.sync_host) elif acc.id not in self.monitors: try: if acc.is_sync_locked and acc.is_killed: acc.sync_unlock() acc.sync_lock() info = provider_info(acc.provider) provider_supports_condstore = info.get("condstore", None) account_supports_condstore = getattr(acc, 'supports_condstore', None) if (provider_supports_condstore or account_supports_condstore): # upgrade generic providers if they support condstore monitor = self.monitor_cls_for['generic_condstore']( acc.id, acc.namespace.id, acc.email_address, acc.provider) else: monitor = self.monitor_cls_for[acc.provider]( acc.id, acc.namespace.id, acc.email_address, acc.provider) self.monitors[acc.id] = monitor monitor.start() # For Gmail accounts, also start contacts and events sync if acc.provider == 'gmail': contact_sync = ContactSync(acc.id) self.contact_sync_monitors[acc.id] = contact_sync contact_sync.start() event_sync = EventSync(acc.id) self.event_sync_monitors[acc.id] = event_sync event_sync.start() acc.start_sync(fqdn) db_session.add(acc) db_session.commit() self.log.info('sync started', account_id=account_id) except Exception as e: self.log.error('error encountered', msg=e.message) else: self.log.info('sync already started', account_id=account_id)
def event_sync(config, db): from inbox.events.remote_sync import EventSync return EventSync(1)
def start_sync(self, account_id): """ Starts a sync for the account with the given account_id. If that account doesn't exist, does nothing. """ with session_scope() as db_session: acc = db_session.query(Account).get(account_id) if acc is None: self.log.error('no such account', account_id=account_id) return fqdn = platform.node() self.log.info('starting sync', account_id=acc.id, email_address=acc.email_address) if acc.sync_host is not None and acc.sync_host != fqdn: self.log.error( 'Sync Host Mismatch', message='account is syncing on another host {}'.format( acc.sync_host), account_id=account_id) elif acc.id not in self.monitors: # Before starting the sync, clear the heartbeat and individual # folder should_run bits. These bits will be flipped to the # correct state by the mailsync monitor. try: for status in acc.foldersyncstatuses: status.sync_should_run = False except Exception as e: self.log.error('Error resetting folder run status', message=str(e.message), account_id=acc.id) try: clear_heartbeat_status(acc.id) except Exception as e: self.log.error('Error clearing heartbeat on sync start', message=str(e.message), account_id=acc.id) try: if acc.is_sync_locked and acc.is_killed: acc.sync_unlock() acc.sync_lock() monitor = self.monitor_cls_for[acc.provider](acc) self.monitors[acc.id] = monitor monitor.start() info = acc.provider_info if info.get('contacts', None) and acc.sync_contacts: contact_sync = ContactSync(acc.email_address, acc.provider, acc.id, acc.namespace.id) self.contact_sync_monitors[acc.id] = contact_sync contact_sync.start() if info.get('events', None) and acc.sync_events: event_sync = EventSync(acc.email_address, acc.provider, acc.id, acc.namespace.id) self.event_sync_monitors[acc.id] = event_sync event_sync.start() acc.sync_started() db_session.add(acc) db_session.commit() self.log.info('Sync started', account_id=account_id, sync_host=fqdn) except Exception as e: self.log.error('sync_error', message=str(e.message), account_id=account_id) else: self.log.info('sync already started', account_id=account_id)
def test_handle_changes(db, generic_account): namespace_id = generic_account.namespace.id event_sync = EventSync(generic_account.email_address, "google", generic_account.id, namespace_id) # Sync calendars/events event_sync.provider.sync_calendars = calendar_response event_sync.provider.sync_events = event_response event_sync.sync() assert (db.session.query(Calendar).filter( Calendar.namespace_id == namespace_id, Calendar.name != "Emailed events").count() == 2) assert (db.session.query(Event).join(Calendar).filter( Event.namespace_id == namespace_id, Calendar.uid == "first_calendar_uid").count() == 3) assert (db.session.query(Event).join(Calendar).filter( Event.namespace_id == namespace_id, Calendar.uid == "second_calendar_uid").count() == 2) # Sync a calendar update with long name event_sync.provider.sync_calendars = calendar_long_name event_sync.sync() long_calendar = (db.session.query(Calendar).filter( Calendar.namespace_id == namespace_id, Calendar.uid == "long_calendar_uid").one()) assert len(long_calendar.name) == MAX_INDEXABLE_LENGTH # Sync a calendar update event_sync.provider.sync_calendars = calendar_response_with_update event_sync.provider.sync_events = event_response event_sync.sync() # Check that we have the same number of calendars and events as before assert (db.session.query(Calendar).filter( Calendar.namespace_id == namespace_id, Calendar.name != "Emailed events").count() == 3) assert (db.session.query(Event).join(Calendar).filter( Event.namespace_id == namespace_id, Calendar.uid == "first_calendar_uid").count() == 3) assert (db.session.query(Event).join(Calendar).filter( Event.namespace_id == namespace_id, Calendar.uid == "second_calendar_uid").count() == 2) assert (db.session.query(Event).join(Calendar).filter( Event.namespace_id == namespace_id, Calendar.uid == "long_calendar_uid").count() == 2) # Check that calendar attribute was updated. first_calendar = (db.session.query(Calendar).filter( Calendar.namespace_id == namespace_id, Calendar.uid == "first_calendar_uid").one()) assert first_calendar.name == "Super Important Meetings" # Sync an event update event_sync.provider.sync_events = event_response_with_update event_sync.sync() # Make sure the update was persisted first_event = (db.session.query(Event).filter( Event.namespace_id == namespace_id, Event.calendar_id == first_calendar.id, Event.uid == "first_event_uid", ).one()) assert first_event.title == "Top Secret Plotting Meeting" # Sync a participant update event_sync.provider.sync_events = event_response_with_participants_update event_sync.sync() # Make sure the update was persisted first_event = (db.session.query(Event).filter( Event.namespace_id == namespace_id, Event.calendar_id == first_calendar.id, Event.uid == "first_event_uid", ).one()) db.session.refresh(first_event) assert first_event.participants == [{ "name": "Johnny Thunders", "email": "*****@*****.**" }] # Sync an event delete event_sync.provider.sync_events = event_response_with_delete event_sync.sync() # Make sure the delete was persisted. first_event = (db.session.query(Event).filter( Event.namespace_id == namespace_id, Event.calendar_id == first_calendar.id, Event.uid == "first_event_uid", ).first()) db.session.refresh(first_event) assert first_event.status == "cancelled" # Sync a calendar delete event_public_ids = [ id_ for id_, in db.session.query(Event.public_id).filter( Event.namespace_id == namespace_id, Event.calendar_id == first_calendar.id) ] event_sync.provider.sync_calendars = calendar_response_with_delete event_sync.sync() assert (db.session.query(Calendar).filter( Calendar.namespace_id == namespace_id, Calendar.uid == "first_calendar_uid").first() is None) # Check that delete transactions are created for events on the deleted # calendar. deleted_event_transactions = (db.session.query(Transaction).filter( Transaction.object_type == "event", Transaction.command == "delete", Transaction.namespace_id == namespace_id, Transaction.object_public_id.in_(event_public_ids), ).all()) assert len(deleted_event_transactions) == 3 # Check that events with the same uid but associated to a different # calendar still survive. assert (db.session.query(Event).filter( Event.namespace_id == namespace_id).count() == 4)
def start_sync(self, account_id): """ Starts a sync for the account with the given account_id. If that account doesn't exist, does nothing. """ with session_scope() as db_session: acc = db_session.query(Account).get(account_id) if acc is None: self.log.error('no such account', account_id=account_id) return fqdn = platform.node() self.log.info('starting sync', account_id=acc.id, email_address=acc.email_address) if acc.sync_host is not None and acc.sync_host != fqdn: self.log.warning('account is syncing on another host', account_id=account_id, email_address=acc.email_address, sync_host=acc.sync_host) elif acc.id not in self.monitors: try: if acc.is_sync_locked and acc.is_killed: acc.sync_unlock() acc.sync_lock() info = provider_info(acc.provider) provider_supports_condstore = info.get("condstore", None) account_supports_condstore = getattr( acc, 'supports_condstore', None) if (provider_supports_condstore or account_supports_condstore): # upgrade generic providers if they support condstore monitor = self.monitor_cls_for['generic_condstore']( acc.id, acc.namespace.id, acc.email_address, acc.provider) else: monitor = self.monitor_cls_for[acc.provider]( acc.id, acc.namespace.id, acc.email_address, acc.provider) self.monitors[acc.id] = monitor monitor.start() # For Gmail accounts, also start contacts and events sync if acc.provider == 'gmail': contact_sync = ContactSync(acc.id) self.contact_sync_monitors[acc.id] = contact_sync contact_sync.start() event_sync = EventSync(acc.id) self.event_sync_monitors[acc.id] = event_sync event_sync.start() acc.start_sync(fqdn) db_session.add(acc) db_session.commit() self.log.info('sync started', account_id=account_id) except Exception as e: self.log.error('error encountered', msg=e.message) else: self.log.info('sync already started', account_id=account_id)
def start_sync(self, account_id): """ Starts a sync for the account with the given account_id. If that account doesn't exist, does nothing. """ with session_scope() as db_session: acc = db_session.query(Account).get(account_id) if acc is None: self.log.error('no such account', account_id=account_id) return fqdn = platform.node() self.log.info('starting sync', account_id=acc.id, email_address=acc.email_address) if acc.sync_host is not None and acc.sync_host != fqdn: self.log.error('Sync Host Mismatch', message='account is syncing on another host {}' .format(acc.sync_host), account_id=account_id) elif acc.id not in self.monitors: # Before starting the sync, clear the heartbeat and individual # folder should_run bits. These bits will be flipped to the # correct state by the mailsync monitor. try: for status in acc.foldersyncstatuses: status.sync_should_run = False except Exception as e: self.log.error('Error resetting folder run status', message=str(e.message), account_id=acc.id) try: clear_heartbeat_status(acc.id) except Exception as e: self.log.error('Error clearing heartbeat on sync start', message=str(e.message), account_id=acc.id) try: if acc.is_sync_locked and acc.is_killed: acc.sync_unlock() acc.sync_lock() monitor = self.monitor_cls_for[acc.provider](acc) self.monitors[acc.id] = monitor monitor.start() info = acc.provider_info if info.get('contacts', None) and acc.sync_contacts: contact_sync = ContactSync(acc.email_address, acc.provider, acc.id, acc.namespace.id) self.contact_sync_monitors[acc.id] = contact_sync contact_sync.start() if info.get('events', None) and acc.sync_events: event_sync = EventSync(acc.email_address, acc.provider, acc.id, acc.namespace.id) self.event_sync_monitors[acc.id] = event_sync event_sync.start() acc.sync_started() db_session.add(acc) db_session.commit() self.log.info('Sync started', account_id=account_id, sync_host=fqdn) except Exception as e: self.log.error('sync_error', message=str(e.message), account_id=account_id) else: self.log.info('sync already started', account_id=account_id)