def __deduplicate_message_download(self, crispin_client, remote_g_metadata,
uids):
"""
Deduplicate message download using X-GM-MSGID.
Returns
-------
list
Deduplicated UIDs.
"""
with mailsync_session_scope() as db_session:
local_g_msgids = g_msgids(self.account_id, db_session,
in_={remote_g_metadata[uid].msgid
for uid in uids if uid in
remote_g_metadata})
full_download, imapuid_only = partition(
lambda uid: uid in remote_g_metadata and
remote_g_metadata[uid].msgid in local_g_msgids,
sorted(uids, key=int))
if imapuid_only:
log.info('skipping already downloaded uids',
count=len(imapuid_only))
# Since we always download messages via All Mail and create the
# relevant All Mail ImapUids too at that time, we don't need to
# create them again here if we're deduping All Mail downloads.
if crispin_client.selected_folder_name != \
crispin_client.folder_names()['all']:
add_new_imapuids(crispin_client, remote_g_metadata,
self.syncmanager_lock, imapuid_only)
return full_download
def deduplicate_message_download(crispin_client, db_session, log,
syncmanager_lock, remote_g_metadata, uids):
""" Deduplicate message download using X-GM-MSGID.
Returns
-------
list
Deduplicated UIDs.
"""
local_g_msgids = set(account.g_msgids(crispin_client.account_id,
db_session,
in_=[remote_g_metadata[uid].msgid for
uid in uids if uid in
remote_g_metadata]))
full_download, imapuid_only = partition(
lambda uid: uid in remote_g_metadata and
remote_g_metadata[uid].msgid in local_g_msgids,
sorted(uids, key=int))
if imapuid_only:
log.info("Skipping {0} uids already downloaded"
.format(len(imapuid_only)))
# Since we always download messages via All Mail and create the
# relevant All Mail ImapUids too at that time, we don't need to create
# them again here if we're deduping All Mail downloads.
if crispin_client.selected_folder_name != \
crispin_client.folder_names()['all']:
add_new_imapuids(crispin_client, log, db_session,
remote_g_metadata, syncmanager_lock, imapuid_only)
return full_download
def index(self, transactions, db_session):
"""
Translate database operations to CloudSearch index operations
and perform them.
"""
docs = []
doc_service = get_doc_service()
add_txns, delete_txns = partition(
lambda trx: trx.command == 'delete', transactions)
delete_docs = [{'type': 'delete', 'id': txn.record_id}
for txn in delete_txns]
add_record_ids = [txn.record_id for txn in add_txns]
add_records = db_session.query(Contact).options(
joinedload("phone_numbers")).filter(
Contact.id.in_(add_record_ids))
add_docs = [{'type': 'add', 'id': obj.id,
'fields': cloudsearch_contact_repr(obj)}
for obj in add_records]
docs = delete_docs + add_docs
if docs:
doc_service.upload_documents(
documents=json.dumps(docs),
contentType='application/json')
self.log.info('docs indexed', adds=len(add_docs),
deletes=len(delete_docs))
def index(self, transactions, db_session):
"""
Translate database operations to CloudSearch index operations
and perform them.
"""
docs = []
doc_service = get_doc_service()
add_txns, delete_txns = partition(lambda trx: trx.command == 'delete',
transactions)
delete_docs = [{
'type': 'delete',
'id': txn.record_id
} for txn in delete_txns]
add_record_ids = [txn.record_id for txn in add_txns]
add_records = db_session.query(Contact).options(
joinedload("phone_numbers")).filter(Contact.id.in_(add_record_ids))
add_docs = [{
'type': 'add',
'id': obj.id,
'fields': cloudsearch_contact_repr(obj)
} for obj in add_records]
docs = delete_docs + add_docs
if docs:
doc_service.upload_documents(documents=json.dumps(docs),
contentType='application/json')
self._report_batch_upload()
self.log.info('docs indexed',
adds=len(add_docs),
deletes=len(delete_docs))
def __deduplicate_message_download(self, crispin_client, remote_g_metadata,
uids):
"""
Deduplicate message download using X-GM-MSGID.
Returns
-------
list
Deduplicated UIDs.
"""
with mailsync_session_scope() as db_session:
local_g_msgids = g_msgids(self.namespace_id, db_session,
in_={remote_g_metadata[uid].msgid
for uid in uids if uid in
remote_g_metadata})
full_download, imapuid_only = partition(
lambda uid: uid in remote_g_metadata and
remote_g_metadata[uid].msgid in local_g_msgids,
sorted(uids, key=int))
if imapuid_only:
log.info('downloading new uids for existing messages',
count=len(imapuid_only))
add_new_imapuids(crispin_client, remote_g_metadata,
self.syncmanager_lock, imapuid_only)
return full_download
def __deduplicate_message_download(self, crispin_client, remote_g_metadata,
uids):
"""
Deduplicate message download using X-GM-MSGID.
Returns
-------
list
Deduplicated UIDs.
"""
with mailsync_session_scope() as db_session:
local_g_msgids = g_msgids(self.namespace_id,
db_session,
in_={
remote_g_metadata[uid].msgid
for uid in uids
if uid in remote_g_metadata
})
full_download, imapuid_only = partition(
lambda uid: uid in remote_g_metadata and remote_g_metadata[uid].
msgid in local_g_msgids, sorted(uids, key=int))
if imapuid_only:
log.info('downloading new uids for existing messages',
count=len(imapuid_only))
add_new_imapuids(crispin_client, remote_g_metadata,
self.syncmanager_lock, imapuid_only)
return full_download
def accounts():
global last_calc_at, ACCOUNTS_INFO
delta = (datetime.datetime.utcnow() - last_calc_at).seconds if \
last_calc_at else None
app.logger.error('In accounts, last_calc_at, delta: {0}, {1}'.format(
str(last_calc_at), str(delta)))
if not last_calc_at or delta >= recalc_after:
app.logger.error('Recalc')
ACCOUNTS_INFO = []
accounts = g.db_session.query(Account).all()
imap, eas = partition(lambda a: a.provider == 'eas', accounts)
if imap:
ACCOUNTS_INFO.extend(calculate_imap_status(g.db_session, imap))
if eas and 'eas' in backend_module_registry:
ACCOUNTS_INFO.extend(calculate_eas_status(g.db_session, eas))
last_calc_at = datetime.datetime.utcnow()
return json.dumps(ACCOUNTS_INFO, cls=DateTimeJSONEncoder)
def __deduplicate_message_download(self, crispin_client, remote_g_metadata,
uids):
"""
Deduplicate message download using X-GM-MSGID.
Returns
-------
list
Deduplicated UIDs.
"""
with mailsync_session_scope() as db_session:
local_g_msgids = g_msgids(self.account_id, db_session,
in_={remote_g_metadata[uid].msgid
for uid in uids if uid in
remote_g_metadata})
full_download, imapuid_only = partition(
lambda uid: uid in remote_g_metadata and
remote_g_metadata[uid].msgid in local_g_msgids,
sorted(uids, key=int))
if imapuid_only:
log.info('skipping already downloaded uids',
count=len(imapuid_only))
# Since we always download messages via All Mail and create the
# relevant All Mail ImapUids too at that time, we don't need to
# create them again here if we're deduping All Mail downloads.
if crispin_client.selected_folder_name != \
crispin_client.folder_names()['all']:
add_new_imapuids(crispin_client, remote_g_metadata,
self.syncmanager_lock, imapuid_only)
return full_download
def accounts():
global last_calc_at, ACCOUNTS_INFO
delta = (datetime.datetime.utcnow() - last_calc_at).seconds if \
last_calc_at else None
app.logger.error('In accounts, last_calc_at, delta: {0}, {1}'.format(
str(last_calc_at), str(delta)))
if not last_calc_at or delta >= recalc_after:
app.logger.error('Recalc')
ACCOUNTS_INFO = []
accounts = g.db_session.query(Account).all()
imap, eas = partition(lambda a: a.provider == 'eas', accounts)
if imap:
ACCOUNTS_INFO.extend(calculate_imap_status(g.db_session, imap))
if eas and 'eas' in backend_module_registry:
ACCOUNTS_INFO.extend(calculate_eas_status(g.db_session, eas))
last_calc_at = datetime.datetime.utcnow()
return json.dumps(ACCOUNTS_INFO, cls=DateTimeJSONEncoder)
def deduplicate_message_download(crispin_client, db_session, log,
syncmanager_lock, remote_g_metadata, uids):
""" Deduplicate message download using X-GM-MSGID.
Returns
-------
list
Deduplicated UIDs.
"""
local_g_msgids = set(account.g_msgids(crispin_client.account_id,
db_session,
in_=[remote_g_metadata[uid].msgid for
uid in uids if uid in
remote_g_metadata]))
full_download, imapuid_only = partition(
lambda uid: uid in remote_g_metadata and
remote_g_metadata[uid].msgid in local_g_msgids,
sorted(uids, key=int))
if imapuid_only:
log.info("Skipping {0} uids already downloaded"
.format(len(imapuid_only)))
# Since we always download messages via All Mail and create the
# relevant All Mail ImapUids too at that time, we don't need to create
# them again here if we're deduping All Mail downloads.
if crispin_client.selected_folder_name != \
crispin_client.folder_names()['all']:
add_new_imapuids(crispin_client, log, db_session,
remote_g_metadata, syncmanager_lock, imapuid_only)
return full_download
def new_or_updated(uids, local_uids):
"""
HIGHESTMODSEQ queries return a list of messages that are *either*
new *or* updated. We do different things with each, so we need to
sort out which is which.
"""
return partition(lambda x: x in local_uids, uids)
def new_or_updated(uids, local_uids):
"""
HIGHESTMODSEQ queries return a list of messages that are *either*
new *or* updated. We do different things with each, so we need to
sort out which is which.
"""
return partition(lambda x: x in local_uids, uids)
def deduplicate_message_download(crispin_client, db_session, log,
remote_g_metadata, uids, c):
""" Deduplicate message download using X-GM-MSGID. """
local_g_msgids = set(account.g_msgids(crispin_client.account_id,
db_session, in_=[remote_g_metadata[uid]['msgid'] for uid in uids]))
full_download, imapuid_only = partition(
lambda uid: remote_g_metadata[uid]['msgid'] in local_g_msgids,
sorted(uids, key=int))
log.info("Skipping {0} uids already downloaded".format(len(imapuid_only)))
if len(imapuid_only) > 0:
add_new_imapuid(crispin_client, db_session, remote_g_metadata,
imapuid_only, c)
return full_download
def index(self, transactions, db_session):
"""
Translate database operations to CloudSearch index operations
and perform them.
"""
docs = []
doc_service = get_doc_service()
add_txns, delete_txns = partition(lambda trx: trx.command == "delete", transactions)
delete_docs = [{"type": "delete", "id": txn.record_id} for txn in delete_txns]
add_record_ids = [txn.record_id for txn in add_txns]
add_records = (
db_session.query(Contact).options(joinedload("phone_numbers")).filter(Contact.id.in_(add_record_ids))
)
add_docs = [{"type": "add", "id": obj.id, "fields": cloudsearch_contact_repr(obj)} for obj in add_records]
docs = delete_docs + add_docs
if docs:
doc_service.upload_documents(documents=json.dumps(docs), contentType="application/json")
self._report_batch_upload()
self.log.info("docs indexed", adds=len(add_docs), deletes=len(delete_docs))