def initiate_collections(cls, since: datetime) -> None:
""" Initiate the visa status storage with the file based data."""
since_midnight = since.replace(hour=0, minute=0, second=0, microsecond=0)
today_midnight = datetime.combine(datetime.now().date(), datetime.min.time())
date_range = [since_midnight + timedelta(days=d) for d in range((today_midnight - since_midnight).days + 1)]
embassy_lst = USEmbassy.get_embassy_lst()
cls.drop()
cls.visa_status.create_index([('write_date', pymongo.ASCENDING)])
for vt in VISA_TYPES:
for emb in embassy_lst:
print()
accumulated_inserted = 0
for date in date_range:
file_path = util.construct_data_file_path(vt, emb.location, date.strftime('%Y/%m/%d'))
if not os.path.exists(file_path):
continue
with open(file_path) as f:
fetched_result_lst = [util.file_line_to_dt(ln) for ln in f.readlines()]
available_dates_arr = [
{'write_time': datetime.combine(date.date(), wt), 'available_date': avai_dt}
for wt, avai_dt in fetched_result_lst
]
cls.visa_status.insert_one(
{
'visa_type': vt,
'embassy_code': emb.code,
'write_date': date,
'available_dates': available_dates_arr
}
)
if len(available_dates_arr) > 0:
earliest_dt = min([d['available_date'] for d in available_dates_arr])
latest_dt = max([d['available_date'] for d in available_dates_arr])
cls.overview.update_one(
{'visa_type': vt, 'embassy_code': emb.code},
{
'$push': {
'overview': {
'write_date': date,
'earliest_date': earliest_dt,
'latest_date': latest_dt,
}
}
},
upsert=True,
)
accumulated_inserted += len(available_dates_arr)
print(
f'Inserted: {vt}-{emb.location}-{date.year}/{date.month}/{date.day}\
\t\t{len(available_dates_arr)}\trecords |\t{accumulated_inserted} in total',
end='\r'
)
def restore_overview(cls) -> None:
""" This method should only be used when `mongorestore` is executed and the
`tuixue.visa_status` collection is restored.
"""
cls.drop('overview')
embassy_lst = USEmbassy.get_embassy_lst()
for visa_type in VISA_TYPES:
for emb in embassy_lst:
print()
avai_dt_cache = defaultdict(list)
all_avai_dt = cls.visa_status.aggregate([
{'$match': {'visa_type': visa_type, 'embassy_code': emb.code}},
{'$unwind': '$available_dates'},
{
'$project': {
'_id': False,
'write_time': '$available_dates.write_time',
'available_date': '$available_dates.available_date'
}
},
])
for adt in all_avai_dt:
write_time_utc = adt['write_time']
available_date = adt['available_date']
write_time_emb = write_time_utc.astimezone(emb.timezone)
write_date_emb = write_time_emb.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None)
avai_dt_cache[write_date_emb].append(available_date)
for write_date, avai_dt_arr in avai_dt_cache.items():
if len(avai_dt_arr) > 0:
earliest_dt, latest_dt = min(avai_dt_arr), max(avai_dt_arr)
cls.overview.update_one(
{'visa_type': visa_type, 'embassy_code': emb.code},
{
'$push': {
'overview': {
'write_date': write_date,
'earliest_date': earliest_dt,
'latest_date': latest_dt,
}
}
},
upsert=True,
)
print(
'Update tuixue.overview: {}\t{}\t\t\t{}'.format(
visa_type,
emb.location,
write_date.strftime('%Y/%m/%d')
),
end='\r'
)
def initiate_latest_written_parallel(cls, sys: str) -> None:
""" write an empty latest_written record for every embassy and visa type.
this method pick the latest `write_date` for a `(visa_type, embassy_code)` pair, then get
the last written record from `available_dates` array of it. And overwrite the whole
`last_written` collection.
"""
embassy_code_lst = [emb.code for emb in USEmbassy.get_embassy_lst() if emb.sys == sys]
query_param = list(cls.visa_status.aggregate([
{
'$group': {
'_id': {'visa_type': '$visa_type', 'embassy_code': '$embassy_code'},
'write_date': {'$max': '$write_date'},
},
},
{'$replaceRoot': {'newRoot': {'$mergeObjects': ['$_id', {'write_date': '$write_date'}]}}},
]))
query_param = [query for query in query_param if query['embassy_code'] in embassy_code_lst]
last_effective_write = cls.visa_status.aggregate([
{'$facet': {'{}{}'.format(q['visa_type'], q['embassy_code']): [
{'$match': q},
{
'$project': {
'_id': False,
'visa_type': True,
'embassy_code': True,
'available_date': {'$slice': ['$available_dates.available_date', -1]},
},
},
{'$unwind': '$available_date'},
] for q in query_param}},
{
'$project': {
'facet_result': {
'$setUnion': ['${}{}'.format(q['visa_type'], q['embassy_code']) for q in query_param],
},
},
},
{'$unwind': '$facet_result'},
{'$replaceRoot': {'newRoot': '$facet_result'}},
{'$set': {'write_time': datetime.now(timezone.utc)}},
], allowDiskUse=True)
cls.latest_written.drop()
cls.latest_written.insert_many(list(last_effective_write))
def send_subscription_confirmation(cls, email: str,
subs_lst: List[DB.EmailSubscription]):
""" Send the email for confirmation of email subscription."""
confirmation_url = URL(
f'https://{FRONTEND_BASE_URI}/visa/email/subscription')
confirmation_url.query_param.set('email', email)
for visa_type, code, till in subs_lst:
confirmation_url.query_param.append('visa_type', visa_type.value)
confirmation_url.query_param.append('code', code.value)
confirmation_url.query_param.append('till', till)
subscription_str = '<ul>\n{}\n</ul>'.format('\n'.join([
'<li>{} Visa at {} till {}.</li>'.format(
VISA_TYPE_DETAILS[vt],
next((e.name_en
for e in USEmbassy.get_embassy_lst() if e.code == ec),
'None'),
tl.strftime('%Y/%m/%d') if tl != datetime.max else 'FOREVER',
) for vt, ec, tl in subs_lst
]))
content = SUBSCRIPTION_CONFIRMATION_CONTENT.format(
user=email.split('@')[0],
email=email,
subscription_str=subscription_str,
confirmation_url=confirmation_url,
)
for _ in range(10): # for robust
sent = cls.send_email(
title=SUBSCRIPTION_CONFIRMATION_TITLE.format(email=email),
content=content,
receivers=[email])
if sent:
break
else:
sent = False
return sent
def initiate_latest_written_sequential(cls, sys: str, backtrack_hr: int = 12) -> None:
""" Initate latest_written in sequentail order."""
embassy_code_lst = [emb.code for emb in USEmbassy.get_embassy_lst() if emb.sys == sys]
now = datetime.now()
start = datetime.combine((now - timedelta(hours=backtrack_hr)).date(), datetime.min.time())
end = datetime.combine(now.date(), datetime.min.time())
dates = [start + timedelta(days=d) for d in range((end - start).days + 1)]
query_param = cls.visa_status.aggregate([
{'$match': {'write_date': {'$in': dates}}},
{
'$group': {
'_id': {'visa_type': '$visa_type', 'embassy_code': '$embassy_code'},
'write_date': {'$max': '$write_date'},
},
},
{'$replaceRoot': {'newRoot': {'$mergeObjects': ['$_id', {'write_date': '$write_date'}]}}},
], allowDiskUse=True)
for query in query_param:
if query['embassy_code'] not in embassy_code_lst:
continue
cursor = cls.visa_status.aggregate([
{'$match': query},
{
'$project': {
'_id': False,
'write_time': datetime.now(timezone.utc),
'available_date': {'$slice': ['$available_dates.available_date', -1]},
},
},
{'$unwind': '$available_date'},
], allowDiskUse=True)
query.pop('write_date')
for last_effective_fetch in cursor:
cls.latest_written.update_one(query, {'$set': last_effective_fetch}, upsert=True)
def send_subscription_confirmation(cls, email: str,
subs_lst: List[DB.EmailSubscription]):
""" Send the email for confirmation of email subscription."""
query_dct = {'visa_type': [], 'code': [], 'till': []}
for visa_type, code, till in subs_lst:
query_dct['visa_type'].append(visa_type)
query_dct['code'].append(code)
query_dct['till'].append(till)
# Construct the redirect frontend url
confirmation_url = urlunsplit(
('https', 'tuixue.online', '/subscription/email',
urlencode(query_dct, doseq=True, quote_via=quote), ''))
subscription_str = '<ul>\n{}\n</ul>'.format('\n'.join([
'<li>{} Visa at {} till {}.</li>'.format(
vt,
next((e.name_en
for e in USEmbassy.get_embassy_lst() if e.code == ec),
'None'),
tl.strftime('%Y/%m/%d') if tl != datetime.max else 'FOREVER',
) for vt, ec, tl in subs_lst
]))
content = SUBSCRIPTION_CONFIRMATION_CONTENT.format(
email=email,
subscription_str=subscription_str,
confirmation_url=confirmation_url,
)
for _ in range(10): # for robust
sent = cls.send_email(
title=SUBSCRIPTION_CONFIRMATION_TITLE.format(email=email),
content=content,
receivers=[email])
if sent:
break
return sent
def send_unsubscription_confirmation(cls, email: str):
""" Send the email for confirmation of email unsubscription. """
subs_lst_by_email = DB.Subscription.get_subscriptions_by_email(email)
if len(subs_lst_by_email
) == 0: # If the user has no subscription/email doesn't exist
for _ in range(10):
sent = cls.send_email(
title=UNSUBSCRIPTION_EMPTY_SUBS_TITLE.format(email=email),
content=UNSUBSCRIPTION_EMPTY_SUBS_CONTENT.format(
user=email.split('@')[0],
email=email,
base_uri=FRONTEND_BASE_URI),
receivers=[email],
)
if sent:
break
else:
sent = False
return sent
unsubs_url = URL(
f'https://{FRONTEND_BASE_URI}/visa/email/unsubscription'
) # Unsubscription confirmation url
unsubs_url.query_param.set('email', email)
unsubs_all_url = unsubs_url.copy()
unsubs_info = []
for subs in subs_lst_by_email:
url = unsubs_url.copy()
url.query_param.set('visa_type', subs['visa_type'])
url.query_param.set('code', subs['embassy_code'])
url.query_param.set('till', subs['till'])
unsubs_info.append((subs['visa_type'], subs['embassy_code'],
subs['till'], subs['expired'], url))
unsubs_all_url.query_param.append('visa_type', subs['visa_type'])
unsubs_all_url.query_param.append('code', subs['embassy_code'])
unsubs_all_url.query_param.append('till', subs['till'])
unsubscription_str = '{}'.format('\n'.join([
'<li>{} Visa at {} {} on {}: click <a href="{}">this link</a> to unsubscribe.</li>'
.format(
VISA_TYPE_DETAILS[vt],
next((e.name_en
for e in USEmbassy.get_embassy_lst() if e.code == ec),
'None'),
'expired' if exp else 'expiring',
tl.strftime('%Y/%m/%d') if tl.year < 9999 else 'FOREVER',
url,
) for vt, ec, tl, exp, url in unsubs_info
]))
content = UNSUBSCRIPTION_CONFIRMATION_CONTENT.format(
user=email.split('@')[0],
email=email,
unsubscription_str=unsubscription_str,
unsubscribe_all_url=unsubs_all_url,
)
for _ in range(10):
sent = cls.send_email(title=UNSUBSCRIPTION_CONFIRMATION_TITLE,
content=content,
receivers=[email])
if sent:
break
else:
sent = False
return sent
def initiate_collections_tz(cls, since: datetime) -> None:
""" Initiate the database with following handling of datetime object regarding timezone.
1. All of the `available_date` data are stored as is. (what we fetch is what we store)
2. All of the `write_time` and `write_date` data in Mongo collections **`visa_status`**
and **`latest_written`** are stored in UTC+0 standard time.
3. **(Very important here)** All of the `write_time` and `write_date` data in Mongo
collection **`overview`** are stored in the time in the local time zone of a given
U.S. Embassy location. e.g. The overview data of U.S. Embassy in Phnom Pend on the
date Oct 10th, 2020 stands for the time range `"2020-10-10T00:00+07:00"` to
`"2020-10-10T23:59+07:00"`, **NOT** `"2020-10-10T00:00+00:00"` to `"2020-10-10T23:59+00:00"`.
4. All time data in a HTTP request from frontend **must be** a UTC standard time. The
`Date.toISOString` is the default way we construct the time related query in a request
url in frontend. FastAPI backend should add a layer of logic that consolidate the received
datetime object must have a `tzinfo` attribute otherwise should return a 422 status code.
"""
since_midnight = since.replace(hour=0, minute=0, second=0, microsecond=0)
today_midnight = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
date_range = [since_midnight + timedelta(days=d) for d in range((today_midnight - since_midnight).days + 1)]
embassy_lst = USEmbassy.get_embassy_lst()
cls.drop()
cls.visa_status.create_index([('write_date', pymongo.ASCENDING)])
for vt in VISA_TYPES:
for emb in embassy_lst:
print() # Go to a new line (inner loop using end='\r')
avai_dt_cache_utc = defaultdict(list)
avai_dt_cache_emb = defaultdict(list)
for date in date_range:
file_path = util.construct_data_file_path(vt, emb.location, date.strftime('%Y/%m/%d'))
if not os.path.exists(file_path):
continue
with open(file_path) as f:
available_dates_arr = [
{'write_time': datetime.combine(date.date(), wt), 'available_date': avai_dt}
for wt, avai_dt in [util.file_line_to_dt(ln) for ln in f.readlines()]
]
for adt in available_dates_arr:
write_time_utc = adt['write_time'].astimezone(tz=None).astimezone(tz=timezone.utc)
write_date_utc = write_time_utc.replace(hour=0, minute=0, second=0, microsecond=0)
write_date_emb = write_time_utc\
.astimezone(emb.timezone)\
.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None)
available_date = adt['available_date']
avai_dt_cache_utc[write_date_utc].append(
{'write_time': write_time_utc, 'available_date': available_date}
)
avai_dt_cache_emb[write_date_emb].append(available_date)
print(' ' * 150, end='\r') # erase previous print
print('Reading: {}-{}-{}'.format(vt, emb.location, date.strftime('%Y/%m/%d')), end='\t\t')
print(
'UTC\t{}: {}'.format(
write_date_utc.strftime('%Y/%m/%d'),
len(avai_dt_cache_utc[write_date_utc])
),
end='\t'
)
print(
'EMB\t{}: {}'.format(
write_date_emb.strftime('%Y/%m/%d'),
len(avai_dt_cache_emb[write_date_emb])
),
end='\t'
)
print(
'|Total:\tUTC-{}\tEMB-{}'.format(
sum([len(cache_len) for cache_len in avai_dt_cache_emb.values()]),
sum([len(cache_len) for cache_len in avai_dt_cache_emb.values()]),
),
end='\r'
)
if len(avai_dt_cache_utc) > 0:
cls.visa_status.insert_many([ # insert all visa status fetch result in one write
{
'visa_type': vt,
'embassy_code': emb.code,
'write_date': write_date,
'available_dates': avai_dt_arr,
} for write_date, avai_dt_arr in avai_dt_cache_utc.items()
])
else:
print('Skipping: {}-{} No records'.format(vt, emb.location), end='\r')
for write_date, avai_dt_arr in avai_dt_cache_emb.items():
if len(avai_dt_arr) > 0:
earliest_dt, latest_dt = min(avai_dt_arr), max(avai_dt_arr)
cls.overview.update_one(
{'visa_type': vt, 'embassy_code': emb.code},
{
'$push': {
'overview': {
'write_date': write_date,
'earliest_date': earliest_dt,
'latest_date': latest_dt,
}
}
},
upsert=True,
)