def save_fetched_visa_status(
cls,
visa_type: VisaType,
embassy_code: EmbassyCode,
write_time: datetime,
available_date: Optional[datetime],
) -> None:
""" The method called when a new fetched result is obtained from crawler backend. The
`'latest_written'` collection will always be modified, whereas the `'available_dates'`
collection will only be modified when available date is not None
"""
embassy = USEmbassy.get_embassy_by_code(embassy_code)
write_time_utc = write_time.astimezone(tz=None).astimezone(tz=timezone.utc)
write_date_utc = write_time_utc.replace(hour=0, minute=0, second=0, microsecond=0)
write_date_emb = write_time_utc.astimezone(embassy.timezone)\
.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None)
query = {'visa_type': visa_type, 'embassy_code': embassy_code}
visa_status_query = {**query, 'write_date': write_date_utc}
overview_query = {**query, 'overview.write_date': write_date_emb}
new_fetch = {'write_time': write_time_utc, 'available_date': available_date}
# udpate document if exists, otherwise insert a new document
cls.latest_written.update_one(query, {'$set': new_fetch}, upsert=True)
if available_date is not None:
cls.visa_status.update_one(visa_status_query, {'$push': {'available_dates': new_fetch}}, upsert=True)
if cls.overview.find_one(overview_query) is None: # $(update) of array can't work with upsert
cls.overview.update_one(
query,
{
'$push': {
'overview': {
'write_date': write_date_emb,
'earliest_date': available_date,
'latest_date': available_date
}
}
},
upsert=True
)
else:
cls.overview.update_one(
overview_query,
{
'$min': {'overview.$.earliest_date': available_date},
'$max': {'overview.$.latest_date': available_date},
}
)
def find_visa_status_past24h_turning_point(
cls,
visa_type: VisaType,
embassy_code: EmbassyCode,
timestamp: datetime,
):
""" Fill in the missing minute and return the visa status detail with consecutive duplicate removed"""
visa_status = cls.find_visa_status_past24h(visa_type, embassy_code, timestamp)
if visa_status is None or len(visa_status['available_dates']) == 0:
return
embassy = USEmbassy.get_embassy_by_code(embassy_code)
interval = CGI_FETCH_TIME_INTERVAL[visa_type] if embassy.sys == 'cgi' else AIS_FETCH_TIME_INTERVAL[visa_type]
interval = (interval + 60) * 1000 # add 1min tolerance
def convert(dt: datetime):
return dt_to_utc(dt, remove_second=True)
available_dates = [{
'write_time': convert(i['write_time']),
'available_date': i['available_date'],
} for i in visa_status['available_dates']]
ts_start, ts_end = list(map(convert, visa_status['time_range']))
purified_available_dates = []
first_dp = available_dates[0]
if first_dp['write_time'] - ts_start > 1:
purified_available_dates = [{'write_time': ts_start, 'available_date': None}]
for i, (prev_dp, next_dp) in enumerate(zip(available_dates[:-1], available_dates[1:])):
if i == 0:
purified_available_dates.append(prev_dp)
if next_dp['write_time'] - prev_dp['write_time'] <= interval:
if prev_dp['available_date'] == next_dp['available_date']:
continue
else:
purified_available_dates.append(next_dp)
else:
purified_available_dates.append({'write_time': prev_dp['write_time'] + 60000, 'available_date': None})
purified_available_dates.append(next_dp)
last_dp = available_dates[-1]
if ts_end - last_dp['write_time'] > interval:
purified_available_dates.append({'write_time': last_dp['write_time'] + 60000, 'available_date': None})
return {
**visa_status,
'time_range': [ts_start, ts_end],
'available_dates': purified_available_dates,
}
def find_visa_status_overview_embtz(
cls,
visa_type: Union[VisaType, List[VisaType]],
embassy_code: Union[EmbassyCode, List[EmbassyCode]],
since_utc: datetime,
to_utc: datetime,
):
""" This method fix the problem of `cls.find_visa_status_overview` as the previous method doesn't
convert the querying date into the embassy timezone.
"""
def dt_to_date(dt: datetime) -> datetime:
return dt.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None)
def utc_to_embtz(dt: datetime, embtz: timezone) -> datetime:
return dt_to_date(dt.astimezone(embtz))
def single_target_query(visa_type: str, embassy_code: str, date_range: List[datetime]) -> List[dict]:
""" construct the sub-pipeline for mongodb aggregation `facet` stage."""
return [
{'$match': {'visa_type': visa_type, 'embassy_code': embassy_code}},
{
'$project': {
'visa_type': '$visa_type',
'embassy_code': '$embassy_code',
'overview': {
'$filter': {
'input': '$overview',
'as': 'ov',
'cond': {'$in': ['$$ov.write_date', date_range]}
}
}
}
},
{'$unwind': '$overview'},
{
'$project': {
'_id': False,
'visa_type': '$visa_type',
'embassy_code': '$embassy_code',
'write_date': '$overview.write_date',
'earliest_date': '$overview.earliest_date',
'latest_date': '$overview.latest_date',
}
},
]
if not isinstance(visa_type, list):
visa_type = [visa_type]
if not isinstance(embassy_code, list):
embassy_code = [embassy_code]
overview_target = [
{
'visa_type': vt,
'embassy_code': emb.code,
'date_range': [
utc_to_embtz(since_utc, emb.timezone) + timedelta(days=d)
for d in range(
(utc_to_embtz(to_utc, emb.timezone) - utc_to_embtz(since_utc, emb.timezone)).days + 1
)
],
} for vt in visa_type for emb in [USEmbassy.get_embassy_by_code(ec) for ec in embassy_code]
]
utc_date_range = [
dt_to_date(since_utc) + timedelta(days=d)
for d in range((dt_to_date(to_utc) - dt_to_date(since_utc)).days + 1)
]
embtz_utc_map = {tgt['embassy_code']: dict(zip(tgt['date_range'], utc_date_range)) for tgt in overview_target}
query = [
{
'$facet': {
'{}{}'.format(tgt['visa_type'], tgt['embassy_code']): single_target_query(**tgt)
for tgt in overview_target
},
},
{
'$project': {
'facet_result': {
'$setUnion': ['${}{}'.format(tgt['visa_type'], tgt['embassy_code']) for tgt in overview_target]
}
},
},
{'$unwind': '$facet_result'},
{'$replaceRoot': {'newRoot': '$facet_result'}}
]
overview_embtz = list(cls.overview.aggregate(query))
overview_utc = [{
**ov,
'write_date': embtz_utc_map[ov['embassy_code']][ov['write_date']],
} for ov in overview_embtz]
ov_groupby_date = defaultdict(list)
for overview in overview_utc:
write_date = overview.pop('write_date')
ov_groupby_date[write_date].append(overview)
return sorted(
[{'date': write_date, 'overview': overview} for write_date, overview in ov_groupby_date.items()],
key=lambda ov: ov['date'],
reverse=True,
)