def _set_fixed_dt(self, fixed_dt):
if not fixed_dt:
return None
fixed_dt = DATE_TIMESPLIT.sub('', fixed_dt)
fixed_dt = pad_timestamp(fixed_dt, PAD_14_DOWN)
fixed_dt = timestamp_to_iso_date(fixed_dt)
return fixed_dt
def _get_referrer(self, params):
ref_url = super(WBMementoIndexSource, self)._get_referrer(params)
if ref_url:
timestamp = params.get('closest', '20')
timestamp = pad_timestamp(timestamp, PAD_14_DOWN)
ref_url = self._get_replay_url(timestamp, ref_url)
ref_url = ref_url.replace('im_/', '/')
return ref_url
def cdx_clamp(cdx_iter, from_ts, to_ts):
"""
Clamp by start and end ts
"""
if from_ts and len(from_ts) < 14:
from_ts = pad_timestamp(from_ts, PAD_14_DOWN)
if to_ts and len(to_ts) < 14:
to_ts = pad_timestamp(to_ts, PAD_14_UP)
for cdx in cdx_iter:
if from_ts and cdx[TIMESTAMP] < from_ts:
continue
if to_ts and cdx[TIMESTAMP] > to_ts:
continue
yield cdx