def segment_source(self):
if self.date_range[0] is None:
return beam.Create([])
dt = datetimeFromTimestamp(self.date_range[0])
ts = timestampFromDatetime(dt - timedelta(days=1))
try:
source = GCPSource(gcp_path=self.options.segments,
first_date_ts=ts,
last_date_ts=ts)
except HttpError as exn:
logging.warn("Segment source not found: %s %s" %
(self.options.segments, dt))
if exn.status_code == 404:
return beam.Create([])
else:
raise
return source
def __init__(self,
table,
dataset=None,
project=None,
first_date_ts=None,
last_date_ts=None):
self.first_date_ts = first_date_ts
self.last_date_ts = last_date_ts
self.table_ref = decode_table_ref(table, dataset, project)
table_id = self.table_ref.tableId
if first_date_ts is not None:
assert last_date_ts is not None, 'Must supply both first_date and last_date, or neither'
dt = datetimeFromTimestamp(first_date_ts)
table_id = '{}{}'.format(table_id, dt.strftime('%Y%m%d'))
client = BigQueryWrapper()
self._table_info = client._get_table(self.table_ref.projectId,
self.table_ref.datasetId,
table_id)
def _date_str(self, date_ts):
"""convert a timestamp to a string date representation"""
return datetimeFromTimestamp(date_ts).strftime(self.DATE_FORMAT)
def _seg_id(ssvid, ts):
ts = datetimeFromTimestamp(ts)
return '{}-{}'.format(ssvid, datetime2str(ts))