def __init__(self, *args, **kwargs):
super(IntervalPullFromCybersourceTask, self).__init__(*args, **kwargs)
# Provide default for output_root at this level.
if self.output_root is None:
self.output_root = self.warehouse_path
path = url_path_join(self.warehouse_path, 'payments')
file_pattern = '*cybersource_{}.tsv'.format(self.merchant_id)
path_targets = PathSetTask([path],
include=[file_pattern],
include_zero_length=True).output()
paths = list(
set([os.path.dirname(target.path) for target in path_targets]))
dates = [path.rsplit('/', 2)[-1] for path in paths]
latest_date = sorted(dates)[-1]
latest_completion_date = datetime.datetime.strptime(
latest_date, "dt=%Y-%m-%d").date()
run_date = latest_completion_date + datetime.timedelta(days=1)
# Limit intervals to merchant account close date(if any).
if self.merchant_close_date:
run_date = min(run_date, self.merchant_close_date)
self.interval_end = min(self.interval_end,
self.merchant_close_date)
self.selection_interval = date_interval.Custom(self.interval_start,
run_date)
self.run_interval = date_interval.Custom(run_date, self.interval_end)
def requires(self):
paypal_interval = date_interval.Custom(self.paypal_interval_start,
self.import_date)
for date in paypal_interval:
url = url_path_join(self.warehouse_path, 'payments',
'dt=' + date.isoformat(), 'paypal.tsv')
yield ExternalURL(url=url)
def __init__(self, *args, **kwargs):
super(IntervalPullFromAffiliateWindowTask,
self).__init__(*args, **kwargs)
# Provide default for output_root at this level.
if self.output_root is None:
self.output_root = url_path_join(self.warehouse_path, 'fees',
'affiliate_window')
path = self.output_root
file_pattern = '*affiliate_window.tsv'
path_targets = PathSetTask([path],
include=[file_pattern],
include_zero_length=True).output()
if path_targets:
paths = list(
set([os.path.dirname(target.path) for target in path_targets]))
dates = [path.rsplit('/', 2)[-1] for path in paths]
latest_date = sorted(dates)[-1]
latest_completion_date = datetime.datetime.strptime(
latest_date, "dt=%Y-%m-%d").date()
self.interval_start = latest_completion_date + datetime.timedelta(
days=1)
print("Found previous reports to {}".format(latest_date))
else:
# If this is the first run, start from the beginning
print(
"Couldn't find last completed date, defaulting to start date: {}"
.format(self.interval_start))
self.run_interval = date_interval.Custom(self.interval_start,
self.interval_end)
print("Running reports from interval {}".format(self.run_interval))
def __init__(self, *args, **kwargs):
super(IntervalPullFromCybersourceTask, self).__init__(*args, **kwargs)
# Provide default for output_root at this level.
if self.output_root is None:
self.output_root = self.warehouse_path
if self.interval is None:
self.interval = date_interval.Custom(self.interval_start, self.interval_end)
def __init__(self, *args, **kwargs):
super(PaypalTransactionsIntervalTask, self).__init__(*args, **kwargs)
# Provide default for output_root at this level.
if self.output_root is None:
self.output_root = self.warehouse_path
path = url_path_join(self.warehouse_path, 'payments')
path_targets = PathSetTask([path], include=['*paypal.tsv']).output()
paths = list(set([os.path.dirname(target.path) for target in path_targets]))
dates = [path.rsplit('/', 2)[-1] for path in paths]
latest_date = sorted(dates)[-1]
latest_completion_date = datetime.datetime.strptime(latest_date, "dt=%Y-%m-%d").date()
run_date = latest_completion_date + datetime.timedelta(days=1)
self.selection_interval = date_interval.Custom(self.interval_start, run_date)
self.run_interval = date_interval.Custom(run_date, self.interval_end)
def __init__(self, *args, **kwargs):
super(PaypalTransactionsIntervalTask, self).__init__(*args, **kwargs)
# Provide default for output_root at this level.
if self.output_root is None:
self.output_root = self.warehouse_path
if self.interval is None:
self.interval = date_interval.Custom(self.interval_start,
self.interval_end)
def requires(self):
config = get_config()
for merchant_id in self.cybersource_merchant_ids:
section_name = 'cybersource:' + merchant_id
interval_start = luigi.DateParameter().parse(config.get(section_name, 'interval_start'))
interval_end = self.import_date
merchant_close_date = config.get(section_name, 'merchant_close_date', '')
if merchant_close_date:
parsed_date = luigi.DateParameter().parse(merchant_close_date)
interval_end = min(self.import_date, parsed_date)
cybersource_interval = date_interval.Custom(interval_start, interval_end)
for date in cybersource_interval:
filename = "cybersource_{}.tsv".format(merchant_id)
url = url_path_join(self.warehouse_path, 'payments', 'dt=' + date.isoformat(), filename)
yield ExternalURL(url=url)