def process_item(self, item, spider): if 'search_results' not in getattr(spider, 'pipelines', []): return item dt = Delorean() item['id'] = item['id'][0] item['auction_id'] = item['auction_id'][0] item['site'] = item['site'][0] item['link'] = item['link'][0] item['name'] = item['name'][0] item['price'] = ','.join( [x.replace(',', '').replace('$', '') for x in item['price']]) if 'price' in item else 0 item['modified'] = int(dt.epoch()) existing_record = Session().query(AuctionItem)\ .filter(AuctionItem.id == item['id'])\ .filter(AuctionItem.auction_id == item['auction_id']).all() item = AuctionItem(**item) if existing_record is not None: item.update() else: item.create() return item
def process_item(self, item, spider): if 'search_results' not in getattr(spider,'pipelines',[]): return item dt = Delorean() item['id'] = item['id'][0] item['auction_id'] = item['auction_id'][0] item['site'] = item['site'][0] item['link'] = item['link'][0] item['name'] = item['name'][0] item['price'] = ','.join([x.replace(',','').replace('$','') for x in item['price']]) if 'price' in item else 0 item['modified'] = int(dt.epoch()) existing_record = Session().query(AuctionItem)\ .filter(AuctionItem.id == item['id'])\ .filter(AuctionItem.auction_id == item['auction_id']).all() item = AuctionItem(**item) if existing_record is not None: item.update() else: item.create() return item
def get(self): now = Delorean().truncate('minute') mtime = self.request.get('mtime') logging.info('match schedule_timestamp query = %f' % now.epoch()) if mtime: logging.debug('manual fire schedule %s ' % mtime) jobs = Schedule.query(Schedule.schedule_timestamp == float(mtime), Schedule.error == None, Schedule.status == '').fetch() elif settings.DEBUG: jobs = Schedule.query(Schedule.schedule_timestamp == 1432634400.0, Schedule.error == None, Schedule.status == '').fetch() else: jobs = Schedule.query(Schedule.schedule_timestamp == now.epoch(), Schedule.error == None, Schedule.status == '').fetch() for job in jobs: logging.info('job schedule found!!, categroy:%s, hour_capacity= %d' % (job.category, job.hour_capacity)) mailer2_async = Mailer2Async(job.key, ['mailer']) tasks.addTask(['mailer'], mailer2_async.run)
def epoch(date): dt = Delorean(datetime=date, timezone='US/Eastern') return int(dt.epoch())
def epoch(date): dt = Delorean(datetime=date, timezone='US/Eastern') return int(dt.epoch())
class DeloreanTests(TestCase): def setUp(self): date1 = datetime(2013, 1, 3, 4, 31, 14, 148546) self.do = Delorean(datetime=date1, timezone="UTC") def test_truncation_hour(self): """ Tests that truncate always works """ self.do.truncate('hour') self.assertEqual(self.do.naive(), datetime(2013, 1, 3, 4, 0)) def test_truncation_second(self): """ Tests that truncate always works """ self.do.truncate('second') self.assertEqual(self.do.naive(), datetime(2013, 1, 3, 4, 31, 14, 0)) def test_truncation_minute(self): """ Tests that truncate always works """ self.do.truncate('minute') self.assertEqual(self.do.naive(), datetime(2013, 1, 3, 4, 31, 0, 0)) def test_truncation_day(self): """ Tests that truncate always works """ self.do.truncate('day') self.assertEqual(self.do.naive(), datetime(2013, 1, 3, 0, 0, 0, 0)) def test_truncation_month(self): """ Tests that truncate always works """ self.do.truncate('month') self.assertEqual(self.do.naive(), datetime(2013, 1, 1, 0, 0, 0, 0)) def test_truncation_year(self): """ Tests that truncate always works """ self.do.truncate('year') self.assertEqual(self.do.naive(), datetime(2013, 1, 1, 0, 0, 0, 0)) def test_date(self): self.assertEqual(self.do.date, date(2013, 1, 3)) def test_datetime(self): self.assertEqual(self.do.naive(), datetime(2013, 1, 3, 4, 31, 14, 148546)) def test_naive(self): dt1 = Delorean() dt_naive = dt1.naive() self.assertEqual(dt_naive.tzinfo, None) def test_naive_timezone(self): dt1 = Delorean(timezone="US/Eastern").truncate('second').naive() dt2 = Delorean().truncate('second').naive() self.assertEqual(dt2, dt1) self.assertEqual(dt1.tzinfo, None) def test_localize(self): dt = datetime.today() utc = timezone("UTC") dt = localize(dt, "UTC") self.assertEqual(dt.tzinfo, utc) def test_failure_truncation(self): self.assertRaises(ValueError, self.do.truncate, "century") def test_normalize(self): dt1 = Delorean() dt2 = Delorean(timezone="US/Eastern") dt1.truncate('minute') dt2.truncate('minute') dt_normalized = normalize(dt1.datetime, "US/Eastern") self.assertEqual(dt2.datetime, dt_normalized) def test_normalize_failure(self): naive_datetime = datetime.today() self.assertRaises(ValueError, normalize, naive_datetime, "US/Eastern") def test_localize_failure(self): dt1 = localize(datetime.utcnow(), "UTC") self.assertRaises(ValueError, localize, dt1, "UTC") def test_timezone(self): utc = timezone('UTC') do_timezone = Delorean().timezone() self.assertEqual(utc, do_timezone) def test_datetime_timezone_default(self): do = Delorean() do.truncate('minute') dt1 = datetime_timezone(UTC) self.assertEqual(dt1.replace(second=0, microsecond=0), do.datetime) def test_datetime_timezone(self): do = Delorean(timezone="US/Eastern") do.truncate("minute") dt1 = datetime_timezone(tz="US/Eastern") self.assertEqual(dt1.replace(second=0, microsecond=0), do.datetime) def test_parse(self): do = parse('Thu Sep 25 10:36:28 BRST 2003') dt1 = utc.localize(datetime(2003, 9, 25, 10, 36, 28)) self.assertEqual(do.datetime, dt1) def test_parse_with_utc_year_fill(self): do = parse('Thu Sep 25 10:36:28') dt1 = utc.localize(datetime(2013, 9, 25, 10, 36, 28)) self.assertEqual(do.datetime, dt1) def test_parse_with_timezone_year_fill(self): do = parse('Thu Sep 25 10:36:28') dt1 = utc.localize(datetime(2013, 9, 25, 10, 36, 28)) self.assertEqual(do.datetime, dt1) self.assertEqual(do._tz, "UTC") def test_move_namedday(self): dt_next = datetime(2013, 1, 4, 4, 31, 14, 148546, tzinfo=utc) dt_next_2 = datetime(2013, 1, 11, 4, 31, 14, 148546, tzinfo=utc) dt_last = datetime(2012, 12, 28, 4, 31, 14, 148546, tzinfo=utc) dt_last_2 = datetime(2012, 12, 21, 4, 31, 14, 148546, tzinfo=utc) d_obj_next = self.do.next_friday() d_obj_next_2 = self.do.next_friday(2) d_obj_last = self.do.last_friday() d_obj_last_2 = self.do.last_friday(2) self.assertEqual(dt_next, d_obj_next.datetime) self.assertEqual(dt_last, d_obj_last.datetime) self.assertEqual(dt_next_2, d_obj_next_2.datetime) self.assertEqual(dt_last_2, d_obj_last_2.datetime) def test_move_namedday_function(self): dt_next = datetime(2013, 1, 4, 4, 31, 14, 148546, tzinfo=utc) dt_last = datetime(2012, 12, 28, 4, 31, 14, 148546, tzinfo=utc) d_obj_next = move_datetime_namedday(self.do.datetime, 'next', 'friday') d_obj_last = move_datetime_namedday(self.do.datetime, 'last', 'friday') self.assertEqual(dt_next, d_obj_next) self.assertEqual(dt_last, d_obj_last) def test_move_week(self): dt_next = datetime(2013, 1, 10, 4, 31, 14, 148546, tzinfo=utc) dt_next_2 = datetime(2013, 1, 17, 4, 31, 14, 148546, tzinfo=utc) dt_last = datetime(2012, 12, 27, 4, 31, 14, 148546, tzinfo=utc) dt_last_2 = datetime(2012, 12, 20, 4, 31, 14, 148546, tzinfo=utc) d_obj_next = self.do.next_week() d_obj_next_2 = self.do.next_week(2) d_obj_last = self.do.last_week() d_obj_last_2 = self.do.last_week(2) self.assertEqual(dt_next, d_obj_next.datetime) self.assertEqual(dt_last, d_obj_last.datetime) self.assertEqual(dt_next_2, d_obj_next_2.datetime) self.assertEqual(dt_last_2, d_obj_last_2.datetime) def test_move_week_function(self): dt_next = datetime(2013, 1, 10, 4, 31, 14, 148546, tzinfo=utc) dt_last = datetime(2012, 12, 27, 4, 31, 14, 148546, tzinfo=utc) d_obj_next = move_datetime_week(self.do.datetime, 'next', 1) d_obj_last = move_datetime_week(self.do.datetime, 'last', 1) self.assertEqual(dt_next, d_obj_next) self.assertEqual(dt_last, d_obj_last) def test_move_month(self): dt_next = datetime(2013, 2, 3, 4, 31, 14, 148546, tzinfo=utc) dt_next_2 = datetime(2013, 3, 3, 4, 31, 14, 148546, tzinfo=utc) dt_last = datetime(2012, 12, 3, 4, 31, 14, 148546, tzinfo=utc) dt_last_2 = datetime(2012, 11, 3, 4, 31, 14, 148546, tzinfo=utc) d_obj_next = self.do.next_month() d_obj_next_2 = self.do.next_month(2) d_obj_last = self.do.last_month() d_obj_last_2 = self.do.last_month(2) self.assertEqual(dt_next, d_obj_next.datetime) self.assertEqual(dt_last, d_obj_last.datetime) self.assertEqual(dt_next_2, d_obj_next_2.datetime) self.assertEqual(dt_last_2, d_obj_last_2.datetime) def test_move_month_function(self): dt_next = datetime(2013, 2, 3, 4, 31, 14, 148546, tzinfo=utc) dt_last = datetime(2012, 12, 3, 4, 31, 14, 148546, tzinfo=utc) d_obj_next = move_datetime_month(self.do.datetime, 'next', 1) d_obj_last = move_datetime_month(self.do.datetime, 'last', 1) self.assertEqual(dt_next, d_obj_next) self.assertEqual(dt_last, d_obj_last) def test_move_year(self): dt_next = datetime(2014, 1, 3, 4, 31, 14, 148546, tzinfo=utc) dt_next_2 = datetime(2015, 1, 3, 4, 31, 14, 148546, tzinfo=utc) dt_last = datetime(2012, 1, 3, 4, 31, 14, 148546, tzinfo=utc) dt_last_2 = datetime(2011, 1, 3, 4, 31, 14, 148546, tzinfo=utc) d_obj_next = self.do.next_year() d_obj_next_2 = self.do.next_year(2) d_obj_last = self.do.last_year() d_obj_last_2 = self.do.last_year(2) self.assertEqual(dt_next, d_obj_next.datetime) self.assertEqual(dt_last, d_obj_last.datetime) self.assertEqual(dt_next_2, d_obj_next_2.datetime) self.assertEqual(dt_last_2, d_obj_last_2.datetime) def test_move_year_function(self): dt_next = datetime(2014, 1, 3, 4, 31, 14, 148546, tzinfo=utc) dt_last = datetime(2012, 1, 3, 4, 31, 14, 148546, tzinfo=utc) d_obj_next = move_datetime_year(self.do.datetime, 'next', 1) d_obj_last = move_datetime_year(self.do.datetime, 'last', 1) self.assertEqual(dt_next, d_obj_next) self.assertEqual(dt_last, d_obj_last) def test_range_count(self): """ tests the range method with count used """ count = list(stops(DAILY, count=5)) self.assertEqual(len(count), 5) def test_range_with_start(self): dates1 = [] for do in stops(DAILY, count=5, start=datetime.utcnow()): do.truncate('minute') dates1.append(do) do = Delorean().truncate('minute') dates2 = [] for x in range(5): dates2.append(do.next_day(x)) self.assertEqual(dates1, dates2) def test_range_with_start_and_stop(self): dates1 = [] tomorrow = datetime.utcnow() + timedelta(days=1) for do in stops(DAILY, start=datetime.utcnow(), until=tomorrow): do.truncate('minute') dates1.append(do) do = Delorean().truncate('minute') dates2 = [] for x in range(2): dates2.append(do.next_day(x)) self.assertEqual(dates1, dates2) def test_range_with_interval(self): dates1 = [] for do in stops(DAILY, interval=2, count=3, start=datetime.utcnow()): do.truncate('minute') dates1.append(do) do = Delorean().truncate('minute') dates2 = [] for x in range(6): if (x % 2) == 0: dates2.append(do.next_day(x)) self.assertEqual(dates1, dates2) def test_delorean_failure(self): dt = datetime.utcnow() dt = utc.localize(dt) self.assertRaises(DeloreanInvalidDatetime, Delorean, datetime=dt) def test_delorean_with_datetime(self): dt = datetime.utcnow() d = Delorean(datetime=dt, timezone=UTC) dt = utc.localize(dt) self.assertEqual(dt, d._dt) self.assertEqual(UTC, d._tz) def test_delorean_with_timezone(self): dt = datetime.utcnow() d = Delorean(datetime=dt, timezone=UTC) d = d.shift("US/Eastern") dt = utc.localize(dt) dt = est.normalize(dt) self.assertEqual(dt, d._dt) self.assertEqual(est, timezone(d._tz)) def test_delorean_with_only_timezone(self): dt = datetime.utcnow() dt = utc.localize(dt) dt = est.normalize(dt) dt = dt.replace(second=0, microsecond=0) d = Delorean(timezone="US/Eastern") d.truncate('minute') self.assertEqual(est, timezone(d._tz)) self.assertEqual(dt, d._dt) def testparse_with_timezone(self): d1 = parse("2011/01/01 00:00:00 -0700") d2 = datetime(2011, 1, 1, 7, 0) d2 = utc.localize(d2) self.assertEqual(d2, d1.datetime) self.assertEqual(utc, timezone(d1._tz)) def test_shift_failure(self): self.assertRaises(DeloreanInvalidTimezone, self.do.shift, "US/Westerrn") def test_epoch(self): unix_time = self.do.epoch() self.assertEqual(unix_time, 1357187474.148546) def test_epoch_creation(self): do = epoch(1357187474.148546) self.assertEqual(self.do, do)
def index(): dt = Delorean() auctions = Session().query(Auction).order_by(Auction.date.desc()).all() return render_template('admin/index.html', auctions=auctions, now=int(dt.epoch()))