def retry_handler(job, exc_type, exc_value, traceback): job.meta.setdefault('failures', 0) job.meta['failures'] += 1 # Too many failures if job.meta['failures'] >= MAX_FAILURES: logger.warn('job %s: failed too many times times - moving to failed queue' % job.id) job.save() return True # Requeue job and stop it from being moved into the failed queue logger.warn('job %s: failed %d times - retrying' % (job.id, job.meta['failures'])) for queue in queues: if queue.name == job.origin: queue.enqueue_job(job, timeout=job.timeout) return False # Can't find queue, which should basically never happen as we only work jobs that match the given queue names and # queues are transient in rq. logger.warn('job %s: cannot find queue %s - moving to failed queue' % (job.id, job.origin)) return True
def get_optimum_time(orig_in, dest_in, min_leave_in, max_leave_in, min_dest_in, max_dest_in, granularity_in, traffic_model_in, tz_in, orig_to_dest_only_in): class DirectionTimeOptimizer: def __init__(self, origin, destination, min_time_to_leave, max_time_to_leave, min_time_at_dest, max_time_at_dest, orig_to_dest_only, search_granularity_mins=15, traffic_model="best_guess"): self.origin = origin self.orig_to_dest_only = orig_to_dest_only self.destination = destination self.min_time_to_leave = self.round_closest_mins(min_time_to_leave) self.max_time_to_leave = self.round_closest_mins(max_time_to_leave) self.search_granularity_mins = int(search_granularity_mins) self.time_to_leave_segments = int( (((self.max_time_to_leave - self.min_time_to_leave ).total_seconds() / 60) / self.search_granularity_mins)) + 1 if not orig_to_dest_only: self.min_time_at_dest = self.round_closest(min_time_at_dest) self.max_time_at_dest = self.round_closest(max_time_at_dest) self.time_at_dest_segments = int( (max_time_at_dest - min_time_at_dest) / search_granularity_mins) + 1 self.traffic_model = traffic_model self.origin_leave_time_lookup = {} def round_closest(self, value, quantization=15): return int(quantization * round(value / float(quantization))) def round_closest_mins(self, time_value, quantization=15): return (time_value + timedelta(minutes=self.round_closest(time_value.minute) - time_value.minute)) def get_duration_from_result(self, result): if (not len(result) > 0 or 'legs' not in result[0] or not len(result[0]['legs']) > 0 or 'duration_in_traffic' not in result[0]['legs'][0]): return None else: return result[0]['legs'][0]['duration_in_traffic'][ 'value'] / 60.0 def calculate_possible_times_to_leave(self): min_set = [ self.min_time_to_leave + timedelta(minutes=i * self.search_granularity_mins) for i in xrange(0, self.time_to_leave_segments) ] for time_to_leave in min_set: gmaps_result = gmaps.directions( self.origin, self.destination, mode="driving", traffic_model=self.traffic_model, departure_time=time_to_leave) result = self.get_duration_from_result(gmaps_result) if result is not None: self.origin_leave_time_lookup[time_to_leave] = { 'orig_to_dest_time': result, 'summary_name': gmaps_result[0]['summary'] } if not self.origin_leave_time_lookup == {}: return True else: return False def calculate_possible_times_to_return(self): if self.origin_leave_time_lookup == {}: return False for leave_time in self.origin_leave_time_lookup.keys(): time_set = [ leave_time + timedelta(minutes=(i * self.search_granularity_mins) + self.round_closest( int(self.origin_leave_time_lookup[leave_time] ['orig_to_dest_time'])) + self.min_time_at_dest) for i in xrange(0, self.time_at_dest_segments) ] for dest_leave_time in time_set: gmaps_result = gmaps.directions( self.destination, self.origin, mode="driving", traffic_model=self.traffic_model, departure_time=dest_leave_time) result = self.get_duration_from_result(gmaps_result) if 'dest_return_time_lookup' not in self.origin_leave_time_lookup[ leave_time] and result is not None: self.origin_leave_time_lookup[leave_time][ 'dest_return_time_lookup'] = {} self.origin_leave_time_lookup[leave_time][ 'dest_return_time_lookup'][dest_leave_time] = { 'dest_to_orig_time': result, 'summary_name': gmaps_result[0]['summary'] } elif 'dest_return_time_lookup' in self.origin_leave_time_lookup[ leave_time] and result is not None: self.origin_leave_time_lookup[leave_time][ 'dest_return_time_lookup'][dest_leave_time] = { 'dest_to_orig_time': result, 'summary_name': gmaps_result[0]['summary'] } return True def determine_optimum_times_both_dirs(self): min_combination = { 'orig_to_dest': None, 'dest_to_orig': None, 'orig_to_dest_summary': None, 'dest_to_orig_summary': None, 'status': 'Success' } min_seen = 10000000 for timestamp, item in self.origin_leave_time_lookup.iteritems(): for sub_ts, sub_item in item[ 'dest_return_time_lookup'].iteritems(): if (item['orig_to_dest_time'] + sub_item['dest_to_orig_time']) < min_seen: min_seen = item['orig_to_dest_time'] + sub_item[ 'dest_to_orig_time'] min_combination['orig_to_dest'] = timestamp min_combination['dest_to_orig'] = sub_ts min_combination['orig_to_dest_time'] = item[ 'orig_to_dest_time'] min_combination['dest_to_orig_time'] = sub_item[ 'dest_to_orig_time'] min_combination['orig_to_dest_summary'] = item[ 'summary_name'] min_combination['dest_to_orig_summary'] = sub_item[ 'summary_name'] return min_combination def determine_optimum_times_one_dir(self): min_combination = { 'orig_to_dest': None, 'orig_to_dest_summary': None, 'status': 'Success' } min_seen = 10000000 for timestamp, item in self.origin_leave_time_lookup.iteritems(): if item['orig_to_dest_time'] < min_seen: min_seen = item['orig_to_dest_time'] min_combination['orig_to_dest'] = timestamp min_combination['orig_to_dest_time'] = item[ 'orig_to_dest_time'] min_combination['orig_to_dest_summary'] = item[ 'summary_name'] return min_combination job = get_current_job() job.meta['status'] = 'create' job.save() my_optimizer = DirectionTimeOptimizer(orig_in, dest_in, min_leave_in, max_leave_in, min_dest_in, max_dest_in, orig_to_dest_only_in, granularity_in, traffic_model_in) job.meta['status'] = 'created' job.save() try: my_optimizer.calculate_possible_times_to_leave() job.meta['status'] = 'times_to_leave_done' job.save() if not orig_to_dest_only_in: my_optimizer.calculate_possible_times_to_return() job.meta['status'] = 'times_to_return_done' job.save() except googlemaps.exceptions.ApiError as e: if 'departure_time is in the past' in str(e): job.meta['status'] = 'api_error_departure_in_the_past' job.save() return None else: job.meta['status'] = 'api_error_unknown' job.save() return None if not orig_to_dest_only_in: result = my_optimizer.determine_optimum_times_both_dirs() else: result = my_optimizer.determine_optimum_times_one_dir() job.meta['status'] = 'complete' job.save() result['requested'] = ({ 'orig_to_dest_only': orig_to_dest_only_in, 'orig_in': orig_in, 'dest_in': dest_in, 'min_leave_in': min_leave_in, 'max_leave_in': max_leave_in, 'min_dest_in': min_dest_in, 'max_dest_in': max_dest_in, 'granularity_in': granularity_in, 'traffic_model_in': traffic_model_in, 'tz_in': tz_in } if not orig_to_dest_only_in else { 'orig_to_dest_only': orig_to_dest_only_in, 'orig_in': orig_in, 'dest_in': dest_in, 'min_leave_in': min_leave_in, 'max_leave_in': max_leave_in, 'granularity_in': granularity_in, 'traffic_model_in': traffic_model_in, 'tz_in': tz_in }) return result
def add_meta(job, *exc_info): job.meta = {'foo': 1} job.save() return True