def get(self, year=None, month=None, day=None, segment_num=None, slug=None):
logging.info("Segment View")
year = helper.parse_int(year)
month = helper.parse_int(month)
day = helper.parse_int(day)
segment_num = helper.parse_int(segment_num)
if not (year and month and day and segment_num is not None):
self.error(404)
else:
segment_date = datetime.date(year, month, day)
segment_key = Segment.calc_key_name(segment_date, segment_num)
segment = Segment.lookup(segment_date, segment_num)
if not segment:
self.error(404)
else:
if not slug:
slug = slugify(segment.description)
path = self.request.path
if not path.endswith('/'):
path = "%s/" % path
self.redirect("%s%s" % (path, slug))
else:
params = {'segment': segment, 'disqus_id': segment_key, 'disqus_permalink': self.request.path}
template_values = helper.init_template_values(params)
self.response.out.write(template.render(helper.get_template_path("segment"),
template_values))
def test_segment_crud(self):
# Create
segment = Segment(name='Foo')
db.session.add(segment)
db.session.commit()
self.assertIn(segment, Segment.query.all())
self.assertIsInstance(segment.created_at, DT.datetime)
self.assertIsInstance(segment.updated_at, DT.datetime)
# Read
segment = Segment.query.filter_by(name='Foo').first()
self.assertEqual(segment.name, 'Foo')
# Update
old_created_at = segment.created_at
old_updated_at = segment.updated_at
segment.name = 'Bar'
segment = Segment.query.filter_by(name='Bar').first()
self.assertIsInstance(segment, Segment)
self.assertEqual('Bar', segment.name)
self.assertEqual(segment.created_at, old_created_at)
self.assertNotEqual(segment.updated_at, old_updated_at)
# Delete
db.session.delete(segment)
count = Segment.query.filter_by(name='Bar').count()
self.assertEqual(0, count)
def persist_segments(segment_props, out=None):
for props in [props for props in segment_props if props]:
printer(out, '%s:%02d (%s) - %s -> ' % \
(props.get('date'), props.get('num'), \
props.get('description'), props.get('is_classic')))
if not Segment.lookup(date=props['date'], num=props['num']):
segment = Segment.create(date=props['date'], num=props['num'], \
url=props['url'], description=props['description'], \
duration=props.get('duration', 0), \
is_classic=props.get('is_classic', False))
logging.debug("created segment %s" % segment)
printer(out, "just created.\n")
else:
printer(out, "already created.\n")
def test_rules_in_groups(self):
segment = Segment(name="Bar")
rule_1 = Rule(group_id=1, field='foo', comparator="MATCH", value='bar')
rule_2 = Rule(group_id=2, field='foo', comparator="MATCH", value='bar')
rule_3 = Rule(group_id=2, field='foo', comparator="MATCH", value='bar')
rule_4 = Rule(group_id=3, field='foo', comparator="MATCH", value='bar')
rule_5 = Rule(group_id=3, field='foo', comparator="MATCH", value='bar')
rule_6 = Rule(group_id=3, field='foo', comparator="MATCH", value='bar')
segment.rules = [rule_1, rule_2, rule_3, rule_4, rule_5, rule_6]
groups = segment.rules_in_groups()
self.assertEqual(len(groups), 3)
self.assertEqual(groups[1], [rule_1])
self.assertEqual(groups[2], [rule_2, rule_3])
self.assertEqual(groups[3], [rule_4, rule_5, rule_6])
def segments(query=None, limit=MAX_RESULTS, grouped_by=None, min_date=None, max_date=None, randomize=False, compact=False, is_ajax=False):
def segment_comparator(s1, s2):
return s1['epoch_time'] > s2['epoch_time'] and -1 or 1
segments = Segment.cache()
if query and len(query):
filtered_segments = [segment for segment in segments if segment['description'].lower().find(query.lower()) != -1]
else:
filtered_segments = segments
min_date_secs = int(time.mktime(min_date.timetuple())) if min_date else -1
max_date_secs = int(time.mktime(max_date.timetuple())) if max_date else sys.maxint
filtered_segments = [seg_props for seg_props in filtered_segments \
if min_date_secs <= seg_props.get('epoch_time', 0) < max_date_secs]
if limit > -1:
filtered_segments = filtered_segments[:limit]
filtered_segments.sort(segment_comparator)
if is_ajax:
for segment in filtered_segments:
del segment['date']
if compact:
# date_format = "%B %d, %Y"
for segment in filtered_segments:
pass
if randomize and len(filtered_segments):
return random.choice(filtered_segments)
else:
return filtered_segments
def hello_world():
if request.method == 'POST':
segment = request.form['segment']
count = request.form['count']
tag = request.form['tag']
inform = Segment(segment=segment, count=count, tag=tag)
# try:
db.session.add(inform)
db.session.commit()
return redirect('/')
# except:
# return 'Error'
else:
auth = requests.get(current_app.config['URL_AUTH'], headers=current_app.config['HEADERS_AUTH'])
if auth.status_code == 200:
tags = requests.get(current_app.config['URL_TAGS'], headers=current_app.config['HEADERS_AUTH'])
tags = json.loads(tags.text)
segments = requests.get(current_app.config['URL_SEGMENTS'], headers=current_app.config['HEADERS_AUTH'])
segments = json.loads(segments.text)
return render_template('main.html', tags=tags, segments=segments)
def get(self, date=None, num=None):
self.response.headers['Content-Type'] = 'text/plain'
if date and num:
key_name = "%s_%s" % (date, num)
logging.info("Listened to segment %s." % key_name)
next_url = None
if key_name:
try:
segment = Segment.get_by_key_name(key_name)
except:
segment = None
self.error(500)
if segment:
try:
segment.listened()
except:
logging.error("Error while calling segment.listened() for segment %s.", key_name)
try:
next_url = segment.url
except:
pass
else:
self.error(404)
if next_url:
self.redirect(next_url, permanent=True)
else:
self.error(404)
else:
self.error(404)
def get(self):
self.response.headers['Content-Type'] = 'text/plain'
for segment in Segment.all():
if segment.description.startswith('Seg') and not segment.description.startswith('Segment '): # or segment.description.startswith('Seg. '):
self.response.out.write("\nOld segment description: %s\n" % segment.description)
m = re.match(r"\bSeg\.?\s?\d*-?\s?(.*)\b", segment.description)
if m:
try:
segment.description = m.group(1)
self.response.out.write("New segment description: %s\n" % segment.description)
segment.put()
except:
pass
def get(self):
self.response.headers['Content-Type'] = 'text/plain'
self.response.out.write("Repeat segments\n")
url_segs = defaultdict(list)
for segment in Segment.cache():
url_segs[segment['u']].append(segment)
for url, segments in url_segs.items():
if len(segments) > 1:
self.response.out.write("\n\n%s" % url)
for seg in segments:
self.response.out.write("\n\t%s" % seg)
def get(self):
self.response.headers['Content-Type'] = 'text/plain'
url_segs = defaultdict(list)
for segment in Segment.cache():
url_segs[segment['url']].append(segment)
keys_to_del = []
for url, segments in url_segs.items():
if len(segments) > 1:
control_segment = segments[0]
for segment in segments[1:]:
if segment['description'] == control_segment['description']:
keys_to_del.append(segment['key'])
self.response.out.write("Keys to delete: %s\n" % keys_to_del)
segs_to_del = [seg for seg in Segment.get(keys_to_del) if seg is not None]
self.response.out.write("Segments to delete: %s\n" % segs_to_del)
if len(segs_to_del):
db.delete(segs_to_del)
Segment.cache(True)
self.response.out.write("\nDeleted marked segments and reset cache.")
else:
self.response.out.write("\nNo segments were deleted.")
def get_segments(self, file, part, video_id):
try:
audio_file = "aud.part" + str(part) + "." + file + ".wav"
fs, s = aIO.readAudioFile(audio_file)
af, _, afn = aF.mtFeatureExtraction(s, fs, int(0.5 * fs),
int(0.5 * fs), int(0.1 * fs),
int(0.1 * fs))
video_file = "vid.part" + str(part) + "." + file
vf, t, vfn = self._video_extractor.extract_features(video_file)
except Exception as e:
print(e)
return None
else:
segment_features = []
# construct segmentfeature
vmean = vf.T.mean(axis=1)
for i, val in enumerate(vmean):
if isinstance(val, np.ndarray):
val = val[0]
feature = SegmentFeatures(value=val,
seq_no=1,
feature_id=self._feature_map[vfn[i]])
segment_features.append(feature)
amean = af.mean(axis=1)
for i, val in enumerate(amean):
feature = SegmentFeatures(value=val,
seq_no=1,
feature_id=self._feature_map[afn[i]])
segment_features.append(feature)
# construct segment with its segmentfeatures
segment = Segment(video_id=video_id,
start_sec=part,
end_sec=self._splitter.get_segment_end(part),
features=segment_features)
return segment
def get(self):
self.response.headers['Content-Type'] = 'application/json'
key = self.request.get('key', '')
segments = Segment.cache()
try:
idx = [segment['key'] for segment in segments].index(key)
except:
idx = -1
try:
limit = int(self.request.get('limit', MAX_RESULTS))
except:
limit = MAX_RESULTS
if limit < -1:
limit = len(segments)
latest_segments = segments[idx + 1:idx + 1 + limit]
self.response.out.write("%s" % json.dumps({'segments': latest_segments}))
def get(self):
segments = Segment.cache()
logging.debug("segments: %s" % segments)
self.response.headers['Content-Type'] = 'text/plain'
def get(self):
self.response.headers['Content-Type'] = 'application/json'
segments = [segment.props() for segment in Segment.all()]
for segment in segments:
segment['date'] = segment['date'].ctime()
self.response.out.write("%s" % json.dumps({'segments': segments}))
def test_segment_matches_data(self):
segment = Segment(name="Foo")
true_rule_1 = Rule(group_id=1, field='sessions_count', comparator="MATCH", value=self.visitor_data['sessions_count'])
false_rule_1 = Rule(group_id=1, field='sessions_count', comparator="MATCH", value=100)
true_rule_2 = Rule(group_id=2, field='sessions_count', comparator="MATCH", value=self.visitor_data['sessions_count'])
false_rule_2 = Rule(group_id=2, field='sessions_count', comparator="MATCH", value=100)
# False
segment.rules = []
self.assertFalse(segment.matches_data(self.visitor_data))
# True
segment.rules = [true_rule_1]
self.assertTrue(segment.matches_data(self.visitor_data))
# False
segment.rules = [false_rule_1]
self.assertFalse(segment.matches_data(self.visitor_data))
# True OR False = True
segment.rules = [true_rule_1, false_rule_1]
self.assertTrue(segment.matches_data(self.visitor_data))
# False OR True = True
segment.rules = [false_rule_1, true_rule_1]
self.assertTrue(segment.matches_data(self.visitor_data))
# True OR True = True
segment.rules = [true_rule_1, true_rule_1]
self.assertTrue(segment.matches_data(self.visitor_data))
# False OR False = False
segment.rules = [false_rule_1, false_rule_1]
self.assertFalse(segment.matches_data(self.visitor_data))
# True AND True = True
segment.rules = [true_rule_1, true_rule_2]
self.assertTrue(segment.matches_data(self.visitor_data))
# True AND False = False
segment.rules = [true_rule_1, false_rule_2]
self.assertFalse(segment.matches_data(self.visitor_data))
# False AND True = False
segment.rules = [false_rule_2, true_rule_1]
self.assertFalse(segment.matches_data(self.visitor_data))
# False AND False = False
segment.rules = [false_rule_1, false_rule_2]
self.assertFalse(segment.matches_data(self.visitor_data))
# True AND (True OR False) = True
segment.rules = [true_rule_1, true_rule_2, false_rule_2]
self.assertTrue(segment.matches_data(self.visitor_data))
# False AND (True OR False) = False
segment.rules = [false_rule_1, true_rule_2, false_rule_2]
self.assertFalse(segment.matches_data(self.visitor_data))
def _parse_response(self, response_dict):
routes = []
for i, route in enumerate(response_dict['routes']):
# filter by transfers
if self._search_request.no_transfers and len(
route['segments']) > 1:
continue
# create route obj
rt = Route(
order=i,
pl_from=self._search_request.req_from,
pl_to=self._search_request.req_to,
from_seg=response_dict['places'][route['depPlace']]
['shortName'],
transfers=len(route['segments']) - 1,
duration=self._parse_duration(route['totalDuration']),
duration_raw=route['totalDuration'],
)
# filter by total price
if not route.get('indicativePrices'):
continue
else:
pr = route['indicativePrices'][-1]
if self._search_request.price_lower_limit and \
int(pr['price']) < self._search_request.price_lower_limit:
continue
if self._search_request.price_upper_limit and \
int(pr['price']) > self._search_request.price_upper_limit:
continue
# parse total price
self._parse_price(route, rt)
# parse segments
segments = []
types_set = set()
for segment in route['segments']:
transport_type = response_dict['vehicles'][
segment['vehicle']]['kind']
types_set.add(transport_type)
if any(not self._names.get(tr) for tr in types_set):
continue
for segment in route['segments']:
# create segment dict
transport_type = response_dict['vehicles'][
segment['vehicle']]['kind']
seg = Segment(
to=self._limit_name(response_dict['places'][
segment['arrPlace']]['shortName']),
to_full=response_dict['places'][segment['arrPlace']]
['shortName'],
transport_type=TransportType(
name=self._names[transport_type],
icon=self._icons[transport_type],
),
)
# parse segment price
if segment.get('indicativePrices'):
self._parse_price(segment, seg)
else:
seg.price = '-'
seg.price_raw = 0
# parsing specific segment type
if segment['segmentKind'] == 'surface':
seg.segment_type = 'surface'
seg.duration = self._parse_duration(
segment['transitDuration'] +
segment['transferDuration'])
seg.duration_raw = segment['transitDuration'] + segment[
'transferDuration']
if segment.get('agencies'):
seg.frequency = self._parse_frequency(
segment['agencies'][0]['frequency'])
links = segment['agencies'][0]['links']
for link in links:
if link['text'] == 'Book at':
seg.book_name = link['displayUrl']
seg.book_url = link['url']
elif link['text'] == 'Schedules at':
seg.schedule_name = link['displayUrl']
seg.schedule_url = link['url']
# end
else:
seg.segment_type = 'air'
if segment.get('outbound'):
duration = 0
leg = segment['outbound'][0]
start_index = leg['hops'][0]['depPlace']
time_start = leg['hops'][0]['depTime']
for hop in leg['hops']:
end_index = hop['arrPlace']
duration += hop['duration']
time_end = hop['arrTime']
seg.airport_start_code = response_dict['places'][
start_index]['code']
seg.airport_start_name = response_dict['places'][
start_index]['shortName']
seg.airport_end_code = response_dict['places'][
end_index]['code']
seg.airport_end_name = response_dict['places'][
end_index]['shortName']
seg.time_start = time_start
seg.time_end = time_end
seg.duration = self._parse_duration(duration)
seg.duration_raw = duration
seg.operating_days = self._parse_days(
leg['operatingDays'])
flights = []
for leg in segment['outbound']:
flight = Flight(operating_days=self._parse_days(
leg['operatingDays']))
if leg.get('indicativePrices'):
self._parse_price(leg, flight)
duration = 0
hops = []
airlines = []
for hop in leg['hops']:
duration += hop['duration']
start_index = hop['depPlace']
end_index = hop['arrPlace']
hp = FlightSegment(
airport_start_code=response_dict['places']
[start_index]['code'],
airport_start_name=response_dict['places']
[start_index]['shortName'],
airport_end_code=response_dict['places']
[end_index]['code'],
airport_end_name=response_dict['places']
[end_index]['shortName'],
time_start=hop['depTime'],
time_end=hop['arrTime'],
duration=self._parse_duration(
hop['duration']),
duration_raw=hop['duration'],
airline_name=response_dict['airlines'][
hop['airline']]['name'],
)
hops.append(hp)
airlines.append(response_dict['airlines'][
hop['airline']]['name'])
flight.flight_segments = hops
flight.duration_raw = duration
flight.duration = self._parse_duration(duration)
flight.airlines = ', '.join(a
for a in set(airlines))
flights.append(flight)
flights_obj = Flights(choices=flights).save()
seg.flights = flights_obj
# end
segments.append(seg)
# end
# parse transport types
transport_types = [
TransportType(
name=self._names[tp],
icon=self._icons[tp],
) for tp in types_set
]
rt.segments = segments
rt.transport_types = transport_types
routes.append(rt)
self._save_best_route(routes)
self._search_request.routes = routes
self._search_request.save()
return self._search_request.result