def get_data_segment_angles(model_id,
driver_id,
repeat,
test=False,
segment_version=1,
extra=((1, 1), 2)):
seed = random.Random(x=driver_id + model_id)
da = DataAccess()
ngram_range, min_df = extra
if test:
set1 = list(da.get_rides_segments(driver_id, version=segment_version))
set2 = list(
da.get_random_rides(settings.BIG_CHUNK_TEST * repeat,
driver_id,
segments=True,
version=segment_version,
seed=seed))
else:
driver_train, driver_test = da.get_rides_split(driver_id,
settings.BIG_CHUNK,
segments=True,
version=segment_version)
other_train = list(
da.get_random_rides(settings.BIG_CHUNK * repeat,
driver_id,
segments=True,
version=segment_version,
seed=seed))
other_test = list(
da.get_random_rides(settings.SMALL_CHUNK,
driver_id,
segments=True,
version=segment_version))
set1 = driver_train + other_train
set2 = driver_test + other_test
# create features for each (segment, angle, segment) tuple
set1 = [[
'%s_%s_%s' % (d[0][i - 1], d[1][i - 1], d[0][i])
for i in xrange(1, len(d[0]))
] for d in set1]
set2 = [[
'%s_%s_%s' % (d[0][i - 1], d[1][i - 1], d[0][i])
for i in xrange(1, len(d[0]))
] for d in set2]
set1 = [util.get_list_string(d) for d in set1]
set2 = [util.get_list_string(d) for d in set2]
vectorizer = CountVectorizer(min_df=min_df, ngram_range=ngram_range)
set1 = vectorizer.fit_transform(set1)
set2 = vectorizer.transform(set2)
return set1, set2
def get_data_segment_lengths(model_id, driver_id, repeat, test=False, segment_version=1, extra=((1,8),1)):
seed = random.Random(x=driver_id+model_id)
da = DataAccess()
ngram_range, min_df = extra
if test:
set1 = list(da.get_rides_segments(driver_id, version=segment_version))
set2 = list(da.get_random_rides(
settings.BIG_CHUNK_TEST * repeat,
driver_id,
segments=True,
version=segment_version,
seed=seed
))
else:
driver_train, driver_test = da.get_rides_split(
driver_id,
settings.BIG_CHUNK,
segments=True,
version=segment_version
)
other_train = list(da.get_random_rides(
settings.BIG_CHUNK * repeat,
driver_id,
segments=True,
version=segment_version,
seed=seed
))
other_test = list(da.get_random_rides(
settings.SMALL_CHUNK,
driver_id,
segments=True,
version=segment_version
))
set1 = driver_train + other_train
set2 = driver_test + other_test
# keep only lengths
set1 = [d[0] for d in set1]
set2 = [d[0] for d in set2]
# convert to text
set1 = [util.get_list_string(d) for d in set1]
set2 = [util.get_list_string(d) for d in set2]
vectorizer = CountVectorizer(min_df=min_df, ngram_range=ngram_range)
set1 = vectorizer.fit_transform(set1)
set2 = vectorizer.transform(set2)
return set1, set2
def get_data_segment_lengths(model_id,
driver_id,
repeat,
test=False,
segment_version=1,
extra=((1, 8), 1)):
seed = random.Random(x=driver_id + model_id)
da = DataAccess()
ngram_range, min_df = extra
if test:
set1 = list(da.get_rides_segments(driver_id, version=segment_version))
set2 = list(
da.get_random_rides(settings.BIG_CHUNK_TEST * repeat,
driver_id,
segments=True,
version=segment_version,
seed=seed))
else:
driver_train, driver_test = da.get_rides_split(driver_id,
settings.BIG_CHUNK,
segments=True,
version=segment_version)
other_train = list(
da.get_random_rides(settings.BIG_CHUNK * repeat,
driver_id,
segments=True,
version=segment_version,
seed=seed))
other_test = list(
da.get_random_rides(settings.SMALL_CHUNK,
driver_id,
segments=True,
version=segment_version))
set1 = driver_train + other_train
set2 = driver_test + other_test
# keep only lengths
set1 = [d[0] for d in set1]
set2 = [d[0] for d in set2]
# convert to text
set1 = [util.get_list_string(d) for d in set1]
set2 = [util.get_list_string(d) for d in set2]
vectorizer = CountVectorizer(min_df=min_df, ngram_range=ngram_range)
set1 = vectorizer.fit_transform(set1)
set2 = vectorizer.transform(set2)
return set1, set2
def get_data_segment_angles_v2(model_id, driver_id, repeat, test=False, segment_version=1, extra=((1,3),1)):
seed = random.Random(x=driver_id+model_id)
da = DataAccess()
ngram_range, min_df = extra
if test:
set1 = list(da.get_rides_segments(driver_id, version=segment_version))
set2 = list(da.get_random_rides(
settings.BIG_CHUNK_TEST * repeat,
driver_id,
segments=True,
version=segment_version,
seed=seed
))
else:
driver_train, driver_test = da.get_rides_split(
driver_id,
settings.BIG_CHUNK,
segments=True,
version=segment_version
)
other_train = list(da.get_random_rides(
settings.BIG_CHUNK * repeat,
driver_id,
segments=True,
version=segment_version,
seed=seed
))
other_test = list(da.get_random_rides(
settings.SMALL_CHUNK,
driver_id,
segments=True,
version=segment_version
))
set1 = driver_train + other_train
set2 = driver_test + other_test
# create features for each (segment, angle, segment) tuple
set1 = [['%s_%s' % (d[0][i-1], d[1][i-1]) for i in xrange(1, len(d[0]))] for d in set1]
set2 = [['%s_%s' % (d[0][i-1], d[1][i-1]) for i in xrange(1, len(d[0]))] for d in set2]
set1 = [util.get_list_string(d) for d in set1]
set2 = [util.get_list_string(d) for d in set2]
vectorizer = CountVectorizer(min_df=min_df, ngram_range=ngram_range)
set1 = vectorizer.fit_transform(set1)
set2 = vectorizer.transform(set2)
return set1, set2
