def _map_position(data):
antenna = Position()
if 'antenna_id' in data:
antenna.antenna = data['antenna_id']
return antenna
if 'latitude' in data and 'longitude' in data:
antenna.position = float(data['latitude']), float(data['longitude'])
return antenna
def _map_position(data):
antenna = Position()
if 'antenna_id' in data:
antenna.antenna = data['antenna_id']
return antenna
if 'latitude' in data and 'longitude' in data:
antenna.position = float(data['latitude']), float(
data['longitude'])
return antenna
def _map_position(data):
antenna = Position()
if 'antenna_id' in data:
antenna.antenna = data['antenna_id']
return antenna
elif 'place_id' in data:
raise NameError("Use field name 'antenna_id' in input files. 'place_id' is deprecated.")
if 'latitude' in data and 'longitude' in data:
antenna.position = float(data['latitude']), float(data['longitude'])
return antenna
def test_weekend_filter(self):
records = [
Record("test_itr", "in", "1", dt(2014, 8, 22), 1, Position()),
Record("test_itr", "in", "1", dt(2014, 8, 31), 1, Position()),
Record("test_itr", "in", "1", dt(2014, 10, 18), 1, Position())
]
user = bc.User()
user.records = records
rv = bc.helper.group.filter_user(user, part_of_week='weekend')
self.assertEqual(rv, [records[1], records[2]])
def test_weekday_filter(self):
records = [
Record("test_itr", "in", "1", datetime.datetime(2014, 8, 22), 1, Position()),
Record("test_itr", "in", "1", datetime.datetime(2014, 8, 31), 1, Position()),
Record("test_itr", "in", "1", datetime.datetime(2014, 10, 18), 1, Position())
]
user = bc.User()
user.records = records
filtered_records = bc.helper.group.filter_user(user, part_of_week='weekday')
self.assertEqual(filtered_records, [records[0]])
def test_weekly_group(self):
records = [
Record("test_itr", "in", "1", dt(2014, 8, 24), 1, Position()),
Record("test_itr", "in", "1", dt(2014, 9, 4), 1, Position()),
Record("test_itr", "in", "1", dt(2014, 9, 11), 1, Position())
]
user = bc.User()
user.records = records
grouping = bc.helper.group.group_records(user.records, groupby='week')
groups = [[r for r in l] for l in grouping]
self.assertEqual(groups, [[records[0]], [records[1]], [records[2]]])
def test_weekend_group(self):
records = [
Record("test_itr", "in", "1", datetime.datetime(2014, 8, 23), 1, Position()),
Record("test_itr", "in", "1", datetime.datetime(2014, 8, 31), 1, Position()),
Record("test_itr", "in", "1", datetime.datetime(2014, 10, 18), 1, Position())
]
user = bc.User()
user.records = records
grouping = bc.helper.group.group_records(user, groupby='week', part_of_week='weekend')
groups = [[r for r in l] for l in grouping]
self.assertEqual(groups, [[records[0]], [records[1]], [records[2]]])
def test_none_group(self):
records = [
Record("call", "in", "1", dt(2014, 9, 4), 1, Position()),
Record("call", "in", "1", dt(2014, 9, 5), 1, Position()),
Record("call", "in", "1", dt(2014, 9, 11), 1, Position()),
Record("call", "in", "1", dt(2014, 9, 12), 1, Position())
]
grouping = bc.helper.group.group_records(records, groupby=None)
self.assertEqual(records, list(next(grouping)))
with self.assertRaises(StopIteration):
next(grouping)
def test_none_group(self):
records = [
Record("call", "in", "1", datetime.datetime(2014, 9, 4), 1, Position()),
Record("call", "in", "1", datetime.datetime(2014, 9, 5), 1, Position()),
Record("call", "in", "1", datetime.datetime(2014, 9, 11), 1, Position()),
Record("call", "in", "1", datetime.datetime(2014, 9, 12), 1, Position())
]
user = bc.User()
user.records = records
grouping = bc.helper.group.group_records(user, groupby=None)
self.assertEqual(records, list(next(grouping)))
self.assertRaises(StopIteration, grouping.next)
def test_daily_filter(self):
records = [
Record("test_itr", "in", "1", datetime.datetime(2014, 8, 22, 10, 00), 1, Position()),
Record("test_itr", "in", "1", datetime.datetime(2014, 8, 23, 10, 00), 1, Position()),
Record("test_itr", "in", "1", datetime.datetime(2014, 9, 7, 11, 00), 1, Position()),
Record("test_itr", "in", "1", datetime.datetime(2014, 10, 18, 2, 00), 1, Position())
]
user = bc.User()
user.records = records
filtered_records = bc.helper.group.filter_user(user, part_of_day='night')
self.assertEqual(filtered_records, [records[3]])
filtered_records = bc.helper.group.filter_user(user, part_of_day='day')
self.assertEqual(filtered_records, [records[0], records[1], records[2]])
def _parse(reader):
records = []
antennas = dict()
for row in reader:
direction = 'out' if row[0] == '1' else 'in'
interaction = 'call' if row[1] in ['11', '12'] else 'text'
contact = row[3]
date = datetime.strptime(row[4], "%Y-%m-%d %H:%M:%S")
call_duration = float(row[5])
lon, lat = float(row[6]), float(row[7])
latlon = (lat, lon)
antenna = None
for key, value in antennas.items():
if latlon == value:
antenna = key
break
if antenna is None:
antenna = len(antennas) + 1
antennas[antenna] = latlon
position = Position(antenna=antenna, location=latlon)
record = Record(direction=direction,
interaction=interaction,
correspondent_id=contact,
call_duration=call_duration,
datetime=date,
position=position)
records.append(record)
return records, antennas
def random_record(**kwargs):
n_users = 48
rate = 1e-4
year = 2012
month = _choice(range(1, 3))
day = _choice(range(1, 28))
# ensures that some correspondents have more interactions than others
correspondent = int(random.random() * n_users)
r = {
'datetime':
datetime.datetime(year, month, day) + datetime.timedelta(
seconds=math.floor(-1 / rate * math.log(random.random()))),
'interaction':
_choice(['text', 'text', 'text', 'call', 'call']),
'correspondent_id':
"correspondent_{}".format(correspondent),
'direction':
_choice(['in', 'in', 'out']),
'call_duration':
int(random.random() * 1000),
'position':
Position(location=(_uniform(-5, 5), _uniform(-5, 5)))
}
if r['interaction'] == "text":
r['call_duration'] = None
r.update(kwargs)
return Record(**r)
def test_daily_group(self):
records = [
Record("test_itr", "in", "1", datetime.datetime(2014, 8, 22, 10, 00), 1, Position()),
Record("test_itr", "in", "1", datetime.datetime(2014, 8, 23, 10, 00), 1, Position()),
Record("test_itr", "in", "1", datetime.datetime(2014, 9, 7, 11, 00), 1, Position()),
Record("test_itr", "in", "1", datetime.datetime(2014, 10, 18, 2, 00), 1, Position())
]
user = bc.User()
user.records = records
grouping = bc.helper.group.group_records(user, groupby='week', part_of_day='night')
groups = [[r for r in l] for l in grouping]
self.assertEqual(groups, [[records[3]]])
grouping = bc.helper.group.group_records(user, groupby='week', part_of_day='day')
groups = [[r for r in l] for l in grouping]
self.assertEqual(groups, [[records[0], records[1]], [records[2]]])
def test_read_orange_stdin(self):
user = bc.io.read_orange(describe=False)
self.assertEqual(
user.records[0],
Record(interaction='text',
direction='out',
correspondent_id='770000005',
datetime=datetime.datetime(2013, 12, 16, 7, 30, 30),
call_duration=0.0,
position=Position(1, (42.3987, -71.575))))
def test_read_csv_no_position(self):
user = bc.read_csv("u_test_no_position", "samples", describe=False)
self.assertEqual(
user.records[1],
Record(interaction='call',
direction='in',
correspondent_id='770000001',
datetime=datetime.datetime(2013, 12, 16, 5, 39, 30),
duration=0,
position=Position()))
def test_weekly_group(self):
records = [
Record("test_itr", "in", "1", datetime.datetime(2014, 8, 24), 1,
Position()),
Record("test_itr", "in", "1", datetime.datetime(2014, 9, 4), 1,
Position()),
Record("test_itr", "in", "1", datetime.datetime(2014, 9, 11), 1,
Position())
]
user = bc.User()
user.records = records
grouping = bc.helper.group.group_records(user, groupby='week')
record = grouping.next()
self.assertTrue(record.next().datetime, records[0].datetime)
record = grouping.next()
self.assertTrue(record.next().datetime, records[1].datetime)
record = grouping.next()
self.assertTrue(record.next().datetime, records[2].datetime)
def test_read_csv_antenna_id_no_places(self):
user = bc.read_csv("u_test_antennas", "samples", describe=False)
self.assertEqual(
user.records[1],
Record(interaction='call',
direction='in',
correspondent_id='770000001',
datetime=datetime.datetime(2013, 12, 16, 5, 39, 30),
call_duration=0,
position=Position('13084', None)))
self.assertEqual(
bc.spatial.radius_of_gyration(user).values(), [None] * 2)
def test_none_group(self):
records = [
Record("call", "in", "1", datetime.datetime(2014, 9, 5), 1,
Position()),
Record("call", "in", "1", datetime.datetime(2014, 9, 4), 1,
Position()),
Record("call", "in", "1", datetime.datetime(2014, 9, 11), 1,
Position()),
Record("call", "in", "1", datetime.datetime(2014, 9, 12), 1,
Position())
]
user = bc.User()
user.records = records
grouping = bc.helper.group.group_records(user, groupby=None)
record = grouping.next()
self.assertTrue(record.next().datetime, records[0].datetime)
self.assertTrue(record.next().datetime, records[1].datetime)
self.assertTrue(record.next().datetime, records[2].datetime)
self.assertTrue(record.next().datetime, records[3].datetime)
self.assertRaises(StopIteration, record.next)
self.assertRaises(StopIteration, grouping.next)
def test_read_csv_antenna_id_no_places(self):
user = bc.read_csv("u_test_antennas", "samples", describe=False)
self.assertEqual(user.records[1],
Record(interaction='call',
direction='in',
correspondent_id='770000001',
datetime=dt(2013, 12, 16, 5, 39, 30),
call_duration=0,
position=Position('13084', None)))
result = {'allweek': {'allday': None}}
radius = bc.spatial.radius_of_gyration(user, groupby=None)
self.assertEqual(radius, result)
def test_read_csv_antenna_id(self):
user = bc.read_csv("u_test_antennas", "samples",
antennas_path="samples/towers.csv", describe=False)
self.assertEqual(user.records[1],
Record(interaction='call',
direction='in',
correspondent_id='770000001',
datetime=dt(2013, 12, 16, 5, 39, 30),
call_duration=0,
position=Position('13084', None)))
radius = bc.spatial.radius_of_gyration(user, groupby=None)
self.assertGreater(radius['allweek']['allday'], 0)
def sample_user(number_records=1960, seed=42):
old_state = random.getstate()
random.seed(42)
towers = {701:(42.3555,-71.099541),
702:(42.359039,-71.094595),
703:(42.360481,-71.087321),
704:(42.361013,-71.097868),
705:(42.370849,-71.114613),
706:(42.3667427,-71.1069847),
707:(42.367589,-71.076537)}
towers_position = [Position(antenna=k, location=v) for k, v in towers.items()]
records = [random_record(position=random.choice(towers_position)) for _ in xrange(number_records)]
user = bc.io.load("sample_user", records, towers, None)
random.setstate(old_state)
return user
def random_record(**kwargs):
n_users = 150
rate = 1e-4
year = random.choice(range(2012, 2015))
month = random.choice(range(1, 12))
day = random.choice(range(1, 28))
r = {'datetime': datetime.datetime(year, month, day) + datetime.timedelta(seconds=math.floor(-1/rate*math.log(random.random()))),
'interaction': random.choice(['text', 'call']),
'correspondent_id': random.randint(1, n_users),
'direction': random.choice(['in', 'out']),
'call_duration': random.randint(1, 1000),
'position': Position(location=(random.uniform(-5, 5), random.uniform(-5, 5)))}
if r['interaction'] == "text":
r['call_duration'] = ''
r.update(kwargs)
return Record(**r)
def parse_record(raw):
direction = parse_direction(raw['CALL_TYPE'].strip())
if direction == 'in':
contact = raw.get('A_PARTY', raw.get('A_NUMBER'))
cell_id = raw['B_CELL']
else:
contact = raw.get('B_PARTY', raw.get('B_NUMBER'))
cell_id = raw['A_CELL']
position = Position(antenna=cell_id, location=cells.get(cell_id))
_date_str = raw.get('CDATE', raw.get('CALL_DATE'))
_time_str = raw.get('CTIME', raw.get('CALL_TIME'))
_datetime = datetime.strptime(_date_str + _time_str, "%Y%m%d%H:%M:%S")
r = Record(interaction='call',
direction=direction,
correspondent_id=contact,
call_duration=float(raw['DURATION'].strip()),
datetime=_datetime,
position=position)
return r
def sample_user(number_records=1482, seed=42, pct_in_network=0.8):
old_state = random.getstate()
try:
random.seed(seed, version=1)
except TypeError:
random.seed(seed)
towers = OrderedDict([(704, (42.361013, -71.097868)),
(705, (42.370849, -71.114613)),
(706, (42.3667427, -71.1069847)),
(707, (42.367589, -71.076537)),
(701, (42.3555, -71.099541)),
(702, (42.359039, -71.094595)),
(703, (42.360481, -71.087321))])
towers_position = [
Position(antenna=k, location=v) for k, v in towers.items()
]
ego_records = [
random_record(position=_choice(towers_position))
for _ in range(number_records)
]
user, _ = bc.io.load("sample_user",
ego_records,
towers,
None,
describe=False)
# create network
correspondents = set([record.correspondent_id for record in ego_records])
correspondents = sorted(list(correspondents))
correspondent_records = {}
connections = {}
n_in_network = int(len(correspondents) * pct_in_network)
n_in_network = (n_in_network // 2) * 2
in_network_correspondents = _sample(correspondents, n_in_network)
def reverse_records(records, current_owner):
for r in records:
r.direction = 'out' if r.direction == 'in' else 'in'
r.correspondent_id = current_owner
return records
# set records from ego
for c_id in sorted(in_network_correspondents):
reciprocal_records = [
r for r in ego_records if r.correspondent_id == c_id
]
reciprocal_records = reverse_records(copy.deepcopy(reciprocal_records),
"sample_user")
correspondent_records[c_id] = reciprocal_records
def generate_group_with_random_links(pct_users_in_group):
n_in_group = int(len(correspondents) * pct_users_in_group)
group = _sample(non_grouped_correspondents, n_in_group)
networkusers_group = list()
for user in group:
if user in in_network_correspondents:
networkusers_group.append(user)
def create_pair(source):
user_pair = [source, _choice(group)]
if user_pair[0] in non_grouped_correspondents:
non_grouped_correspondents.remove(user_pair[0])
if user_pair[1] in non_grouped_correspondents:
non_grouped_correspondents.remove(user_pair[1])
extra_records = [
random_record(position=_choice(towers_position),
interaction=_choice(['text', 'call', 'call']),
correspondent_id=user_pair[1])
for _ in range(_randint(25, 150))
]
correspondent_records[user_pair[0]].extend(extra_records)
if (user_pair[1] in in_network_correspondents):
rr = reverse_records(copy.deepcopy(extra_records),
user_pair[0])
correspondent_records[user_pair[1]].extend(rr)
# create pairs of users
for i in range(len(networkusers_group)):
create_pair(networkusers_group[i])
if random.random() > 0.5:
create_pair(networkusers_group[i])
non_grouped_correspondents = copy.deepcopy(correspondents)
for i in range(4):
generate_group_with_random_links(pct_users_in_group=0.4 - i * 0.1)
# create user object
for c_id in sorted(correspondents):
if c_id in in_network_correspondents:
correspondent_user, _ = bc.io.load(c_id,
correspondent_records[c_id],
towers,
None,
describe=False)
else:
correspondent_user = None
connections[c_id] = correspondent_user
# return the network dictionary sorted by key
user.network = OrderedDict(sorted(connections.items(), key=lambda t: t[0]))
user.recompute_missing_neighbors()
random.setstate(old_state)
return user
