def _map_position(data): antenna = Position() if 'antenna_id' in data: antenna.antenna = data['antenna_id'] return antenna if 'latitude' in data and 'longitude' in data: antenna.position = float(data['latitude']), float(data['longitude']) return antenna
def _map_position(data): antenna = Position() if 'antenna_id' in data: antenna.antenna = data['antenna_id'] return antenna if 'latitude' in data and 'longitude' in data: antenna.position = float(data['latitude']), float( data['longitude']) return antenna
def _map_position(data): antenna = Position() if 'antenna_id' in data: antenna.antenna = data['antenna_id'] return antenna elif 'place_id' in data: raise NameError("Use field name 'antenna_id' in input files. 'place_id' is deprecated.") if 'latitude' in data and 'longitude' in data: antenna.position = float(data['latitude']), float(data['longitude']) return antenna
def test_weekend_filter(self): records = [ Record("test_itr", "in", "1", dt(2014, 8, 22), 1, Position()), Record("test_itr", "in", "1", dt(2014, 8, 31), 1, Position()), Record("test_itr", "in", "1", dt(2014, 10, 18), 1, Position()) ] user = bc.User() user.records = records rv = bc.helper.group.filter_user(user, part_of_week='weekend') self.assertEqual(rv, [records[1], records[2]])
def test_weekday_filter(self): records = [ Record("test_itr", "in", "1", datetime.datetime(2014, 8, 22), 1, Position()), Record("test_itr", "in", "1", datetime.datetime(2014, 8, 31), 1, Position()), Record("test_itr", "in", "1", datetime.datetime(2014, 10, 18), 1, Position()) ] user = bc.User() user.records = records filtered_records = bc.helper.group.filter_user(user, part_of_week='weekday') self.assertEqual(filtered_records, [records[0]])
def test_weekly_group(self): records = [ Record("test_itr", "in", "1", dt(2014, 8, 24), 1, Position()), Record("test_itr", "in", "1", dt(2014, 9, 4), 1, Position()), Record("test_itr", "in", "1", dt(2014, 9, 11), 1, Position()) ] user = bc.User() user.records = records grouping = bc.helper.group.group_records(user.records, groupby='week') groups = [[r for r in l] for l in grouping] self.assertEqual(groups, [[records[0]], [records[1]], [records[2]]])
def test_weekend_group(self): records = [ Record("test_itr", "in", "1", datetime.datetime(2014, 8, 23), 1, Position()), Record("test_itr", "in", "1", datetime.datetime(2014, 8, 31), 1, Position()), Record("test_itr", "in", "1", datetime.datetime(2014, 10, 18), 1, Position()) ] user = bc.User() user.records = records grouping = bc.helper.group.group_records(user, groupby='week', part_of_week='weekend') groups = [[r for r in l] for l in grouping] self.assertEqual(groups, [[records[0]], [records[1]], [records[2]]])
def test_none_group(self): records = [ Record("call", "in", "1", dt(2014, 9, 4), 1, Position()), Record("call", "in", "1", dt(2014, 9, 5), 1, Position()), Record("call", "in", "1", dt(2014, 9, 11), 1, Position()), Record("call", "in", "1", dt(2014, 9, 12), 1, Position()) ] grouping = bc.helper.group.group_records(records, groupby=None) self.assertEqual(records, list(next(grouping))) with self.assertRaises(StopIteration): next(grouping)
def test_none_group(self): records = [ Record("call", "in", "1", datetime.datetime(2014, 9, 4), 1, Position()), Record("call", "in", "1", datetime.datetime(2014, 9, 5), 1, Position()), Record("call", "in", "1", datetime.datetime(2014, 9, 11), 1, Position()), Record("call", "in", "1", datetime.datetime(2014, 9, 12), 1, Position()) ] user = bc.User() user.records = records grouping = bc.helper.group.group_records(user, groupby=None) self.assertEqual(records, list(next(grouping))) self.assertRaises(StopIteration, grouping.next)
def test_daily_filter(self): records = [ Record("test_itr", "in", "1", datetime.datetime(2014, 8, 22, 10, 00), 1, Position()), Record("test_itr", "in", "1", datetime.datetime(2014, 8, 23, 10, 00), 1, Position()), Record("test_itr", "in", "1", datetime.datetime(2014, 9, 7, 11, 00), 1, Position()), Record("test_itr", "in", "1", datetime.datetime(2014, 10, 18, 2, 00), 1, Position()) ] user = bc.User() user.records = records filtered_records = bc.helper.group.filter_user(user, part_of_day='night') self.assertEqual(filtered_records, [records[3]]) filtered_records = bc.helper.group.filter_user(user, part_of_day='day') self.assertEqual(filtered_records, [records[0], records[1], records[2]])
def _parse(reader): records = [] antennas = dict() for row in reader: direction = 'out' if row[0] == '1' else 'in' interaction = 'call' if row[1] in ['11', '12'] else 'text' contact = row[3] date = datetime.strptime(row[4], "%Y-%m-%d %H:%M:%S") call_duration = float(row[5]) lon, lat = float(row[6]), float(row[7]) latlon = (lat, lon) antenna = None for key, value in antennas.items(): if latlon == value: antenna = key break if antenna is None: antenna = len(antennas) + 1 antennas[antenna] = latlon position = Position(antenna=antenna, location=latlon) record = Record(direction=direction, interaction=interaction, correspondent_id=contact, call_duration=call_duration, datetime=date, position=position) records.append(record) return records, antennas
def random_record(**kwargs): n_users = 48 rate = 1e-4 year = 2012 month = _choice(range(1, 3)) day = _choice(range(1, 28)) # ensures that some correspondents have more interactions than others correspondent = int(random.random() * n_users) r = { 'datetime': datetime.datetime(year, month, day) + datetime.timedelta( seconds=math.floor(-1 / rate * math.log(random.random()))), 'interaction': _choice(['text', 'text', 'text', 'call', 'call']), 'correspondent_id': "correspondent_{}".format(correspondent), 'direction': _choice(['in', 'in', 'out']), 'call_duration': int(random.random() * 1000), 'position': Position(location=(_uniform(-5, 5), _uniform(-5, 5))) } if r['interaction'] == "text": r['call_duration'] = None r.update(kwargs) return Record(**r)
def test_daily_group(self): records = [ Record("test_itr", "in", "1", datetime.datetime(2014, 8, 22, 10, 00), 1, Position()), Record("test_itr", "in", "1", datetime.datetime(2014, 8, 23, 10, 00), 1, Position()), Record("test_itr", "in", "1", datetime.datetime(2014, 9, 7, 11, 00), 1, Position()), Record("test_itr", "in", "1", datetime.datetime(2014, 10, 18, 2, 00), 1, Position()) ] user = bc.User() user.records = records grouping = bc.helper.group.group_records(user, groupby='week', part_of_day='night') groups = [[r for r in l] for l in grouping] self.assertEqual(groups, [[records[3]]]) grouping = bc.helper.group.group_records(user, groupby='week', part_of_day='day') groups = [[r for r in l] for l in grouping] self.assertEqual(groups, [[records[0], records[1]], [records[2]]])
def test_read_orange_stdin(self): user = bc.io.read_orange(describe=False) self.assertEqual( user.records[0], Record(interaction='text', direction='out', correspondent_id='770000005', datetime=datetime.datetime(2013, 12, 16, 7, 30, 30), call_duration=0.0, position=Position(1, (42.3987, -71.575))))
def test_read_csv_no_position(self): user = bc.read_csv("u_test_no_position", "samples", describe=False) self.assertEqual( user.records[1], Record(interaction='call', direction='in', correspondent_id='770000001', datetime=datetime.datetime(2013, 12, 16, 5, 39, 30), duration=0, position=Position()))
def test_weekly_group(self): records = [ Record("test_itr", "in", "1", datetime.datetime(2014, 8, 24), 1, Position()), Record("test_itr", "in", "1", datetime.datetime(2014, 9, 4), 1, Position()), Record("test_itr", "in", "1", datetime.datetime(2014, 9, 11), 1, Position()) ] user = bc.User() user.records = records grouping = bc.helper.group.group_records(user, groupby='week') record = grouping.next() self.assertTrue(record.next().datetime, records[0].datetime) record = grouping.next() self.assertTrue(record.next().datetime, records[1].datetime) record = grouping.next() self.assertTrue(record.next().datetime, records[2].datetime)
def test_read_csv_antenna_id_no_places(self): user = bc.read_csv("u_test_antennas", "samples", describe=False) self.assertEqual( user.records[1], Record(interaction='call', direction='in', correspondent_id='770000001', datetime=datetime.datetime(2013, 12, 16, 5, 39, 30), call_duration=0, position=Position('13084', None))) self.assertEqual( bc.spatial.radius_of_gyration(user).values(), [None] * 2)
def test_none_group(self): records = [ Record("call", "in", "1", datetime.datetime(2014, 9, 5), 1, Position()), Record("call", "in", "1", datetime.datetime(2014, 9, 4), 1, Position()), Record("call", "in", "1", datetime.datetime(2014, 9, 11), 1, Position()), Record("call", "in", "1", datetime.datetime(2014, 9, 12), 1, Position()) ] user = bc.User() user.records = records grouping = bc.helper.group.group_records(user, groupby=None) record = grouping.next() self.assertTrue(record.next().datetime, records[0].datetime) self.assertTrue(record.next().datetime, records[1].datetime) self.assertTrue(record.next().datetime, records[2].datetime) self.assertTrue(record.next().datetime, records[3].datetime) self.assertRaises(StopIteration, record.next) self.assertRaises(StopIteration, grouping.next)
def test_read_csv_antenna_id_no_places(self): user = bc.read_csv("u_test_antennas", "samples", describe=False) self.assertEqual(user.records[1], Record(interaction='call', direction='in', correspondent_id='770000001', datetime=dt(2013, 12, 16, 5, 39, 30), call_duration=0, position=Position('13084', None))) result = {'allweek': {'allday': None}} radius = bc.spatial.radius_of_gyration(user, groupby=None) self.assertEqual(radius, result)
def test_read_csv_antenna_id(self): user = bc.read_csv("u_test_antennas", "samples", antennas_path="samples/towers.csv", describe=False) self.assertEqual(user.records[1], Record(interaction='call', direction='in', correspondent_id='770000001', datetime=dt(2013, 12, 16, 5, 39, 30), call_duration=0, position=Position('13084', None))) radius = bc.spatial.radius_of_gyration(user, groupby=None) self.assertGreater(radius['allweek']['allday'], 0)
def sample_user(number_records=1960, seed=42): old_state = random.getstate() random.seed(42) towers = {701:(42.3555,-71.099541), 702:(42.359039,-71.094595), 703:(42.360481,-71.087321), 704:(42.361013,-71.097868), 705:(42.370849,-71.114613), 706:(42.3667427,-71.1069847), 707:(42.367589,-71.076537)} towers_position = [Position(antenna=k, location=v) for k, v in towers.items()] records = [random_record(position=random.choice(towers_position)) for _ in xrange(number_records)] user = bc.io.load("sample_user", records, towers, None) random.setstate(old_state) return user
def random_record(**kwargs): n_users = 150 rate = 1e-4 year = random.choice(range(2012, 2015)) month = random.choice(range(1, 12)) day = random.choice(range(1, 28)) r = {'datetime': datetime.datetime(year, month, day) + datetime.timedelta(seconds=math.floor(-1/rate*math.log(random.random()))), 'interaction': random.choice(['text', 'call']), 'correspondent_id': random.randint(1, n_users), 'direction': random.choice(['in', 'out']), 'call_duration': random.randint(1, 1000), 'position': Position(location=(random.uniform(-5, 5), random.uniform(-5, 5)))} if r['interaction'] == "text": r['call_duration'] = '' r.update(kwargs) return Record(**r)
def parse_record(raw): direction = parse_direction(raw['CALL_TYPE'].strip()) if direction == 'in': contact = raw.get('A_PARTY', raw.get('A_NUMBER')) cell_id = raw['B_CELL'] else: contact = raw.get('B_PARTY', raw.get('B_NUMBER')) cell_id = raw['A_CELL'] position = Position(antenna=cell_id, location=cells.get(cell_id)) _date_str = raw.get('CDATE', raw.get('CALL_DATE')) _time_str = raw.get('CTIME', raw.get('CALL_TIME')) _datetime = datetime.strptime(_date_str + _time_str, "%Y%m%d%H:%M:%S") r = Record(interaction='call', direction=direction, correspondent_id=contact, call_duration=float(raw['DURATION'].strip()), datetime=_datetime, position=position) return r
def sample_user(number_records=1482, seed=42, pct_in_network=0.8): old_state = random.getstate() try: random.seed(seed, version=1) except TypeError: random.seed(seed) towers = OrderedDict([(704, (42.361013, -71.097868)), (705, (42.370849, -71.114613)), (706, (42.3667427, -71.1069847)), (707, (42.367589, -71.076537)), (701, (42.3555, -71.099541)), (702, (42.359039, -71.094595)), (703, (42.360481, -71.087321))]) towers_position = [ Position(antenna=k, location=v) for k, v in towers.items() ] ego_records = [ random_record(position=_choice(towers_position)) for _ in range(number_records) ] user, _ = bc.io.load("sample_user", ego_records, towers, None, describe=False) # create network correspondents = set([record.correspondent_id for record in ego_records]) correspondents = sorted(list(correspondents)) correspondent_records = {} connections = {} n_in_network = int(len(correspondents) * pct_in_network) n_in_network = (n_in_network // 2) * 2 in_network_correspondents = _sample(correspondents, n_in_network) def reverse_records(records, current_owner): for r in records: r.direction = 'out' if r.direction == 'in' else 'in' r.correspondent_id = current_owner return records # set records from ego for c_id in sorted(in_network_correspondents): reciprocal_records = [ r for r in ego_records if r.correspondent_id == c_id ] reciprocal_records = reverse_records(copy.deepcopy(reciprocal_records), "sample_user") correspondent_records[c_id] = reciprocal_records def generate_group_with_random_links(pct_users_in_group): n_in_group = int(len(correspondents) * pct_users_in_group) group = _sample(non_grouped_correspondents, n_in_group) networkusers_group = list() for user in group: if user in in_network_correspondents: networkusers_group.append(user) def create_pair(source): user_pair = [source, _choice(group)] if user_pair[0] in non_grouped_correspondents: non_grouped_correspondents.remove(user_pair[0]) if user_pair[1] in non_grouped_correspondents: non_grouped_correspondents.remove(user_pair[1]) extra_records = [ random_record(position=_choice(towers_position), interaction=_choice(['text', 'call', 'call']), correspondent_id=user_pair[1]) for _ in range(_randint(25, 150)) ] correspondent_records[user_pair[0]].extend(extra_records) if (user_pair[1] in in_network_correspondents): rr = reverse_records(copy.deepcopy(extra_records), user_pair[0]) correspondent_records[user_pair[1]].extend(rr) # create pairs of users for i in range(len(networkusers_group)): create_pair(networkusers_group[i]) if random.random() > 0.5: create_pair(networkusers_group[i]) non_grouped_correspondents = copy.deepcopy(correspondents) for i in range(4): generate_group_with_random_links(pct_users_in_group=0.4 - i * 0.1) # create user object for c_id in sorted(correspondents): if c_id in in_network_correspondents: correspondent_user, _ = bc.io.load(c_id, correspondent_records[c_id], towers, None, describe=False) else: correspondent_user = None connections[c_id] = correspondent_user # return the network dictionary sorted by key user.network = OrderedDict(sorted(connections.items(), key=lambda t: t[0])) user.recompute_missing_neighbors() random.setstate(old_state) return user