if hasattr(td, 'total_seconds'):

# python 2.7

return td.total_seconds()

# I don't care about microsecond

"""Return a 5 points GeoJSON polygon based on the bottom left and upper

right coordinates of bbox [lat_bl, long_bl, lat_ur, long_ur]

(5 because the polygon needs to be closed, see:

https://groups.google.com/d/msg/mongodb-user/OPouYFHS_zU/cS21L0XAMkkJ )

Or in other words ;(

http://toblerity.org/shapely/manual.html#shapely.geometry.box

>>> bbox_to_polygon([37, -122, 35, -120])

{'type': 'Polygon', 'coordinates': [[[-122, 37], [-120, 37], [-120, 35], [-122, 35], [-122, 37]]]}

"""

assert(len(bbox) == 4)

if latitude_first:

lat_bl, long_bl, lat_ur, long_ur = bbox

else:

long_bl, lat_bl, long_ur, lat_ur = bbox

r = {}

r['type'] = 'Polygon'

r['coordinates'] = [[[long_bl, lat_bl], [long_ur, lat_bl],

[long_ur, lat_ur], [long_bl, lat_ur],

[long_bl, lat_bl]]]

"""Return a query operator that selects all date between start_year and

end_year and in the months corresponding to season"""

first_month = {'winter': 12, 'spring': 3, 'summer': 6, 'fall': 9}[season]

last_month = (first_month + 3) % 12

req = {'$or': []}

for year in range(start_year, end_year+1):

start = datetime.datetime(year, first_month, 15)

if season == 'winter':

end = datetime.datetime(year+1, last_month, 15)

else:

end = datetime.datetime(year, last_month, 15)

req['$or'].append({'taken': {'$gte': start, '$lte': end}})

web_url = u"http://www.flickr.com/photos/{}/{}".format(p['uid'], p['_id'])

BASE = u"http://farm{}.staticflickr.com/{}/{}_{}_{}.jpg"

url = BASE.format(p['farm'], p['server'], p['_id'], p['secret'], size)

if webpage:

return url, web_url

tourist_status=False, uploaded=False, user=None):

"""Return a list of [long, lat] for each photo taken between start and end

(or uploaded) in bbox which has tag. extra_info is a list of other fields

to include after. It can also adds tourist status as [long, lat,

is_tourist]"""

query = {}

field = {'loc': 1, '_id': 0}

if extra_info is not None:

field.update(zip(extra_info, len(extra_info)*[1, ]))

else:

extra_info = []

if tourist_status:

user_stat = get_user_status()

field['uid'] = 1

# if bbox is None:

# query['hint'] = 'sf'

# else:

# query['loc'] = inside_bbox(bbox)

query['hint'] = 'helsinki'

if tag is not None:

query['ntags'] = tag

if user is not None:

query['uid'] = user

time_field = 'upload' if uploaded else 'taken'

query[time_field] = {'$gte': start, '$lte': end}

# query.update(season_query(2008, 2013, 'winter'))

cursor = collection.find(query, field)

def format_photo(p):

loc = p['loc']['coordinates']

tourist = [user_stat[p['uid']]] if tourist_status else []

extra = [p[attr] for attr in extra_info]

return loc + tourist + extra

now = datetime.datetime.now()

if interval is None:

interval = now - start

num_period = 1

else:

num_period = total_seconds(now - start)

num_period = int(num_period/total_seconds(interval))

schema = {'geometry': 'Point', 'properties': {}}

# schema = {'geometry': 'Point', 'properties': {'tourist': 'int'}}

for i in range(num_period):

places = map(lambda p: {'geometry': mapping(Point(p[0], p[1])),

'properties': {}},

# 'properties': {'tourist': int(p[2])}},

tag_location(collection, tag, bbox,

start + i * interval,

start + (i+1) * interval,

tourist_status=True))

print('{} - {}: {}'.format(start + i * interval,

start + (i+1) * interval, len(places)))

name = u'{}_{}.shp'.format(tag, i+1)

with fiona.collection(name, "w", "ESRI Shapefile", schema) as f:

long_step = (bbox[3] - bbox[1])/k

lat_step = (bbox[2] - bbox[0])/k

assert(offset <= min(long_step, lat_step))

hoffset = offset

voffset = 0

x = bbox[1]

y = bbox[0]

region = []

while y + voffset < bbox[2] - 0.5*lat_step:

while x + hoffset < bbox[3] - 0.5*long_step:

region.append([x + hoffset, y + voffset,

x + hoffset + long_step, y + voffset + lat_step])

x += long_step

y += lat_step

x = bbox[1]

def coord2region_index(coords):

longitude, latitude = coords

# TODO handle correctly edge case

x = longitude - hoffset - bbox[1] - 1e-8

y = latitude - voffset - bbox[0] - 1e-8

if x < 0 or y < 0:

return -1

r = k*int(floor(y/lat_step)) + int(floor(x/long_step))

if r > k*k - 1:

return - 1

return r

def region_index2bbox(r):

i = r/k

j = r%k

x = bbox[1] + j * long_step

y = bbox[0] + i * lat_step

return [x + hoffset, y + voffset,

x + hoffset + long_step, y + voffset + lat_step]

""" Return D(tag || all_photos) given count of tag (including the 0

region) on a grid of size k×k."""

from math import log

Nt = float(sum(count[1:]))

d = sio.loadmat('mfreq/freq_{}__background.mat'.format(k))

N = np.sum(d['c'])

ratio = log(N/Nt)

KL = float(np.sum([(p/Nt)*(log(float(p)/q)+ratio)

for p, q in zip(count[1:], d['c'].flat) if p > 0 and q > 0]))

nb_inter=19, exclude_zero=True, plot=True):

"""split bbox in k^2 rectangles and compute the frequency of tag in each of

them. Return a list of list of Polygon, grouped by similar frequency

into nb_inter bucket (potentially omitting the zero one for clarity)."""

freq_name = u'mfreq/freq_{}_{}.mat'.format(k, '_background' if tag is None else tag)

count = (k*k+1)*[0, ]

try:

count[1:] = sio.loadmat(freq_name)['c']

KL_div = 0 if tag is None else compute_KL(count, k)

except IOError:

collection = DB.photos if collection is None else collection

# coords = tag_location(collection, tag, bbox, start, end,

# tourist_status=True)

coords = tag_location(collection, tag, bbox, start, end)

# r, f, _ = k_split_bbox(bbox, k)

r, f = RECTANGLES, RECT_TO_INDEX

# count[0] is for potential points that do not fall in any region (it

# must only happens because of rounding inprecision)

count = (len(r)+1)*[0, ]

# count = (len(r)+1)*[(0, 0), ]

for loc in coords:

# rloc = loc[0:2]

# prev = count[f(rloc)+1]

# count[f(rloc)+1] = (prev[0] + int(loc[2]), prev[1]+1)

count[f(loc)+1] += 1

# save_var('alltourist', count)

# count = load_var('alltourist')

# N = len(coords)

if tag is None:

tag = '_background'

KL_div = 0

else:

KL_div = compute_KL(count, k)

sio.savemat(freq_name, {'c': np.array(count[1:])}, do_compression=True)

entropy = compute_entropy(count[1:])

print(u'Entropy and KL of {}: {:.4f}, {:.4f}'.format(tag, entropy, KL_div))

if not plot:

return entropy, KL_div

# freq = np.array(count[1:])/(1.0*N)

log_freq = np.maximum(0, np.log(count[1:]))

# log_freq = [0 if p[1] == 0 else (1.0*p[0]/p[1] - 0.5) for p in count[1:]]

# entropy = 0

# sio.savemat('tourist_ratio', {'c': np.array(log_freq)})

maxv = np.max(log_freq)

minv = np.min(log_freq)

interval_size = (maxv - minv)/nb_inter

bucket = []

for i in range(nb_inter):

bucket.append([])

for i, v in enumerate(log_freq):

if not (exclude_zero and v < 1e-8):

poly = {'geometry': mapping(shape(bbox_to_polygon(r[i], False))),

'properties': {}}

index = (min(nb_inter - 1, int(floor(v / interval_size))))

bucket[index].append(poly)

plot_polygons(bucket, tag, nb_inter, minv, maxv)

schema = {'geometry': 'Polygon', 'properties': {}}

colormap_ = matplotlib.cm.ScalarMappable(cmap='YlOrBr')

vrange = np.linspace(minv, maxv, nb_inter)

colormap = [to_css_hex(c) for c in colormap_.to_rgba(vrange)]

style = []

for i in range(nb_inter):

if len(bucket[i]) > 0:

name = u'{}_freq_{:03}'.format(tag, i+1)

KARTO_CONFIG['layers'][name] = {'src': name+'.shp'}

style.append(CSS.format(name, colormap[i], colormap[i]))

with fiona.collection(name+'.shp', "w",

"ESRI Shapefile", schema) as f:

f.writerecords(bucket[i])

with open('photos.json', 'w') as f:

json.dump(KARTO_CONFIG, f)

with open('photos.css', 'w') as f:

places = tag_location(collection, tag, bbox, start, end)

p = np.array(zip(map(itemgetter(1), places), map(itemgetter(0), places)))

grav = np.mean(p, 0)

tmp = p - grav

dst = np.sum(tmp**2, 1)

sio.savemat(u'grav_' + tag, {'grav': dst})

mu_grav = np.mean(dst)

sigma_grav = np.std(dst)

H_grav = compute_entropy(dst)

H_pair = -1

try:

if len(dst) < 22000:

dst = pdist(p)

h, b = np.histogram(dst, 200)

start = clock()

H_pair = compute_entropy(dst)

print(u'H_pair_{}: {}s'.format(tag, clock() - start))

# outplot('pairwise_' + tag, ['', ''], h, b[1:])

except MemoryError:

logging.warn(u'{} is too big for pdist'.format(tag))

# pool = Pool(4)

res = map(fixed_tag_metrics, tags)

# pool.close()

outplot('e_grav.dat', ['H', 'tags'],

[v[2] for v in res], [v[4] for v in res])

outplot('e_pair.dat', ['H', 'tags'],

[v[3] for v in res], [v[4] for v in res])

mus = np.array([v[0] for v in res])

sigmas = np.array([v[1] for v in res])

sio.savemat('mu_sigma', {'A': np.vstack([mus, sigmas]).T})

name = 'user_status' + ('_full' if with_count else '')

fields = {'tourist': 1}

if with_count:

fields.update({'count': 1})

try:

d = load_var(name)

except IOError:

users = list(DB.users.find(fields=fields))

if with_count:

d = dict([(u['_id'], (u['count'], u['tourist'])) for u in users])

else:

d = dict([(u['_id'], u['tourist']) for u in users])

save_var(name, d)

users_collection = db['users']

users_from_photos = db['photos'].aggregate([

{"$match": {"hint": "sf"}},

{'$project': {'_id': 0, 'upload': 1, 'user': '$uid'}},

{'$group': {'_id': '$user',

'first': {'$min': '$upload'},

'last': {'$max': '$upload'},

"count": {"$sum": 1}}}

])['result']

month = total_seconds(datetime.timedelta(days=365.24/12))

users = []

autochthons = 0

for u in users_from_photos:

timespan = total_seconds(u['last'] - u['first'])

u['tourist'] = timespan < month

if not u['tourist']:

autochthons += 1

users.append(u)

print(100*autochthons/len(users))

try:

users_collection.insert(users, continue_on_error=True)

except pymongo.errors.DuplicateKeyError:

print('duplicate')

r = DB.photos.aggregate([

{"$match": {"hint": "sf", "ntags": tag}},

{"$project": {"uid": 1}},

{"$group": {"_id": "$uid", "count": {"$sum": 1}}},

{"$sort": SON([("count", -1), ("_id", -1)])}

])

places = tag_location(DB.photos, tag, None, FIRST_TIME, LAST_TIME,

extra_info=['taken'])

res = []

for days in PERIOD:

time_step = days*24*3600

times = [int(total_seconds(p[2] - FIRST_TIME)/time_step)

for p in places]

res.append(float(compute_entropy(Counter(times))))