def getCountry(self, lat, lon):
cc = countries.CountryChecker(config.worldBordersShapeFile)
country = cc.getCountry(countries.Point(lat, lon))
if country == None:
return "None"
else:
return country.iso
def get_country(self, lat, lon):
logger.debug(f"[database] get_country lat:{lat} lon:{lon}")
country = None
try:
cc = countries.CountryChecker(config.worldBordersShapeFile)
country = cc.getCountry(countries.Point(lat, lon))
except Exception as e:
logger.error(f"Error in get_country: {repr(e)}")
logger.debug(f"[database] country for lat:{lat} lon:{lon}: {country}")
if country == None:
return "None"
else:
return country.iso
from collections import OrderedDict
import numpy as np
import sys
#Logging framework
import logging
logging.basicConfig(format='%(levelname)s: %(message)s',level=logging.INFO)
log = logging.getLogger('LocHist')
#Progress bar
from tqdm import tqdm
tqdm.monitor_interval = 0 #per https://github.com/tqdm/tqdm/issues/481
from countries import countries
cc = countries.CountryChecker('./countries/TM_WORLD_BORDERS-0.3.shp')
from optparse import OptionParser
parser = OptionParser()
parser.add_option("-s", dest="startdate",
help="Date to start analyzing in Format YYYY-Mo-DD \n Example: 2013-Jan-30")
parser.add_option("-r", "--resolution", default=6,
help="Resolution of location data in hours (Default is 6hrs)")
parseOpts, args = parser.parse_args()
if parseOpts.startdate == None:
log.error('Must provide a start date')
parser.print_help()
sys.exit(-1)
import argparse
import json
import glob
from datetime import datetime, timedelta
import dateutil.parser
import pandas as pd
import numpy as np
import xarray as xr
import plotly.graph_objects as go
# local
from countries import countries
GEOMETRY = None
CC = countries.CountryChecker('500cities/cities.shp')
PLACES = [
{ 'city': 'New Orleans', 'state': 'LA' },
{ 'city': 'Houston', 'state': 'TX' },
{ 'city': 'Baltimore', 'state': 'MD' },
{ 'city': 'Norfolk', 'state': 'VA' },
{ 'city': 'Seattle', 'state': 'WA' },
{ 'city': 'Portland', 'state': 'OR' }
]
def within_area(latlon):
global GEOMETRY
lat = float(latlon[0])
lon = float(latlon[1])
# longitude values in the data are from 0 to 360
# but longitude values in the shapefile are -180 to 180
if lon > 180:
lon = lon - 360
from decimal import Decimal
from Levenshtein import ratio
from polyglot.text import Text
from django.conf import settings
from django.utils import timezone
from dataman.elastic import search, tokenize, FilterConverter, QueryConverter, \
ES_INDEX_MAPPING, ES_KEYWORDS
from countries import countries
from core.utils import RecordDict, get_val_by_path, flatten_dict, \
get_place_coords, avg_coords_list, meters, get_parsed_datetime, \
build_filters_geo, build_filters_time, ensure_dict, \
UnsupportedValueError, MissingDataError
cc = countries.CountryChecker(settings.WORLD_BORDERS)
LOG = logging.getLogger("tweet")
SIMILAR_PREDICTION_EDIT_DISTANCE_MAX = 0.8
def normalize_aggressive(text):
"""
Performs aggressive normalization of text
"""
# Ampersand
text = re.sub(r'\s+&?\s+', ' and ', text)
# User mentions
text = re.sub(r'@[A-Za-z0-9_]+\b', '_USER_ ', text)
# Time
text = re.sub(r"\b\d\d?:\d\d\s*[ap]\.?m\.?\b",