def clean(self):
try:
country = countries.lookup(self.name)
except Exception:
raise ValidationError('{} is not a valid country'.format(self.name))
else:
self.name = country.name
def get(cls, country):
try:
c = countries.lookup(country)
return Country(c.alpha_2, c.alpha_3, c.numeric, c.name,
getattr(c, "official_name", c.name))
except (LookupError, KeyError):
raise LookupError(f"Invalid country code: {country}")
def to_internal_value(self, data):
country_name = data.get("country")
if not country_name:
raise serializers.ValidationError(
{"country": [self.error_messages["required"]]})
try:
query_data = {"id": int(country_name)}
except ValueError:
country = countries.lookup(country_name)
query_data = {"name": country.name}
finally:
try:
country = models.Country.objects.get(**query_data)
except Exception:
raise serializers.ValidationError({
"country": [
f'Invalid country "{country_name}" - object does not exist.'
]
})
data_ = data.copy()
data_["country"] = country.name
if not data_.get("name"):
data_["name"] = data_.get("centre_name")
internal_value = super().to_internal_value(data_)
return internal_value
def query_timed(firstYear,lastYear, pattern= ''):
year = firstYear
yearList = []
firstAdded = False
countryList = []
timeList = []
query_response = otx.getall()
#query_response = query_response['results']
while year <= lastYear:
yearList.append(year)
year = year + 1
for pulse in query_response:
countriesPulse = []
for tag in pulse["tags"]:
tryCountry = False
if len(tag) >= 3:
try:
checkCountry = countries.lookup(tag)
tryCountry = True
except:
pass
if tryCountry:
if tag not in pulse["targeted_countries"]:
countriesPulse.append(countries.lookup(tag).name)
if pulse["targeted_countries"]:
for target in pulse["targeted_countries"]:
countriesPulse.append(countries.lookup(target).name)
if countriesPulse:
for ioc in pulse["indicators"]:
# if fnmatch.filter(ioc["tags], pattern):
date = ioc["created"]
date = dateutil.parser.isoparse(date)
if int(date.year) in yearList:
countryList.append(countriesPulse)
timeList.append(ioc["created"])
targetedCountries = dict()
for group in countryList:
for target in group:
if target in targetedCountries:
targetedCountries[target] += 1
else:
targetedCountries[target] = 1
def test_country_create(self, mock_verify_token):
mock_verify_token.return_value = {'email': self.admin_user.email}
data = {"name": countries.lookup('Rwanda').name}
response = client.post(
self.country_url,
data=data,
HTTP_AUTHORIZATION="Token {}".format(self.admin_user),
)
self.assertIn("name", response.data.keys())
self.assertEqual(response.status_code, 201)
async def bind_country(self, ctx, *, country="reset"):
# Check the country name
if country != "reset":
country = countries.lookup(country)
if country:
country = country.name
else:
raise commands.BadArgument
await rest_api.bind_user_param(self.bot,
ctx.author.id,
country=country)
async def bind_country(self, ctx, *, country="reset"):
# Check the country name
if country != "reset":
country = countries.lookup(country)
if country:
country = country.name
else:
raise commands.BadArgument
await util_users.change_user_info(self.bot,
ctx.author.id,
country=country)
def get(cls, country):
try:
if PYCOUNTRY:
c = countries.lookup(country)
return Country(c.alpha_2, c.alpha_3, c.numeric, c.name,
getattr(c, "official_name", c.name))
else:
c = countries.get(country)
return Country(c.alpha2, c.alpha3, c.numeric, c.name,
c.apolitical_name)
except (LookupError, KeyError):
raise LookupError("Invalid country code: {0}".format(country))
def lookup_country_code(country):
country = country.strip()
if not country:
return ''
# Try the pycountry database first
try:
return countries.lookup(country).alpha_3
except LookupError:
pass
# If that fails, try common names that are not in the pycountry database
country = country.lower()
if country in additional_common_names:
return additional_common_names[country]
return ''
def scrape_info(submitter_id):
submitter_name = ''
country_code = ''
try:
with urlopen('https://www.ncbi.nlm.nih.gov/clinvar/submitters/' +
str(submitter_id) + '/') as f:
root = html5lib.parse(
f, transport_encoding=f.info().get_content_charset())
submitter_el = root.find(
'.//html:div[@id="maincontent"]//html:div[@class="submitter_main indented"]',
ns)
name_el = submitter_el.find('./html:h2', ns)
contact_info_el = submitter_el.find(
'.//html:div[@class="indented"]/html:div[@class="indented"]',
ns)
if name_el != None and name_el.text: #submitter 1 has no name
submitter_name = re.sub(r'\s+', ' ', name_el.text.strip())
if contact_info_el != None: #submitter 1 has no contact information
contact_info = list(contact_info_el.itertext())[1:]
contact_info = list(
filter(
lambda info: not re.match(
'http://|https://|Organization ID:', info),
contact_info))
if len(contact_info) >= 1:
country_and_zip = re.match('(.+) - (.+)', contact_info[-1])
country_name = country_and_zip.group(
1) if country_and_zip else contact_info[-1]
try:
country_code = countries.lookup(country_name).alpha_3
except LookupError: #not a real country
pass
except HTTPError as err:
if err.code == 404:
print('\r\033[KNo information for submitter ' + str(submitter_id))
else:
raise err
return [submitter_id, submitter_name, country_code]
def to_internal_value(self, data):
country_name = data.get('country')
if not country_name:
raise serializers.ValidationError(
{'country': [self.error_messages['required']]})
try:
query_data = {'id': int(country_name)}
except ValueError:
country = countries.lookup(country_name)
query_data = {'name': country.name}
finally:
try:
country = models.Country.objects.get(**query_data)
except Exception:
raise serializers.ValidationError({
'country': [
f'Invalid country \"{country_name}\" - object does not exist.'
]
})
data_ = data.copy()
data_['country'] = country.name
internal_value = super().to_internal_value(data_)
return internal_value
def return_figures():
''' Fuction to create plotly visualizations
Input: None
Output: list (dict): list containing the plotly visualizations
'''
# Load Johns Hopkins University Data
# new URLs and data structure
url_confirmed = 'https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_confirmed_global.csv'
url_death = 'https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_deaths_global.csv'
#url_confirmed = 'https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Confirmed.csv'
#url_death = 'https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Deaths.csv'
#url_recovered = 'https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Recovered.csv'
df_confirmed = pd.read_csv(url_confirmed, error_bad_lines=False)
df_death = pd.read_csv(url_death, error_bad_lines=False)
#df_recovered = pd.read_csv(url_recovered, error_bad_lines=False)
# Load population data from the World Bank
df_pop_2018 = pd.read_pickle('covidapp/data/df_pop_2018.pkl')
df_pop_2018 = df_pop_2018.drop(['Country Name'], axis=1)
#df_confirmed.to_pickle('data/covid-19_confirmed.pkl')
#df_death.to_pickle('data/covid-19_death.pkl')
#df_recovered.to_pickle('data/covid-19_recovered.pkl')
last_col_confirmed = df_confirmed.columns[-1]
cases = defaultdict(int)
for i in range(len(df_confirmed)):
cases[df_confirmed['Country/Region']
[i]] += df_confirmed[last_col_confirmed][i]
last_col_death = df_death.columns[-1]
deaths = defaultdict(int)
for i in range(len(df_death)):
deaths[df_death['Country/Region'][i]] += df_death[last_col_death][i]
'''
# Gruppiertes Balkendiagramm
graph_one = []
x = []
y = []
for key, value in sorted(cases.items(), key=lambda item: item[1], reverse=True):
x.append(key)
y.append(value)
z = []
for key, value in sorted(deaths.items(), key=lambda item: item[1], reverse=True):
z.append(value)
x = x[0:15]
y = y[0:15]
z = z[0:15]
graph_one.append(
go.Bar(x = x,
y = y,
name = 'cases',)
)
graph_one.append(
go.Bar(x = x,
y = z,
name = 'deaths',
marker_color = 'grey',)
)
layout_one = dict(title = 'Confirmed Cases | Top 15 Countries',
xaxis = dict(title = 'Country',),
yaxis = dict(title = 'Cases'),
barmode = 'group',
)
'''
# 1. Graph: Bar chart of confirmed cases per top country
graph_one = []
x = []
y = []
for key, value in sorted(cases.items(),
key=lambda item: item[1],
reverse=True):
x.append(key)
y.append(value)
# Create dataframe for later
df_country_cases = {'Country/Region': x, 'Cases': y}
df_country_cases = pd.DataFrame(df_country_cases)
x = x[0:15]
y = y[0:15]
graph_one.append(go.Bar(
x=x,
y=y,
))
layout_one = dict(
title='Confirmed Cases | Top 15 Countries',
#xaxis = dict(title = 'Country',),
yaxis=dict(title='Cases'),
separators=',.',
)
# 2. Graph: Bar chart of deaths per top country
graph_two = []
x = []
y = []
for key, value in sorted(deaths.items(),
key=lambda item: item[1],
reverse=True):
x.append(key)
y.append(value)
# Create dataframe for later
df_country_deaths = {'Country/Region': x, 'Deaths': y}
df_country_deaths = pd.DataFrame(df_country_deaths)
x = x[0:15]
y = y[0:15]
graph_two.append(go.Bar(
x=x,
y=y,
marker_color='grey',
))
layout_two = dict(
title='Fatal Cases | Top 15 Countries',
#xaxis = dict(title = 'Country',),
yaxis=dict(title='Deaths'),
separators=',.',
)
# 3. Graph: Line chart of new infections per day for top countries
graph_three = []
x = []
y = []
# Create DataFrame with new cases
arr_confirmed = df_confirmed.iloc[:, 4:].to_numpy()
arr_confirmed_sub = np.append(arr=np.zeros(
(len(arr_confirmed), 1)).astype(int),
values=arr_confirmed,
axis=1)
arr_confirmed_sub = arr_confirmed_sub[:, :-1]
arr_new = arr_confirmed - arr_confirmed_sub
df_new_values = pd.DataFrame(arr_new)
df_new_values.reset_index(inplace=True)
df_new = df_confirmed.iloc[:, :4]
df_new.reset_index(inplace=True)
df_new = df_new.merge(df_new_values, on='index')
df_new = df_new.iloc[:, 1:]
df_new.columns = df_confirmed.columns.tolist()
# Create dictionary with new cases of the last 7 days
countrylist = []
new_last7 = defaultdict(int)
for i in range(len(df_new)):
new_last7[df_new['Country/Region'][i]] += df_new.iloc[:, -7:].sum(
axis=1)[i]
# Create countrylist with top 15 countries
countrylist = []
for key, value in sorted(new_last7.items(),
key=lambda item: item[1],
reverse=True):
countrylist.append(key)
countrylist = countrylist[:12]
datelist = df_new.columns[-28:].tolist() # last 28 days
# Create graph
for country in countrylist:
x = datelist
y = []
for date in datelist:
y.append(df_new[date][df_new['Country/Region'] == country].sum())
graph_three.append(go.Scatter(x=x, y=y, mode='lines', name=country))
layout_three = dict(
title=
'Infections per Day <br>Top 12 Countries by most infections in last 7 days',
xaxis=dict( #title = 'Date',
autotick=True),
yaxis=dict(title='Cases'),
separators=',.',
)
# 4. Graph: Scatter confirmed cases vs. deaths
graph_four = []
# Create countrylist with top countries by total cases
countrylist = []
count = 0
for key, value in sorted(cases.items(),
key=lambda item: item[1],
reverse=True):
if count < 12:
countrylist.append(key)
count += 1
else:
break
for country in countrylist:
x = [cases[country]]
y = [deaths[country]]
#text = str(country) + ' ' + str(x) + '/' + str(y)
graph_four.append(
go.Scatter(
x=x,
y=y,
mode='markers',
#text = text,
name=country,
textposition='top center'))
layout_four = dict(
title='Confirmed Cases & Deaths <br>Top 12 Countries by total cases',
xaxis=dict(title='Cases'),
yaxis=dict(title='Deaths'),
separators=',.',
)
# 5. Graph: Bubble map with confirmed cases per country
'''
graph_five = []
#cases_deaths = []
for latitude in df_confirmed['Lat']:
df_lat = df_confirmed[df_confirmed['Lat'] == latitude]
df_death_lat = df_death[df_death['Lat'] == latitude]
for longitude in df_lat['Long']:
cases = df_lat[last_col_confirmed][df_lat['Long'] == longitude].sum()
deaths = df_death[last_col_death][df_death['Long'] == longitude].sum()
country_name = str(df_lat['Country/Region'][df_lat['Long'] == longitude].unique().tolist()[0])
#cases_deaths.append([country, cases, deaths])
if cases > 0:
if str(df_lat['Province/State'][df_lat['Long'] == longitude].unique().tolist()[0]) == 'nan':
bubble_text = (
country_name
+ '<br>Cases: ' + str(cases)
+ '<br>Deaths: ' + str(deaths)
)
else:
bubble_text = (
country_name + ', '
+ str(df_lat['Province/State'][df_lat['Long'] == longitude].unique().tolist()[0])
+ '<br>Cases: ' + str(cases)
+ '<br>Deaths: ' + str(deaths)
)
graph_five.append(
go.Scattergeo(
locationmode = 'country names', #USA-states',
lat = [latitude],
lon = [longitude],
text = bubble_text,
marker = dict(
size = cases/1000,
#color = ,
line_color = 'rgb(40, 40, 40)',
line_width = 0.5,
sizemode = 'area'
)
#, name =
)
)
else:
break
layout_five = dict(title = 'Confirmed Cases & Deaths | Bubble Map (all cases)',
showlegend = False,
geo = dict(
projection = dict(type = 'natural earth'),
showland = True,
landcolor = 'rgb(217, 217, 217)',
#showocean = True,
#oceancolor = 'rgb(204,229,255)',
showcountries = True
)
)
'''
# 6. Graph: Colored Map (Chroropleth Map) with Cases / Population
# Create dataframe with population, cases and death columns
# Match Country Names with Pycountry library
no_pycountry = []
matched_country_dict = defaultdict(str)
manual_country_dict = {
'Brunei': 'BRN',
'Congo (Brazzaville)': 'COG',
'Congo (Kinshasa)': 'COD',
"Cote d'Ivoire": 'CIV',
'Holy See': 'VAT',
'Iran': 'IRN',
'Korea, South': 'KOR',
'Kosovo': 'UNK',
'Reunion': 'REU',
'Russia': 'RUS',
'Taiwan*': 'TWN',
'The Bahamas': 'BHS',
'occupied Palestinian territory': 'PSE'
}
for country in df_confirmed['Country/Region'].drop_duplicates(
).sort_values():
try:
matched_country_dict[country] = countries.lookup(country).alpha_3
except:
try:
matched_country_dict[country] = manual_country_dict[country]
except:
#print(country, ' not found')
no_pycountry.append(country)
# Create Dataframe with cases, deaths and population
df_cases_deaths = df_country_cases.copy()
df_cases_deaths = df_cases_deaths.merge(df_country_deaths,
on='Country/Region')
df_cases_deaths['Country Code'] = df_cases_deaths['Country/Region'].apply(
lambda x: matched_country_dict[x])
df_cases_deaths = df_cases_deaths.merge(df_pop_2018, on='Country Code')
df_cases_deaths.columns = [
'Country/Region', 'Cases', 'Deaths', 'Country Code', 'Population'
]
# Calculate ratios
df_cases_deaths['Mortality Rate'] = df_cases_deaths[
'Deaths'] / df_cases_deaths['Cases'] * 100 # percentage
df_cases_deaths['Cases / Population'] = df_cases_deaths[
'Cases'] / df_cases_deaths['Population'] * 100 # percentage
df_cases_deaths['Infection Rate'] = df_cases_deaths[
'Cases'] / df_cases_deaths['Population'] * 10000
# Drop unneeded columns & save to pickle (pickle will be used to show HTML table)
df_cases_deaths.drop(['Country Code', 'Population', 'Infection Rate'],
axis=1).to_pickle('covidapp/data/df_cases_deaths.pkl')
# Create Choropleth Map
graph_six = []
text = []
for country in df_cases_deaths['Country/Region']:
text.append(
str(
str(country) + '<br> Cases: ' +
'{:,}'.format(df_cases_deaths['Cases'][
df_cases_deaths['Country/Region'] == country].tolist()
[0]).replace(',', '.') + '<br> Deaths: ' +
'{:,}'.format(df_cases_deaths['Deaths'][
df_cases_deaths['Country/Region'] == country].tolist()
[0]).replace(',', '.') +
'<br> Mortality Rate: ' +
'{:,.2f}'.format(df_cases_deaths['Mortality Rate'][
df_cases_deaths['Country/Region'] == country].tolist()
[0]).replace(',', '.') + ' %' +
'<br> Cases / Population: ' +
'{:,.3f}'.format(df_cases_deaths['Cases / Population'][
df_cases_deaths['Country/Region'] == country].tolist()
[0]).replace(',', '.') + ' %'))
graph_six.append(
go.Choropleth(
locations=df_cases_deaths['Country Code'],
z=df_cases_deaths['Infection Rate'],
#colorscale = 'Reds',
colorscale=[
'#f9ebea',
#'#f2d7d5',
#'#e6b0aa',
'#cd6155',
'#c0392b',
'#a93226',
'#922b21',
'#7b241c',
'#641e16'
],
autocolorscale=False,
showscale=False,
#colorbar_title = 'Cases / Population'
text=text,
hoverinfo='text',
hoverlabel=dict(bgcolor=' #ebedef')))
layout_six = dict(
title='Most Affected Countries (Cases / Population)',
showlegend=False,
showcolorbar=False,
geo=dict(
projection=dict(type='natural earth'),
showframe=True,
#showcoastline = False,
showcountries=True))
# Append all charts to the figures list
figures = []
figures.append(dict(data=graph_one, layout=layout_one))
figures.append(dict(data=graph_two, layout=layout_two))
figures.append(dict(data=graph_three, layout=layout_three))
figures.append(dict(data=graph_four, layout=layout_four))
#figures.append(dict(data=graph_five, layout=layout_five))
figures.append(dict(data=graph_six, layout=layout_six))
return figures
"Cote d'Ivoire" : 'CIV',
'Holy See' : 'VAT',
'Iran' : 'IRN',
'Korea, South' : 'KOR',
'Kosovo' : 'UNK',
'Reunion' : 'REU',
'Russia' : 'RUS',
'Taiwan*' : 'TWN',
'The Bahamas' : 'BHS',
'occupied Palestinian territory' : 'PSE'
}
for country in df_confirmed['Country/Region'].drop_duplicates().sort_values():
try:
matched_country_dict[country] = countries.lookup(country).alpha_3
except:
try:
matched_country_dict[country] = manual_country_dict[country]
except:
print(country, ' not found')
no_pycountry.append(country)
'''
## Load the World Development Indicators by the World Bank and create data pickle for population data
# Comment: I created the pickle file only once to reduce server load
df_wdi = pd.read_csv('../covidapp/data/WDIData.csv')
# Extract the population data for 2018 (2019 has to many NaN values)
def is_valid_country(c: str) -> bool:
try:
countries.lookup(c)
return True
except LookupError:
return False
class Misc(commands.Cog):
def __init__(self, bot, config):
self.bot = bot
self.config = config
@cog_ext.cog_slash(
description="Translate the provided Japanese text into English via Google Translate.",
guild_ids=hkd.get_all_guild_ids(),
options=[
create_option(
name="text",
description="The text to translate.",
option_type=3,
required=True
)
]
)
async def translate(self, ctx: SlashContext, text: str):
await ctx.defer()
await ctx.send(embed=hkd.create_embed(description=Translator().translate(text, src='ja', dest='en').text))
@cog_ext.cog_slash(
description="Convert currency from one type to another.",
guild_ids=hkd.get_all_guild_ids(),
options=[
create_option(
name="conversion",
description="Convert amount of currency-a to currency-b.",
option_type=3,
required=True
)
]
)
async def currency(self, ctx: SlashContext, conversion: str):
await ctx.defer()
conversion_split = conversion.split()
if len(conversion_split) == 4 and conversion_split[2].lower() == 'to':
with suppress(Exception):
result = CurrencyRates().convert(conversion_split[1].upper(), conversion_split[3].upper(), Decimal(conversion_split[0]))
await ctx.send(embed=hkd.create_embed(title='{0} {1}'.format('{:f}'.format(result).rstrip('0').rstrip('.'), conversion_split[3].upper())))
return
await ctx.send(embed=hkd.create_embed(description="Couldn't convert. Please follow this format for converting currency: **/currency** 12.34 AUD to USD.", colour=Colour.red()))
@cog_ext.cog_slash(
description="Show weather information for the specified location.",
guild_ids=hkd.get_all_guild_ids(),
options=[
create_option(
name="location",
description="The location to show the weather for.",
option_type=3,
required=True
)
]
)
async def weather(self, ctx: SlashContext, location: str):
await ctx.defer()
if len(query := location.split(',')) > 1:
with suppress(Exception):
query[1] = countries.get(name=query[1].strip().title()).alpha_2
with suppress(Exception):
result = requests.get('http://api.openweathermap.org/data/2.5/weather', params={'q': ','.join(query), 'APPID': self.config['weather_api_key']}).json()
timezone = pytz.timezone(TimezoneFinder().timezone_at(lat=result['coord']['lat'], lng=result['coord']['lon']))
embed_fields = []
embed_fields.append(('Weather', '{0}'.format(result['weather'][0]['description'].title())))
embed_fields.append(('Temperature', '{0} °C, {1} °F'.format('{0:.2f}'.format(float(result['main']['temp']) - 273.15), '{0:.2f}'.format((1.8 * (float(result['main']['temp']) - 273.15)) + 32.0))))
embed_fields.append(('Humidity', '{0}%'.format(result['main']['humidity'])))
embed_fields.append(('Wind Speed', '{0} m/s'.format(result['wind']['speed'])))
embed_fields.append(('Sunrise', '{0:%I}:{0:%M} {0:%p}'.format(datetime.fromtimestamp(result['sys']['sunrise'], tz=timezone))))
embed_fields.append(('Sunset', '{0:%I}:{0:%M} {0:%p}'.format(datetime.fromtimestamp(result['sys']['sunset'], tz=timezone))))
embed_fields.append(('Pressure', '{0} hPa'.format(result['main']['pressure'])))
await ctx.send(content='**Weather for {0}, {1}**'.format(result['name'], countries.lookup(result['sys']['country']).name), embed=hkd.create_embed(fields=embed_fields, inline=True))
return
await ctx.send(embed=hkd.create_embed(description="Couldn't get weather. Please follow this format for checking the weather: **/weather** Melbourne, Australia.", colour=Colour.red()))
# Run this code cell to install and import the pycountry library
#!pip install pycountry
from pycountry import countries
import pandas as pd
# Run this code cell to see an example of how the library works
countries.get(name='Spain')
# Run this code cell to see how you can also look up countries without specifying the key
countries.lookup('Kingdom of Spain')
#encoding
from encodings.aliases import aliases
alias_values = set(aliases.values())
# This code finds the encodings that works for the file
for encoding in set(aliases.values()):
try:
df = pd.read_csv("mystery.csv", encoding=encoding)
print('successful', encoding)
except:
pass
# Fill null
# Fill with mean of a group
df_melt = pd.read_csv('gdp_data.csv')
df_melt['GDP_filled'] = df_melt.groupby('Country Name')['GDP'].transform(
lambda x: x.fillna(x.mean()))
def parse_file(self, file):
"""
Parse a single address book file
"""
prin = pprint.PrettyPrinter()
self.stats = {}
for vob in vobject.readComponents(open(file)):
content = vob.contents
if self.args.verbose:
logging.debug("parsing %s:", file)
prin.pprint(content)
if "n" not in content:
self.stats["noname"] = self.stats.get("noname", 0) + 1
return
if "email" not in content or not re.match(
r"^[^@]+@[^@]+\.[^@]+$", content["email"][0].value):
self.stats["noemail"] = self.stats.get("noemail", 0) + 1
return
self.stats["valid"] = self.stats.get("valid", 0) + 1
# aggregate stats what kind of fields we have available
for i in content:
# if i in c:
self.stats[i] = self.stats.get(i, 0) + 1
emails = []
for email in content.get("email", []):
if re.match(r"^[^@äöü]+@[^@]+\.[^@]+$", email.value):
emails.append(email.value)
profile_list = []
for email in emails:
profile = self.hatchbuck.search_email(email)
if profile:
profile_list.append(profile)
else:
continue
# No contacts found
if not profile_list:
# create new contact
profile = dict()
profile["firstName"] = content["n"][0].value.given
profile["lastName"] = content["n"][0].value.family
if "title" in content:
profile["title"] = content["title"][0].value
if "org" in content:
profile["company"] = content["org"][0].value
profile["subscribed"] = True
profile["status"] = {"name": "Lead"}
if self.args.source:
profile["source"] = {"id": self.args.source}
# override hatchbuck sales rep username if set
# (default: api key owner)
if self.args.user:
profile["salesRep"] = {"username": self.args.user}
profile["emails"] = []
for email in content.get("email", []):
if not re.match(r"^[^@äöü]+@[^@]+\.[^@]+$", email.value):
continue
if "WORK" in email.type_paramlist:
kind = "Work"
elif "HOME" in email.type_paramlist:
kind = "Home"
else:
kind = "Other"
profile["emails"].append({
"address": email.value,
"type": kind
})
profile = self.hatchbuck.create(profile)
logging.info("added contact: %s", profile)
for profile in profile_list:
if profile["firstName"] == "" or "@" in profile["firstName"]:
profile = self.hatchbuck.profile_add(
profile, "firstName", None,
content["n"][0].value.given)
if profile["lastName"] == "" or "@" in profile["lastName"]:
profile = self.hatchbuck.profile_add(
profile, "lastName", None,
content["n"][0].value.family)
if "title" in content and profile.get("title", "") == "":
profile = self.hatchbuck.profile_add(
profile, "title", None, content["title"][0].value)
if "company" in profile:
if "org" in content and profile.get("company", "") == "":
profile = self.hatchbuck.profile_add(
profile, "company", None, content["org"][0].value)
if profile["company"] == "":
# empty company name ->
# maybe we can guess the company name from the email
# address?
# logging.warning("empty company with emails: %s",
# profile['emails'])
pass
# clean up company name
if re.match(r";$", profile["company"]):
logging.warning("found unclean company name: %s",
profile["company"])
if re.match(r"\|", profile["company"]):
logging.warning("found unclean company name: %s",
profile["company"])
for addr in content.get("adr", []):
address = {
"street": addr.value.street,
"zip_code": addr.value.code,
"city": addr.value.city,
"country": addr.value.country,
}
try:
if "WORK" in addr.type_paramlist:
kind = "Work"
elif "HOME" in addr.type_paramlist:
kind = "Home"
else:
kind = "Other"
except AttributeError:
# if there is no type at all
kind = "Other"
logging.debug("adding address %s %s", address, profile)
profile = self.hatchbuck.profile_add_address(
profile, address, kind)
for telefon in content.get("tel", []):
# number cleanup
number = telefon.value
for rep in "()-\xa0":
# clean up number
number = number.replace(rep, "")
number = number.replace("+00", "+").replace("+0", "+")
try:
if "WORK" in telefon.type_paramlist:
kind = "Work"
elif "HOME" in telefon.type_paramlist:
kind = "Home"
else:
kind = "Other"
except AttributeError:
# if there is no type at all
kind = "Other"
redundant = False
try:
phonenumber = phonenumbers.parse(number, None)
pformatted = phonenumbers.format_number(
phonenumber,
phonenumbers.PhoneNumberFormat.INTERNATIONAL)
except phonenumbers.phonenumberutil.NumberParseException:
# number could not be parsed, e.g. because it is a
# local number without country code
logging.warning(
"could not parse number %s as %s in %s, "
"trying to guess country from address",
telefon.value,
number,
self.hatchbuck.short_contact(profile),
)
pformatted = number
# try to guess the country from the addresses
countries_found = []
for addr in profile.get("addresses", []):
if (addr.get("country", False) and addr["country"]
not in countries_found):
countries_found.append(addr["country"])
logging.debug("countries found %s", countries_found)
if len(countries_found) == 1:
# lets try to parse the number with the country
countrycode = countries.lookup(
countries_found[0]).alpha_2
logging.debug("countrycode %s", countrycode)
try:
phonenumber = phonenumbers.parse(
number, countrycode)
pformatted = phonenumbers.format_number(
phonenumber,
phonenumbers.PhoneNumberFormat.
INTERNATIONAL,
)
logging.debug("guess %s", pformatted)
profile = self.hatchbuck.profile_add(
profile,
"phones",
"number",
pformatted,
{"type": kind},
)
# if we got here we now have a full number
continue
except phonenumbers.phonenumberutil.NumberParseException:
logging.warning(
"could not parse number %s as %s using country %s in %s",
telefon.value,
number,
countrycode,
self.hatchbuck.short_contact(profile),
)
pformatted = number
# check that there is not an international/longer
# number there already
# e.g. +41 76 4000 464 compared to 0764000464
# skip the 0 in front
num = number.replace(" ", "")[1:]
for tel2 in profile["phones"]:
# check for suffix match
if tel2["number"].replace(" ", "").endswith(num):
logging.warning(
"not adding number %s from %s because it "
"is a suffix of existing %s",
num,
self.hatchbuck.short_contact(profile),
tel2["number"],
)
redundant = True
break
if not redundant:
profile = self.hatchbuck.profile_add(
profile, "phones", "number", pformatted,
{"type": kind})
# clean & deduplicate all phone numbers
profile = self.hatchbuck.clean_all_phone_numbers(profile)
for skype in content.get("x-skype", []):
profile = self.hatchbuck.profile_add(
profile,
"instantMessaging",
"address",
skype.value,
{"type": "Skype"},
)
for msn in content.get("x-msn", []):
profile = self.hatchbuck.profile_add(
profile,
"instantMessaging",
"address",
msn.value,
{"type": "Messenger"},
)
for msn in content.get("x-msnim", []):
profile = self.hatchbuck.profile_add(
profile,
"instantMessaging",
"address",
msn.value,
{"type": "Messenger"},
)
for twitter in content.get("x-twitter", []):
if "twitter.com" in twitter.value:
value = twitter.value
else:
value = "http://twitter.com/" + twitter.value.replace(
"@", "")
profile = self.hatchbuck.profile_add(
profile, "socialNetworks", "address", value,
{"type": "Twitter"})
for url in content.get("url", []) + content.get(
"x-socialprofile", []):
value = url.value
if not value.startswith("http"):
value = "http://" + value
if "facebook.com" in value:
profile = self.hatchbuck.profile_add(
profile,
"socialNetworks",
"address",
value,
{"type": "Facebook"},
)
elif "twitter.com" in value:
profile = self.hatchbuck.profile_add(
profile,
"socialNetworks",
"address",
value,
{"type": "Twitter"},
)
else:
profile = self.hatchbuck.profile_add(
profile, "website", "websiteUrl", value)
for bday in content.get("bday", []):
date = {
"year": bday.value[0:4],
"month": bday.value[5:7],
"day": bday.value[8:10],
}
profile = self.hatchbuck.profile_add_birthday(
profile, date)
if self.args.tag:
if not self.hatchbuck.profile_contains(
profile, "tags", "name", self.args.tag):
self.hatchbuck.add_tag(profile["contactId"],
self.args.tag)
# get the list of unique contacts IDs to detect if there are
# multiple contacts in hatchbuck for this one contact in CardDAV
profile_contactids = []
message = ""
for profile in profile_list:
if profile["contactId"] not in profile_contactids:
profile_contactids.append(profile["contactId"])
email_profile = " "
for email_add in profile.get("emails", []):
email_profile = email_add["address"] + " "
number_profile = " "
for phone_number in profile.get("phones", []):
number_profile = phone_number["number"] + " "
message += ("{0} {1} ({2}, {3}, {4})".format(
profile["firstName"],
profile["lastName"],
email_profile,
number_profile,
profile["contactUrl"],
) + ", ")
if len(profile_contactids) > 1:
# there are duplicates
NotificationService().send_message(
"Duplicates: %s from file: %s" % (message[:-2], file))
#!/usr/bin/env python3
import csv
from pycountry import countries
from urllib.request import urlopen
submitter_info = {}
for row in csv.reader(open('submitter_info.tsv'), delimiter='\t'):
submitter_id = row[0]
submitter_name = row[1]
country_code = row[2]
submitter_info[submitter_id] = [submitter_name, country_code]
with open('organization_summary.txt') as f:
next(f) #skip header
for row in csv.reader(f, delimiter='\t'):
submitter_name = row[0]
submitter_id = row[1]
try:
country_code = countries.lookup(row[4]).alpha_3
except LookupError: #not a real country
country_code = ''
if not country_code and submitter_id in submitter_info:
country_code = submitter_info[submitter_id][1]
submitter_info[submitter_id] = [submitter_name, country_code]
with open('submitter_info.tsv', 'w') as f:
writer = csv.writer(f, delimiter='\t', lineterminator='\n')
for submitter_id in sorted(submitter_info.keys(), key=int):
writer.writerow([submitter_id] + submitter_info[submitter_id])
def get_country_name_from_code(self, code):
country = pyc.lookup(code)
return country.name
def _clean_address(self, address): # pylint: disable=too-many-branches
"""
clean up an address
:param address: address dict
:return: cleaned up address
"""
if address.get("country", ""):
# check if the country is a valid country name
# this also fixes two letter country codes
try:
country = countries.search_fuzzy(
self._clean_country_name(address["country"]))[0].name
except LookupError:
country = False
if country and country != address["country"]:
address["country"] = country
if re.match(r"^[a-zA-Z]{2}-[0-9]{4,6}$", address.get("zip", "")):
# zipcode contains country code (e.g. CH-8005)
countrycode, zipcode = address["zip"].split("-")
try:
address["country"] = countries.lookup(countrycode).name
address["zip"] = zipcode
except LookupError:
pass
if (not address.get("street", "") and not address.get("zip", "")
and address.get("city", "")):
# there is a city but no street/zip
# logging.warning(address["city"].split(","))
if len(address["city"].split(",")) == 3:
# Copenhagen Area, Capital Region, Denmark
# Lausanne, Canton de Vaud, Suisse
# Vienna, Vienna, Austria
try:
address["country"] = countries.search_fuzzy(
self._clean_country_name(
address["city"].split(",")[2]))[0].name
address["city"] = self._clean_city_name(
address["city"].split(",")[0])
except LookupError:
pass
elif len(address["city"].split(",")) == 2:
# Zürich und Umgebung, Schweiz
# Currais Novos e Região, Brasil
# München und Umgebung, Deutschland
# Région de Genève, Suisse
# Zürich Area, Svizzera
try:
address["country"] = countries.search_fuzzy(
self._clean_country_name(
address["city"].split(",")[1]))[0].name
address["city"] = self._clean_city_name(
address["city"].split(",")[0])
except LookupError:
pass
elif len(address["city"].split(",")) == 1:
# United States
# Switzerland
# Luxembourg
try:
address["country"] = countries.search_fuzzy(
self._clean_country_name(
self._clean_city_name(address["city"])))[0].name
# if the lookup is successful this is a country name, not city name
address["city"] = ""
except LookupError:
pass
if address.get("type", "") not in ["Work", "Home", "Other"]:
address["type"] = "Other"
# ensure all relevant fields are set
address["street"] = self._clean_street_name(address.get("street", ""))
address["city"] = self._clean_city_name(address.get("city", ""))
address["zip"] = address.get("zip", "").strip()
address["state"] = address.get("state", "").strip()
address["country"] = self._clean_country_name(
address.get("country", ""))
return address
def lambda_handler(event, context):
logger.info(f'event = {event}')
# get the region from queryStringParameters, default set as Taiwan
if "queryStringParameters" in event.keys(
) & "region" in event["queryStringParameters"].keys():
region = countries.lookup(event["queryStringParameters"]['region'])
else:
region = countries.lookup("Taiwan")
if datetime.today().year == 2021:
data = pd.read_csv(
'./110中華民國政府行政機關辦公日曆表.csv',
# os.path.abspath('./lambda-handler/110中華民國政府行政機關辦公日曆表.csv'),
index_col='西元日期',
encoding='big5')
elif datetime.today().year == 2022:
data = pd.read_csv(
'./111年中華民國政府行政機關辦公日曆表.csv',
# os.path.abspath('./lambda-handler/111年中華民國政府行政機關辦公日曆表.csv'),
index_col='西元日期',
encoding='big5')
# print(data.columns)
# print(data.index)
tz = pytz.timezone('Asia/Taipei')
today = datetime.now(tz)
# print(today)
# tomorrow = today + timedelta(days = 1)
# print(tomorrow)
day = data.loc[int(today.strftime("%Y%m%d"))]
# day = data.loc[int(tomorrow.strftime("%Y%m%d"))]
if day['是否放假'] == 0:
data = {
# 'result': "明天要上班哦～",
'result': "今天要上班哦～",
'region': region.common_name,
'workingDay': True,
'datetime': today.isoformat()
}
elif day['是否放假'] == 2:
data = {
# 'result': "明天放假囉！",
'result': "今天放假囉!",
'region': region.common_name,
'workingDay': False,
'datetime': today.isoformat()
}
else:
data = {'result': "出現了一些錯誤兒...", 'datetime': today.isoformat()}
logger.info(f'data = {data}')
result = {
'statusCode': 200,
'body': json.dumps(data),
# deepcode ignore TooPermissiveCors: the api allow global access
'headers': {
'Content-Type': 'application/json',
'Access-Control-Allow-Origin': '*'
}
}
# result = json.dumps(data)
return result
