def clean(self): try: country = countries.lookup(self.name) except Exception: raise ValidationError('{} is not a valid country'.format(self.name)) else: self.name = country.name
def get(cls, country): try: c = countries.lookup(country) return Country(c.alpha_2, c.alpha_3, c.numeric, c.name, getattr(c, "official_name", c.name)) except (LookupError, KeyError): raise LookupError(f"Invalid country code: {country}")
def to_internal_value(self, data): country_name = data.get("country") if not country_name: raise serializers.ValidationError( {"country": [self.error_messages["required"]]}) try: query_data = {"id": int(country_name)} except ValueError: country = countries.lookup(country_name) query_data = {"name": country.name} finally: try: country = models.Country.objects.get(**query_data) except Exception: raise serializers.ValidationError({ "country": [ f'Invalid country "{country_name}" - object does not exist.' ] }) data_ = data.copy() data_["country"] = country.name if not data_.get("name"): data_["name"] = data_.get("centre_name") internal_value = super().to_internal_value(data_) return internal_value
def query_timed(firstYear,lastYear, pattern= ''): year = firstYear yearList = [] firstAdded = False countryList = [] timeList = [] query_response = otx.getall() #query_response = query_response['results'] while year <= lastYear: yearList.append(year) year = year + 1 for pulse in query_response: countriesPulse = [] for tag in pulse["tags"]: tryCountry = False if len(tag) >= 3: try: checkCountry = countries.lookup(tag) tryCountry = True except: pass if tryCountry: if tag not in pulse["targeted_countries"]: countriesPulse.append(countries.lookup(tag).name) if pulse["targeted_countries"]: for target in pulse["targeted_countries"]: countriesPulse.append(countries.lookup(target).name) if countriesPulse: for ioc in pulse["indicators"]: # if fnmatch.filter(ioc["tags], pattern): date = ioc["created"] date = dateutil.parser.isoparse(date) if int(date.year) in yearList: countryList.append(countriesPulse) timeList.append(ioc["created"]) targetedCountries = dict() for group in countryList: for target in group: if target in targetedCountries: targetedCountries[target] += 1 else: targetedCountries[target] = 1
def test_country_create(self, mock_verify_token): mock_verify_token.return_value = {'email': self.admin_user.email} data = {"name": countries.lookup('Rwanda').name} response = client.post( self.country_url, data=data, HTTP_AUTHORIZATION="Token {}".format(self.admin_user), ) self.assertIn("name", response.data.keys()) self.assertEqual(response.status_code, 201)
async def bind_country(self, ctx, *, country="reset"): # Check the country name if country != "reset": country = countries.lookup(country) if country: country = country.name else: raise commands.BadArgument await rest_api.bind_user_param(self.bot, ctx.author.id, country=country)
async def bind_country(self, ctx, *, country="reset"): # Check the country name if country != "reset": country = countries.lookup(country) if country: country = country.name else: raise commands.BadArgument await util_users.change_user_info(self.bot, ctx.author.id, country=country)
def get(cls, country): try: if PYCOUNTRY: c = countries.lookup(country) return Country(c.alpha_2, c.alpha_3, c.numeric, c.name, getattr(c, "official_name", c.name)) else: c = countries.get(country) return Country(c.alpha2, c.alpha3, c.numeric, c.name, c.apolitical_name) except (LookupError, KeyError): raise LookupError("Invalid country code: {0}".format(country))
def lookup_country_code(country): country = country.strip() if not country: return '' # Try the pycountry database first try: return countries.lookup(country).alpha_3 except LookupError: pass # If that fails, try common names that are not in the pycountry database country = country.lower() if country in additional_common_names: return additional_common_names[country] return ''
def scrape_info(submitter_id): submitter_name = '' country_code = '' try: with urlopen('https://www.ncbi.nlm.nih.gov/clinvar/submitters/' + str(submitter_id) + '/') as f: root = html5lib.parse( f, transport_encoding=f.info().get_content_charset()) submitter_el = root.find( './/html:div[@id="maincontent"]//html:div[@class="submitter_main indented"]', ns) name_el = submitter_el.find('./html:h2', ns) contact_info_el = submitter_el.find( './/html:div[@class="indented"]/html:div[@class="indented"]', ns) if name_el != None and name_el.text: #submitter 1 has no name submitter_name = re.sub(r'\s+', ' ', name_el.text.strip()) if contact_info_el != None: #submitter 1 has no contact information contact_info = list(contact_info_el.itertext())[1:] contact_info = list( filter( lambda info: not re.match( 'http://|https://|Organization ID:', info), contact_info)) if len(contact_info) >= 1: country_and_zip = re.match('(.+) - (.+)', contact_info[-1]) country_name = country_and_zip.group( 1) if country_and_zip else contact_info[-1] try: country_code = countries.lookup(country_name).alpha_3 except LookupError: #not a real country pass except HTTPError as err: if err.code == 404: print('\r\033[KNo information for submitter ' + str(submitter_id)) else: raise err return [submitter_id, submitter_name, country_code]
def to_internal_value(self, data): country_name = data.get('country') if not country_name: raise serializers.ValidationError( {'country': [self.error_messages['required']]}) try: query_data = {'id': int(country_name)} except ValueError: country = countries.lookup(country_name) query_data = {'name': country.name} finally: try: country = models.Country.objects.get(**query_data) except Exception: raise serializers.ValidationError({ 'country': [ f'Invalid country \"{country_name}\" - object does not exist.' ] }) data_ = data.copy() data_['country'] = country.name internal_value = super().to_internal_value(data_) return internal_value
def return_figures(): ''' Fuction to create plotly visualizations Input: None Output: list (dict): list containing the plotly visualizations ''' # Load Johns Hopkins University Data # new URLs and data structure url_confirmed = 'https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_confirmed_global.csv' url_death = 'https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_deaths_global.csv' #url_confirmed = 'https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Confirmed.csv' #url_death = 'https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Deaths.csv' #url_recovered = 'https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Recovered.csv' df_confirmed = pd.read_csv(url_confirmed, error_bad_lines=False) df_death = pd.read_csv(url_death, error_bad_lines=False) #df_recovered = pd.read_csv(url_recovered, error_bad_lines=False) # Load population data from the World Bank df_pop_2018 = pd.read_pickle('covidapp/data/df_pop_2018.pkl') df_pop_2018 = df_pop_2018.drop(['Country Name'], axis=1) #df_confirmed.to_pickle('data/covid-19_confirmed.pkl') #df_death.to_pickle('data/covid-19_death.pkl') #df_recovered.to_pickle('data/covid-19_recovered.pkl') last_col_confirmed = df_confirmed.columns[-1] cases = defaultdict(int) for i in range(len(df_confirmed)): cases[df_confirmed['Country/Region'] [i]] += df_confirmed[last_col_confirmed][i] last_col_death = df_death.columns[-1] deaths = defaultdict(int) for i in range(len(df_death)): deaths[df_death['Country/Region'][i]] += df_death[last_col_death][i] ''' # Gruppiertes Balkendiagramm graph_one = [] x = [] y = [] for key, value in sorted(cases.items(), key=lambda item: item[1], reverse=True): x.append(key) y.append(value) z = [] for key, value in sorted(deaths.items(), key=lambda item: item[1], reverse=True): z.append(value) x = x[0:15] y = y[0:15] z = z[0:15] graph_one.append( go.Bar(x = x, y = y, name = 'cases',) ) graph_one.append( go.Bar(x = x, y = z, name = 'deaths', marker_color = 'grey',) ) layout_one = dict(title = 'Confirmed Cases | Top 15 Countries', xaxis = dict(title = 'Country',), yaxis = dict(title = 'Cases'), barmode = 'group', ) ''' # 1. Graph: Bar chart of confirmed cases per top country graph_one = [] x = [] y = [] for key, value in sorted(cases.items(), key=lambda item: item[1], reverse=True): x.append(key) y.append(value) # Create dataframe for later df_country_cases = {'Country/Region': x, 'Cases': y} df_country_cases = pd.DataFrame(df_country_cases) x = x[0:15] y = y[0:15] graph_one.append(go.Bar( x=x, y=y, )) layout_one = dict( title='Confirmed Cases | Top 15 Countries', #xaxis = dict(title = 'Country',), yaxis=dict(title='Cases'), separators=',.', ) # 2. Graph: Bar chart of deaths per top country graph_two = [] x = [] y = [] for key, value in sorted(deaths.items(), key=lambda item: item[1], reverse=True): x.append(key) y.append(value) # Create dataframe for later df_country_deaths = {'Country/Region': x, 'Deaths': y} df_country_deaths = pd.DataFrame(df_country_deaths) x = x[0:15] y = y[0:15] graph_two.append(go.Bar( x=x, y=y, marker_color='grey', )) layout_two = dict( title='Fatal Cases | Top 15 Countries', #xaxis = dict(title = 'Country',), yaxis=dict(title='Deaths'), separators=',.', ) # 3. Graph: Line chart of new infections per day for top countries graph_three = [] x = [] y = [] # Create DataFrame with new cases arr_confirmed = df_confirmed.iloc[:, 4:].to_numpy() arr_confirmed_sub = np.append(arr=np.zeros( (len(arr_confirmed), 1)).astype(int), values=arr_confirmed, axis=1) arr_confirmed_sub = arr_confirmed_sub[:, :-1] arr_new = arr_confirmed - arr_confirmed_sub df_new_values = pd.DataFrame(arr_new) df_new_values.reset_index(inplace=True) df_new = df_confirmed.iloc[:, :4] df_new.reset_index(inplace=True) df_new = df_new.merge(df_new_values, on='index') df_new = df_new.iloc[:, 1:] df_new.columns = df_confirmed.columns.tolist() # Create dictionary with new cases of the last 7 days countrylist = [] new_last7 = defaultdict(int) for i in range(len(df_new)): new_last7[df_new['Country/Region'][i]] += df_new.iloc[:, -7:].sum( axis=1)[i] # Create countrylist with top 15 countries countrylist = [] for key, value in sorted(new_last7.items(), key=lambda item: item[1], reverse=True): countrylist.append(key) countrylist = countrylist[:12] datelist = df_new.columns[-28:].tolist() # last 28 days # Create graph for country in countrylist: x = datelist y = [] for date in datelist: y.append(df_new[date][df_new['Country/Region'] == country].sum()) graph_three.append(go.Scatter(x=x, y=y, mode='lines', name=country)) layout_three = dict( title= 'Infections per Day <br>Top 12 Countries by most infections in last 7 days', xaxis=dict( #title = 'Date', autotick=True), yaxis=dict(title='Cases'), separators=',.', ) # 4. Graph: Scatter confirmed cases vs. deaths graph_four = [] # Create countrylist with top countries by total cases countrylist = [] count = 0 for key, value in sorted(cases.items(), key=lambda item: item[1], reverse=True): if count < 12: countrylist.append(key) count += 1 else: break for country in countrylist: x = [cases[country]] y = [deaths[country]] #text = str(country) + ' ' + str(x) + '/' + str(y) graph_four.append( go.Scatter( x=x, y=y, mode='markers', #text = text, name=country, textposition='top center')) layout_four = dict( title='Confirmed Cases & Deaths <br>Top 12 Countries by total cases', xaxis=dict(title='Cases'), yaxis=dict(title='Deaths'), separators=',.', ) # 5. Graph: Bubble map with confirmed cases per country ''' graph_five = [] #cases_deaths = [] for latitude in df_confirmed['Lat']: df_lat = df_confirmed[df_confirmed['Lat'] == latitude] df_death_lat = df_death[df_death['Lat'] == latitude] for longitude in df_lat['Long']: cases = df_lat[last_col_confirmed][df_lat['Long'] == longitude].sum() deaths = df_death[last_col_death][df_death['Long'] == longitude].sum() country_name = str(df_lat['Country/Region'][df_lat['Long'] == longitude].unique().tolist()[0]) #cases_deaths.append([country, cases, deaths]) if cases > 0: if str(df_lat['Province/State'][df_lat['Long'] == longitude].unique().tolist()[0]) == 'nan': bubble_text = ( country_name + '<br>Cases: ' + str(cases) + '<br>Deaths: ' + str(deaths) ) else: bubble_text = ( country_name + ', ' + str(df_lat['Province/State'][df_lat['Long'] == longitude].unique().tolist()[0]) + '<br>Cases: ' + str(cases) + '<br>Deaths: ' + str(deaths) ) graph_five.append( go.Scattergeo( locationmode = 'country names', #USA-states', lat = [latitude], lon = [longitude], text = bubble_text, marker = dict( size = cases/1000, #color = , line_color = 'rgb(40, 40, 40)', line_width = 0.5, sizemode = 'area' ) #, name = ) ) else: break layout_five = dict(title = 'Confirmed Cases & Deaths | Bubble Map (all cases)', showlegend = False, geo = dict( projection = dict(type = 'natural earth'), showland = True, landcolor = 'rgb(217, 217, 217)', #showocean = True, #oceancolor = 'rgb(204,229,255)', showcountries = True ) ) ''' # 6. Graph: Colored Map (Chroropleth Map) with Cases / Population # Create dataframe with population, cases and death columns # Match Country Names with Pycountry library no_pycountry = [] matched_country_dict = defaultdict(str) manual_country_dict = { 'Brunei': 'BRN', 'Congo (Brazzaville)': 'COG', 'Congo (Kinshasa)': 'COD', "Cote d'Ivoire": 'CIV', 'Holy See': 'VAT', 'Iran': 'IRN', 'Korea, South': 'KOR', 'Kosovo': 'UNK', 'Reunion': 'REU', 'Russia': 'RUS', 'Taiwan*': 'TWN', 'The Bahamas': 'BHS', 'occupied Palestinian territory': 'PSE' } for country in df_confirmed['Country/Region'].drop_duplicates( ).sort_values(): try: matched_country_dict[country] = countries.lookup(country).alpha_3 except: try: matched_country_dict[country] = manual_country_dict[country] except: #print(country, ' not found') no_pycountry.append(country) # Create Dataframe with cases, deaths and population df_cases_deaths = df_country_cases.copy() df_cases_deaths = df_cases_deaths.merge(df_country_deaths, on='Country/Region') df_cases_deaths['Country Code'] = df_cases_deaths['Country/Region'].apply( lambda x: matched_country_dict[x]) df_cases_deaths = df_cases_deaths.merge(df_pop_2018, on='Country Code') df_cases_deaths.columns = [ 'Country/Region', 'Cases', 'Deaths', 'Country Code', 'Population' ] # Calculate ratios df_cases_deaths['Mortality Rate'] = df_cases_deaths[ 'Deaths'] / df_cases_deaths['Cases'] * 100 # percentage df_cases_deaths['Cases / Population'] = df_cases_deaths[ 'Cases'] / df_cases_deaths['Population'] * 100 # percentage df_cases_deaths['Infection Rate'] = df_cases_deaths[ 'Cases'] / df_cases_deaths['Population'] * 10000 # Drop unneeded columns & save to pickle (pickle will be used to show HTML table) df_cases_deaths.drop(['Country Code', 'Population', 'Infection Rate'], axis=1).to_pickle('covidapp/data/df_cases_deaths.pkl') # Create Choropleth Map graph_six = [] text = [] for country in df_cases_deaths['Country/Region']: text.append( str( str(country) + '<br> Cases: ' + '{:,}'.format(df_cases_deaths['Cases'][ df_cases_deaths['Country/Region'] == country].tolist() [0]).replace(',', '.') + '<br> Deaths: ' + '{:,}'.format(df_cases_deaths['Deaths'][ df_cases_deaths['Country/Region'] == country].tolist() [0]).replace(',', '.') + '<br> Mortality Rate: ' + '{:,.2f}'.format(df_cases_deaths['Mortality Rate'][ df_cases_deaths['Country/Region'] == country].tolist() [0]).replace(',', '.') + ' %' + '<br> Cases / Population: ' + '{:,.3f}'.format(df_cases_deaths['Cases / Population'][ df_cases_deaths['Country/Region'] == country].tolist() [0]).replace(',', '.') + ' %')) graph_six.append( go.Choropleth( locations=df_cases_deaths['Country Code'], z=df_cases_deaths['Infection Rate'], #colorscale = 'Reds', colorscale=[ '#f9ebea', #'#f2d7d5', #'#e6b0aa', '#cd6155', '#c0392b', '#a93226', '#922b21', '#7b241c', '#641e16' ], autocolorscale=False, showscale=False, #colorbar_title = 'Cases / Population' text=text, hoverinfo='text', hoverlabel=dict(bgcolor=' #ebedef'))) layout_six = dict( title='Most Affected Countries (Cases / Population)', showlegend=False, showcolorbar=False, geo=dict( projection=dict(type='natural earth'), showframe=True, #showcoastline = False, showcountries=True)) # Append all charts to the figures list figures = [] figures.append(dict(data=graph_one, layout=layout_one)) figures.append(dict(data=graph_two, layout=layout_two)) figures.append(dict(data=graph_three, layout=layout_three)) figures.append(dict(data=graph_four, layout=layout_four)) #figures.append(dict(data=graph_five, layout=layout_five)) figures.append(dict(data=graph_six, layout=layout_six)) return figures
"Cote d'Ivoire" : 'CIV', 'Holy See' : 'VAT', 'Iran' : 'IRN', 'Korea, South' : 'KOR', 'Kosovo' : 'UNK', 'Reunion' : 'REU', 'Russia' : 'RUS', 'Taiwan*' : 'TWN', 'The Bahamas' : 'BHS', 'occupied Palestinian territory' : 'PSE' } for country in df_confirmed['Country/Region'].drop_duplicates().sort_values(): try: matched_country_dict[country] = countries.lookup(country).alpha_3 except: try: matched_country_dict[country] = manual_country_dict[country] except: print(country, ' not found') no_pycountry.append(country) ''' ## Load the World Development Indicators by the World Bank and create data pickle for population data # Comment: I created the pickle file only once to reduce server load df_wdi = pd.read_csv('../covidapp/data/WDIData.csv') # Extract the population data for 2018 (2019 has to many NaN values)
def is_valid_country(c: str) -> bool: try: countries.lookup(c) return True except LookupError: return False
class Misc(commands.Cog): def __init__(self, bot, config): self.bot = bot self.config = config @cog_ext.cog_slash( description="Translate the provided Japanese text into English via Google Translate.", guild_ids=hkd.get_all_guild_ids(), options=[ create_option( name="text", description="The text to translate.", option_type=3, required=True ) ] ) async def translate(self, ctx: SlashContext, text: str): await ctx.defer() await ctx.send(embed=hkd.create_embed(description=Translator().translate(text, src='ja', dest='en').text)) @cog_ext.cog_slash( description="Convert currency from one type to another.", guild_ids=hkd.get_all_guild_ids(), options=[ create_option( name="conversion", description="Convert amount of currency-a to currency-b.", option_type=3, required=True ) ] ) async def currency(self, ctx: SlashContext, conversion: str): await ctx.defer() conversion_split = conversion.split() if len(conversion_split) == 4 and conversion_split[2].lower() == 'to': with suppress(Exception): result = CurrencyRates().convert(conversion_split[1].upper(), conversion_split[3].upper(), Decimal(conversion_split[0])) await ctx.send(embed=hkd.create_embed(title='{0} {1}'.format('{:f}'.format(result).rstrip('0').rstrip('.'), conversion_split[3].upper()))) return await ctx.send(embed=hkd.create_embed(description="Couldn't convert. Please follow this format for converting currency: **/currency** 12.34 AUD to USD.", colour=Colour.red())) @cog_ext.cog_slash( description="Show weather information for the specified location.", guild_ids=hkd.get_all_guild_ids(), options=[ create_option( name="location", description="The location to show the weather for.", option_type=3, required=True ) ] ) async def weather(self, ctx: SlashContext, location: str): await ctx.defer() if len(query := location.split(',')) > 1: with suppress(Exception): query[1] = countries.get(name=query[1].strip().title()).alpha_2 with suppress(Exception): result = requests.get('http://api.openweathermap.org/data/2.5/weather', params={'q': ','.join(query), 'APPID': self.config['weather_api_key']}).json() timezone = pytz.timezone(TimezoneFinder().timezone_at(lat=result['coord']['lat'], lng=result['coord']['lon'])) embed_fields = [] embed_fields.append(('Weather', '{0}'.format(result['weather'][0]['description'].title()))) embed_fields.append(('Temperature', '{0} °C, {1} °F'.format('{0:.2f}'.format(float(result['main']['temp']) - 273.15), '{0:.2f}'.format((1.8 * (float(result['main']['temp']) - 273.15)) + 32.0)))) embed_fields.append(('Humidity', '{0}%'.format(result['main']['humidity']))) embed_fields.append(('Wind Speed', '{0} m/s'.format(result['wind']['speed']))) embed_fields.append(('Sunrise', '{0:%I}:{0:%M} {0:%p}'.format(datetime.fromtimestamp(result['sys']['sunrise'], tz=timezone)))) embed_fields.append(('Sunset', '{0:%I}:{0:%M} {0:%p}'.format(datetime.fromtimestamp(result['sys']['sunset'], tz=timezone)))) embed_fields.append(('Pressure', '{0} hPa'.format(result['main']['pressure']))) await ctx.send(content='**Weather for {0}, {1}**'.format(result['name'], countries.lookup(result['sys']['country']).name), embed=hkd.create_embed(fields=embed_fields, inline=True)) return await ctx.send(embed=hkd.create_embed(description="Couldn't get weather. Please follow this format for checking the weather: **/weather** Melbourne, Australia.", colour=Colour.red()))
# Run this code cell to install and import the pycountry library #!pip install pycountry from pycountry import countries import pandas as pd # Run this code cell to see an example of how the library works countries.get(name='Spain') # Run this code cell to see how you can also look up countries without specifying the key countries.lookup('Kingdom of Spain') #encoding from encodings.aliases import aliases alias_values = set(aliases.values()) # This code finds the encodings that works for the file for encoding in set(aliases.values()): try: df = pd.read_csv("mystery.csv", encoding=encoding) print('successful', encoding) except: pass # Fill null # Fill with mean of a group df_melt = pd.read_csv('gdp_data.csv') df_melt['GDP_filled'] = df_melt.groupby('Country Name')['GDP'].transform( lambda x: x.fillna(x.mean()))
def parse_file(self, file): """ Parse a single address book file """ prin = pprint.PrettyPrinter() self.stats = {} for vob in vobject.readComponents(open(file)): content = vob.contents if self.args.verbose: logging.debug("parsing %s:", file) prin.pprint(content) if "n" not in content: self.stats["noname"] = self.stats.get("noname", 0) + 1 return if "email" not in content or not re.match( r"^[^@]+@[^@]+\.[^@]+$", content["email"][0].value): self.stats["noemail"] = self.stats.get("noemail", 0) + 1 return self.stats["valid"] = self.stats.get("valid", 0) + 1 # aggregate stats what kind of fields we have available for i in content: # if i in c: self.stats[i] = self.stats.get(i, 0) + 1 emails = [] for email in content.get("email", []): if re.match(r"^[^@äöü]+@[^@]+\.[^@]+$", email.value): emails.append(email.value) profile_list = [] for email in emails: profile = self.hatchbuck.search_email(email) if profile: profile_list.append(profile) else: continue # No contacts found if not profile_list: # create new contact profile = dict() profile["firstName"] = content["n"][0].value.given profile["lastName"] = content["n"][0].value.family if "title" in content: profile["title"] = content["title"][0].value if "org" in content: profile["company"] = content["org"][0].value profile["subscribed"] = True profile["status"] = {"name": "Lead"} if self.args.source: profile["source"] = {"id": self.args.source} # override hatchbuck sales rep username if set # (default: api key owner) if self.args.user: profile["salesRep"] = {"username": self.args.user} profile["emails"] = [] for email in content.get("email", []): if not re.match(r"^[^@äöü]+@[^@]+\.[^@]+$", email.value): continue if "WORK" in email.type_paramlist: kind = "Work" elif "HOME" in email.type_paramlist: kind = "Home" else: kind = "Other" profile["emails"].append({ "address": email.value, "type": kind }) profile = self.hatchbuck.create(profile) logging.info("added contact: %s", profile) for profile in profile_list: if profile["firstName"] == "" or "@" in profile["firstName"]: profile = self.hatchbuck.profile_add( profile, "firstName", None, content["n"][0].value.given) if profile["lastName"] == "" or "@" in profile["lastName"]: profile = self.hatchbuck.profile_add( profile, "lastName", None, content["n"][0].value.family) if "title" in content and profile.get("title", "") == "": profile = self.hatchbuck.profile_add( profile, "title", None, content["title"][0].value) if "company" in profile: if "org" in content and profile.get("company", "") == "": profile = self.hatchbuck.profile_add( profile, "company", None, content["org"][0].value) if profile["company"] == "": # empty company name -> # maybe we can guess the company name from the email # address? # logging.warning("empty company with emails: %s", # profile['emails']) pass # clean up company name if re.match(r";$", profile["company"]): logging.warning("found unclean company name: %s", profile["company"]) if re.match(r"\|", profile["company"]): logging.warning("found unclean company name: %s", profile["company"]) for addr in content.get("adr", []): address = { "street": addr.value.street, "zip_code": addr.value.code, "city": addr.value.city, "country": addr.value.country, } try: if "WORK" in addr.type_paramlist: kind = "Work" elif "HOME" in addr.type_paramlist: kind = "Home" else: kind = "Other" except AttributeError: # if there is no type at all kind = "Other" logging.debug("adding address %s %s", address, profile) profile = self.hatchbuck.profile_add_address( profile, address, kind) for telefon in content.get("tel", []): # number cleanup number = telefon.value for rep in "()-\xa0": # clean up number number = number.replace(rep, "") number = number.replace("+00", "+").replace("+0", "+") try: if "WORK" in telefon.type_paramlist: kind = "Work" elif "HOME" in telefon.type_paramlist: kind = "Home" else: kind = "Other" except AttributeError: # if there is no type at all kind = "Other" redundant = False try: phonenumber = phonenumbers.parse(number, None) pformatted = phonenumbers.format_number( phonenumber, phonenumbers.PhoneNumberFormat.INTERNATIONAL) except phonenumbers.phonenumberutil.NumberParseException: # number could not be parsed, e.g. because it is a # local number without country code logging.warning( "could not parse number %s as %s in %s, " "trying to guess country from address", telefon.value, number, self.hatchbuck.short_contact(profile), ) pformatted = number # try to guess the country from the addresses countries_found = [] for addr in profile.get("addresses", []): if (addr.get("country", False) and addr["country"] not in countries_found): countries_found.append(addr["country"]) logging.debug("countries found %s", countries_found) if len(countries_found) == 1: # lets try to parse the number with the country countrycode = countries.lookup( countries_found[0]).alpha_2 logging.debug("countrycode %s", countrycode) try: phonenumber = phonenumbers.parse( number, countrycode) pformatted = phonenumbers.format_number( phonenumber, phonenumbers.PhoneNumberFormat. INTERNATIONAL, ) logging.debug("guess %s", pformatted) profile = self.hatchbuck.profile_add( profile, "phones", "number", pformatted, {"type": kind}, ) # if we got here we now have a full number continue except phonenumbers.phonenumberutil.NumberParseException: logging.warning( "could not parse number %s as %s using country %s in %s", telefon.value, number, countrycode, self.hatchbuck.short_contact(profile), ) pformatted = number # check that there is not an international/longer # number there already # e.g. +41 76 4000 464 compared to 0764000464 # skip the 0 in front num = number.replace(" ", "")[1:] for tel2 in profile["phones"]: # check for suffix match if tel2["number"].replace(" ", "").endswith(num): logging.warning( "not adding number %s from %s because it " "is a suffix of existing %s", num, self.hatchbuck.short_contact(profile), tel2["number"], ) redundant = True break if not redundant: profile = self.hatchbuck.profile_add( profile, "phones", "number", pformatted, {"type": kind}) # clean & deduplicate all phone numbers profile = self.hatchbuck.clean_all_phone_numbers(profile) for skype in content.get("x-skype", []): profile = self.hatchbuck.profile_add( profile, "instantMessaging", "address", skype.value, {"type": "Skype"}, ) for msn in content.get("x-msn", []): profile = self.hatchbuck.profile_add( profile, "instantMessaging", "address", msn.value, {"type": "Messenger"}, ) for msn in content.get("x-msnim", []): profile = self.hatchbuck.profile_add( profile, "instantMessaging", "address", msn.value, {"type": "Messenger"}, ) for twitter in content.get("x-twitter", []): if "twitter.com" in twitter.value: value = twitter.value else: value = "http://twitter.com/" + twitter.value.replace( "@", "") profile = self.hatchbuck.profile_add( profile, "socialNetworks", "address", value, {"type": "Twitter"}) for url in content.get("url", []) + content.get( "x-socialprofile", []): value = url.value if not value.startswith("http"): value = "http://" + value if "facebook.com" in value: profile = self.hatchbuck.profile_add( profile, "socialNetworks", "address", value, {"type": "Facebook"}, ) elif "twitter.com" in value: profile = self.hatchbuck.profile_add( profile, "socialNetworks", "address", value, {"type": "Twitter"}, ) else: profile = self.hatchbuck.profile_add( profile, "website", "websiteUrl", value) for bday in content.get("bday", []): date = { "year": bday.value[0:4], "month": bday.value[5:7], "day": bday.value[8:10], } profile = self.hatchbuck.profile_add_birthday( profile, date) if self.args.tag: if not self.hatchbuck.profile_contains( profile, "tags", "name", self.args.tag): self.hatchbuck.add_tag(profile["contactId"], self.args.tag) # get the list of unique contacts IDs to detect if there are # multiple contacts in hatchbuck for this one contact in CardDAV profile_contactids = [] message = "" for profile in profile_list: if profile["contactId"] not in profile_contactids: profile_contactids.append(profile["contactId"]) email_profile = " " for email_add in profile.get("emails", []): email_profile = email_add["address"] + " " number_profile = " " for phone_number in profile.get("phones", []): number_profile = phone_number["number"] + " " message += ("{0} {1} ({2}, {3}, {4})".format( profile["firstName"], profile["lastName"], email_profile, number_profile, profile["contactUrl"], ) + ", ") if len(profile_contactids) > 1: # there are duplicates NotificationService().send_message( "Duplicates: %s from file: %s" % (message[:-2], file))
#!/usr/bin/env python3 import csv from pycountry import countries from urllib.request import urlopen submitter_info = {} for row in csv.reader(open('submitter_info.tsv'), delimiter='\t'): submitter_id = row[0] submitter_name = row[1] country_code = row[2] submitter_info[submitter_id] = [submitter_name, country_code] with open('organization_summary.txt') as f: next(f) #skip header for row in csv.reader(f, delimiter='\t'): submitter_name = row[0] submitter_id = row[1] try: country_code = countries.lookup(row[4]).alpha_3 except LookupError: #not a real country country_code = '' if not country_code and submitter_id in submitter_info: country_code = submitter_info[submitter_id][1] submitter_info[submitter_id] = [submitter_name, country_code] with open('submitter_info.tsv', 'w') as f: writer = csv.writer(f, delimiter='\t', lineterminator='\n') for submitter_id in sorted(submitter_info.keys(), key=int): writer.writerow([submitter_id] + submitter_info[submitter_id])
def get_country_name_from_code(self, code): country = pyc.lookup(code) return country.name
def _clean_address(self, address): # pylint: disable=too-many-branches """ clean up an address :param address: address dict :return: cleaned up address """ if address.get("country", ""): # check if the country is a valid country name # this also fixes two letter country codes try: country = countries.search_fuzzy( self._clean_country_name(address["country"]))[0].name except LookupError: country = False if country and country != address["country"]: address["country"] = country if re.match(r"^[a-zA-Z]{2}-[0-9]{4,6}$", address.get("zip", "")): # zipcode contains country code (e.g. CH-8005) countrycode, zipcode = address["zip"].split("-") try: address["country"] = countries.lookup(countrycode).name address["zip"] = zipcode except LookupError: pass if (not address.get("street", "") and not address.get("zip", "") and address.get("city", "")): # there is a city but no street/zip # logging.warning(address["city"].split(",")) if len(address["city"].split(",")) == 3: # Copenhagen Area, Capital Region, Denmark # Lausanne, Canton de Vaud, Suisse # Vienna, Vienna, Austria try: address["country"] = countries.search_fuzzy( self._clean_country_name( address["city"].split(",")[2]))[0].name address["city"] = self._clean_city_name( address["city"].split(",")[0]) except LookupError: pass elif len(address["city"].split(",")) == 2: # Zürich und Umgebung, Schweiz # Currais Novos e Região, Brasil # München und Umgebung, Deutschland # Région de Genève, Suisse # Zürich Area, Svizzera try: address["country"] = countries.search_fuzzy( self._clean_country_name( address["city"].split(",")[1]))[0].name address["city"] = self._clean_city_name( address["city"].split(",")[0]) except LookupError: pass elif len(address["city"].split(",")) == 1: # United States # Switzerland # Luxembourg try: address["country"] = countries.search_fuzzy( self._clean_country_name( self._clean_city_name(address["city"])))[0].name # if the lookup is successful this is a country name, not city name address["city"] = "" except LookupError: pass if address.get("type", "") not in ["Work", "Home", "Other"]: address["type"] = "Other" # ensure all relevant fields are set address["street"] = self._clean_street_name(address.get("street", "")) address["city"] = self._clean_city_name(address.get("city", "")) address["zip"] = address.get("zip", "").strip() address["state"] = address.get("state", "").strip() address["country"] = self._clean_country_name( address.get("country", "")) return address
def lambda_handler(event, context): logger.info(f'event = {event}') # get the region from queryStringParameters, default set as Taiwan if "queryStringParameters" in event.keys( ) & "region" in event["queryStringParameters"].keys(): region = countries.lookup(event["queryStringParameters"]['region']) else: region = countries.lookup("Taiwan") if datetime.today().year == 2021: data = pd.read_csv( './110中華民國政府行政機關辦公日曆表.csv', # os.path.abspath('./lambda-handler/110中華民國政府行政機關辦公日曆表.csv'), index_col='西元日期', encoding='big5') elif datetime.today().year == 2022: data = pd.read_csv( './111年中華民國政府行政機關辦公日曆表.csv', # os.path.abspath('./lambda-handler/111年中華民國政府行政機關辦公日曆表.csv'), index_col='西元日期', encoding='big5') # print(data.columns) # print(data.index) tz = pytz.timezone('Asia/Taipei') today = datetime.now(tz) # print(today) # tomorrow = today + timedelta(days = 1) # print(tomorrow) day = data.loc[int(today.strftime("%Y%m%d"))] # day = data.loc[int(tomorrow.strftime("%Y%m%d"))] if day['是否放假'] == 0: data = { # 'result': "明天要上班哦~", 'result': "今天要上班哦~", 'region': region.common_name, 'workingDay': True, 'datetime': today.isoformat() } elif day['是否放假'] == 2: data = { # 'result': "明天放假囉!", 'result': "今天放假囉!", 'region': region.common_name, 'workingDay': False, 'datetime': today.isoformat() } else: data = {'result': "出現了一些錯誤兒...", 'datetime': today.isoformat()} logger.info(f'data = {data}') result = { 'statusCode': 200, 'body': json.dumps(data), # deepcode ignore TooPermissiveCors: the api allow global access 'headers': { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' } } # result = json.dumps(data) return result