def write_field_data_to_db(field_name, field_data):
for country_name in field_data.keys():
# get country if it exists; create it if it doesn't.
country_slug = slugify(country_name)
try:
country = Country.objects.get(url_name=country_slug)
except Country.DoesNotExist:
country = Country(url_name=country_slug)
country.CIAWFB_name_short = country_name
country.save()
# Get CIA WFB Entry if it exists; create it if it doesn't.
try:
CIAWFB_object = CIAWFBEntry.objects.get(country__id=country.id)
except CIAWFBEntry.DoesNotExist:
CIAWFB_object = CIAWFBEntry(country=country, date_entered=timezone.now())
CIAWFB_object.save()
# Now update the field we've got for that CIAWFB entry
db_name = slugify(field_name).replace('-', '_')
try:
setattr(CIAWFB_object, db_name, field_data[country_name])
CIAWFB_object.save()
except DatabaseError:
print('Unable to write field "%s" (country "%s"). Size to write was %s.' %
(db_name, country_name, len(field_data[country_name])))
longest_field = 0
for cname in field_data.keys():
len_data = len(field_data[cname])
if len_data > longest_field:
longest_field = len_data
print("Field: %s; Max Length: %s" % (field_name, longest_field))
raise DatabaseError
def editImmigrant(imm, firstName, lastName, gender, date, country, continent,
ethnicity, spokenLang, processLocation, destCity, destState):
imm = getImmigrant(imm)
imm.immfirstname = firstName
imm.immlastname = lastName
imm.immgender = gender
imm.immdate = date
cont = Continent.objects.get(cname=continent)
try:
# Attempts to find that country
c = Country.objects.get(cname=country)
except Country.DoesNotExist:
# If that country is not found, creates it
c = Country(cname=country, ccontinent=cont)
c.save()
try:
# Attempts to find that ethnicity
e = Ethnicity.objects.get(ename=ethnicity)
except Ethnicity.DoesNotExist:
# If that combination is not found, creates it
e = Ethnicity(ename=ethnicity)
e.save()
try:
# Attempts to find that ethnicity
sl = Languages.objects.get(lname=spokenLang)
except Languages.DoesNotExist:
# If that combination is not found, creates it
sl = Languages(lname=spokenLang)
sl.save()
try:
# Attempts to find that specific ethnic background combination
eb = Ethnicbackground.objects.get(country=country,
ethnicity=ethnicity,
spokenlang=spokenLang)
except Ethnicbackground.DoesNotExist:
# If that combination is not found, creates it
eb = Ethnicbackground(country=c, ethnicity=e, spokenlang=sl)
eb.save()
imm.immeb = eb
imm.immprocloc = Processlocation.objects.get(plname=processLocation)
state = State.objects.get(sname=destState)
try:
# Tries to find the city (that matches the state)
city = City.objects.get(cname=destCity, cstate=state)
except City.DoesNotExist:
# If that city doesn't exist, creates it
city = City(cname=destCity, cstate=state)
city.save()
imm.immdestcity = city
imm.save()
def handle_country(self, data_set):
print("-------handle_country-------")
item = Country()
item.name = data_set["name"]
item.currency = data_set["currency"]
item.save()
def handle(self, *args, **kwargs):
self.stdout.write(self.style.SUCCESS("Lancement du programme"))
self.key = key = os.environ.get("UNSPLASH_KEY")
all_countries = list(pycountry.countries)
clist = []
temp_dict = TempClass()
temp_dict = temp_dict.get_datas()
for i in all_countries:
clist.append(i)
for country in tqdm(clist):
time.sleep(35)
try:
country_resume = wikipedia.summary(country.name, sentences=1)
except:
country_resume = "No resume avaible for now..."
try:
query = country.name.lower()
url = f"https://unsplash.com/napi/search?query={country.name}&xp=&per_page=1"
headers = {"authorization": f"Client-ID {self.key}"}
response = requests.get(url, headers=headers)
urls = []
resp = response.json()
for img in resp["photos"]["results"]:
try:
urls.append((img.get("urls")["small"]))
except:
try:
urls.append((img.get("urls")["thumbnail"]))
except:
urls.append((img.get("urls")["raw"]))
response = requests.get(urls[0])
pic = ("data:" + response.headers["Content-Type"] + ";" +
"base64," +
base64.b64encode(response.content).decode("utf-8"))
except:
self.stdout.write(
self.style.WARNING("WARNING NOT FOUND IMAGE"))
pic = "No image found"
if country.alpha_3 in temp_dict:
c = Country(
name=country.name,
alpha_2=country.alpha_2,
alpha_3=country.alpha_3,
temp_averges=temp_dict[country.alpha_3],
flag=f"../static/img/flags/flag-{country.alpha_2}.jpg",
resume=country_resume,
picture=pic,
)
c.save()
a = self.style.WARNING(f"{country.name}")
b = self.style.SUCCESS(" [OK] ")
self.stdout.write(a + b)
self.stdout.write(self.style.SUCCESS("Opération terminée: [OK]"))
def handle(self, *args, **options):
# TODO: need to load from csv
for c in Country.objects.all():
c.delete()
with open("data/countryInfo.txt", "r") as country_file:
reader = csv.DictReader(country_file, delimiter='\t')
for row in reader:
country = Country(name=row['Country'], iso_code=row['ISO'])
country.save()
print "{country_count} countries saved ".format(country_count=Country.objects.all().count())
def handle(self, *args, **options):
# TODO: need to load from csv
for c in Country.objects.all():
c.delete()
with open("data/countryInfo.txt", "r") as country_file:
reader = csv.DictReader(country_file, delimiter='\t')
for row in reader:
country = Country(name=row['Country'], iso_code=row['ISO'])
country.save()
print "{country_count} countries saved ".format(
country_count=Country.objects.all().count())
def parse(self, response):
Country.objects.all().delete()
City.objects.all().delete()
country_list = response.xpath(
'//*[@id="country"]//option/text()').extract()[1:40]
for country in country_list:
sleep(randint(4, 9))
new_country = Country(name=country)
new_country.save()
yield scrapy.Request(
"https://numbeo.com/cost-of-living/country_result.jsp?country={}"
.format(country),
callback=self.parse_city,
meta={"country_id": new_country.id})
def countreis():
print '\n'
print 'Import/Update Country'
base, prog = get_data()
for row in base:
try:
country = Country.objects(name_eu = row[3])[0]
except:
country = Country(name_eu = row[3])
country.name_ru = row[3]
country.slug = '%s' % (str(row[3]).replace(' ','_'))
country.save()
prog.increment_amount()
print prog, '\r',
sys.stdout.flush()
def setUp(self):
credentials = {"username": "******", "password": "******"}
User.objects.create_superuser(email="*****@*****.**", **credentials)
self.client.login(**credentials)
country = Country(code="CY", name="Country")
country.save()
cc = CooperativeCenter(code="AB12.3", country=country)
cc.save()
cc2 = CooperativeCenter(code="CD45.6", country=country)
cc2.save()
john = User.objects.create_user(username="******", email="*****@*****.**")
john.profile.cooperative_center = cc
john.profile.save()
jane = User.objects.create_user(username="******", email="*****@*****.**")
jane.profile.cooperative_center = cc2
jane.profile.save()
settings.ITEMS_PER_PAGE = 3
# Takes the csv file: countries_webscrape.csv and populates the counties model.
# This code was executed using the django shell.
# Based on code from http://abhishekchhibber.com/django-importing-a-csv-file-to-database-models/
import csv
from main.models import Country
with open('webscrape/countries_webscrape.csv') as webscrapecsv:
csv_file = csv.DictReader(webscrapecsv)
i = 0
for row in csv_file:
country_result = Country(name=row['name'], code=i)
country_result.save()
i += 1
from main.models import Country, State, Holiday
for name in ['Brazil', 'Sweden', 'United States']:
country = Country(name=name)
country.save()