def createCitiesCollection(self):
print "createCitiesCollection()..."
#
collVoiv = self.mongoDB["voivodeships"]
collCounties = self.mongoDB["counties"]
collCities = self.mongoDB["cities"]
#
xmlFile = "./GUS/SIMC.xml"
print "Czytam plik: [%s]..." % xmlFile
tree = etree.parse(xmlFile)
rows = tree.findall("//row")
for row in rows:
# print '============='
# cols = row.findall("col")
tr = self._getRowData(row)
# print tr
tVoiv = collVoiv.find_one({"terc": tr["WOJ"]})
tCounty = collCounties.find_one({"teryt_woj": tr["WOJ"], "teryt_pow": tr["POW"]})
# print tCounty
inCity = {
"name": tr["NAZWA"],
"slug": slughifi(tr["NAZWA"]),
"teryt": "%s%s%s%s" % (tr["WOJ"], tr["POW"], tr["GMI"], tr["RODZ_GMI"]),
"teryt_woj": tr["WOJ"],
"teryt_pow": tr["POW"],
"teryt_gmi": tr["GMI"],
"teryt_rodz_gmi": tr["RODZ_GMI"],
"voiv_id": tVoiv["_id"],
"voiv_name": tVoiv["name"],
"county_id": tCounty["_id"],
"county_name": tCounty["name"],
"has_school": False,
}
print inCity["voiv_name"], "->", inCity["name"]
#
collCities.insert(inCity)
def readDataFromXLS(self):
for xlsFile in voivList:
tmpFile = os.path.join("./MEN", xlsFile )
print "Czytam dane z pliku:[%s]" % xlsFile
wb = xlrd.open_workbook( tmpFile )
sh = wb.sheet_by_index(0)
schoolCounter=0
#print sh.nrows
# city_id = 0
#county_id_counter = 0
czy_sa_dane = False
for rownum in range(sh.nrows):
#print sh.row_values(rownum)[colCity]
cWOJ = sh.row_values(rownum)[colWOJ]
cPOW = sh.row_values(rownum)[colPOW]
cGM = sh.row_values(rownum)[colGM]
#
city_name = sh.row_values(rownum)[colCity] # tutaj nei moze byc strip()
city_slug = slughifi(unicode(city_name))
#
school_type = sh.row_values(rownum)[colSchoolType]
if czy_sa_dane:
if len(city_slug)>0 and school_type in valid_school_types:
tCity = self.collCities.find_one({"slug":city_slug, "teryt_woj":cWOJ})
#print city_name,city_slug,", CITY=",tCity
tSchoolType = self.collSchoolType.find_one({"type":school_type})
#print "SchoolType=",tSchoolType
#print "SchoolType=",school_type
if tCity==None or tCity=="" or tSchoolType==None:
msg = "Brak miasta [%s] lub typu szkoly [%s] w DB" % (slughifi(city_name),school_type)
#open("./brakujace_miasta.log","a").write("%s -> woj: %s, pow: %s\n" % (msg, tCity['voiv_name'],tCity['county_name']))
open("./brakujace_miasta.log","a").write("%s -> woj: , pow: \n" % (msg))
#sys.exit(msg)
else:
#
#school_name = sh.row_values(rownum)[colSchool]
#
# znalazlem dane wiec je zapisze
#print "SchoolType=",tSchoolType
schoolGroup = {
"id":"",
"name": sh.row_values(rownum)[colSchool],
"slug": slughifi(sh.row_values(rownum)[colSchool]),
"school_info": {
"school_type_id": tSchoolType['_id'],
"school_type": tSchoolType['type'],
"school_type_name": tSchoolType['name'].lower(),
"address": "%s %s" % (sh.row_values(rownum)[colStreet].strip(),sh.row_values(rownum)[colHouseNum].strip()),
"postcode": sh.row_values(rownum)[colPostcode].strip(),
"voivodeship_id": tCity['voiv_id'],
"voivodeship": tCity['voiv_name'],
"county_id": tCity['county_id'],
"county": tCity['county_name'],
"city_id": tCity['_id'],
"city": tCity['name'],
"phone": sh.row_values(rownum)[colPhone].strip().replace(" ",""),
"www": sh.row_values(rownum)[colWWW].strip().replace(" ",""),
"email": "",
"note": "Wczytane dnia %s na podst. listy szkół MEN" % time.strftime("%Y-%m-%d"),
"patron": sh.row_values(rownum)[colPatron].strip(),
},
"is_school_class": False,
"group_type": "school",
"creation_dt": time.strftime("%Y-%m-%d %H:%M:%S"),
"edited_dt": time.strftime("%Y-%m-%d %H:%M:%S")
}
#
#print schoolGroup
#
#self.collSchools.insert(inSchool)
sgID = self.collServiceGroup.insert(schoolGroup)
#print "\n\nsgID=",sgID
self.collServiceGroup.update({"_id": sgID}, {"$set": {"id": sgID}})
schoolCounter+=1
self.collCities.update({"_id":tCity['_id']},{"$set":{"has_school":True}})
"""
#
db.save(inSchool)
#
db.test.update({"x": "y"}, {"has_school": True})
"""
#
if city_slug=="50":
czy_sa_dane=True
#
print "\twczytano szkół: ",schoolCounter
def slugify(self, tag):
return slughifi(tag)
def slugify(self, nombre):
return slughifi(nombre)
def upload(request):
if request.method == "POST":
form = forms.DocumentsUploadForm(request.POST, request.FILES)
if form.is_valid():
import slughifi
if request.user.is_authenticated():
creator = request.user
else:
creator = None
zip = form.cleaned_data['zip']
skipped_list = []
ok_list = []
error_list = []
slugs = {}
existing = [book.slug for book in Book.objects.all()]
for filename in zip.namelist():
if filename[-1] == '/':
continue
title = os.path.basename(filename)[:-4]
slug = slughifi(title)
if not (slug and filename.endswith('.xml')):
skipped_list.append(filename)
elif slug in slugs:
error_list.append((filename, slug, _('Slug already used for %s' % slugs[slug])))
elif slug in existing:
error_list.append((filename, slug, _('Slug already used in repository.')))
else:
try:
zip.read(filename).decode('utf-8') # test read
ok_list.append((filename, slug, title))
except UnicodeDecodeError:
error_list.append((filename, title, _('File should be UTF-8 encoded.')))
slugs[slug] = filename
if not error_list:
for filename, slug, title in ok_list:
book = Book.create(
text=zip.read(filename).decode('utf-8'),
creator=creator,
slug=slug,
title=title,
)
return direct_to_template(request, "catalogue/document_upload.html", extra_context={
"form": form,
"ok_list": ok_list,
"skipped_list": skipped_list,
"error_list": error_list,
"logout_to": '/',
})
else:
form = forms.DocumentsUploadForm()
return direct_to_template(request, "catalogue/document_upload.html", extra_context={
"form": form,
"logout_to": '/',
})
