def uploadfile(username, agencydata, comments, md5sum, sizeoffile, bounds):
## todo: cache
if model.Message.all().filter('md5sum =', md5sum).count() > 0:
raise UploadError('This file has previously been uploaded')
## todo: cache
if model.SkipMd5.all().filter('md5sum =', md5sum).count() > 0:
raise UploadError('This file has previously been uploaded')
raw_agencies = utils.readfile(agencydata)
if not raw_agencies:
raise UploadError(
"zip file did not contain any valid agencies in agency.txt.")
## save our msg
new_message = model.Message(user=username, content=comments)
new_message.hasFile = True
memcache.delete('Message.recent')
# new_message.filename = filename
new_message.md5sum = md5sum
new_message.size = sizeoffile
new_message.max_lat = None
new_message.max_lng = None
new_message.min_lat = None
new_message.min_lng = None
if bounds:
bounds_list = bounds.split("|")
try:
new_message.max_lat = float(bounds_list[0])
new_message.max_lng = float(bounds_list[1])
new_message.min_lat = float(bounds_list[2])
new_message.min_lng = float(bounds_list[3])
except ValueError:
logging.error('failed to set bounds from %s' % bounds)
new_message.put()
d = datetime.datetime.now()
datestr = d.strftime('%Y%m%d_%H%M')
seen_agencies = []
for ag in raw_agencies:
## get from the db
## lookup by url first
a = None
if ag.get('agency_url', '').strip():
## try to get via url first as it's more unique
url = ag['agency_url'].strip()
try:
# TODO: use urlnorm
url_parsed = urlparse.urlparse(url)
if not url_parsed.path:
url += '/'
except:
logging.exception('unable to parse url')
a = model.Agency.all().filter('url =', url).get()
if not a:
slug = model.slugify(ag['agency_name'].strip())
s = utils.lookup_agency_alias(slug)
if s:
slug = s
a = memcache.get('Agency.slug.%s' % slug)
if not a:
a = model.Agency.all().filter('slug =', slug).get()
if a:
a.messagecount += 1
a.lastupdate = datetime.datetime.now()
a.put()
memcache.set('Agency.slug.%s' % a.slug, a)
if not a:
a = model.Agency()
a.name = ag['agency_name'].strip()
a.url = ag.get('agency_url', '')
a.messagecount = 1
a.put()
memcache.delete('Agency.recent')
utils.incrAgencyCount()
if len(raw_agencies) == 1:
new_message.filename = '%s_%s.zip' % (a.slug, datestr)
new_message.put()
# some zip files have the same url several times; only capture the first time that url is used
if a in seen_agencies:
continue
seen_agencies.append(a)
ma = model.MessageAgency()
ma.agency = a
ma.message = new_message
ma.hasFile = True
ma.put()
memcache.delete('Agency.all') # because it has the cached last-update
if not new_message.filename:
new_message.filename = '%s_%s.zip' % (username.nickname(), datestr)
new_message.put()
# TODO: can we even hit this, since upload should only be called at a rate of once a minute anyway?
recentFiles = model.Message.all().filter('hasFile =', True).filter(
'date >=', d.replace(second=0, microsecond=0)).count()
if recentFiles > 1: # note we already saved *this* filename
new_message.filename = new_message.filename.replace(
'.zip', '_%d.zip' % recentFiles)
new_message.put()
## send email to user ?
return new_message.filename
def test_slugify(self):
name = 'Test Post with-&-a--lot---of----hyphens-' + UCHAR
slug = model.slugify(name)
assert slug == 'test-post-with-a-lot-of-hyphens'
def uploadfile(username, agencydata, comments, md5sum, sizeoffile, bounds):
## todo: cache
if model.Message.all().filter('md5sum =', md5sum).count() >0:
raise UploadError('This file has previously been uploaded')
## todo: cache
if model.SkipMd5.all().filter('md5sum =', md5sum).count() >0:
raise UploadError('This file has previously been uploaded')
raw_agencies = utils.readfile(agencydata)
if not raw_agencies:
raise UploadError("zip file did not contain any valid agencies in agency.txt.")
## save our msg
new_message = model.Message(user=username, content=comments)
new_message.hasFile = True
memcache.delete('Message.recent')
# new_message.filename = filename
new_message.md5sum = md5sum
new_message.size = sizeoffile
new_message.max_lat = None
new_message.max_lng = None
new_message.min_lat = None
new_message.min_lng = None
if bounds:
bounds_list = bounds.split("|")
try:
new_message.max_lat = float(bounds_list[0])
new_message.max_lng = float(bounds_list[1])
new_message.min_lat = float(bounds_list[2])
new_message.min_lng = float(bounds_list[3])
except ValueError:
logging.error('failed to set bounds from %s' % bounds)
new_message.put()
d = datetime.datetime.now()
datestr = d.strftime('%Y%m%d_%H%M')
seen_agencies = []
for ag in raw_agencies:
## get from the db
## lookup by url first
a = None
if ag.get('agency_url', '').strip():
## try to get via url first as it's more unique
url = ag['agency_url'].strip()
try:
# TODO: use urlnorm
url_parsed = urlparse.urlparse(url)
if not url_parsed.path:
url += '/'
except:
logging.exception('unable to parse url')
a = model.Agency.all().filter('url =', url).get()
if not a:
slug = model.slugify(ag['agency_name'].strip())
s = utils.lookup_agency_alias(slug)
if s:
slug = s
a = memcache.get('Agency.slug.%s' % slug)
if not a:
a = model.Agency.all().filter('slug =', slug).get()
if a:
a.messagecount +=1
a.lastupdate = datetime.datetime.now()
a.put()
memcache.set('Agency.slug.%s' % a.slug, a)
if not a:
a = model.Agency()
a.name = ag['agency_name'].strip()
a.url = ag.get('agency_url', '')
a.messagecount = 1
a.put()
memcache.delete('Agency.recent')
utils.incrAgencyCount()
if len(raw_agencies) == 1:
new_message.filename = '%s_%s.zip' % (a.slug, datestr)
new_message.put()
# some zip files have the same url several times; only capture the first time that url is used
if a in seen_agencies:
continue
seen_agencies.append(a)
ma= model.MessageAgency()
ma.agency = a
ma.message = new_message
ma.hasFile=True
ma.put()
memcache.delete('Agency.all') # because it has the cached last-update
if not new_message.filename:
new_message.filename = '%s_%s.zip' % (username.nickname(), datestr)
new_message.put()
# TODO: can we even hit this, since upload should only be called at a rate of once a minute anyway?
recentFiles = model.Message.all().filter('hasFile =', True).filter('date >=', d.replace(second=0, microsecond=0)).count()
if recentFiles > 1: # note we already saved *this* filename
new_message.filename = new_message.filename.replace('.zip', '_%d.zip' % recentFiles)
new_message.put()
## send email to user ?
return new_message.filename
def test_slugify(self):
name = 'Test Post with-&-a--lot---of----hyphens-' + UCHAR
slug = model.slugify(name)
assert slug == 'test-post-with-a-lot-of-hyphens'