oldArt = db.GqlQuery("Select * from Artifact where jar_name = :1", artifact.jar_name).get()
# update the existing one
if oldArt:
oldArt.artifactId = artifact.artifactId
oldArt.groupId = artifact.groupId
oldArt.modelVersion = artifact.modelVersion
oldArt.version = artifact.version
oldArt.size = artifact.size
oldArt.md5 = artifact.md5
oldArt.sha1 = artifact.sha1
oldArt.mvnModified = artifact.mvnModified
# oldArt.jar = artifact.jar
key = oldArt.put()
logging.info("updating..")
# insert the new one
else:
key = artifact.put()
logging.info("inserting..")
old_location = db.GqlQuery("Select * from Location where url = :1", entity["url"]).get()
if not old_location:
location = Location(name=entity["url"], url=entity["url"], repository=entity["repository"], artifact=key)
location.put()
if is_new:
return jar
if __name__ == "__main__":
bulkload.main(ArtifactLoader())
def loadAirportRoute(request):
bulkload.main(AirportRouteLoader())
return HttpResponse ('<h1>loadAirportRoute</h1>')
entity['updated'] = datetime.datetime.now()
entity['bookmark'] = utils.bookmark_for_kind(self.kind, 'user.name', entity['updated'])
return entity
if __name__ == '__main__':
bulkload.main(MetadataLoader(),
EntityMetadataLoader(),
EntityPropertyMetadataLoader(),
TypeInfoLoader(),
EnumerationLoader(),
EnumerationValueLoader(),
EntityRelationLoader(),
MetadataCommitLoader(),
DeleteAllLoader(),
UserLoader(),
GroupLoader(),
PermissionLoader(),
RoleLoader(),
RolePermissionsLoader(),
AppLoader(),
OrganisationLoader(),
L10nLoader(),
ViewDefinitionLoader(),
ViewMapLoader(),
DataQueryLoader(),
SynchSubscriptionLoader(),
ImportProfileLoader(),
UserLocationLoader())
from google.appengine.ext import bulkload
from google.appengine.api import datastore_types
from google.appengine.api import datastore
from google.appengine.ext import search
class AltitudeLoader(bulkload.Loader):
def __init__(self):
bulkload.Loader.__init__(self, 'Altitude',
[('pos', int),
('alt', int)
])
def HandleEntity(self, entity):
name="P" + str(entity['pos'])
del entity['pos']
newent = datastore.Entity('Altitude',name=name)
newent.update(entity)
return newent
if __name__ == '__main__':
bulkload.main(AltitudeLoader())
from google.appengine.api import users
from google.appengine.ext import bulkload
from google.appengine.api import datastore_types
from google.appengine.ext import search
class ExpressionLoader(bulkload.Loader):
def __init__(self):
# Our 'Expression' entity contains a affyid string and an expression float data
bulkload.Loader.__init__(self, 'Expression',
[('affy_id', str),
('gene_symbol', str),
('entrezid', str),
('gene_name', str),
('evector_day0', float),
('evector_day2', float),
('evector_day4', float),
('evector_day10', float),
('ppargox_day0', float),
('ppargox_day2', float),
('ppargox_day4', float),
('ppargox_day10', float),
])
def HandleEntity(self, entity):
ent = search.SearchableEntity(entity)
return ent
if __name__ == '__main__':
bulkload.main(ExpressionLoader())
from datetime import datetime
from google.appengine.ext import bulkload
from google.appengine.api import datastore_types
class AreaLoader(bulkload.Loader):
def __init__(self):
bulkload.Loader.__init__(self, 'Area',
[('id', int),
('name', str),
('url', datastore_types.Link),
('updated_at', lambda x: datatime.strptime(x, '%Y-%m-%d %H:%M:$S')),
('created_at', lambda x: datatime.strptime(x, '%Y-%m-%d %H:%M:$S')),
])
if __name__ == '__main__':
bulkload.main(AreaLoader())
from google.appengine.ext import bulkload
from google.appengine.api import datastore_types
class ProfileLoader(bulkload.Loader):
def __init__(self):
# Loads management profiles
bulkload.Loader.__init__(self, 'Profile',
[('first_Name', str),
('last_Name', str),
('role', str),
('country', str),
('state', str),
('city', str),
('zip_Code', str),
('phone', str),
('email', datastore_types.Email),
])
if __name__ == '__main__':
bulkload.main(ProfileLoader())
#TEMP removed to check performance
#entity['country']=countryObj.key()
del entity['country_code']
point = entity['point']
entity['pointLat'] = int(point.lat)
entity['pointLon'] = int(point.lon)
entity = search.SearchableEntity(entity)
#required in order to retrieve more than 1000 elements
entity['sequence_nr'] = self.sequence_nr
self.sequence_nr = self.sequence_nr + 1
logging.error("entity right before return %s ", entity)
return entity
#def HandleEntity(self, entity):
# event = datastore_entities.Event(entity.title)
# event.update(entity)
# creator = event['creator']
# if creator:
# contact = datastore.Query('Contact', {'title': creator}).Get(1)
# if not contact:
# contact = [datastore_entities.Contact(creator)]
# datastore.Put(contact[0])
# event['author'] = contact[0].key()
# return event
if __name__ == '__main__':
bulkload.main(AirportLoader())
contract_exists = Contract.get_by_key_name(key_name)
# if not contract_exists:
# logging.debug('found a new contract with reference number:'+entity['reference_number'])
newent = datastore.Entity('Contract', name=key_name)
newent.update(entity)
ent = search.SearchableEntity(newent)
#XXX setup a parent for the entity?
def increment_aggregates(key, count, value):
obj = db.get(key)
obj.contract_count += count
obj.contract_value += float(value)
obj.put()
if not contract_exists:
# increment counter for the Agency
agency = Agency.get_or_insert(entity['agency_name'], name=entity['agency_name'])
db.run_in_transaction(increment_aggregates, agency.key(), 1, entity['contract_value'])
# increment counter for the Vendor.
# prepend a string to avoid BadArgumentError: Names may not begin with a digit; received 1414421 Ontario Inc
vendor_name = 'vendor '+entity['vendor_name']
vendor = Vendor.get_or_insert(vendor_name, name=vendor_name)
db.run_in_transaction(increment_aggregates, vendor.key(), 1, entity['contract_value'])
sleep 1 # maybe this will solve the OverQuotaError
return ent
if __name__ == '__main__':
bulkload.main(ContractLoader())
from google.appengine.ext import bulkload
from google.appengine.api import datastore_types
class ProductLoader(bulkload.Loader):
def __init__(self):
# Loads products
bulkload.Loader.__init__(self, 'Product',
[('description', str),
('price', float),
])
if __name__ == '__main__':
bulkload.main(ProductLoader())
"""This module enables bulk uploading of puzzle packs to swyzl."""
from google.appengine.ext import bulkload
class PackDescriptionsLoader(bulkload.Loader):
def __init__(self):
bulkload.Loader.__init__(self, 'PackOfPuzzles',
[('index', int),
('title', str),
('thumbnail_url_part', str),
('introduction', str)])
if __name__ == '__main__':
bulkload.main(PackDescriptionsLoader())
#logging.info(entity['name'])
entity = utils.handle_entity_with_name_key(entity, self.kind,
self.props)
latitude = entity['latitude']
longitude = entity['longitude']
if latitude and longitude:
hash = str(geohash.Geohash((float(longitude), float(latitude))))
#logging.info(hash)
entity['bbhash1'] = hash[:2]
entity['bbhash2'] = hash[:4]
entity['bbhash'] = hash
entity['updated'] = datetime.datetime.now()
entity['bookmark'] = utils.bookmark_for_kind(self.kind, 'user.name',
entity['updated'])
return entity
if __name__ == '__main__':
bulkload.main(MetadataLoader(), EntityMetadataLoader(),
EntityPropertyMetadataLoader(), TypeInfoLoader(),
EnumerationLoader(), EnumerationValueLoader(),
EntityRelationLoader(), MetadataCommitLoader(),
DeleteAllLoader(), UserLoader(), GroupLoader(),
PermissionLoader(), RoleLoader(), RolePermissionsLoader(),
AppLoader(), OrganisationLoader(), L10nLoader(),
ViewDefinitionLoader(), ViewMapLoader(), DataQueryLoader(),
SynchSubscriptionLoader(), ImportProfileLoader(),
UserLocationLoader())
del entity['_work_osis_id']
del entity['_book_osis_id']
return entity
class VerseLoader(bulkload.Loader):
def __init__(self):
bulkload.Loader.__init__(self, 'Verse',
[('_work_osis_id', str),
('_book_osis_id', str),
('_chapter_osis_id', str),
('osis_id', str),
('number', long),
('text', str),
])
def HandleEntity(self, entity):
work = Work.gql('WHERE osis_id = :1', entity['_work_osis_id']).fetch(1)[0]
book = work.books.filter('osis_id =', entity['_book_osis_id']).fetch(1)[0]
chapter = book.chapters.filter('osis_id =', entity['_chapter_osis_id']).fetch(1)[0]
entity['chapter'] = chapter.key()
del entity['_work_osis_id']
del entity['_book_osis_id']
del entity['_chapter_osis_id']
return search.SearchableEntity(entity)
if __name__ == '__main__':
bulkload.main(WorkLoader(), BookLoader(), ChapterLoader(), VerseLoader())
logging.error("FIPS %s is invalid" % zone.fips)
return 0
class ZoneLoader(bulkload.Loader):
def __init__(self):
bulkload.Loader.__init__(self, 'Zone',
[
('state', str),
('zone' , str),
('cwa' , str),
('name' , str),
('state_zone', str),
('county', str),
('fips' , str),
('time_zone', str),
('fe_area', str),
('latitude', float),
('longitude', float)
])
""" Not Implemented """
def _HandleEntity(self, entity):
logging.debug(entity)
return None
if __name__ == '__main__':
bulkload.main(ZoneLoader())
entity.state = get_state_by_name(entity.state)
return entity
def get_state_by_name(name):
return getByName(name=name,
modelClass=State)
class CityLoader(bulkload.Loader):
def __init__(self):
bulkload.Loader.__init__(self, 'City',
[('name', str),
('state', City.get_by_name),
])
def get_city_by_name(name):
return getByName(name=name,
modelClass=City)
class CountryLoader(bulkload.Loader):
def __init__(self):
bulkload.Loader.__init__(self, 'Country',
[('name', str),
])
if __name__ == '__main__':
bulkload.main(CountryLoader(),
StateLoader(),
CityLoader()
)
"""This module enables bulk loading of individual puzzles."""
from google.appengine.ext import bulkload
class PuzzlesLoader(bulkload.Loader):
def __init__(self):
bulkload.Loader.__init__(self, 'Puzzle',
[('name', str),
('cipher_text', str),
('solution_text', str),
('short_clue', str),
('pack_title', str)])
if __name__ == '__main__':
bulkload.main(PuzzlesLoader())
from datetime import datetime
from google.appengine.ext import bulkload
from google.appengine.api import datastore_types
from models import Area
class ShopLoader(bulkload.Loader):
def __init__(self):
bulkload.Loader.__init__(self, 'Shop',
[('id', int),
('area', self.get_area_key),
('name', str),
('address', str),
('geo', datastore_types.GeoPt),
('extra', datastore_types.Text),
('updated_at', lambda x: datatime.strptime(x, '%Y-%m-%d %H:%M:$S')),
('created_at', lambda x: datatime.strptime(x, '%Y-%m-%d %H:%M:$S')),
])
def get_area_key(self, name):
area = Area.all().filter('name = ', name).get()
if area:
return area.key()
else:
return ''
if __name__ == '__main__':
bulkload.main(ShopLoader())
from google.appengine.ext import bulkload
from google.appengine.api import datastore_types
from google.appengine.ext import search
class TramInfoLoader(bulkload.Loader):
def __init__(self):
bulkload.Loader.__init__(self, 'TramInfo',
[ ('tracker_id', str), ('location', str) ])
def HandleEntity(self, entity):
ent = search.SearchableEntity(entity)
return ent
if __name__ == '__main__':
bulkload.main(TramInfoLoader())
('cc_ip',str),
('cc_when', lambda x: datetime.datetime.strptime(x,'%Y-%m-%d %H:%M:%S') ),
('usr_angle', int),
('usr_speed', int),
])
def HandleEntity(self, entity):
game = Game.get_by_key_name('Sapus Tongue')
country = getGeoIPCode( entity['cc_ip'])
game.nro_scores += 1
game.put()
query = db.Query(ScoresCountry)
query.ancestor( game )
query.filter('country_code =', country)
result = query.fetch(limit=1)
if query.count() > 0:
rec = result[0]
rec.quantity += 1
else:
rec = ScoresCountry( parent=game, game=game, country_code=country, quantity=1 )
rec.put()
return None
if __name__ == '__main__':
bulkload.main(ScoreLoader())
#!/usr/bin/env python
# -*- coding=utf-8 -*-
from google.appengine.ext import bulkload
from google.appengine.api import datastore_types
from gae.index import getByName
class CountryLoader(bulkload.Loader):
def __init__(self):
bulkload.Loader.__init__(self, 'Country',
[('name', str),
])
if __name__ == '__main__':
bulkload.main(CountryLoader())
#!/usr/bin/env python
"""
Client for uploading books from a CSV file. Is run as follows:
bulkload_client.py
--filename ws/books.csv
--kind Book
--url http://localhost:8080/load
--cookie='dev_appserver_login="******"'
"""
from google.appengine.ext import bulkload
from google.appengine.ext import search
class BookLoader(bulkload.Loader):
"Book bulkloader"
def __init__(self):
"Set the expected data structure for the CSV file"
bulkload.Loader.__init__(self, 'Book',
[('title', str), ('ident', str), ('url', str),])
def HandleEntity(self, entity):
"Make the entities searchable"
ent = search.SearchableEntity(entity)
return ent
if __name__ == '__main__':
# make the script importable
bulkload.main(BookLoader())
--url=http://localhost:8080/load-ponto --kind=Ponto --cookie "dev_appserver_login=\"[email protected]:True:185804764220139124118\"" """ from google.appengine.ext import bulkload from google.appengine.api import datastore_types from google.appengine.ext import search from models import Ponto, Linha import geohash class PontoLoader(bulkload.Loader): def __init__(self): bulkload.Loader.__init__( self, "Ponto", [("nome", lambda x: x.decode("utf-8")), ("ordem", int), ("lat", float), ("lng", float)] ) def HandleEntity(self, entity): linha = Linha.all().filter("nome =", entity["nome"]).fetch(1)[0] ponto = Ponto(ordem=entity["ordem"], linha=linha, lat=entity["lat"], lng=entity["lng"]) pontosAnt = Ponto.all().filter("linha =", linha).filter("ordem =", ponto.ordem - 1).fetch(1) if pontosAnt: pontoAnt = pontosAnt[0] ponto.setNearhash(pontoAnt) ponto.put() return None if __name__ == "__main__": bulkload.main(PontoLoader())
from google.appengine.ext import bulkload
from google.appengine.api import datastore_types
from google.appengine.ext import search
class GameLoader(bulkload.Loader):
def __init__(self):
# Our 'Game' entity contains a a lot of parameters
bulkload.Loader.__init__(self, 'Game',
[
('created_on', str),
])
def HandleEntity(self, entity):
ent = search.SearchableEntity(entity)
return ent
if __name__ == '__main__':
bulkload.main(GameLoader())
('to_icao_id',str),
])
def HandleEntity(self, entity):
logging.error ("HandleEntity %s " ,entity)
to_icao_id=entity['to_icao_id']
if to_icao_id:
bTry=True
while bTry:
try:
airportObj = models.Airport.gql("Where icao_id=:1",to_icao_id).get()
bTry=False
except OSError:
logging.error("Got OS error")
#Add the key
entity['airport_to']=airportObj.key()
del entity['to_icao_id']
entity['sequence_nr']=self.sequence_nr
self.sequence_nr= self.sequence_nr +1
logging.error ("entity right before return %s " ,entity)
return entity
if __name__ == '__main__':
bulkload.main(AirportRouteLoader())
from datetime import datetime
from google.appengine.ext import bulkload
from google.appengine.api import datastore_types
class StatusLoader(bulkload.Loader):
def __init__(self):
bulkload.Loader.__init__(self, 'Status',
[('user', datastore_types.users.User),
('beer', str),
('amount', int),
('status', datastore_types.Text),
('updated_at', lambda x: datetime.strptime(x, '%Y-%m-%d %H:%M:%S')),
])
if __name__ == '__main__':
bulkload.main(StatusLoader())
from google.appengine.ext import bulkload
from google.appengine.ext import db
from google.appengine.api import datastore_types
from google.appengine.ext import search
import model
class EntryLoader(bulkload.Loader):
def __init__(self):
bulkload.Loader.__init__(self, 'FetchedEntry', [
('group_id', str),
('name', str),
('package', str),
('type', str),
])
def HandleEntity(self, entity):
group = model.Group.get(db.Key(encoded=entity['group_id']))
name = entity['name']
package = entity['package']
type = entity['type']
entry = model.Entry.CreateAndSave(group, name, package, type)
return None
if __name__ == '__main__':
bulkload.main(EntryLoader())
("urn", int),
("local_authority_id", int),
("local_authority_name", str),
("estab", int),
("school_name", str),
("street", str),
("locality", str),
("address_3", str),
("town", str),
("county", str),
("postcode", str),
("telephone_std", str),
("telephone_number", str),
("head_title", str),
("head_first_name", str),
("head_last_name", str),
("head_honours", str),
("establishment_type", str),
("education_phase", str),
("statutory_lowest_age", int),
("statutory_highest_age", int),
])
def HandleEntity(self, entity):
ent = search.SearchableEntity(entity)
models.increment_counter("school_total")
return ent
if __name__ == '__main__':
bulkload.main(SchoolLoader())
após um login, e a URL para a do servidor se for o caso): python2.5 /usr/local/bin/bulkload_client.py --filename=linhas.csv --url=http://localhost:8080/load-linha --kind=Linha --cookie "dev_appserver_login=\"[email protected]:True:185804764220139124118\"" """ from google.appengine.ext import bulkload from google.appengine.api import datastore_types from google.appengine.ext import search from models import Ponto, Linha import geohash from google.appengine.api.datastore_types import Text class LinhaLoader(bulkload.Loader): def __init__(self): bulkload.Loader.__init__(self, 'Linha', [('nome', lambda x: x.decode('utf-8')), ('url', str)]) def HandleEntity(self, entity): for linha in Linha.all().filter("nome =", entity["nome"]).fetch(999): for ponto in linha.pontos: ponto.delete() linha.delete() return entity if __name__ == '__main__': bulkload.main(LinhaLoader())
class ClazzLoader( bulkload.Loader ):
def __init__( self ):
bulkload.Loader.__init__( self, 'Clazz',
[
( 'jar_name', str ),
( 'class_name' , str )
]
)
def HandleEntity( self, entity ):
#Obtain the Jar entity associated to this model
j_name = entity[ 'jar_name' ]
jar = datastore.Query( 'Jar', { 'jar_name': j_name } ).Get( 1 )
if len( jar ) > 0:
old_class = datastore.Query( 'Clazz', { 'class_name': entity[ 'class_name' ] } ).Get( 1 )
if old_class:
logging.info("ya existe")
else:
logging.info("creando")
newent = datastore.Entity( 'Clazz' )
newent[ 'jar' ] = jar[0].key()
newent[ 'class_name' ] = entity[ 'class_name' ]
ent = search.SearchableEntity( newent )
return ent
else:
logging.error( "Failed to insert class " + entity[ 'class_name' ] + " parent jar does not exist: " + j_name )
if __name__ == '__main__':
bulkload.main( ClazzLoader() )
from google.appengine.ext import bulkload
from google.appengine.api import datastore_types
from google.appengine.ext import search
class TopicLoader(bulkload.Loader):
def __init__(self):
bulkload.Loader.__init__(self, 'FreebaseTopic',
[('topic', str),
('type', str),
])
def HandleEntity(self, entity):
ent = search.SearchableEntity(entity)
return ent
if __name__ == '__main__':
bulkload.main(TopicLoader())
from datetime import datetime
from google.appengine.ext import bulkload
from google.appengine.api import datastore_types
class TotalAmountLoader(bulkload.Loader):
def __init__(self):
bulkload.Loader.__init__(self, 'TotalAmount',
[('user', datastore_types.users.User),
('amount', int),
('nama_cup', int),
('nama_small', int),
('nama_middle', int),
('nama_big', int),
('kan_350', int),
('kan_500', int),
('updated_at', lambda x: datetime.strptime(x, '%Y-%m-%d %H:%M:%S')),
])
if __name__ == '__main__':
bulkload.main(TotalAmountLoader())
from google.appengine.ext import bulkload
from google.appengine.api import datastore_types
from google.appengine.ext import db
from google.appengine.api import users
import datetime
class OrderLoader(bulkload.Loader):
def __init__(self):
# Loads orders
bulkload.Loader.__init__(self, 'Order',
[('client', str),
('status', str),
('createdOn', lambda x: datetime.datetime.strptime(x, "%d/%m/%Y %H:%M")),
])
if __name__ == '__main__':
bulkload.main(OrderLoader())
