def perform(self, url):
#Download the file
r = requests.get(url)
self.update_state(state='DOWNLOADED')
if r.status_code != 200:
raise Exception("Bad status code {} {}".format(url, r.status_code))
#Save it in a temp file
d = tempfile.mkdtemp()
filename = os.path.join(d, "gtfs.zip")
with open(filename, "wb") as f:
f.write(r.content)
#Load it
accumulator = Accumulator()
problems = transitfeed.ProblemReporter(accumulator)
gtfs_factory = transitfeed.GetGtfsFactory()
loader = gtfs_factory.Loader(filename,
problems=problems,
extra_validation=False,
gtfs_factory=gtfs_factory)
schedule = loader.Load() #this is a transport schedule
self.update_state(state='LOADED')
#Remove tempfiles
os.remove(filename)
os.rmdir(d)
#Run validation & return results
schedule.Validate(validate_children=False)
self.update_state(state=states.SUCCESS)
accumulator = problems.GetAccumulator()
validations = {
key: [{
"type": str(type(value)),
"dict": value.__dict__
} for value in getattr(accumulator, key)]
for key in ["errors", "warnings", "notices"]
}
#Get dataset end date
end_date = max([v.end_date for v in schedule.service_periods.itervalues()])
dataset = app.backend.database['datasets'].find_one(
{"celery_task_id": self.request.id})
anomalies = set(dataset['anomalies'])
if end_date < datetime.date.today().strftime("%Y%m%d"):
anomalies = anomalies.add("out_of_date")
elif "out_of_date" in anomalies:
anomalies = anomalies.remove("out_of_date")
if anomalies is None:
anomalies = set()
app.backend.database['datasets'].find_one_and_update(
{"celery_task_id": self.request.id},
{"$set": {
"validations": validations,
"anomalies": list(anomalies)
}})
return {"validations": validations}
def testClasses(self):
class FakeGtfsFactory(object):
pass
factory = transitfeed.GetGtfsFactory()
gtfs_class_instances = [factory.Shape("id"), factory.ShapePoint()]
gtfs_class_instances += [
factory.GetGtfsClassByFileName(filename)()
for filename in factory.GetLoadingOrder()
]
for instance in gtfs_class_instances:
self.AssertDefaultFactoryIsReturnedIfNoneIsSet(instance)
self.AssertFactoryIsSavedAndReturned(instance, FakeGtfsFactory())
def MakeLoaderAndLoad(self,
problems=None,
extra_validation=True,
gtfs_factory=None):
"""Returns a Schedule loaded with the contents of the file dict."""
if gtfs_factory is None:
gtfs_factory = transitfeed.GetGtfsFactory()
if problems is None:
problems = self.problems
self.CreateZip()
self.loader = transitfeed.Loader(problems=problems,
extra_validation=extra_validation,
zip=self.zip,
gtfs_factory=gtfs_factory)
return self.loader.Load()
def GetGtfsFactory(factory=None):
if not factory:
factory = transitfeed.GetGtfsFactory()
# Agency class extension
factory.UpdateClass('Agency', agency.Agency)
# FareAttribute class extension
factory.UpdateClass('FareAttribute', fareattribute.FareAttribute)
# Route class extension
factory.UpdateClass('Route', route.Route)
# Stop class extension
factory.UpdateClass('Stop', stop.Stop)
return factory
def GetGtfsFactory(factory=None):
if not factory:
factory = transitfeed.GetGtfsFactory()
# Agency class extension
factory.UpdateClass("Agency", agency.Agency)
# FareAttribute class extension
factory.UpdateClass("FareAttribute", fareattribute.FareAttribute)
# Route class extension
factory.UpdateClass("Route", route.Route)
# Stop class extension
factory.UpdateClass("Stop", stop.Stop)
return factory
def setUp(self):
self._factory = transitfeed.GetGtfsFactory()
def setUp(self):
super(DeprecatedFieldNamesTestCase, self).setUp()
# init a new gtfs_factory instance and update its class mappings
self.gtfs_factory = transitfeed.GetGtfsFactory()
self.gtfs_factory.UpdateClass("Agency", self.Agency)
self.gtfs_factory.UpdateClass("Stop", self.Stop)