def update(update_log=None, slug=None, trigger=None):
if slug:
update_definition = UpdateDefinition.objects.get(slug=slug)
update_definition.queue(silent=True, trigger=trigger)
return
elif not update_log:
raise ValueError("One of update_log and slug needs to be provided.")
# A mapping from Store to graph names
store_graphs = collections.defaultdict(set)
variables = update_log.update_definition.variables.all()
variables = dict((v.name, v.value) for v in variables)
with logged(update_log):
for pipeline in update_log.update_definition.pipelines.all():
for store in pipeline.stores.all():
output_directory = tempfile.mkdtemp()
transform_manager = TransformManager(update_log,
output_directory,
variables,
force=update_log.forced,
store_graphs=store_graphs,
store=store)
try:
transform = evaluate_pipeline(pipeline.value.strip())
except SyntaxError:
raise ValueError("Couldn't parse the given pipeline: %r" % pipeline.value.strip())
try:
transform(transform_manager)
except NotChanged:
logger.info("Aborted update as data hasn't changed")
except TransformException:
logger.exception("Transform failed.")
except Exception:
logger.exception("Transform failed, perhaps ungracefully.")
finally:
shutil.rmtree(output_directory)
updated = _time_zone.localize(datetime.datetime.now())
store_graphs = dict((store, frozenset(store_graphs[store])) for store in store_graphs)
for store in store_graphs:
graphs_updated.send(update,
store=store,
graphs=store_graphs[store],
when=updated)
update_completed.send(update,
update_definition=update_log.update_definition,
store_graphs=store_graphs,
when=updated)
def update(update_log=None, slug=None, trigger=None):
if slug:
update_definition = UpdateDefinition.objects.get(slug=slug)
update_definition.queue(silent=True, trigger=trigger)
return
elif not update_log:
raise ValueError("One of update_log and slug needs to be provided.")
graphs_touched = collections.defaultdict(set)
variables = update_log.update_definition.variables.all()
variables = dict((v.name, v.value) for v in variables)
with logged(update_log):
for pipeline in update_log.update_definition.pipelines.all():
for store in pipeline.stores.all():
output_directory = tempfile.mkdtemp()
transform_manager = TransformManager(update_log,
output_directory,
variables,
force=update_log.forced,
graphs_touched=graphs_touched,
store=store)
try:
transform = evaluate_pipeline(pipeline.value.strip())
except SyntaxError:
raise ValueError("Couldn't parse the given pipeline: %r" % pipeline.value.strip())
try:
transform(transform_manager)
except NotChanged:
logger.info("Aborted update as data hasn't changed")
except TransformException, e:
logger.exception("Transform failed.")
except Exception, e:
logger.exception("Transform failed, perhaps ungracefully.")
finally:
def save(self, *args, **kwargs):
try:
evaluate_pipeline(self.value)
except (SyntaxError, NameError), e:
raise ValueError(e)
def save(self, *args, **kwargs):
try:
evaluate_pipeline(self.value)
except (SyntaxError, NameError), e:
raise ValueError(e)