def execute(self,job,previous_state):
p = job.publish
#import ipdb;ipdb.set_trace()
# Write JSON output file
json_out = {}
json_out["name"] = p.table_name
json_out["job_id"] = job.id
json_out["job_batch_id"] = job.batch_id
json_out["schema"] = p.workspace.publish_schema
json_out["data_schema"] = p.workspace.publish_data_schema
json_out["outdated_schema"] = p.workspace.publish_outdated_schema
json_out["workspace"] = p.workspace.name
json_out["channel"] = p.workspace.publish_channel.name
json_out["spatial_data"] = SpatialTable.check_spatial(job.publish.spatial_type)
json_out["spatial_type"] = SpatialTable.get_spatial_type_desc(job.publish.spatial_type)
json_out["sync_postgres_data"] = p.workspace.publish_channel.sync_postgres_data
json_out["sync_geoserver_data"] = p.workspace.publish_channel.sync_geoserver_data
json_out["dump_path"] = "{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, p.pgdump_file.path)
json_out["data_md5"] = file_md5(p.pgdump_file.path)
json_out["preview_path"] = "{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, settings.PREVIEW_ROOT)
json_out["applications"] = ["{0}:{1}".format(o.application,o.order) for o in Application_Layers.objects.filter(publish=p)]
json_out["title"] = p.title
json_out["abstract"] = p.abstract
json_out["allow_authenticated"] = p.workspace.allow_authenticated
if p.geoserver_setting:
json_out["geoserver_setting"] = json.loads(p.geoserver_setting)
if p.workspace.publish_channel.sync_geoserver_data and p.style_file:
json_out["style_path"] = "{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, p.style_file.path)
json_out["style_md5"] = file_md5(p.style_file.path)
#bbox
if SpatialTable.check_spatial(job.publish.spatial_type):
cursor=connection.cursor()
st = SpatialTable.get_instance(cursor,p.workspace.schema,p.table_name,True)
if st.geometry_columns:
json_out["bbox"] = st.geometry_columns[0][2]
elif st.geography_columns:
json_out["bbox"] = st.geography_columns[0][2]
#create the dir if required
if not os.path.exists(os.path.dirname(p.output_filename_abs)):
os.makedirs(os.path.dirname(p.output_filename_abs))
with open(p.output_filename_abs, "wb") as output:
json.dump(json_out, output, indent=4)
# Try and add file to repository, if no changes then continue
hg = hglib.open(BorgConfiguration.BORG_STATE_REPOSITORY)
try:
hg.add(files=[p.output_filename_abs])
hg.commit(include=[p.output_filename_abs],addremove=True, user=BorgConfiguration.BORG_STATE_USER, message="{} - updated {}.{}".format(p.job_batch_id, p.workspace.name, p.name))
except hglib.error.CommandError as e:
if e.out != "nothing changed\n":
return (HarvestStateOutcome.failed, self.get_exception_message())
finally:
hg.close()
return (HarvestStateOutcome.succeed, None)
def update_catalogue_service(self,md5=False,extra_datas=None):
meta_data = self.builtin_metadata
if extra_datas:
meta_data.update(extra_datas)
bbox = meta_data.get("bounding_box",None)
crs = meta_data.get("crs",None)
#update catalog service
res = requests.post("{}/catalogue/api/records/?style_content=true".format(settings.CSW_URL),json=meta_data,auth=(settings.CSW_USER,settings.CSW_PASSWORD))
if 400 <= res.status_code < 600 and res.content:
res.reason = "{}({})".format(res.reason,res.content)
res.raise_for_status()
meta_data = res.json()
#process styles
styles = meta_data.get("styles",[])
#filter out qml and lyr styles
sld_styles = [s for s in meta_data.get("styles",[]) if s["format"].lower() == "sld"]
meta_data["styles"] = {}
style_dump_dir = BorgConfiguration.LIVE_LAYER_DIR
if not os.path.exists(style_dump_dir):
os.makedirs(style_dump_dir)
for style in sld_styles:
if style["default"]:
#default sld file
meta_data["default_style"] = style["name"]
#write the style into file system
style_file = os.path.join(style_dump_dir,"{}.{}.{}.sld".format(self.datasource.workspace.name,self.kmi_name,style["name"]))
with open(style_file,"wb") as f:
f.write(style["raw_content"].decode("base64"))
if md5:
meta_data["styles"][style["name"]] = {"file":"{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, style_file),"default":style["default"],"md5":file_md5(style_file)}
else:
meta_data["styles"][style["name"]] = {"file":"{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, style_file),"default":style["default"]}
#add extra data to meta data
meta_data["workspace"] = self.datasource.workspace.name
meta_data["schema"] = self.datasource.schema
meta_data["name"] = self.kmi_name
meta_data["table"] = self.table
meta_data["datastore"] = self.datasource.name
meta_data["auth_level"] = self.datasource.workspace.auth_level
meta_data["preview_path"] = "{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, BorgConfiguration.PREVIEW_DIR)
meta_data["spatial_data"] = SpatialTable.check_spatial(self.spatial_type)
meta_data["spatial_type"] = SpatialTable.get_spatial_type_desc(self.spatial_type)
meta_data["channel"] = self.datasource.workspace.publish_channel.name
meta_data["sync_geoserver_data"] = self.datasource.workspace.publish_channel.sync_geoserver_data
if self.geoserver_setting:
meta_data["geoserver_setting"] = json.loads(self.geoserver_setting)
#bbox
if "bounding_box" in meta_data:
del meta_data["bounding_box"]
meta_data["bbox"] = bbox
meta_data["crs"] = crs
return meta_data
def _post_clean(self):
super(PublishForm,self)._post_clean()
if self.errors:
return
#populate the value of the relation columns
if 'dependents' in self.cleaned_data:
sorted_dependents = self.cleaned_data['dependents'].order_by('pk')
else:
sorted_dependents = []
self.instance.init_relations()
pos = 0
normal_table_pos = 0
length = len(sorted_dependents)
for relation in (self.instance.relations):
normal_table_pos = 0
for normal_table in relation.normal_tables:
relation.set_normal_table(normal_table_pos, sorted_dependents[pos] if pos < length else None)
pos += 1
normal_table_pos += 1
if self.instance and SpatialTable.check_spatial(self.instance.spatial_type):
self.set_setting_to_model()
def builtin_metadata(self):
meta_data = {}
meta_data["workspace"] = self.datasource.workspace.name
meta_data["name"] = self.kmi_name
meta_data["service_type"] = "WMS"
if SpatialTable.check_normal(self.spatial_type) or not self.datasource.workspace.publish_channel.sync_geoserver_data:
meta_data["service_type"] = ""
elif SpatialTable.check_raster(self.spatial_type):
meta_data["service_type"] = "WMS"
meta_data["service_type_version"] = self.datasource.workspace.publish_channel.wms_version
else:
meta_data["service_type"] = "WFS"
meta_data["service_type_version"] = self.datasource.workspace.publish_channel.wfs_version
meta_data["modified"] = (self.last_modify_time or self.last_refresh_time).astimezone(timezone.get_default_timezone()).strftime("%Y-%m-%d %H:%M:%S.%f")
#bbox
meta_data["bounding_box"] = self.bbox or None
meta_data["crs"] = self.crs or None
#ows resource
meta_data["ows_resource"] = {}
if meta_data["service_type"] == "WFS" and self.datasource.workspace.publish_channel.wfs_endpoint:
meta_data["ows_resource"]["wfs"] = True
meta_data["ows_resource"]["wfs_version"] = self.datasource.workspace.publish_channel.wfs_version
meta_data["ows_resource"]["wfs_endpoint"] = self.datasource.workspace.publish_channel.wfs_endpoint
if meta_data["service_type"] in ("WFS","WMS") and self.datasource.workspace.publish_channel.wfs_endpoint:
meta_data["ows_resource"]["wms"] = True
meta_data["ows_resource"]["wms_version"] = self.datasource.workspace.publish_channel.wms_version
meta_data["ows_resource"]["wms_endpoint"] = self.datasource.workspace.publish_channel.wms_endpoint
geo_settings = json.loads(self.geoserver_setting) if self.geoserver_setting else {}
if geo_settings.get("create_cache_layer",False) and self.datasource.workspace.publish_channel.gwc_endpoint:
meta_data["ows_resource"]["gwc"] = True
meta_data["ows_resource"]["gwc_endpoint"] = self.datasource.workspace.publish_channel.gwc_endpoint
return meta_data
def refresh(self,time=None):
self.try_begin_transaction("livelayer_refresh")
try:
time = time or timezone.now()
if Layer.is_system_table(self.table):
return False
st = SpatialTable.get_instance(self.datasource.schema,self.table,refresh=True,bbox=True,crs=True,dbUtil=self.datasource.dbUtil)
self.last_refresh_time = time
self.crs = st.crs
self.bbox = json.dumps(st.bbox)
self.spatial_type = st.spatial_type
sql = self.datasource.dbUtil.get_create_table_sql(self.table,self.datasource.schema)
if not self.sql or self.sql != sql:
self.last_modify_time = time
self.status = self.next_status(ResourceAction.UPDATE)
self.sql = sql
self.save()
return True
finally:
self.try_clear_transaction("livelayer_refresh")
def spatial_type_desc(self,o):
return SpatialTable.get_spatial_type_desc(o.spatial_type)
def get_form(self, request, obj=None, **kwargs):
# Proper kwargs are form, fields, exclude, formfield_callback
if obj and SpatialTable.check_normal(obj.spatial_type):
return NormalTablePublishForm
else:
return PublishForm
def get_fields(self, request, obj=None):
if obj and SpatialTable.check_normal(obj.spatial_type):
base_fields = ['name','workspace','interval','status','input_table','dependents','priority','sql','create_extra_index_sql']
else:
base_fields = ['name','workspace','interval','status','input_table','dependents','priority','sql','create_extra_index_sql',"create_cache_layer","server_cache_expire","client_cache_expire"]
return base_fields + list(self.get_readonly_fields(request, obj))
def spatial_type_desc(self, o):
return SpatialTable.get_spatial_type_desc(o.spatial_type)
def get_form(self, request, obj=None, **kwargs):
# Proper kwargs are form, fields, exclude, formfield_callback
if obj and SpatialTable.check_normal(obj.spatial_type):
return NormalTablePublishForm
else:
return PublishForm