def execute(self,job,previous_state): p = job.publish #import ipdb;ipdb.set_trace() # Write JSON output file json_out = {} json_out["name"] = p.table_name json_out["job_id"] = job.id json_out["job_batch_id"] = job.batch_id json_out["schema"] = p.workspace.publish_schema json_out["data_schema"] = p.workspace.publish_data_schema json_out["outdated_schema"] = p.workspace.publish_outdated_schema json_out["workspace"] = p.workspace.name json_out["channel"] = p.workspace.publish_channel.name json_out["spatial_data"] = SpatialTable.check_spatial(job.publish.spatial_type) json_out["spatial_type"] = SpatialTable.get_spatial_type_desc(job.publish.spatial_type) json_out["sync_postgres_data"] = p.workspace.publish_channel.sync_postgres_data json_out["sync_geoserver_data"] = p.workspace.publish_channel.sync_geoserver_data json_out["dump_path"] = "{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, p.pgdump_file.path) json_out["data_md5"] = file_md5(p.pgdump_file.path) json_out["preview_path"] = "{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, settings.PREVIEW_ROOT) json_out["applications"] = ["{0}:{1}".format(o.application,o.order) for o in Application_Layers.objects.filter(publish=p)] json_out["title"] = p.title json_out["abstract"] = p.abstract json_out["allow_authenticated"] = p.workspace.allow_authenticated if p.geoserver_setting: json_out["geoserver_setting"] = json.loads(p.geoserver_setting) if p.workspace.publish_channel.sync_geoserver_data and p.style_file: json_out["style_path"] = "{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, p.style_file.path) json_out["style_md5"] = file_md5(p.style_file.path) #bbox if SpatialTable.check_spatial(job.publish.spatial_type): cursor=connection.cursor() st = SpatialTable.get_instance(cursor,p.workspace.schema,p.table_name,True) if st.geometry_columns: json_out["bbox"] = st.geometry_columns[0][2] elif st.geography_columns: json_out["bbox"] = st.geography_columns[0][2] #create the dir if required if not os.path.exists(os.path.dirname(p.output_filename_abs)): os.makedirs(os.path.dirname(p.output_filename_abs)) with open(p.output_filename_abs, "wb") as output: json.dump(json_out, output, indent=4) # Try and add file to repository, if no changes then continue hg = hglib.open(BorgConfiguration.BORG_STATE_REPOSITORY) try: hg.add(files=[p.output_filename_abs]) hg.commit(include=[p.output_filename_abs],addremove=True, user=BorgConfiguration.BORG_STATE_USER, message="{} - updated {}.{}".format(p.job_batch_id, p.workspace.name, p.name)) except hglib.error.CommandError as e: if e.out != "nothing changed\n": return (HarvestStateOutcome.failed, self.get_exception_message()) finally: hg.close() return (HarvestStateOutcome.succeed, None)
def update_catalogue_service(self,md5=False,extra_datas=None): meta_data = self.builtin_metadata if extra_datas: meta_data.update(extra_datas) bbox = meta_data.get("bounding_box",None) crs = meta_data.get("crs",None) #update catalog service res = requests.post("{}/catalogue/api/records/?style_content=true".format(settings.CSW_URL),json=meta_data,auth=(settings.CSW_USER,settings.CSW_PASSWORD)) if 400 <= res.status_code < 600 and res.content: res.reason = "{}({})".format(res.reason,res.content) res.raise_for_status() meta_data = res.json() #process styles styles = meta_data.get("styles",[]) #filter out qml and lyr styles sld_styles = [s for s in meta_data.get("styles",[]) if s["format"].lower() == "sld"] meta_data["styles"] = {} style_dump_dir = BorgConfiguration.LIVE_LAYER_DIR if not os.path.exists(style_dump_dir): os.makedirs(style_dump_dir) for style in sld_styles: if style["default"]: #default sld file meta_data["default_style"] = style["name"] #write the style into file system style_file = os.path.join(style_dump_dir,"{}.{}.{}.sld".format(self.datasource.workspace.name,self.kmi_name,style["name"])) with open(style_file,"wb") as f: f.write(style["raw_content"].decode("base64")) if md5: meta_data["styles"][style["name"]] = {"file":"{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, style_file),"default":style["default"],"md5":file_md5(style_file)} else: meta_data["styles"][style["name"]] = {"file":"{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, style_file),"default":style["default"]} #add extra data to meta data meta_data["workspace"] = self.datasource.workspace.name meta_data["schema"] = self.datasource.schema meta_data["name"] = self.kmi_name meta_data["table"] = self.table meta_data["datastore"] = self.datasource.name meta_data["auth_level"] = self.datasource.workspace.auth_level meta_data["preview_path"] = "{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, BorgConfiguration.PREVIEW_DIR) meta_data["spatial_data"] = SpatialTable.check_spatial(self.spatial_type) meta_data["spatial_type"] = SpatialTable.get_spatial_type_desc(self.spatial_type) meta_data["channel"] = self.datasource.workspace.publish_channel.name meta_data["sync_geoserver_data"] = self.datasource.workspace.publish_channel.sync_geoserver_data if self.geoserver_setting: meta_data["geoserver_setting"] = json.loads(self.geoserver_setting) #bbox if "bounding_box" in meta_data: del meta_data["bounding_box"] meta_data["bbox"] = bbox meta_data["crs"] = crs return meta_data
def spatial_type_desc(self,o): return SpatialTable.get_spatial_type_desc(o.spatial_type)
def spatial_type_desc(self, o): return SpatialTable.get_spatial_type_desc(o.spatial_type)