Exemple #1
0
    def execute(self,job,previous_state):
        p = job.publish

        #import ipdb;ipdb.set_trace()
        # Write JSON output file
        json_out = {}
        json_out["name"] = p.table_name
        json_out["job_id"] = job.id
        json_out["job_batch_id"] = job.batch_id
        json_out["schema"] = p.workspace.publish_schema
        json_out["data_schema"] = p.workspace.publish_data_schema
        json_out["outdated_schema"] = p.workspace.publish_outdated_schema
        json_out["workspace"] = p.workspace.name
        json_out["channel"] = p.workspace.publish_channel.name
        json_out["spatial_data"] = SpatialTable.check_spatial(job.publish.spatial_type)
        json_out["spatial_type"] = SpatialTable.get_spatial_type_desc(job.publish.spatial_type)
        json_out["sync_postgres_data"] = p.workspace.publish_channel.sync_postgres_data
        json_out["sync_geoserver_data"] = p.workspace.publish_channel.sync_geoserver_data
        json_out["dump_path"] = "{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, p.pgdump_file.path)
        json_out["data_md5"] = file_md5(p.pgdump_file.path)
        json_out["preview_path"] = "{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, settings.PREVIEW_ROOT)
        json_out["applications"] = ["{0}:{1}".format(o.application,o.order) for o in Application_Layers.objects.filter(publish=p)]
        json_out["title"] = p.title
        json_out["abstract"] = p.abstract
        json_out["allow_authenticated"] = p.workspace.allow_authenticated

        if p.geoserver_setting:
            json_out["geoserver_setting"] = json.loads(p.geoserver_setting)
        if p.workspace.publish_channel.sync_geoserver_data and p.style_file:
            json_out["style_path"] = "{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, p.style_file.path)
            json_out["style_md5"] = file_md5(p.style_file.path)

        #bbox
        if SpatialTable.check_spatial(job.publish.spatial_type):
            cursor=connection.cursor()
            st = SpatialTable.get_instance(cursor,p.workspace.schema,p.table_name,True)
            if st.geometry_columns:
                json_out["bbox"] = st.geometry_columns[0][2]
            elif st.geography_columns:
                json_out["bbox"] = st.geography_columns[0][2]

        #create the dir if required
        if not os.path.exists(os.path.dirname(p.output_filename_abs)):
            os.makedirs(os.path.dirname(p.output_filename_abs))

        with open(p.output_filename_abs, "wb") as output:
            json.dump(json_out, output, indent=4)

        # Try and add file to repository, if no changes then continue
        hg = hglib.open(BorgConfiguration.BORG_STATE_REPOSITORY)
        try:
            hg.add(files=[p.output_filename_abs])
            hg.commit(include=[p.output_filename_abs],addremove=True, user=BorgConfiguration.BORG_STATE_USER, message="{} - updated {}.{}".format(p.job_batch_id, p.workspace.name, p.name))
        except hglib.error.CommandError as e:
            if e.out != "nothing changed\n":
                return (HarvestStateOutcome.failed, self.get_exception_message())
        finally:
            hg.close()

        return (HarvestStateOutcome.succeed, None)
    def update_catalogue_service(self,md5=False,extra_datas=None):
        meta_data = self.builtin_metadata
        if extra_datas:
            meta_data.update(extra_datas)
        bbox = meta_data.get("bounding_box",None)
        crs = meta_data.get("crs",None)
        #update catalog service
        res = requests.post("{}/catalogue/api/records/?style_content=true".format(settings.CSW_URL),json=meta_data,auth=(settings.CSW_USER,settings.CSW_PASSWORD))
        if 400 <= res.status_code < 600 and res.content:
            res.reason = "{}({})".format(res.reason,res.content)
        res.raise_for_status()
        meta_data = res.json()

        #process styles
        styles = meta_data.get("styles",[])
        #filter out qml and lyr styles
        sld_styles = [s for s in meta_data.get("styles",[]) if s["format"].lower() == "sld"]
        meta_data["styles"] = {}
        style_dump_dir = BorgConfiguration.LIVE_LAYER_DIR
        if not os.path.exists(style_dump_dir):
            os.makedirs(style_dump_dir)

        for style in sld_styles:
            if style["default"]:
                #default sld file
                meta_data["default_style"] = style["name"]
            #write the style into file system
            style_file = os.path.join(style_dump_dir,"{}.{}.{}.sld".format(self.datasource.workspace.name,self.kmi_name,style["name"]))
            with open(style_file,"wb") as f:
                f.write(style["raw_content"].decode("base64"))
            if md5:
                meta_data["styles"][style["name"]] = {"file":"{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, style_file),"default":style["default"],"md5":file_md5(style_file)}
            else:
                meta_data["styles"][style["name"]] = {"file":"{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, style_file),"default":style["default"]}

        #add extra data to meta data
        meta_data["workspace"] = self.datasource.workspace.name
        meta_data["schema"] = self.datasource.schema
        meta_data["name"] = self.kmi_name
        meta_data["table"] = self.table
        meta_data["datastore"] = self.datasource.name
        meta_data["auth_level"] = self.datasource.workspace.auth_level
        meta_data["preview_path"] = "{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, BorgConfiguration.PREVIEW_DIR)
        meta_data["spatial_data"] = SpatialTable.check_spatial(self.spatial_type)
        meta_data["spatial_type"] = SpatialTable.get_spatial_type_desc(self.spatial_type)

        meta_data["channel"] = self.datasource.workspace.publish_channel.name
        meta_data["sync_geoserver_data"] = self.datasource.workspace.publish_channel.sync_geoserver_data

        if self.geoserver_setting:
            meta_data["geoserver_setting"] = json.loads(self.geoserver_setting)
                
        #bbox
        if "bounding_box" in meta_data:
            del meta_data["bounding_box"]
        meta_data["bbox"] = bbox
        meta_data["crs"] = crs

        return meta_data
    def _post_clean(self):
        super(PublishForm,self)._post_clean()
        if self.errors:
            return

        #populate the value of the relation columns
        if 'dependents' in self.cleaned_data:
            sorted_dependents = self.cleaned_data['dependents'].order_by('pk')
        else:
            sorted_dependents = []

        self.instance.init_relations()

        pos = 0
        normal_table_pos = 0
        length = len(sorted_dependents)
        for relation in (self.instance.relations):
            normal_table_pos = 0
            for normal_table in relation.normal_tables:
                relation.set_normal_table(normal_table_pos, sorted_dependents[pos] if pos < length else None)
                pos += 1
                normal_table_pos += 1
    
        if self.instance and SpatialTable.check_spatial(self.instance.spatial_type):
            self.set_setting_to_model()
    def builtin_metadata(self):
        meta_data = {}
        meta_data["workspace"] = self.datasource.workspace.name
        meta_data["name"] = self.kmi_name
        meta_data["service_type"] = "WMS"
        if SpatialTable.check_normal(self.spatial_type) or not self.datasource.workspace.publish_channel.sync_geoserver_data:
            meta_data["service_type"] = ""
        elif SpatialTable.check_raster(self.spatial_type):
            meta_data["service_type"] = "WMS"
            meta_data["service_type_version"] = self.datasource.workspace.publish_channel.wms_version
        else:
            meta_data["service_type"] = "WFS"
            meta_data["service_type_version"] = self.datasource.workspace.publish_channel.wfs_version

        meta_data["modified"] = (self.last_modify_time or self.last_refresh_time).astimezone(timezone.get_default_timezone()).strftime("%Y-%m-%d %H:%M:%S.%f")

        #bbox
        meta_data["bounding_box"] = self.bbox or None
        meta_data["crs"] = self.crs or None

        #ows resource
        meta_data["ows_resource"] = {}
        if meta_data["service_type"] == "WFS" and self.datasource.workspace.publish_channel.wfs_endpoint:
            meta_data["ows_resource"]["wfs"] = True
            meta_data["ows_resource"]["wfs_version"] = self.datasource.workspace.publish_channel.wfs_version
            meta_data["ows_resource"]["wfs_endpoint"] = self.datasource.workspace.publish_channel.wfs_endpoint

        if meta_data["service_type"] in ("WFS","WMS") and self.datasource.workspace.publish_channel.wfs_endpoint:
            meta_data["ows_resource"]["wms"] = True
            meta_data["ows_resource"]["wms_version"] = self.datasource.workspace.publish_channel.wms_version
            meta_data["ows_resource"]["wms_endpoint"] = self.datasource.workspace.publish_channel.wms_endpoint

            geo_settings = json.loads(self.geoserver_setting) if self.geoserver_setting else {}
            if geo_settings.get("create_cache_layer",False) and self.datasource.workspace.publish_channel.gwc_endpoint:
                meta_data["ows_resource"]["gwc"] = True
                meta_data["ows_resource"]["gwc_endpoint"] = self.datasource.workspace.publish_channel.gwc_endpoint

        return meta_data
    def refresh(self,time=None):
        self.try_begin_transaction("livelayer_refresh")
        try:
            time = time or timezone.now()
            if Layer.is_system_table(self.table):
                return False

            st = SpatialTable.get_instance(self.datasource.schema,self.table,refresh=True,bbox=True,crs=True,dbUtil=self.datasource.dbUtil)
            self.last_refresh_time = time
    
            self.crs = st.crs
            self.bbox = json.dumps(st.bbox)
            self.spatial_type = st.spatial_type
            sql = self.datasource.dbUtil.get_create_table_sql(self.table,self.datasource.schema)
            if not self.sql or self.sql != sql:
                self.last_modify_time = time
                self.status = self.next_status(ResourceAction.UPDATE)
                self.sql = sql
            self.save()
            return True
        finally:
            self.try_clear_transaction("livelayer_refresh")
Exemple #6
0
 def spatial_type_desc(self,o):
     return SpatialTable.get_spatial_type_desc(o.spatial_type)
Exemple #7
0
 def get_form(self, request, obj=None, **kwargs):
     # Proper kwargs are form, fields, exclude, formfield_callback
     if obj and SpatialTable.check_normal(obj.spatial_type):
         return NormalTablePublishForm
     else:
         return PublishForm
 def get_fields(self, request, obj=None):
     if obj and SpatialTable.check_normal(obj.spatial_type):
         base_fields = ['name','workspace','interval','status','input_table','dependents','priority','sql','create_extra_index_sql']
     else:
         base_fields = ['name','workspace','interval','status','input_table','dependents','priority','sql','create_extra_index_sql',"create_cache_layer","server_cache_expire","client_cache_expire"]
     return base_fields + list(self.get_readonly_fields(request, obj))
Exemple #9
0
 def spatial_type_desc(self, o):
     return SpatialTable.get_spatial_type_desc(o.spatial_type)
Exemple #10
0
 def get_form(self, request, obj=None, **kwargs):
     # Proper kwargs are form, fields, exclude, formfield_callback
     if obj and SpatialTable.check_normal(obj.spatial_type):
         return NormalTablePublishForm
     else:
         return PublishForm