def execute(self, job, previous_state):
"""
dump the full table data into download folder
"""
if 'data' in job.metadict:
del job.metadict['data']
#create the dir if required
if not os.path.exists(job.dump_dir):
#dump dir does not exist, create it
os.makedirs(job.dump_dir)
file_name = job.publish.table_name + ".db"
dump_file = os.path.join(job.dump_dir, file_name)
cmd = self.dump_cmd + [
"-t", job.publish.workspace.publish_data_schema + "." +
job.publish.table_name, "-f", dump_file
]
cursor = connection.cursor()
if not previous_state.is_error_state:
#table with same name maybe published by previous job. drop it if have.
cursor.execute('drop table if exists "{0}"."{1}" cascade'.format(
job.publish.workspace.publish_data_schema,
job.publish.table_name))
#move table to publish schema for dump
cursor.execute('alter table "{0}"."{1}" set schema {2}'.format(
job.publish.workspace.schema, job.publish.table_name,
job.publish.workspace.publish_data_schema))
try:
#import ipdb;ipdb.set_trace()
output = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self.env).communicate()
logger.debug("execute ({0})\nstdin:{1}\nstdout:{2}".format(
cmd, output[0], output[1]))
finally:
#move table back to original schema
cursor.execute('alter table "{0}"."{1}" set schema "{2}"'.format(
job.publish.workspace.publish_data_schema,
job.publish.table_name, job.publish.workspace.schema))
if output[1].strip():
return (HarvestStateOutcome.failed, output[1])
else:
job.metadict['data'] = {
"file":
"{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, dump_file),
"md5":
file_md5(dump_file)
}
return (HarvestStateOutcome.succeed, None)
def unpublish(self):
try_set_push_owner("wmsserver")
hg = None
try:
meta_data = {}
meta_data["name"] = self.name
meta_data["workspace"] = self.workspace.name
#write meta data file
file_name = "{}.meta.json".format(self.name)
meta_file = os.path.join(BorgConfiguration.UNPUBLISH_DIR,
self.workspace.publish_channel.name,
self.workspace.name, "stores", file_name)
#create the dir if required
if not os.path.exists(os.path.dirname(meta_file)):
os.makedirs(os.path.dirname(meta_file))
with open(meta_file, "wb") as output:
json.dump(meta_data, output, indent=4)
json_out = {}
json_out['meta'] = {
"file":
"{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, meta_file),
"md5":
file_md5(meta_file)
}
json_out['action'] = 'remove'
json_out["remove_time"] = timezone.localtime(
timezone.now()).strftime("%Y-%m-%d %H:%M:%S.%f")
json_filename = self.json_filename_abs('unpublish')
#create the dir if required
if not os.path.exists(os.path.dirname(json_filename)):
os.makedirs(os.path.dirname(json_filename))
with open(json_filename, "wb") as output:
json.dump(json_out, output, indent=4)
hg = hglib.open(BorgConfiguration.BORG_STATE_REPOSITORY)
hg.commit(include=[json_filename],
addremove=True,
user="******",
message="Unpublish wms store {}.{}".format(
self.workspace.name, self.name))
increase_committed_changes()
try_push_to_repository("wmsserver", hg)
finally:
if hg: hg.close()
try_clear_push_owner("wmsserver")
def execute(self, job, previous_state):
p = job.publish
meta_data = p.update_catalogue_service(
style_dump_dir=job.dump_dir,
md5=True,
extra_datas={
"publication_date":
datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")
})
#write meta data file
file_name = "{}.meta.json".format(p.table_name)
meta_file = os.path.join(job.dump_dir, file_name)
with open(meta_file, "wb") as output:
json.dump(meta_data, output, indent=4)
job.metadict['meta'] = {
"file": "{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX,
meta_file),
"md5": file_md5(meta_file)
}
return (HarvestStateOutcome.succeed, None)
def publish(self):
"""
publish layer's json reference (if exists) to the repository,
"""
json_filename = self.json_filename_abs('publish');
try_set_push_owner("livelayer")
hg = None
try:
meta_data = self.update_catalogue_service(md5=True,extra_datas={"publication_date":datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")})
#write meta data file
file_name = "{}.{}.meta.json".format(self.datasource.workspace.name,self.kmi_name)
meta_file = os.path.join(BorgConfiguration.LIVE_LAYER_DIR,file_name)
#create the dir if required
if not os.path.exists(os.path.dirname(meta_file)):
os.makedirs(os.path.dirname(meta_file))
with open(meta_file,"wb") as output:
json.dump(meta_data, output, indent=4)
json_out = {}
json_out['meta'] = {"file":"{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, meta_file),"md5":file_md5(meta_file)}
json_out['action'] = "publish"
json_out["publish_time"] = timezone.localtime(timezone.now()).strftime("%Y-%m-%d %H:%M:%S.%f")
#create the dir if required
if not os.path.exists(os.path.dirname(json_filename)):
os.makedirs(os.path.dirname(json_filename))
with open(json_filename, "wb") as output:
json.dump(json_out, output, indent=4)
hg = hglib.open(BorgConfiguration.BORG_STATE_REPOSITORY)
#remove other related json files
json_files = [ self.json_filename_abs(action) for action in [ 'empty_gwc' ] ]
#get all existing files.
json_files = [ f for f in json_files if os.path.exists(f) ]
if json_files:
hg.remove(files=json_files)
json_files.append(json_filename)
hg.commit(include=json_files,addremove=True, user="******", message="update live layer {}.{}".format(self.datasource.workspace.name, self.kmi_name))
increase_committed_changes()
try_push_to_repository("livelayer",hg)
finally:
if hg: hg.close()
try_clear_push_owner("livelayer")
def update_catalogue_service(self,md5=False,extra_datas=None):
meta_data = self.builtin_metadata
if extra_datas:
meta_data.update(extra_datas)
bbox = meta_data.get("bounding_box",None)
crs = meta_data.get("crs",None)
#update catalog service
res = requests.post("{}/catalogue/api/records/?style_content=true".format(settings.CSW_URL),json=meta_data,auth=(settings.CSW_USER,settings.CSW_PASSWORD))
if 400 <= res.status_code < 600 and res.content:
res.reason = "{}({})".format(res.reason,res.content)
res.raise_for_status()
meta_data = res.json()
#process styles
styles = meta_data.get("styles",[])
#filter out qml and lyr styles
sld_styles = [s for s in meta_data.get("styles",[]) if s["format"].lower() == "sld"]
meta_data["styles"] = {}
style_dump_dir = BorgConfiguration.LIVE_LAYER_DIR
if not os.path.exists(style_dump_dir):
os.makedirs(style_dump_dir)
for style in sld_styles:
if style["default"]:
#default sld file
meta_data["default_style"] = style["name"]
#write the style into file system
style_file = os.path.join(style_dump_dir,"{}.{}.{}.sld".format(self.datasource.workspace.name,self.kmi_name,style["name"]))
with open(style_file,"wb") as f:
f.write(style["raw_content"].decode("base64"))
if md5:
meta_data["styles"][style["name"]] = {"file":"{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, style_file),"default":style["default"],"md5":file_md5(style_file)}
else:
meta_data["styles"][style["name"]] = {"file":"{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, style_file),"default":style["default"]}
#add extra data to meta data
meta_data["workspace"] = self.datasource.workspace.name
meta_data["schema"] = self.datasource.schema
meta_data["name"] = self.kmi_name
meta_data["table"] = self.table
meta_data["datastore"] = self.datasource.name
meta_data["auth_level"] = self.datasource.workspace.auth_level
meta_data["preview_path"] = "{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, BorgConfiguration.PREVIEW_DIR)
meta_data["spatial_data"] = SpatialTable.check_spatial(self.spatial_type)
meta_data["spatial_type"] = SpatialTable.get_spatial_type_desc(self.spatial_type)
meta_data["channel"] = self.datasource.workspace.publish_channel.name
meta_data["sync_geoserver_data"] = self.datasource.workspace.publish_channel.sync_geoserver_data
if self.geoserver_setting:
meta_data["geoserver_setting"] = json.loads(self.geoserver_setting)
#bbox
if "bounding_box" in meta_data:
del meta_data["bounding_box"]
meta_data["bbox"] = bbox
meta_data["crs"] = crs
return meta_data
def publish(self):
"""
publish store's json reference (if exists) to the repository,
"""
try_set_push_owner("liveserver")
hg = None
try:
meta_data = {}
meta_data["name"] = self.name
meta_data["host"] = self.host
meta_data["port"] = self.port
meta_data["database"] = self.db_name
meta_data["user"] = self.user
meta_data["passwd"] = self.password
meta_data["schema"] = self.schema
meta_data["workspace"] = self.workspace.name
if self.geoserver_setting:
meta_data["geoserver_setting"] = json.loads(self.geoserver_setting)
#write meta data file
file_name = "{}.{}.meta.json".format(self.workspace.name,self.name)
meta_file = os.path.join(BorgConfiguration.LIVE_STORE_DIR,file_name)
#create the dir if required
if not os.path.exists(os.path.dirname(meta_file)):
os.makedirs(os.path.dirname(meta_file))
with open(meta_file,"wb") as output:
json.dump(meta_data, output, indent=4)
json_out = {}
json_out['meta'] = {"file":"{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, meta_file),"md5":file_md5(meta_file)}
json_out['action'] = 'publish'
json_out["publish_time"] = timezone.localtime(timezone.now()).strftime("%Y-%m-%d %H:%M:%S.%f")
json_filename = self.json_filename_abs('publish');
#create the dir if required
if not os.path.exists(os.path.dirname(json_filename)):
os.makedirs(os.path.dirname(json_filename))
with open(json_filename, "wb") as output:
json.dump(json_out, output, indent=4)
hg = hglib.open(BorgConfiguration.BORG_STATE_REPOSITORY)
hg.commit(include=[json_filename],addremove=True, user="******", message="Update live store {}.{}".format(self.workspace.name, self.name))
increase_committed_changes()
try_push_to_repository("liveserver",hg)
finally:
if hg: hg.close()
try_clear_push_owner("liveserver")
def unpublish(self):
"""
unpublish layer group
"""
#remove it from catalogue service
res = requests.delete("{}/catalogue/api/records/{}:{}/".format(
settings.CSW_URL, self.workspace.name, self.name),
auth=(settings.CSW_USER, settings.CSW_PASSWORD))
if res.status_code != 404:
res.raise_for_status()
json_filename = self.json_filename_abs('unpublish')
try_set_push_owner("layergroup")
hg = None
try:
meta_data = {}
#add extra data to meta data
meta_data["workspace"] = self.workspace.name
meta_data["name"] = self.name
meta_data["native_name"] = self.name
meta_data["auth_level"] = self.workspace.auth_level
meta_data["spatial_data"] = True
meta_data["channel"] = self.workspace.publish_channel.name
meta_data[
"sync_geoserver_data"] = self.workspace.publish_channel.sync_geoserver_data
#write meta data file
file_name = "{}.meta.json".format(self.name)
meta_file = os.path.join(BorgConfiguration.UNPUBLISH_DIR,
self.workspace.publish_channel.name,
self.workspace.name, "layergroups",
file_name)
#create the dir if required
if not os.path.exists(os.path.dirname(meta_file)):
os.makedirs(os.path.dirname(meta_file))
with open(meta_file, "wb") as output:
json.dump(meta_data, output, indent=4)
json_out = {}
json_out['meta'] = {
"file":
"{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, meta_file),
"md5":
file_md5(meta_file)
}
json_out['action'] = "remove"
json_out["remove_time"] = timezone.localtime(
timezone.now()).strftime("%Y-%m-%d %H:%M:%S.%f")
#create the dir if required
if not os.path.exists(os.path.dirname(json_filename)):
os.makedirs(os.path.dirname(json_filename))
with open(json_filename, "wb") as output:
json.dump(json_out, output, indent=4)
hg = hglib.open(BorgConfiguration.BORG_STATE_REPOSITORY)
#remove other related json files
json_files = [
self.json_filename_abs(action) for action in ['empty_gwc']
]
#get all existing files.
json_files = [f for f in json_files if os.path.exists(f)]
if json_files:
hg.remove(files=json_files)
json_files.append(json_filename)
hg.commit(include=json_files,
user="******",
addremove=True,
message="Unpublish layer group {}.{}".format(
self.workspace.name, self.name))
increase_committed_changes()
try_push_to_repository("layergroup", hg)
finally:
if hg: hg.close()
try_clear_push_owner("layergroup")
def publish(self):
"""
publish layer's json reference (if exists) to the repository,
"""
json_filename = self.json_filename_abs('publish')
try_set_push_owner("wmslayer")
hg = None
try:
meta_data = self.update_catalogue_service(
extra_datas={
"publication_date":
datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")
})
#write meta data file
file_name = "{}.{}.meta.json".format(self.server.workspace.name,
self.kmi_name)
meta_file = os.path.join(BorgConfiguration.WMS_LAYER_DIR,
file_name)
#create the dir if required
if not os.path.exists(os.path.dirname(meta_file)):
os.makedirs(os.path.dirname(meta_file))
with open(meta_file, "wb") as output:
json.dump(meta_data, output, indent=4)
json_out = {}
json_out['meta'] = {
"file":
"{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, meta_file),
"md5":
file_md5(meta_file)
}
json_out['action'] = "publish"
json_out["publish_time"] = timezone.localtime(
timezone.now()).strftime("%Y-%m-%d %H:%M:%S.%f")
#create the dir if required
if not os.path.exists(os.path.dirname(json_filename)):
os.makedirs(os.path.dirname(json_filename))
with open(json_filename, "wb") as output:
json.dump(json_out, output, indent=4)
hg = hglib.open(BorgConfiguration.BORG_STATE_REPOSITORY)
#remove other related json files
json_files = [
self.json_filename_abs(action) for action in ['empty_gwc']
]
#get all existing files.
json_files = [f for f in json_files if os.path.exists(f)]
if json_files:
hg.remove(files=json_files)
json_files.append(json_filename)
hg.commit(include=json_files,
addremove=True,
user="******",
message="update wms layer {}.{}".format(
self.server.workspace.name, self.name))
increase_committed_changes()
try_push_to_repository("wmslayer", hg)
finally:
if hg: hg.close()
try_clear_push_owner("wmslayer")
def publish(self):
"""
publish store's json reference (if exists) to the repository,
"""
try_set_push_owner("wmsserver")
hg = None
try:
meta_data = {}
meta_data["name"] = self.name
meta_data["capability_url"] = self.get_capability_url
meta_data["channel"] = self.workspace.publish_channel.name
meta_data["username"] = self.user or ""
meta_data["password"] = self.password or ""
meta_data["workspace"] = self.workspace.name
meta_data[
"sync_geoserver_data"] = self.workspace.publish_channel.sync_geoserver_data
if self.geoserver_setting:
meta_data["geoserver_setting"] = json.loads(
self.geoserver_setting)
#write meta data file
file_name = "{}.{}.meta.json".format(self.workspace.name,
self.name)
meta_file = os.path.join(BorgConfiguration.WMS_STORE_DIR,
file_name)
#create the dir if required
if not os.path.exists(os.path.dirname(meta_file)):
os.makedirs(os.path.dirname(meta_file))
with open(meta_file, "wb") as output:
json.dump(meta_data, output, indent=4)
json_out = {}
json_out['meta'] = {
"file":
"{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, meta_file),
"md5":
file_md5(meta_file)
}
json_out['action'] = 'publish'
json_out["publish_time"] = timezone.localtime(
timezone.now()).strftime("%Y-%m-%d %H:%M:%S.%f")
json_filename = self.json_filename_abs('publish')
#create the dir if required
if not os.path.exists(os.path.dirname(json_filename)):
os.makedirs(os.path.dirname(json_filename))
with open(json_filename, "wb") as output:
json.dump(json_out, output, indent=4)
hg = hglib.open(BorgConfiguration.BORG_STATE_REPOSITORY)
hg.commit(include=[json_filename],
addremove=True,
user="******",
message="Update wms store {}.{}".format(
self.workspace.name, self.name))
increase_committed_changes()
try_push_to_repository("wmsserver", hg)
finally:
if hg: hg.close()
try_clear_push_owner("wmsserver")
def execute(self,job,previous_state):
p = job.publish
meta_data = p.update_catalogue_service(style_dump_dir=job.dump_dir,md5=True,extra_datas={"publication_date":datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")})
#write meta data file
file_name = "{}.meta.json".format(p.table_name)
meta_file = os.path.join(job.dump_dir,file_name)
with open(meta_file,"wb") as output:
json.dump(meta_data, output, indent=4)
job.metadict['meta'] = {"file":"{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, meta_file),"md5":file_md5(meta_file)}
return (HarvestStateOutcome.succeed,None)
def execute(self,job,previous_state):
"""
dump the full table data into download folder
"""
if 'data' in job.metadict:
del job.metadict['data']
#create the dir if required
if not os.path.exists(job.dump_dir):
#dump dir does not exist, create it
os.makedirs(job.dump_dir)
file_name = job.publish.table_name + ".tar"
dump_file = os.path.join(job.dump_dir,file_name)
cmd = self.dump_cmd + ["-t", job.publish.workspace.publish_data_schema + "." + job.publish.table_name, "-f", dump_file]
cursor=connection.cursor()
if not previous_state.is_error_state:
#table with same name maybe published by previous job. drop it if have.
cursor.execute('drop table if exists "{0}"."{1}" cascade'.format(job.publish.workspace.publish_data_schema,job.publish.table_name))
#move table to publish schema for dump
cursor.execute('alter table "{0}"."{1}" set schema {2}'.format(job.publish.workspace.schema,job.publish.table_name,job.publish.workspace.publish_data_schema))
try:
#import ipdb;ipdb.set_trace()
output = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE, env=self.env).communicate()
logger.debug("execute ({0})\nstdin:{1}\nstdout:{2}".format(cmd,output[0],output[1]))
finally:
#move table back to original schema
cursor.execute('alter table "{0}"."{1}" set schema "{2}"'.format(job.publish.workspace.publish_data_schema,job.publish.table_name,job.publish.workspace.schema))
if output[1].strip() :
return (HarvestStateOutcome.failed,output[1])
else:
job.metadict['data'] = {"file":"{}{}".format(BorgConfiguration.MASTER_PATH_PREFIX, dump_file),"md5":file_md5(dump_file)}
return (HarvestStateOutcome.succeed,None)
