def export_mapping(self, plugin_name, folder):
res = {
"mapping": {
"status": None,
"file": None,
"message": None,
"origin": None
}
}
# first check if plugin defines a custom mapping in manifest
# if that's the case, we don't need to export mapping there
# as it'll be exported with "uploader" code
plugindoc = get_data_plugin().find_one({"_id": plugin_name})
assert plugindoc, "Can't find plugin named '%s'" % plugin_name
plugin_folder = plugindoc.get("download", {}).get("data_folder")
assert plugin_folder, "Can't find plugin folder for '%s'" % plugin_name
try:
manifest = json.load(
open(os.path.join(plugin_folder, "manifest.json")))
if "mapping" in manifest.get("uploader", {}):
res["mapping"][
"message"] = "Custom mapping included in uploader export"
res["mapping"]["status"] = "warning"
res["mapping"]["origin"] = "custom"
return res
except Exception as e:
self.logger.error("Can't read manifest while exporting code: %s" %
e)
# try to export mapping from src_master (official)
doc = get_src_master().find_one({"_id": plugin_name})
if doc:
mapping = doc.get("mapping")
res["mapping"]["origin"] = "registered"
else:
doc = get_src_dump().find_one({"_id": plugin_name})
mapping = doc and doc.get("inspect", {}).get("jobs", {}).get(plugin_name, {}).get("inspect", {}).\
get("results", {}).get("mapping")
res["mapping"]["origin"] = "inspection"
if not mapping:
res["mapping"]["origin"] = None
res["mapping"]["status"] = "warning"
res["mapping"][
"message"] = "Can't find registered or generated (inspection) mapping"
return res
else:
ufile = os.path.join(folder, "upload.py")
strmap, _ = yapf_api.FormatCode(pprint.pformat(mapping))
with open(ufile, "a") as fout:
fout.write("""
@classmethod
def get_mapping(klass):
return %s\n""" % textwrap.indent((strmap), prefix=" " * 2))
res["mapping"]["file"] = ufile
res["mapping"]["status"] = "ok"
return res
def __init__(self, source_list, dump_manager, upload_manager,
data_plugin_manager):
self._orig_source_list = source_list
self.source_list = None
self.dump_manager = dump_manager
self.upload_manager = upload_manager
self.data_plugin_manager = data_plugin_manager
self.reload()
self.src_master = get_src_master()
self.src_dump = get_src_dump()
# honoring BaseSourceManager interface (gloups...-
self.register = {}
def prepare(self, state={}):
"""Sync uploader information with database (or given state dict)"""
if self.prepared:
return
if state:
# let's be explicit, _state takes what it wants
for k in self._state:
self._state[k] = state[k]
return
self._state["conn"] = get_src_conn()
self._state["db"] = self._state["conn"][self.__class__.__database__]
self._state["collection"] = self._state["db"][self.collection_name]
self._state["src_dump"] = self.prepare_src_dump()
self._state["src_master"] = get_src_master()
self._state["logger"], self.logfile = self.setup_log()
self.data_folder = self.src_doc.get("download",{}).get("data_folder") or \
self.src_doc.get("data_folder")
# flag ready
self.prepared = True
def export_mapping(self, plugin_name, folder):
res = {
"mapping": {
"status": None,
"file": None,
"message": None,
"origin": None
}
}
# first try to export mapping from src_master (official)
doc = get_src_master().find_one({"_id": plugin_name})
if doc:
mapping = doc.get("mapping")
res["mapping"]["origin"] = "registered"
else:
doc = get_src_dump().find_one({"_id": plugin_name})
mapping = doc and doc.get("inspect",{}).get("jobs",{}).get(plugin_name,{}).get("inspect",{}).\
get("results",{}).get("mapping")
res["mapping"]["origin"] = "inspection"
if not mapping:
res["mapping"]["origin"] = None
res["mapping"]["status"] = "warning"
res["mapping"][
"message"] = "Can't find registered or generated (inspection) mapping"
return res
else:
ufile = os.path.join(folder, "upload.py")
strmap, _ = yapf_api.FormatCode(pprint.pformat(mapping))
with open(ufile, "a") as fout:
fout.write("""
@classmethod
def get_mapping(klass):
return %s\n""" % textwrap.indent((strmap), prefix=" " * 2))
res["mapping"]["file"] = ufile
res["mapping"]["status"] = "ok"
return res