def __init__(self, withname, mode='file'):
"""Object representing the metadata of filename.
If mode == 'mad' - the filename is assumed to be the
madfile. If mode == 'file', the metadata file is assumed to be
the filename plus .mad.
Note - started to ignore the 'mode' parameter - will determine
that automatically (if the file ends with a .mad or not)
"""
if withname[-4:] == '.mad':
#this is the .mad file
self.filename = withname[:-4]
self.metafile = withname
else:
self.filename = withname
self.metafile = withname + '.mad'
if (not os.path.exists(self.filename)
and not os.path.exists(self.metafile)):
mad.util.exer("Neither filename not metadata file " +
"exist - do not know what to do")
G.debug("created a metadata file for file %s" % self.filename)
G.debug("metadata file is %s" % self.metafile)
self.metadata = {}
self.load()
def _POST(url, data):
data = urllib.urlencode({'value': data})
G.debug('data encoded %s' % data)
G.debug('contacting %s' % url)
opener = urllib2.build_opener(urllib2.HTTPHandler)
request = urllib2.Request(url, data)
request.get_method = lambda: 'POST'
F = opener.open(request)
return F.read()
def run(args):
""" Sync data types from SMW """
G.critical("Not quite implemented yet - ask Mark")
sys.exit(0)
site = madpy.smw.get_site(args)
for page in site.Categories['MadProperty']:
print '-' * 80
text = page.edit()
print text
def hook_metadata_prepare(mdf):
if 'gid' in mdf.metadata:
G.debug("metadata has a gid")
return True
if not 'Sha1sum' in mdf.metadata:
G.debug("metadata does not have a shasum")
return True
gid = _get_gid(mdf.metadata['Sha1sum'])
mdf.metadata['Gid'] = gid
def load(self):
"""
Check if there is a metadata file for *filename*.
If so - load
if not - return an empty dict
"""
if os.path.exists(self.metafile):
G.debug("loading metadata")
with open(self.metafile) as F:
self.metadata = yaml.load(F)
plugin.run('post_metadata_load', self)
def run(args):
"""
Upload the pages to SMW
"""
files = args.file
hostname = socket.gethostname()
G.info("this is: %s" % hostname)
for f in files:
full_path = os.path.abspath(f)
md = metadata.Metadata(f)
G.info("Processing file %s" % f)
md.metadata['Hostname'] = hostname
md.metadata['FileLocation'] = full_path
pfridge.save(md)
def run(args):
"""
Upload the pages to SMW
"""
files = args.file
hostname = socket.gethostname()
G.info("this is: %s" % hostname)
site = madpy.smw.get_site(args)
for f in files:
full_path = os.path.abspath(f)
md = metadata.Metadata(f, mode=args.madmode)
if not 'Gid' in md:
G.critical("No Gid found - run 'mad init %s'" % f)
sys.exit()
G.info("Processing file %s" % f)
pname = "%s" % (md.metadata['Gid'])
page = site.Pages[pname]
old_page_text = page.edit()
separator = '<!-- MAD - Do not change anything above this line -->'
if separator in old_page_text:
old_page_text = old_page_text.split(separator)[1]
md.metadata['Hostname'] = hostname
md.metadata['FileLocation'] = full_path
ppropps = []
for k in md.metadata.keys():
v = md.metadata[k]
dti = conf.datatypes[k]
if type(v) == type([]):
for sv in v:
ppropps.append("[[%s::%s| ]]" % (k, sv))
else:
ppropps.append("[[%s::%s| ]]" % (k, v))
if args.attach:
basename = os.path.basename(f)
ufn = basename
G.warning("Uploading file %s (%s)" % (ufn, full_path))
with open(full_path, 'rb') as F:
site.upload(file = F,
filename = ufn,
description = "Mad uploaded file for %s" % pname,
ignore=True)
ppropps.append("[[AttachedFile::File:%s| ]]" % ufn)
if args.force:
new_page = "\n" + " ".join(ppropps) + "[[Category:MadObject]]\n{{" + \
args.template + "}}\n" + separator
else:
new_page = "\n" + " ".join(ppropps) + "[[Category:MadObject]]\n{{" + \
args.template + "}}\n" + separator + old_page_text
#print new_page
page.save(new_page, summary="Mad created/refreshed object data")
def run(args):
"""
add a key value pair to a file's metadata
"""
files = args.file
G.info('found %d files' % len(files))
for filename in files:
if args.madmode == 'mad':
madpy.util.exer('Cannot run mad init in --mad mode')
mdf = Metadata(filename)
metadata = {}
if args.gid:
metadata['gid'] = args.gid
G.info("Processing file %s" % filename)
mtime = datetime.fromtimestamp(os.path.getmtime(filename)).isoformat()
metadata['FileSize'] = os.path.getsize(filename)
if not mdf.has_key('Sha1sum') and args.sha1sum:
G.warning("Generating Shasum for %s" % filename)
metadata['Sha1sum'] = _get_hash('sha1sum', filename)
metadata['LastModified'] = mtime
mdf.update(metadata)
mdf.save()
def get_site(args):
if args.password:
pwd = args.password
else:
pwd = getpass.getpass("SMW Password: "******"Connecting to SMW")
if conf.smw.https:
site = mwclient.Site(('https', conf.smw.host), path=conf.smw.path)
try:
site = mwclient.Site(('http', conf.smw.host), path=conf.smw.path)
except:
try:
site = mwclient.Site(('https', conf.smw.host), path=conf.smw.path)
except:
G.critical("Cannot login the swm site")
raise
try:
site.login(args.username, pwd)
except mwclient.errors.LoginError:
exer("Can not login. (Wrong password?)")
G.info("Connected to SMW")
return site
def run(args):
"""
add a key value pair to parse_argument file's metadata
"""
metadata = {}
for parse_argument in args.setargs:
#is this parse_argument file or parse_argument key=value pair?
if '=' in parse_argument:
#it seems this is a key=value pair
key, value = parse_argument.split('=', 1)
if (not key in conf.datatypes.keys()):
G.critical("Invalid key %s" % key)
print "Available keys are:"
print ", ".join(conf.datatypes.keys())
sys.exit(-1)
else:
mup = mvars.check(key, value)
print mup
else:
G.critical("Please specifiy a key=value pair")
def run(args):
""" Run SMW variable sync """
site = madpy.smw.get_site(args)
for dt in conf.datatypes:
dti = conf.datatypes[dt]
prop_name = "Property:%s" % dt
page = site.Pages[prop_name]
if args.for_real:
ptxt = page.edit()
if (not args.force) and len(ptxt) > 0:
G.warning("Ignoring %s - property page already has content" % prop_name)
continue
ptxt = []
if dti.has_key('help'):
ptxt.append(dti.help + "\n")
if dti.has_key('url'):
ptxt.append("* See also:: %s" % dti.url)
if dti.has_key('smw_data_type'):
ptxt.append("* SMW datatype: [[Has type::%s]]" % dti.smw_data_type)
if dti.has_key('mad_data_type'):
ptxt.append("* Mad data type: [[Mad type::%s]]" % dti.mad_data_type)
if dti.has_key('help'):
ptxt.append("* Mad help text: [[Mad help::%s]]" % dti.help)
ptxt.append("[[Category:MadProperty]]")
if args.for_real:
page.save("\n".join(ptxt), summary="Mad autogenerated Property page")
G.warning("regenerated %s" % prop_name)
else:
print '-' * 80
print "would have saved to %s" % prop_name
print "\n".join(ptxt)
print
def run(args):
"""
add a key value pair to parse_argument file's metadata
"""
files = []
remove_keys = []
kvs = []
for parse_argument in args.setargs:
#is this parse_argument file or parse_argument key=value pair?
if '=' in parse_argument:
#it seems this is a key=value pair
key, value = parse_argument.split('=', 1)
if (not args.force) and (not key in conf.datatypes.keys()):
G.critical("Invalid key %s" % key)
print "Available keys are:"
print ", ".join(conf.datatypes.keys())
sys.exit(-1)
if not value:
remove_keys.append(key)
G.info("Preparing %s for removal" % key)
else:
kvs.append((key, value))
G.debug("adding key, value pair %s, %s" % (key, value))
else:
fmd = Metadata(parse_argument)
files.append(fmd)
G.info('found %d key value pairs' % len(kvs))
G.info('found %d files' % len(files))
if len(files) == 0:
exerr("Must specify at least one file")
for madfile in files:
G.info("Processing file %s" % madfile)
for k, v in kvs:
G.debug("applying %s, %s" % (k, v))
madfile.apply_kv(k, v)
madfile.remove_keys(remove_keys)
madfile.save()
def save(md):
G.debug('pfridge save')
if not 'Gid' in md.keys():
G.critical("error - no gid defined")
sys.exit(-1)
if not 'Pfridge' in md.keys():
url = conf.pfridge.gidserver
url += 'register/%s' % md['Gid']
md['pfridge'] = md['Gid']
G.debug("pfridge GID register")
G.debug("calling %s" % url)
for k in md.keys():
if k == 'Gid':
continue
if k == 'Pfridge':
continue
v = md[k]
url = conf.pfridge.metaserver + 'set/%s/%s' % (md['Gid'], k)
G.debug("pfridge metadata store: %s %s" % (k, v))
rv = _POST(url, v)
G.debug("rv %s" % rv)