def main():
db = metakit.storage(sys.argv[1], 0)
views = []
sofar = []
depth = 0
for c in db.description():
if c == '[': depth += 1
elif c == ']': depth -= 1
if depth == 0 and c == ',':
views.append("".join(sofar))
sofar[:] = []
else:
sofar.append(c)
views.append("".join(sofar))
di = dict(zip(views, [view_to_list(db.getas(v)) for v in views]))
print "converted succesfully"
#####
db = metakit.storage('out.db', 1)
for viewdesc, data in di.iteritems():
view = db.getas(viewdesc)
fill(view, data)
db.commit()
print "converted succesfully"
def main():
db = metakit.storage(sys.argv[1], 0)
views = []
sofar = []
depth = 0
for c in db.description():
if c == '[': depth += 1
elif c == ']': depth -= 1
if depth == 0 and c == ',':
views.append("".join(sofar))
sofar[:] = []
else:
sofar.append(c)
views.append("".join(sofar))
di = dict(zip(views, [view_to_list(db.getas(v)) for v in views]))
print "converted succesfully"
#####
db = metakit.storage('out.db', 1)
for viewdesc, data in di.iteritems():
view = db.getas(viewdesc)
fill(view, data)
db.commit()
print "converted succesfully"
def __init__(self, filename=None):
if isinstance(filename, unicode):
filename = filename.encode(sys.getfilesystemencoding())
self.filename = filename
if self.filename is None:
# We initially create an in-memory database.
# When we save to a file, we will reopen the database from the file.
self.db = metakit.storage()
# self.filename = 'defau.gt'
# self.db = metakit.storage(self.filename, 1)
# for desc in storage_desc.values():
# self.db.getas(desc)
# self.db.commit()
else:
self.db = metakit.storage(self.filename, 1)
self.cleanup()
# self.aside = metakit.storage('grafity-storage.mka', 1)
# self.db.aside(self.aside)
# print >>sys.stderr, "project created"
self._modified = False
action_list.connect('added', self.on_action_added)
self.items = {}
self.deleted = {}
self._dict = {}
self.save_dict = {}
# Create top folder.
# - it must be created before all other items
# - it must be created with _isroot=True, to set itself as its parent folder
try:
fv = self.db.getas(storage_desc[Folder])
row = fv.select(name='top')[0]
self.top = self.items[row.id] = Folder(self, location=(fv, row, row.id), _isroot=True)
except IndexError:
# can't find it in the database, create a new one.
self.top = Folder(self, 'top', _isroot=True)
self.here = self.top
self.this = None
# create objects
for cls, desc in [(i, storage_desc[i]) for i in (Folder, grafity.worksheet.Worksheet, grafity.graph.Graph)]:
view = self.db.getas(desc)
for i, row in enumerate(view):
if row.id != self.top.id:
if not row.id.startswith('-'):
# print 'loading', cls, row.id,
self.items[row.id] = cls(self, location=(view, row, row.id))
# print 'end'
else:
self.deleted[row.id] = cls(self, location=(view, row, row.id))
def CreateDb(dbName, modelName='default'):
# database version
db = metakit.storage(dbName, 1)
vw = db.getas(MetaKit.VERSION_VIEW)
vw.append(version=MetaKit.VERSION)
db.commit()
return GetDb(dbName, modelName)
def __init__(self, description):
self.storage = metakit.storage()
self.v = self.storage.getas(description)
self.arr = []
self.failure = Failure()
self.fail = self.failure.fail
self.columns = map(lambda c: string.split(c, ':')[0], string.split(description[string.index(description, '[') + 1:-1], ','))
def __init__(self, directory, dbName, cache = 128):
log.message("Opening database", dbName)
self.dbName = dbName
self.nextID = 10000
self.cacheSize = cache
#
try:
os.makedirs(directory)
except OSError:
pass
# open db
self.storage = metakit.storage(os.path.join(directory, dbName), 1)
view = self.storage.getas(self.dbSchema).blocked()
map = self.storage.getas("map[_H:I,_R:I]")
self.view = view.hash(map, 1)
# cache
self.cache = {}
self.cacheLinks = {
"__first__": [None, "__last__"],
"__last__": ["__first__", None],
}
# stats
self.statCount = 0
self.statHit = 0
self.statSwap = 0
self.statMiss = 0
self.statCleanSwap = 0
def read_feeds_opml():
"""Opens Akregator feedlistbackup archive and returns lxml-parsed OPML xml."""
db = metakit.storage(os.path.join(ak_archive_path, 'feedlistbackup.mk4'),
0)
vw = db.getas(db.description())
assert len(vw) == 1, "Expecting one row in feedlistbackup."
return et.fromstring(vw[0].feedList)
def __init__(self, filename):
self.db = metakit.storage(filename, 1)
if os.path.exists(filename+'.swp'):
os.remove(filename+'.swp')
self.filename = filename
self.undodb = metakit.storage(filename+'.swp', 1)
self.items = {}
self.oplist = []
self.action_name = None
self.storage = self
for name in dir(type(self)):
attr = getattr(self, name)
if isinstance(attr, Container):
attr.name = name
attr.storage = self
itemtypes[name] = attr.cls
def readfeed(file):
# Open a Metakit file, one file per blog, fetch contents
global idxarticle
blogdb = mk.storage(file,0)
blogvw = blogdb.getas(blogdb.description())
for i in blogvw:
writedb_article(i)
idxarticle = idxarticle + 1
return
def __init__(self, description):
self.storage = metakit.storage()
self.v = self.storage.getas(description)
self.arr = []
self.failure = Failure()
self.fail = self.failure.fail
self.columns = map(
lambda c: string.split(c, ':')[0],
string.split(description[string.index(description, '[') + 1:-1],
','))
def initialize_connection (self):
# identical to parent class, except that we open the DB read-only
debug('using file: %s'%self.file,1)
self.file = str(os.path.expandvars(os.path.expanduser(self.file)))
mydir = os.path.split(self.file)[0]
# create the directory if it does not yet exist
if not os.path.exists(mydir):
os.mkdir(mydir)
self.db = metakit.storage(self.file,0) #filename must be string, not unicode
self.contentview=self.db.contents()
def initialize_connection (self):
# identical to parent class, except that we open the DB read-only
debug('using file: %s'%self.file,1)
self.file = str(os.path.expandvars(os.path.expanduser(self.file)))
mydir = os.path.split(self.file)[0]
# create the directory if it does not yet exist
if not os.path.exists(mydir):
os.mkdir(mydir)
self.db = metakit.storage(self.file,0) #filename must be string, not unicode
self.contentview=self.db.contents()
def initialize_connection (self):
debug('using file: %s'%self.file,1)
self.file = os.path.expandvars(os.path.expanduser(self.file))
mydir = os.path.split(self.file)[0]
# create the directory if it does not yet exist
if not os.path.exists(mydir):
os.mkdir(mydir)
self.db = metakit.storage(str(self.file),1) #filename must be string, not unicode
self.contentview=self.db.contents()
#self.load()
self.db.autocommit()
def initialize_connection (self):
debug('using file: %s'%self.file,1)
self.file = os.path.expandvars(os.path.expanduser(self.file))
mydir = os.path.split(self.file)[0]
# create the directory if it does not yet exist
if not os.path.exists(mydir):
os.mkdir(mydir)
self.db = metakit.storage(str(self.file),1) #filename must be string, not unicode
self.contentview=self.db.contents()
#self.load()
self.db.autocommit()
def establishConnection(self, fullSpecifier): """Connect using the fully specified specifier fullSpecifier -- a specifier with all arguments unified and ready to be connected. This specifier should include everything required to do the actual connection (including passwords or the like). All sub-classes must override this method! """ filename = fullSpecifier.dsn return metakit.storage( filename, 1 )
def __init__(self, connection):
"Setup the database connection"
self._system_tables=[]
# Not providing a db name is guaranteed to ruin our connection
if not connection['databasename']:
raise ValueError
filename = os.path.join(connection['directory'], connection['databasename'])
# not sure about the mode to use, I assume we just want read-only
self._db = metakit.storage(filename, 0)
self._cursor='placeholder'
# This one is used in getRow
self._tableName=''
def main(filepath):
db = metakit.storage(filepath, 0)
meta = db.contents()
for table in meta.structure():
view = db.view(table.name)
columns = [column.name for column in view.structure()]
data = [{column: getattr(row, column)
for column in columns} for row in view]
with open('%s.json' % table.name, 'w') as fp:
json.dump(data, fp, indent=2)
def update_feed_articles(outline, feed_id):
articles = []
feed_url = outline.get('xmlUrl')
html_url = outline.get('htmlUrl')
feed_title = outline.get('title')
feed_file = feed_url.replace(':', '_').replace('/', '_') + '.mk4'
fdb = metakit.storage(os.path.join(ak_archive_path, feed_file), 0)
updates, fails = 0, 0
for a in fdb.getas(fdb.description()):
# Akregator status bits: Deleted = 0x01, Trash = 0x02, New = 0x04, Read = 0x08, Keep = 0x10
if a.status & 0x3:
continue # skip articles marked as "deleted" or "thrash"
is_read = bool(a.status & 0x08)
is_fav = bool(a.status & 0x10)
guid = a.link if a.guid.startswith('hash:') else a.guid
try:
# try to update id opportunistically so next step (fix_article_order) won't have to update the same row again
found_rows = c.execute(
'UPDATE ' + entry_table +
' SET id=%s*1000000+id%%1000000, date=%s, is_read=%s, is_favorite=%s WHERE id_feed=%s AND guid=%s',
(a.pubDate, a.pubDate, is_read, is_fav, feed_id,
uesc(guid)))
except MySQLdb.IntegrityError:
# conflict with another row having the target ID value... leave it to be resolved in fix_article_order()
found_rows = c.execute(
'UPDATE ' + entry_table +
' SET date=%s, is_read=%s, is_favorite=%s WHERE id_feed=%s AND guid=%s',
(a.pubDate, is_read, is_fav, feed_id, uesc(guid)))
if found_rows:
updates += 1
else:
fails += 1
if fails:
print(
u"ERROR: failed to update {} articles in {} (article not found in FreshRSS database)"
.format(fails, feed_title))
else:
print(u"Successfully updated {} articles in feed {}".format(
updates, feed_title))
c.execute(
'SELECT COUNT(id) FROM ' + entry_table +
' WHERE id_feed=%s AND is_read=0', (feed_id, ))
return updates, fails, c.fetchone()[0]
def update_schema(self):
storFn = pycs_paths.DATADIR + "/settings.dat"
print "Connecting to MetaKit database in %s" % storFn
import metakit
mkdb = metakit.storage(storFn, 1)
print "Updating PostgreSQL database schema"
# See if the database is completely new
try:
self.fetchone("SELECT db_id FROM pycs_meta")
except DBE, e:
if e.args[0].find("does not exist") != -1:
print "Creating pycs_meta table"
self.execute("""CREATE TABLE pycs_meta (db_id INT)""")
self.execute("INSERT INTO pycs_meta (db_id) VALUES (0)")
else:
raise
def openFile(self, filename):
import metakit
self.commonOpenFileInit(filename)
#print filename
if os.path.exists(filename):
# need to keep a reference to the storage
# or it will be garbage collected
self._db = metakit.storage(filename, 0)
# there should only be one view, but this
# needs to be made more robust, perhaps only
# grabbing the first view?
viewName = self._db.description().split('[')[0]
# *** the view is hard-code below but shouldn't be
# I just want the default view
# self.records = self._db.view('companies')
self.records = self._db.view(viewName)
#print "len(self.records)", len(self.records)
if self.current == -1 and len(self.records) > 0:
self.displayRecord(0)
def test_db():
gl_vlist = VocabList()
log('searching directory: %s' % FEED_DIR)
for dir in os.listdir(FEED_DIR):
if '.mk4' in dir[-4:]:
log('found database: %s' % dir)
# open database
db = metakit.storage(os.path.join(FEED_DIR, dir), 0)
data = read_database(db)
if len(data) > 0:
# feed content in database
log('create library')
lib = Library()
for feed in data:
lib.add_document(read_data(feed))
vlist = lib.gen_vocablist()
vlist.clean(5)
gl_vlist.merge(vlist)
db = None # close database
print gl_vlist
def __init__(self, filename=None):
if isinstance(filename, unicode):
filename = filename.encode(sys.getfilesystemencoding())
self.filename = filename
if self.filename is None:
# We initially create an in-memory database.
# When we save to a file, we will reopen the database from the file.
self.db = metakit.storage()
# self.filename = 'defau.gt'
# self.db = metakit.storage(self.filename, 1)
# for desc in storage_desc.values():
# self.db.getas(desc)
# self.db.commit()
else:
self.db = metakit.storage(self.filename, 1)
self.cleanup()
# self.aside = metakit.storage('grafit-storage.mka', 1)
# self.db.aside(self.aside)
# print >>sys.stderr, "project created"
self._modified = False
action_list.connect('added', self.on_action_added)
self.items = {}
self.deleted = {}
self._dict = {}
self.save_dict = {}
# Create top folder.
# - it must be created before all other items
# - it must be created with _isroot=True, to set itself as its parent folder
try:
fv = self.db.getas(storage_desc[Folder])
row = fv.select(name='top')[0]
self.top = self.items[row.id] = Folder(self,
location=(fv, row, row.id),
_isroot=True)
except IndexError:
# can't find it in the database, create a new one.
self.top = Folder(self, 'top', _isroot=True)
self.here = self.top
self.this = None
# create objects
for cls, desc in [(i, storage_desc[i])
for i in (Folder, grafit.worksheet.Worksheet,
grafit.graph.Graph)]:
view = self.db.getas(desc)
for i, row in enumerate(view):
if row.id != self.top.id:
if not row.id.startswith('-'):
# print 'loading', cls, row.id,
self.items[row.id] = cls(self,
location=(view, row, row.id))
# print 'end'
else:
self.deleted[row.id] = cls(self,
location=(view, row,
row.id))
# 22222 0
# [Property('S', 's'), Property('V', 'sub')]
# 00000 1
# 11111 1
# 22222 0
# [Property('S', 's'), Property('I', 'i'), Property('V', 'sub')]
# ok
import os
try:
os.remove("_selfref.mk")
except:
pass
import metakit
db = metakit.storage('_selfref.mk', 1)
vw = db.getas('data[s:S,sub[^]]')
print len(vw)
v = vw
for i in range(3):
v.append(s=` i ` * 5)
v = v[0].sub
def show(vw):
for v in [vw[0], vw[0].sub[0], vw[0].sub[0].sub[0]]:
print v.s, len(v.sub)
print vw[0].sub[0].sub[0].sub.structure()
assert len(vw[0].sub[0].sub[0].sub) == 0
m = md5.new()
m.update(article.description)
idx = m.hexdigest()
insert = (idxarticle, idxfeed, article.guid, article.title, article.link, article.comments, article.commentsLink, article.status, article.pubDate, article.author)
sqdb.execute("insert into article (id, idfeed, guid, title, url, comments, commentsLink, status, pubDate, author) values (?,?,?,?,?,?,?,?,?,?)", insert)
insert2 = (idxarticle,idx)
sqdb.execute("insert into contentindex(id, id_content) values(?, ?)", insert2)
if idx not in hash:
insert = (idx, zlib.compress(article.description, 9))
sqdb.execute("insert into content(id, content) values (?,?)", insert)
hash[idx]=1
nondupes = nondupes+1
return
print "Starting conversion:"
db=mk.storage(indexfile,0)
vw=db.getas(db.description())
print "Opening up database files"
sqdb = sq3.connect(sq3file)
initdb()
for i in vw:
getfeed(i)
print "Imported "+str(idxarticle)+" articles (ignored "+str(idxarticle-nondupes)+" duplicates) from "+str(idxfeed)+" blogs"
print "Creating indexes:"
print " Indexing article publishing dates..."
sqdb.execute("CREATE INDEX a_pubdate on article (pubDate ASC)")
print " Indexing article's author names..."
sqdb.execute("CREATE INDEX a_author on article (author ASC)")
print " Indexing articles...."
# 00000 1
# 11111 1
# 22222 0
# [Property('S', 's'), Property('V', 'sub')]
# 00000 1
# 11111 1
# 22222 0
# [Property('S', 's'), Property('I', 'i'), Property('V', 'sub')]
# ok
import os
try: os.remove("_selfref.mk")
except: pass
import metakit
db = metakit.storage('_selfref.mk',1)
vw = db.getas('data[s:S,sub[^]]')
print len(vw)
v = vw
for i in range(3):
v.append(s=`i`*5)
v = v[0].sub
def show(vw):
for v in [vw[0], vw[0].sub[0], vw[0].sub[0].sub[0]]:
print v.s, len(v.sub)
print vw[0].sub[0].sub[0].sub.structure()
assert len(vw[0].sub[0].sub[0].sub) == 0
try:
def Run(inputPath, outputPath):
success = False
conn = None
cursor = None
isOverwriting = False
existingPath = None
#check for create curise file
# if not os.path.exists(createSQLFilePath):
# log.lg("high", "ERROR resource file CruiseCreate.sql not found")
#
#
# success = False
# return success
#check output file extention
if not str.lower(outputPath[-7:]) == '.cruise':
log.lg("high", "ERROR outputFile has bad extention")
success = False
return success
#check if overwriting existing filename
# if(os.path.exists(outputPath)):
# log.lg('norm', 'existing file found: overwriting')
# isOverwriting = True
# existingPath = outputPath
# outputPath = outputPath + '.new'
#check input file and extention
if not str.lower(inputPath[-4:]) == '.crz':
log.lg("high", "ERROR file to convert has wrong extention")
success = False
return success
if not os.path.exists(inputPath):
log.lg("high", "ERROR Couldn't find file to convert")
success = False
return success
#load input file
mk = metakit.storage(inputPath, 0)
#check crz file version of the input file
if not isCRZCurrent(mk):
log.lg('high', 'ERROR CRZ file not current version')
success = False
return success
#try:
#create database connecting and cursor
print(outputPath)
dal = DAL(outputPath, True)
dal.Create()
# cursor = conn.cursor()
#cereate database
# import codecs
# createSQLFile = codecs.open(createSQLFilePath, 'rb','utf-8-sig')# http://stackoverflow.com/questions/2359832/dealing-with-utf-8-numbers-in-python
# #createSQLFile = open(createSQLFilePath, 'rb')
# createDBFile(conn, cursor, createSQLFile)
incrementProgress()
#process CRZ file and fill db file
log.lg("norm", 'processing metakit file')
processCRZFile(mk, dal)
#clean up overwritten file of overwriting
# if isOverwriting:
# os.remove(existingPath)
# os.rename(outputPath, existingPath)
dal.Path = outputPath
success = True
# except Exception as er:
# log.lg('high', 'PROGRAM ERROR program ended in error, exception args: ' + repr(er.args))
# log.lg('high', 'PROGRAM ERROR program ended in error, exception message: ' + repr(er.message))
# #log.lg('high', sys.exc_info())
# #os.remove(outputPath)#filling database failed remove it
# success = False
#
# finally:
# pass
# if not cursor is None:
# cursor.close()
# if not conn is None:
# conn.close()
return success
# Properties are case-insensitive, but this can lead to some
# surprising behavior: the first way a property is used will
# determine how it ends up in the global symbol table.
#
# Sample output:
# Property('S', 'HeLLo') Property('S', 'HeLLo')
# 2
# 2
# 135099576
# 135033272
# 0
import metakit
db = metakit.storage()
v1 = db.getas('lo[HeLLo:S]')
v2 = db.getas('hi[hello:S]')
# surprise: this prints two mixed-case names
print v1.HeLLo, v2.hello
# this shows that the MetaKit property is the same for both
# reason: there is a single global case-insensitive symbol table
print metakit.property('S','HeLLo').id
print metakit.property('S','hello').id
# this shows that the Python objects differ
# reason: these are two wrapper objects around the same thing
print id(metakit.property('S','HeLLo'))
print id(metakit.property('S','hello'))
# this causes a mismatch, it will have to be fixed one day
def __init__ (self, filename, mode):
self.db = metakit.storage( filename, 1)
view = self.db.getas("data[key:S,value:S]")
hashvw = self.db.getas("__test_hash__[_H:I,_R:I]")
self.view = view.hash(hashvw, 1)
def Run(inputPath, outputPath):
success = False
conn = None
cursor = None
isOverwriting = False
existingPath = None
#check for create curise file
# if not os.path.exists(createSQLFilePath):
# log.lg("high", "ERROR resource file CruiseCreate.sql not found")
#
#
# success = False
# return success
#check output file extention
if not str.lower(outputPath[-7:]) == '.cruise':
log.lg("high", "ERROR outputFile has bad extention")
success = False
return success
#check if overwriting existing filename
# if(os.path.exists(outputPath)):
# log.lg('norm', 'existing file found: overwriting')
# isOverwriting = True
# existingPath = outputPath
# outputPath = outputPath + '.new'
#check input file and extention
if not str.lower(inputPath[-4:]) == '.crz':
log.lg("high", "ERROR file to convert has wrong extention")
success = False
return success
if not os.path.exists(inputPath):
log.lg("high", "ERROR Couldn't find file to convert")
success = False
return success
#load input file
mk = metakit.storage(inputPath,0)
#check crz file version of the input file
if not isCRZCurrent(mk):
log.lg('high', 'ERROR CRZ file not current version')
success = False
return success
#try:
#create database connecting and cursor
print(outputPath)
dal = DAL(outputPath, True)
dal.Create()
# cursor = conn.cursor()
#cereate database
# import codecs
# createSQLFile = codecs.open(createSQLFilePath, 'rb','utf-8-sig')# http://stackoverflow.com/questions/2359832/dealing-with-utf-8-numbers-in-python
# #createSQLFile = open(createSQLFilePath, 'rb')
# createDBFile(conn, cursor, createSQLFile)
incrementProgress()
#process CRZ file and fill db file
log.lg("norm", 'processing metakit file')
processCRZFile(mk, dal)
#clean up overwritten file of overwriting
# if isOverwriting:
# os.remove(existingPath)
# os.rename(outputPath, existingPath)
dal.Path = outputPath
success = True
# except Exception as er:
# log.lg('high', 'PROGRAM ERROR program ended in error, exception args: ' + repr(er.args))
# log.lg('high', 'PROGRAM ERROR program ended in error, exception message: ' + repr(er.message))
# #log.lg('high', sys.exc_info())
# #os.remove(outputPath)#filling database failed remove it
# success = False
#
# finally:
# pass
# if not cursor is None:
# cursor.close()
# if not conn is None:
# conn.close()
return success
def _change_row_types (self, changes):
"""Change row named 'name' from type 'old' to type 'new'
changes = [(table, name, old, new, converter),...]
We have to make all changes at once or bad things will happen.
"""
# This is quite a complex little problem in pymetakit. I got
# help from Brian Kelley <*****@*****.**> over the
# metakit mailing list on this one. Basically, in order to
# change the datatype of a column we have to add a dummy
# column, drop the old column, add the old column back, and
# copy the information over from our dummy. This is made even
# stranger by the fact that to drop a column in metakit, you
# have to call getas() without the column and then call
# commit() and nuke any reference to the db.
# if we don't have this table yet, then we don't need to do anything
DUMMIES = {}
change_dic = {}
default_descs = {}
dummy_descs = {}
for table,name,old,new,converter in changes:
if not hasattr(self.contentview[0],table):
continue
if not default_descs.has_key(table):
default_descs[table]=self.db.description(table)
if not dummy_descs.has_key(table):
dummy_descs[table]=default_descs[table]
if self._row_type(table,name) == self.type_to_metakit_type(new).lower():
continue
self._backup_database()
# we create a view with our old self and a new temporary self
DUMMIES[name] = 'TMP%s'%name
if not change_dic.has_key(table):
change_dic[table]={}
change_dic[table][name]={}
change_dic[table][name]['old']=old
change_dic[table][name]['new']=new
change_dic[table][name]['converter']=converter
self._move_row(table,(name,old),(DUMMIES[name],old))
# We're going to modify our default setup arguments to
# drop the proper column and add the new one in its sted
# before committing. This allows us to "drop" the old
# version of column 'name'
dummy_descs[table] = re.sub("(,|^)%s:"%re.escape(name),
r"\1%s:"%re.escape(DUMMIES[name]),
dummy_descs[table])
if DUMMIES:
# Drop our old columns...
for table,dummy_desc in dummy_descs.items():
self.db.getas("%s[%s]"%(table,dummy_desc))
debug('dropping columns by committing database',3)
self.db.commit()
debug('deleting reference to our db',3)
del self.db
debug('reinitialize our connection to a new db',3)
#self.initialize_connection() # reinitialize ourselves
#with our new DUMMYNAME column
self.db = metakit.storage(self.file,1)
self.contentview = self.db.contents()
# now we get our new self as a new datatype and copy our
# new information over...
# Loop through the changes we have to make
for table,cd in change_dic.items():
for name,change in cd.items():
newvw = self.setup_table(table,[(name,change['new']),(DUMMIES[name],change['old'])])
vw = newvw.filter(lambda x: getattr(x,DUMMIES[name]))
to_move_vw = newvw.remapwith(vw)
debug('converting attributes',4)
for n,r in enumerate(to_move_vw):
# convert our attributes
setattr(r,name,change['converter'](getattr(r,DUMMIES[name])))
# erase our temporary/holder attribute
setattr(r,DUMMIES[name],None)
debug('moved attribute %s times'%n,3)
default_descs[table] = re.sub(
"(,|^)%s:%s"%(name,self.type_to_metakit_type(change['old'])),
r"\1%s:%s"%(name,self.type_to_metakit_type(change['new'])),
default_descs[table]
)
for table,finished_desc in default_descs.items():
self.db.getas("%s[%s]"%(table,finished_desc)) #setup our table with the right attrs
self.db.commit() # and drop our dummy column
# Demo of derived view dynamics
#
# Output:
# [1, 2, 3, 4, 5, 6]
# [2, 3, 4, 5]
# [2, 4, 5]
# [2, 5]
# [1, 2, 5, 6]
import metakit
db = metakit.storage()
vw = db.getas('data[value:I]')
def fill(l):
vw[:] = []
for i in l:
vw.append(value=i)
def show(v):
print map((lambda x: x.value), v)
fill([1, 2, 3, 4, 5, 6])
show(vw)
# select values in range [2..5]
v2 = vw.select({'value': 2}, {'value': 5})
show(v2)
def _change_row_types(self, changes):
"""Change row named 'name' from type 'old' to type 'new'
changes = [(table, name, old, new, converter),...]
We have to make all changes at once or bad things will happen.
"""
# This is quite a complex little problem in pymetakit. I got
# help from Brian Kelley <*****@*****.**> over the
# metakit mailing list on this one. Basically, in order to
# change the datatype of a column we have to add a dummy
# column, drop the old column, add the old column back, and
# copy the information over from our dummy. This is made even
# stranger by the fact that to drop a column in metakit, you
# have to call getas() without the column and then call
# commit() and nuke any reference to the db.
# if we don't have this table yet, then we don't need to do anything
DUMMIES = {}
change_dic = {}
default_descs = {}
dummy_descs = {}
for table, name, old, new, converter in changes:
if not hasattr(self.contentview[0], table):
continue
if not default_descs.has_key(table):
default_descs[table] = self.db.description(table)
if not dummy_descs.has_key(table):
dummy_descs[table] = default_descs[table]
if self._row_type(table,
name) == self.type_to_metakit_type(new).lower():
continue
self._backup_database()
# we create a view with our old self and a new temporary self
DUMMIES[name] = 'TMP%s' % name
if not change_dic.has_key(table):
change_dic[table] = {}
change_dic[table][name] = {}
change_dic[table][name]['old'] = old
change_dic[table][name]['new'] = new
change_dic[table][name]['converter'] = converter
self._move_row(table, (name, old), (DUMMIES[name], old))
# We're going to modify our default setup arguments to
# drop the proper column and add the new one in its sted
# before committing. This allows us to "drop" the old
# version of column 'name'
dummy_descs[table] = re.sub("(,|^)%s:" % re.escape(name),
r"\1%s:" % re.escape(DUMMIES[name]),
dummy_descs[table])
if DUMMIES:
# Drop our old columns...
for table, dummy_desc in dummy_descs.items():
self.db.getas("%s[%s]" % (table, dummy_desc))
debug('dropping columns by committing database', 3)
self.db.commit()
debug('deleting reference to our db', 3)
del self.db
debug('reinitialize our connection to a new db', 3)
#self.initialize_connection() # reinitialize ourselves
#with our new DUMMYNAME column
self.db = metakit.storage(self.file, 1)
self.contentview = self.db.contents()
# now we get our new self as a new datatype and copy our
# new information over...
# Loop through the changes we have to make
for table, cd in change_dic.items():
for name, change in cd.items():
newvw = self.setup_table(table,
[(name, change['new']),
(DUMMIES[name], change['old'])])
vw = newvw.filter(lambda x: getattr(x, DUMMIES[name]))
to_move_vw = newvw.remapwith(vw)
debug('converting attributes', 4)
for n, r in enumerate(to_move_vw):
# convert our attributes
setattr(r, name,
change['converter'](getattr(r, DUMMIES[name])))
# erase our temporary/holder attribute
setattr(r, DUMMIES[name], None)
debug('moved attribute %s times' % n, 3)
default_descs[table] = re.sub(
"(,|^)%s:%s" %
(name, self.type_to_metakit_type(change['old'])),
r"\1%s:%s" %
(name, self.type_to_metakit_type(change['new'])),
default_descs[table])
for table, finished_desc in default_descs.items():
self.db.getas(
"%s[%s]" %
(table,
finished_desc)) #setup our table with the right attrs
self.db.commit() # and drop our dummy column
def __init__(self, filename, mode):
self.db = metakit.storage(filename, 1)
view = self.db.getas("data[key:S,value:S]")
hashvw = self.db.getas("__test_hash__[_H:I,_R:I]")
self.view = view.hash(hashvw, 1)
def write_feed_json(outline, outnum, output_dir):
"""Writes articles from specified feed (outline) into a JSON file."""
articles = OrderedDict()
feed_url = outline.get('xmlUrl')
html_url = outline.get('htmlUrl')
feed_title = outline.get('title')
feed_file = feed_url.replace(':', '_').replace('/', '_') + '.mk4'
fdb = metakit.storage(os.path.join(ak_archive_path, feed_file), 0)
for a in fdb.getas(fdb.description()):
# skip articles marked as "deleted" or "thrash"
if a.status & 0x3:
continue
# If article guid is a hash, then it means that feed XML does not specify a guid value.
# Akregator does and keeps a hash value, but FreshRSS uses article link.
# Convert to avoid duplicate entries when feed is fetched by FreshRSS.
guid = a.link if a.guid.startswith('hash:') else a.guid
# if guid is duplicate, drop the old article (we are iterating from older to newer;
# so the new one should take precedence, but in case of duplicated guids in JSON
# FreshRSS would import first and ignore all following entries...)
articles.pop(guid, None)
# format article enclosure for appending to contents just like FreshRSS does
if a.hasEnclosure:
enclosure = ['\n<div class="enclosure">']
if a.enclosureType.startswith(
'video/') or a.enclosureType.startswith('audio/'):
tag = 'video' if a.enclosureType.startswith(
'video/') else 'audio'
enclosure.append('<p class="enclosure-content"><')
enclosure.append(tag)
enclosure.append(' preload="none" src="')
enclosure.append(a.enclosureUrl)
enclosure.append('" controls="controls"></')
enclosure.append(tag)
enclosure.append('> <a download="" href="')
enclosure.append(a.enclosureUrl)
enclosure.append('">💾</a></p>')
elif a.enclosureType.startswith('image/'):
enclosure.append('<p class="enclosure-content"><img src="')
enclosure.append(a.enclosureUrl)
enclosure.append('" alt="" /></p>')
elif a.enclosureType.startswith(
'application/') or a.enclosureType.startswith('text/'):
enclosure.append(
'<p class="enclosure-content"><a download="" href="')
enclosure.append(a.enclosureUrl)
enclosure.append('">💾</a></p>')
enclosure.append('</div>')
enclosure = ''.join(enclosure)
else:
enclosure = ''
# format article content
if a.content or a.description or enclosure:
content = '\n' + (a.content or a.description) + enclosure + '\n'
else:
content = ''
articles[guid] = OrderedDict([
('id', uesc(guid)),
(
'categories', [tag for tag in a.tags]
), # akregator does not seem to support tags, although fields exists in database...
('title', uesc(a.title)),
('author', uesc(a.authorName or a.authorEMail)),
('published', a.pubDate),
('updated', a.pubDate),
('alternate',
[OrderedDict([
('href', uesc(a.link)),
('type', 'text/html'),
])]),
('content', {
'content': u(content),
}),
('origin',
OrderedDict([
('streamId', outnum),
('title', uesc(feed_title)),
('htmlUrl', uesc(html_url)),
('feedUrl', uesc(feed_url)),
])),
])
dump_data = OrderedDict([
('id', 'akregator2zip/dump/feed/{}'.format(outnum)),
('title', u'List of {} articles'.format(uesc(feed_title))),
('author', 'akregator2zip dumper'),
('items', list(articles.itervalues())),
])
dump_file = 'feed_{}_{:04d}.json'.format(
datetime.today().strftime('%Y-%m-%d'), outnum)
with codecs.open(os.path.join(output_dir, dump_file), 'w', 'utf-8') as f:
json.dump(dump_data,
f,
ensure_ascii=False,
indent=4,
separators=(',', ': '))
return len(articles)
