Exemple #1
0
def main():
    db = metakit.storage(sys.argv[1], 0)

    views = []

    sofar = []
    depth = 0
    for c in db.description():
        if c == '[': depth += 1
        elif c == ']': depth -= 1
        if depth == 0 and c == ',':
            views.append("".join(sofar))
            sofar[:] = []
        else:
            sofar.append(c)
    views.append("".join(sofar))

    di = dict(zip(views, [view_to_list(db.getas(v)) for v in views]))

    print "converted succesfully"

#####

    db = metakit.storage('out.db', 1)


    for viewdesc, data in di.iteritems():
        view = db.getas(viewdesc)
        fill(view, data)

    db.commit()

    print "converted succesfully"
Exemple #2
0
def main():
    db = metakit.storage(sys.argv[1], 0)

    views = []

    sofar = []
    depth = 0
    for c in db.description():
        if c == '[': depth += 1
        elif c == ']': depth -= 1
        if depth == 0 and c == ',':
            views.append("".join(sofar))
            sofar[:] = []
        else:
            sofar.append(c)
    views.append("".join(sofar))

    di = dict(zip(views, [view_to_list(db.getas(v)) for v in views]))

    print "converted succesfully"

    #####

    db = metakit.storage('out.db', 1)

    for viewdesc, data in di.iteritems():
        view = db.getas(viewdesc)
        fill(view, data)

    db.commit()

    print "converted succesfully"
    def __init__(self, filename=None):
        if isinstance(filename, unicode):
            filename = filename.encode(sys.getfilesystemencoding())
        self.filename = filename


        if self.filename is None:
            # We initially create an in-memory database.
            # When we save to a file, we will reopen the database from the file.
            self.db = metakit.storage()
#            self.filename = 'defau.gt'
#            self.db = metakit.storage(self.filename, 1)
#            for desc in storage_desc.values():
#                self.db.getas(desc)
#            self.db.commit()
        else:
            self.db = metakit.storage(self.filename, 1)
            self.cleanup()

#        self.aside = metakit.storage('grafity-storage.mka', 1)
#        self.db.aside(self.aside)
#        print >>sys.stderr, "project created"

        self._modified = False

        action_list.connect('added', self.on_action_added)

        self.items = {}
        self.deleted = {}
        self._dict = {}
        self.save_dict = {}

        # Create top folder.
        # - it must be created before all other items
        # - it must be created with _isroot=True, to set itself as its parent folder
        try:
            fv = self.db.getas(storage_desc[Folder])
            row = fv.select(name='top')[0]
            self.top = self.items[row.id] = Folder(self, location=(fv, row, row.id), _isroot=True)
        except IndexError:
            # can't find it in the database, create a new one.
            self.top = Folder(self, 'top', _isroot=True)

        self.here = self.top
        self.this = None

        # create objects
        for cls, desc in [(i, storage_desc[i]) for i in (Folder, grafity.worksheet.Worksheet, grafity.graph.Graph)]:
            view = self.db.getas(desc)
            for i, row in enumerate(view):
                if row.id != self.top.id:
                    if not row.id.startswith('-'):
#                        print 'loading', cls, row.id,
                        self.items[row.id] = cls(self, location=(view, row, row.id))
#                        print 'end'
                    else:
                        self.deleted[row.id] = cls(self, location=(view, row, row.id))
Exemple #4
0
def CreateDb(dbName, modelName='default'):
    # database version
    db = metakit.storage(dbName, 1)
    vw = db.getas(MetaKit.VERSION_VIEW)
    vw.append(version=MetaKit.VERSION)
    db.commit()
    return GetDb(dbName, modelName)
 def __init__(self, description):
     self.storage = metakit.storage()
     self.v = self.storage.getas(description)
     self.arr = []
     self.failure = Failure()
     self.fail = self.failure.fail
     self.columns = map(lambda c: string.split(c, ':')[0], string.split(description[string.index(description, '[') + 1:-1], ','))
Exemple #6
0
	def __init__(self, directory, dbName, cache = 128):
		log.message("Opening database", dbName)
		self.dbName = dbName
		self.nextID = 10000
		self.cacheSize = cache
		#
		try:
			os.makedirs(directory)
		except OSError:
			pass
		# open db
		self.storage = metakit.storage(os.path.join(directory, dbName), 1)
		view = self.storage.getas(self.dbSchema).blocked()
		map = self.storage.getas("map[_H:I,_R:I]")
		self.view = view.hash(map, 1)
		# cache
		self.cache = {}
		self.cacheLinks = {
			"__first__": [None, "__last__"],
			"__last__": ["__first__", None],
		}
		# stats
		self.statCount = 0
		self.statHit = 0
		self.statSwap = 0
		self.statMiss = 0
		self.statCleanSwap = 0
def read_feeds_opml():
    """Opens Akregator feedlistbackup archive and returns lxml-parsed OPML xml."""
    db = metakit.storage(os.path.join(ak_archive_path, 'feedlistbackup.mk4'),
                         0)
    vw = db.getas(db.description())
    assert len(vw) == 1, "Expecting one row in feedlistbackup."
    return et.fromstring(vw[0].feedList)
 def __init__(self, filename):
     self.db = metakit.storage(filename, 1)
     if os.path.exists(filename+'.swp'):
         os.remove(filename+'.swp')
     self.filename = filename
     self.undodb = metakit.storage(filename+'.swp', 1)
     self.items = {}
     self.oplist = []
     self.action_name = None
     self.storage = self
     for name in dir(type(self)):
         attr = getattr(self, name)
         if isinstance(attr, Container):
             attr.name = name
             attr.storage = self
             itemtypes[name] = attr.cls
Exemple #9
0
def readfeed(file):
    # Open a Metakit file, one file per blog, fetch contents
    global idxarticle
    blogdb = mk.storage(file,0)
    blogvw = blogdb.getas(blogdb.description())
    for i in blogvw:
        writedb_article(i)
        idxarticle = idxarticle + 1
    return
Exemple #10
0
 def __init__(self, description):
     self.storage = metakit.storage()
     self.v = self.storage.getas(description)
     self.arr = []
     self.failure = Failure()
     self.fail = self.failure.fail
     self.columns = map(
         lambda c: string.split(c, ':')[0],
         string.split(description[string.index(description, '[') + 1:-1],
                      ','))
Exemple #11
0
 def initialize_connection (self):
     # identical to parent class, except that we open the DB read-only
     debug('using file: %s'%self.file,1)
     self.file = str(os.path.expandvars(os.path.expanduser(self.file)))
     mydir = os.path.split(self.file)[0]
     # create the directory if it does not yet exist
     if not os.path.exists(mydir):
         os.mkdir(mydir)
     self.db = metakit.storage(self.file,0) #filename must be string, not unicode
     self.contentview=self.db.contents()
Exemple #12
0
 def initialize_connection (self):
     # identical to parent class, except that we open the DB read-only
     debug('using file: %s'%self.file,1)
     self.file = str(os.path.expandvars(os.path.expanduser(self.file)))
     mydir = os.path.split(self.file)[0]
     # create the directory if it does not yet exist
     if not os.path.exists(mydir):
         os.mkdir(mydir)
     self.db = metakit.storage(self.file,0) #filename must be string, not unicode
     self.contentview=self.db.contents()
Exemple #13
0
 def initialize_connection (self):
     debug('using file: %s'%self.file,1)
     self.file = os.path.expandvars(os.path.expanduser(self.file))
     mydir = os.path.split(self.file)[0]
     # create the directory if it does not yet exist
     if not os.path.exists(mydir):
         os.mkdir(mydir)
     self.db = metakit.storage(str(self.file),1) #filename must be string, not unicode
     self.contentview=self.db.contents()
     #self.load()
     self.db.autocommit()
Exemple #14
0
 def initialize_connection (self):
     debug('using file: %s'%self.file,1)
     self.file = os.path.expandvars(os.path.expanduser(self.file))
     mydir = os.path.split(self.file)[0]
     # create the directory if it does not yet exist
     if not os.path.exists(mydir):
         os.mkdir(mydir)
     self.db = metakit.storage(str(self.file),1) #filename must be string, not unicode
     self.contentview=self.db.contents()
     #self.load()
     self.db.autocommit()
Exemple #15
0
	def establishConnection(self, fullSpecifier):
		"""Connect using the fully specified specifier

		fullSpecifier -- a specifier with all arguments unified
			and ready to be connected.  This specifier should
			include everything required to do the actual
			connection (including passwords or the like).

		All sub-classes must override this method!
		"""
		filename = fullSpecifier.dsn
		return metakit.storage( filename, 1 )
Exemple #16
0
 def __init__(self, connection):
     "Setup the database connection"
     self._system_tables=[]
     # Not providing a db name is guaranteed to ruin our connection
     if not connection['databasename']:
         raise ValueError
     filename = os.path.join(connection['directory'], connection['databasename'])
     # not sure about the mode to use, I assume we just want read-only
     self._db = metakit.storage(filename, 0)
     self._cursor='placeholder'
     # This one is used in getRow
     self._tableName=''
def main(filepath):
    db = metakit.storage(filepath, 0)
    meta = db.contents()

    for table in meta.structure():
        view = db.view(table.name)
        columns = [column.name for column in view.structure()]
        data = [{column: getattr(row, column)
                 for column in columns} for row in view]

        with open('%s.json' % table.name, 'w') as fp:
            json.dump(data, fp, indent=2)
    def update_feed_articles(outline, feed_id):
        articles = []
        feed_url = outline.get('xmlUrl')
        html_url = outline.get('htmlUrl')
        feed_title = outline.get('title')
        feed_file = feed_url.replace(':', '_').replace('/', '_') + '.mk4'
        fdb = metakit.storage(os.path.join(ak_archive_path, feed_file), 0)
        updates, fails = 0, 0
        for a in fdb.getas(fdb.description()):
            # Akregator status bits: Deleted = 0x01, Trash = 0x02, New = 0x04, Read = 0x08, Keep = 0x10
            if a.status & 0x3:
                continue  # skip articles marked as "deleted" or "thrash"
            is_read = bool(a.status & 0x08)
            is_fav = bool(a.status & 0x10)
            guid = a.link if a.guid.startswith('hash:') else a.guid
            try:
                # try to update id opportunistically so next step (fix_article_order) won't have to update the same row again
                found_rows = c.execute(
                    'UPDATE ' + entry_table +
                    ' SET id=%s*1000000+id%%1000000, date=%s, is_read=%s, is_favorite=%s WHERE id_feed=%s AND guid=%s',
                    (a.pubDate, a.pubDate, is_read, is_fav, feed_id,
                     uesc(guid)))
            except MySQLdb.IntegrityError:
                # conflict with another row having the target ID value... leave it to be resolved in fix_article_order()
                found_rows = c.execute(
                    'UPDATE ' + entry_table +
                    ' SET date=%s, is_read=%s, is_favorite=%s WHERE id_feed=%s AND guid=%s',
                    (a.pubDate, is_read, is_fav, feed_id, uesc(guid)))
            if found_rows:
                updates += 1
            else:
                fails += 1

        if fails:
            print(
                u"ERROR: failed to update {} articles in {} (article not found in FreshRSS database)"
                .format(fails, feed_title))
        else:
            print(u"Successfully updated {} articles in feed {}".format(
                updates, feed_title))

        c.execute(
            'SELECT COUNT(id) FROM ' + entry_table +
            ' WHERE id_feed=%s AND is_read=0', (feed_id, ))
        return updates, fails, c.fetchone()[0]
Exemple #19
0
    def update_schema(self):
        storFn = pycs_paths.DATADIR + "/settings.dat"
        print "Connecting to MetaKit database in %s" % storFn
        import metakit
        mkdb = metakit.storage(storFn, 1)
        
        print "Updating PostgreSQL database schema"

        # See if the database is completely new
        try:
            self.fetchone("SELECT db_id FROM pycs_meta")
        except DBE, e:
            if e.args[0].find("does not exist") != -1:
                print "Creating pycs_meta table"
                self.execute("""CREATE TABLE pycs_meta (db_id INT)""")
                self.execute("INSERT INTO pycs_meta (db_id) VALUES (0)")
            else:
                raise
Exemple #20
0
    def openFile(self, filename):
        import metakit
        self.commonOpenFileInit(filename)

        #print filename
        if os.path.exists(filename):
            # need to keep a reference to the storage
            # or it will be garbage collected
            self._db = metakit.storage(filename, 0)
            # there should only be one view, but this
            # needs to be made more robust, perhaps only
            # grabbing the first view?
            viewName = self._db.description().split('[')[0]
            # *** the view is hard-code below but shouldn't be
            # I just want the default view
            # self.records = self._db.view('companies')
            self.records = self._db.view(viewName)
            #print "len(self.records)", len(self.records)
            if self.current == -1 and len(self.records) > 0:
                self.displayRecord(0)
Exemple #21
0
Fichier : rss.py Projet : stes/nbot
def test_db():
    gl_vlist = VocabList()
    log('searching directory: %s' % FEED_DIR)
    for dir in os.listdir(FEED_DIR):
        if '.mk4' in dir[-4:]:
            log('found database: %s' % dir)
            # open database
            db = metakit.storage(os.path.join(FEED_DIR, dir), 0)
            data = read_database(db)
            if len(data) > 0:
                # feed content in database
                log('create library')
                lib = Library()
                for feed in data:
                    lib.add_document(read_data(feed))
                vlist = lib.gen_vocablist()
                vlist.clean(5)
                gl_vlist.merge(vlist)
            db = None # close database
    print gl_vlist
    def __init__(self, filename=None):
        if isinstance(filename, unicode):
            filename = filename.encode(sys.getfilesystemencoding())
        self.filename = filename

        if self.filename is None:
            # We initially create an in-memory database.
            # When we save to a file, we will reopen the database from the file.
            self.db = metakit.storage()
#            self.filename = 'defau.gt'
#            self.db = metakit.storage(self.filename, 1)
#            for desc in storage_desc.values():
#                self.db.getas(desc)
#            self.db.commit()
        else:
            self.db = metakit.storage(self.filename, 1)
            self.cleanup()

#        self.aside = metakit.storage('grafit-storage.mka', 1)
#        self.db.aside(self.aside)
#        print >>sys.stderr, "project created"

        self._modified = False

        action_list.connect('added', self.on_action_added)

        self.items = {}
        self.deleted = {}
        self._dict = {}
        self.save_dict = {}

        # Create top folder.
        # - it must be created before all other items
        # - it must be created with _isroot=True, to set itself as its parent folder
        try:
            fv = self.db.getas(storage_desc[Folder])
            row = fv.select(name='top')[0]
            self.top = self.items[row.id] = Folder(self,
                                                   location=(fv, row, row.id),
                                                   _isroot=True)
        except IndexError:
            # can't find it in the database, create a new one.
            self.top = Folder(self, 'top', _isroot=True)

        self.here = self.top
        self.this = None

        # create objects
        for cls, desc in [(i, storage_desc[i])
                          for i in (Folder, grafit.worksheet.Worksheet,
                                    grafit.graph.Graph)]:
            view = self.db.getas(desc)
            for i, row in enumerate(view):
                if row.id != self.top.id:
                    if not row.id.startswith('-'):
                        #                        print 'loading', cls, row.id,
                        self.items[row.id] = cls(self,
                                                 location=(view, row, row.id))
#                        print 'end'
                    else:
                        self.deleted[row.id] = cls(self,
                                                   location=(view, row,
                                                             row.id))
Exemple #23
0
# 22222 0
# [Property('S', 's'), Property('V', 'sub')]
# 00000 1
# 11111 1
# 22222 0
# [Property('S', 's'), Property('I', 'i'), Property('V', 'sub')]
# ok

import os
try:
    os.remove("_selfref.mk")
except:
    pass

import metakit
db = metakit.storage('_selfref.mk', 1)
vw = db.getas('data[s:S,sub[^]]')

print len(vw)

v = vw
for i in range(3):
    v.append(s=` i ` * 5)
    v = v[0].sub


def show(vw):
    for v in [vw[0], vw[0].sub[0], vw[0].sub[0].sub[0]]:
        print v.s, len(v.sub)
    print vw[0].sub[0].sub[0].sub.structure()
    assert len(vw[0].sub[0].sub[0].sub) == 0
Exemple #24
0
    m = md5.new()
    m.update(article.description)
    idx = m.hexdigest()
    insert = (idxarticle, idxfeed, article.guid, article.title, article.link, article.comments, article.commentsLink, article.status, article.pubDate, article.author)
    sqdb.execute("insert into article (id, idfeed, guid, title, url, comments, commentsLink, status, pubDate, author) values (?,?,?,?,?,?,?,?,?,?)", insert)
    insert2 = (idxarticle,idx)
    sqdb.execute("insert into contentindex(id, id_content) values(?, ?)", insert2)
    if idx not in hash:
    	insert = (idx, zlib.compress(article.description, 9))
        sqdb.execute("insert into content(id, content) values (?,?)", insert)
        hash[idx]=1        
	nondupes = nondupes+1
    return

print "Starting conversion:"
db=mk.storage(indexfile,0)
vw=db.getas(db.description())
print "Opening up database files"
sqdb = sq3.connect(sq3file)
initdb()

for i in vw:
    getfeed(i)

print "Imported "+str(idxarticle)+" articles (ignored "+str(idxarticle-nondupes)+" duplicates) from "+str(idxfeed)+" blogs"
print "Creating indexes:"
print "    Indexing article publishing dates..."
sqdb.execute("CREATE INDEX a_pubdate on article (pubDate ASC)")
print "    Indexing article's author names..."
sqdb.execute("CREATE INDEX a_author on article (author ASC)")
print "    Indexing articles...."
Exemple #25
0
# 00000 1
# 11111 1
# 22222 0
# [Property('S', 's'), Property('V', 'sub')]
# 00000 1
# 11111 1
# 22222 0
# [Property('S', 's'), Property('I', 'i'), Property('V', 'sub')]
# ok

import os
try: os.remove("_selfref.mk")
except: pass

import metakit
db = metakit.storage('_selfref.mk',1)
vw = db.getas('data[s:S,sub[^]]')

print len(vw)

v = vw
for i in range(3):
  v.append(s=`i`*5)
  v = v[0].sub

def show(vw):
  for v in [vw[0], vw[0].sub[0], vw[0].sub[0].sub[0]]:
    print v.s, len(v.sub)
  print vw[0].sub[0].sub[0].sub.structure()
  assert len(vw[0].sub[0].sub[0].sub) == 0
  try:
def Run(inputPath, outputPath):
    success = False
    conn = None
    cursor = None
    isOverwriting = False
    existingPath = None

    #check for create curise file
    #     if not os.path.exists(createSQLFilePath):
    #         log.lg("high", "ERROR resource file CruiseCreate.sql not found")
    #
    #
    #         success = False
    #         return success

    #check output file extention
    if not str.lower(outputPath[-7:]) == '.cruise':
        log.lg("high", "ERROR outputFile has bad extention")
        success = False
        return success

    #check if overwriting existing filename


#     if(os.path.exists(outputPath)):
#         log.lg('norm', 'existing file found: overwriting')
#         isOverwriting = True
#         existingPath = outputPath
#         outputPath = outputPath + '.new'

#check input file and extention
    if not str.lower(inputPath[-4:]) == '.crz':
        log.lg("high", "ERROR file to convert has wrong extention")
        success = False
        return success

    if not os.path.exists(inputPath):
        log.lg("high", "ERROR Couldn't find file to convert")
        success = False
        return success

    #load input file
    mk = metakit.storage(inputPath, 0)
    #check crz file version of the input file
    if not isCRZCurrent(mk):
        log.lg('high', 'ERROR CRZ file not current version')
        success = False
        return success

    #try:
    #create database connecting and cursor
    print(outputPath)
    dal = DAL(outputPath, True)
    dal.Create()
    #         cursor = conn.cursor()
    #cereate database
    #         import codecs
    #         createSQLFile = codecs.open(createSQLFilePath, 'rb','utf-8-sig')# http://stackoverflow.com/questions/2359832/dealing-with-utf-8-numbers-in-python
    #         #createSQLFile = open(createSQLFilePath, 'rb')
    #         createDBFile(conn, cursor, createSQLFile)
    incrementProgress()

    #process CRZ file and fill db file
    log.lg("norm", 'processing metakit file')
    processCRZFile(mk, dal)

    #clean up overwritten file of overwriting
    #         if isOverwriting:
    #             os.remove(existingPath)
    #             os.rename(outputPath, existingPath)
    dal.Path = outputPath
    success = True

    #     except Exception as er:
    #         log.lg('high', 'PROGRAM ERROR program ended in error, exception args: ' + repr(er.args))
    #         log.lg('high', 'PROGRAM ERROR program ended in error, exception message: ' + repr(er.message))
    #         #log.lg('high', sys.exc_info())
    #         #os.remove(outputPath)#filling database failed remove it
    #         success = False
    #
    #     finally:
    #         pass
    #         if not cursor is None:
    #             cursor.close()
    #         if not conn is None:
    #             conn.close()

    return success
Exemple #27
0
# Properties are case-insensitive, but this can lead to some
# surprising behavior: the first way a property is used will
# determine how it ends up in the global symbol table.
#
# Sample output:
#   Property('S', 'HeLLo') Property('S', 'HeLLo')
#   2
#   2
#   135099576
#   135033272
#   0

import metakit
db = metakit.storage()
v1 = db.getas('lo[HeLLo:S]')
v2 = db.getas('hi[hello:S]')

# surprise: this prints two mixed-case names
print v1.HeLLo, v2.hello

# this shows that the MetaKit property is the same for both
# reason: there is a single global case-insensitive symbol table
print metakit.property('S','HeLLo').id
print metakit.property('S','hello').id

# this shows that the Python objects differ
# reason: these are two wrapper objects around the same thing
print id(metakit.property('S','HeLLo'))
print id(metakit.property('S','hello'))

# this causes a mismatch, it will have to be fixed one day
Exemple #28
0
 def __init__ (self, filename, mode):
     self.db = metakit.storage( filename, 1) 
     view = self.db.getas("data[key:S,value:S]")
     hashvw = self.db.getas("__test_hash__[_H:I,_R:I]")
     self.view = view.hash(hashvw, 1)
def Run(inputPath, outputPath):
    success = False
    conn = None
    cursor = None
    isOverwriting = False 
    existingPath = None
    
    #check for create curise file
#     if not os.path.exists(createSQLFilePath):
#         log.lg("high", "ERROR resource file CruiseCreate.sql not found")
#         
#         
#         success = False
#         return success
    
    #check output file extention
    if not str.lower(outputPath[-7:]) == '.cruise':
        log.lg("high", "ERROR outputFile has bad extention")
        success = False
        return success
        
    #check if overwriting existing filename
#     if(os.path.exists(outputPath)):
#         log.lg('norm', 'existing file found: overwriting')
#         isOverwriting = True
#         existingPath = outputPath
#         outputPath = outputPath + '.new'
        
    #check input file and extention
    if not str.lower(inputPath[-4:]) == '.crz':
        log.lg("high", "ERROR file to convert has wrong extention")
        success = False
        return success
    
    if not os.path.exists(inputPath):
        log.lg("high", "ERROR Couldn't find file to convert")
        success = False
        return success
    
    #load input file 
    mk = metakit.storage(inputPath,0)
    #check crz file version of the input file
    if not isCRZCurrent(mk):
        log.lg('high', 'ERROR CRZ file not current version')
        success = False
        return success
    
    #try:
        #create database connecting and cursor
    print(outputPath)
    dal = DAL(outputPath, True)
    dal.Create()
#         cursor = conn.cursor()
    #cereate database
#         import codecs 
#         createSQLFile = codecs.open(createSQLFilePath, 'rb','utf-8-sig')# http://stackoverflow.com/questions/2359832/dealing-with-utf-8-numbers-in-python
#         #createSQLFile = open(createSQLFilePath, 'rb')
#         createDBFile(conn, cursor, createSQLFile)
    incrementProgress()
    
    #process CRZ file and fill db file
    log.lg("norm", 'processing metakit file')
    processCRZFile(mk, dal)
    
    #clean up overwritten file of overwriting
#         if isOverwriting:
#             os.remove(existingPath)
#             os.rename(outputPath, existingPath)
    dal.Path = outputPath
    success = True
         
#     except Exception as er: 
#         log.lg('high', 'PROGRAM ERROR program ended in error, exception args: ' + repr(er.args))
#         log.lg('high', 'PROGRAM ERROR program ended in error, exception message: ' + repr(er.message))
#         #log.lg('high', sys.exc_info())
#         #os.remove(outputPath)#filling database failed remove it
#         success = False
# 
#     finally:
#         pass
#         if not cursor is None: 
#             cursor.close()
#         if not conn is None:
#             conn.close()
        

        
    return success
Exemple #30
0
    def _change_row_types (self, changes):
        """Change row named 'name' from type 'old' to type 'new'

        changes = [(table, name, old, new, converter),...]

        We have to make all changes at once or bad things will happen.
        """
        # This is quite a complex little problem in pymetakit. I got
        # help from Brian Kelley <*****@*****.**> over the
        # metakit mailing list on this one.  Basically, in order to
        # change the datatype of a column we have to add a dummy
        # column, drop the old column, add the old column back, and
        # copy the information over from our dummy. This is made even
        # stranger by the fact that to drop a column in metakit, you
        # have to call getas() without the column and then call
        # commit() and nuke any reference to the db.
        
        # if we don't have this table yet, then we don't need to do anything        
        DUMMIES = {}
        change_dic = {}
        default_descs = {}
        dummy_descs = {}
        for table,name,old,new,converter in changes:
            if not hasattr(self.contentview[0],table):
                continue
            if not default_descs.has_key(table):
                default_descs[table]=self.db.description(table)
            if not dummy_descs.has_key(table):
                dummy_descs[table]=default_descs[table]
            if self._row_type(table,name) == self.type_to_metakit_type(new).lower():
                continue
            self._backup_database()
            # we create a view with our old self and a new temporary self
            DUMMIES[name] = 'TMP%s'%name
            if not change_dic.has_key(table):
                change_dic[table]={}
            change_dic[table][name]={}
            change_dic[table][name]['old']=old
            change_dic[table][name]['new']=new
            change_dic[table][name]['converter']=converter
            self._move_row(table,(name,old),(DUMMIES[name],old))
            # We're going to modify our default setup arguments to
            # drop the proper column and add the new one in its sted
            # before committing. This allows us to "drop" the old
            # version of column 'name'
            dummy_descs[table] = re.sub("(,|^)%s:"%re.escape(name),
                                        r"\1%s:"%re.escape(DUMMIES[name]),
                                        dummy_descs[table])
        if DUMMIES:
            # Drop our old columns...
            for table,dummy_desc in dummy_descs.items():
                self.db.getas("%s[%s]"%(table,dummy_desc))
            debug('dropping columns by committing database',3)
            self.db.commit()
            debug('deleting reference to our db',3)
            del self.db
            debug('reinitialize our connection to a new db',3)
            #self.initialize_connection() # reinitialize ourselves
            #with our new DUMMYNAME column
            self.db = metakit.storage(self.file,1)
            self.contentview = self.db.contents()
            # now we get our new self as a new datatype and copy our
            # new information over...
            # Loop through the changes we have to make
            for table,cd in change_dic.items():
                for name,change in cd.items():
                    newvw = self.setup_table(table,[(name,change['new']),(DUMMIES[name],change['old'])])
                    vw = newvw.filter(lambda x: getattr(x,DUMMIES[name]))
                    to_move_vw = newvw.remapwith(vw)
                    debug('converting attributes',4)
                    for n,r in enumerate(to_move_vw):
                        # convert our attributes
                        setattr(r,name,change['converter'](getattr(r,DUMMIES[name])))
                        # erase our temporary/holder attribute
                        setattr(r,DUMMIES[name],None)
                    debug('moved attribute  %s times'%n,3)
                default_descs[table] = re.sub(
                    "(,|^)%s:%s"%(name,self.type_to_metakit_type(change['old'])),
                    r"\1%s:%s"%(name,self.type_to_metakit_type(change['new'])),
                    default_descs[table]
                    )
            for table,finished_desc in default_descs.items():
                self.db.getas("%s[%s]"%(table,finished_desc)) #setup our table with the right attrs
            self.db.commit() # and drop our dummy column
Exemple #31
0
# Demo of derived view dynamics
#
# Output:
# [1, 2, 3, 4, 5, 6]
# [2, 3, 4, 5]
# [2, 4, 5]
# [2, 5]
# [1, 2, 5, 6]

import metakit
db = metakit.storage()
vw = db.getas('data[value:I]')


def fill(l):
    vw[:] = []
    for i in l:
        vw.append(value=i)


def show(v):
    print map((lambda x: x.value), v)


fill([1, 2, 3, 4, 5, 6])
show(vw)

# select values in range [2..5]
v2 = vw.select({'value': 2}, {'value': 5})
show(v2)
Exemple #32
0
    def _change_row_types(self, changes):
        """Change row named 'name' from type 'old' to type 'new'

        changes = [(table, name, old, new, converter),...]

        We have to make all changes at once or bad things will happen.
        """
        # This is quite a complex little problem in pymetakit. I got
        # help from Brian Kelley <*****@*****.**> over the
        # metakit mailing list on this one.  Basically, in order to
        # change the datatype of a column we have to add a dummy
        # column, drop the old column, add the old column back, and
        # copy the information over from our dummy. This is made even
        # stranger by the fact that to drop a column in metakit, you
        # have to call getas() without the column and then call
        # commit() and nuke any reference to the db.

        # if we don't have this table yet, then we don't need to do anything
        DUMMIES = {}
        change_dic = {}
        default_descs = {}
        dummy_descs = {}
        for table, name, old, new, converter in changes:
            if not hasattr(self.contentview[0], table):
                continue
            if not default_descs.has_key(table):
                default_descs[table] = self.db.description(table)
            if not dummy_descs.has_key(table):
                dummy_descs[table] = default_descs[table]
            if self._row_type(table,
                              name) == self.type_to_metakit_type(new).lower():
                continue
            self._backup_database()
            # we create a view with our old self and a new temporary self
            DUMMIES[name] = 'TMP%s' % name
            if not change_dic.has_key(table):
                change_dic[table] = {}
            change_dic[table][name] = {}
            change_dic[table][name]['old'] = old
            change_dic[table][name]['new'] = new
            change_dic[table][name]['converter'] = converter
            self._move_row(table, (name, old), (DUMMIES[name], old))
            # We're going to modify our default setup arguments to
            # drop the proper column and add the new one in its sted
            # before committing. This allows us to "drop" the old
            # version of column 'name'
            dummy_descs[table] = re.sub("(,|^)%s:" % re.escape(name),
                                        r"\1%s:" % re.escape(DUMMIES[name]),
                                        dummy_descs[table])
        if DUMMIES:
            # Drop our old columns...
            for table, dummy_desc in dummy_descs.items():
                self.db.getas("%s[%s]" % (table, dummy_desc))
            debug('dropping columns by committing database', 3)
            self.db.commit()
            debug('deleting reference to our db', 3)
            del self.db
            debug('reinitialize our connection to a new db', 3)
            #self.initialize_connection() # reinitialize ourselves
            #with our new DUMMYNAME column
            self.db = metakit.storage(self.file, 1)
            self.contentview = self.db.contents()
            # now we get our new self as a new datatype and copy our
            # new information over...
            # Loop through the changes we have to make
            for table, cd in change_dic.items():
                for name, change in cd.items():
                    newvw = self.setup_table(table,
                                             [(name, change['new']),
                                              (DUMMIES[name], change['old'])])
                    vw = newvw.filter(lambda x: getattr(x, DUMMIES[name]))
                    to_move_vw = newvw.remapwith(vw)
                    debug('converting attributes', 4)
                    for n, r in enumerate(to_move_vw):
                        # convert our attributes
                        setattr(r, name,
                                change['converter'](getattr(r, DUMMIES[name])))
                        # erase our temporary/holder attribute
                        setattr(r, DUMMIES[name], None)
                    debug('moved attribute  %s times' % n, 3)
                default_descs[table] = re.sub(
                    "(,|^)%s:%s" %
                    (name, self.type_to_metakit_type(change['old'])),
                    r"\1%s:%s" %
                    (name, self.type_to_metakit_type(change['new'])),
                    default_descs[table])
            for table, finished_desc in default_descs.items():
                self.db.getas(
                    "%s[%s]" %
                    (table,
                     finished_desc))  #setup our table with the right attrs
            self.db.commit()  # and drop our dummy column
Exemple #33
0
 def __init__(self, filename, mode):
     self.db = metakit.storage(filename, 1)
     view = self.db.getas("data[key:S,value:S]")
     hashvw = self.db.getas("__test_hash__[_H:I,_R:I]")
     self.view = view.hash(hashvw, 1)
def write_feed_json(outline, outnum, output_dir):
    """Writes articles from specified feed (outline) into a JSON file."""
    articles = OrderedDict()
    feed_url = outline.get('xmlUrl')
    html_url = outline.get('htmlUrl')
    feed_title = outline.get('title')
    feed_file = feed_url.replace(':', '_').replace('/', '_') + '.mk4'
    fdb = metakit.storage(os.path.join(ak_archive_path, feed_file), 0)
    for a in fdb.getas(fdb.description()):
        # skip articles marked as "deleted" or "thrash"
        if a.status & 0x3:
            continue

        # If article guid is a hash, then it means that feed XML does not specify a guid value.
        # Akregator does and keeps a hash value, but FreshRSS uses article link.
        # Convert to avoid duplicate entries when feed is fetched by FreshRSS.
        guid = a.link if a.guid.startswith('hash:') else a.guid

        # if guid is duplicate, drop the old article (we are iterating from older to newer;
        # so the new one should take precedence, but in case of duplicated guids in JSON
        # FreshRSS would import first and ignore all following entries...)
        articles.pop(guid, None)

        # format article enclosure for appending to contents just like FreshRSS does
        if a.hasEnclosure:
            enclosure = ['\n<div class="enclosure">']
            if a.enclosureType.startswith(
                    'video/') or a.enclosureType.startswith('audio/'):
                tag = 'video' if a.enclosureType.startswith(
                    'video/') else 'audio'
                enclosure.append('<p class="enclosure-content"><')
                enclosure.append(tag)
                enclosure.append(' preload="none" src="')
                enclosure.append(a.enclosureUrl)
                enclosure.append('" controls="controls"></')
                enclosure.append(tag)
                enclosure.append('> <a download="" href="')
                enclosure.append(a.enclosureUrl)
                enclosure.append('">💾</a></p>')
            elif a.enclosureType.startswith('image/'):
                enclosure.append('<p class="enclosure-content"><img src="')
                enclosure.append(a.enclosureUrl)
                enclosure.append('" alt="" /></p>')
            elif a.enclosureType.startswith(
                    'application/') or a.enclosureType.startswith('text/'):
                enclosure.append(
                    '<p class="enclosure-content"><a download="" href="')
                enclosure.append(a.enclosureUrl)
                enclosure.append('">💾</a></p>')
            enclosure.append('</div>')
            enclosure = ''.join(enclosure)
        else:
            enclosure = ''

        # format article content
        if a.content or a.description or enclosure:
            content = '\n' + (a.content or a.description) + enclosure + '\n'
        else:
            content = ''

        articles[guid] = OrderedDict([
            ('id', uesc(guid)),
            (
                'categories', [tag for tag in a.tags]
            ),  # akregator does not seem to support tags, although fields exists in database...
            ('title', uesc(a.title)),
            ('author', uesc(a.authorName or a.authorEMail)),
            ('published', a.pubDate),
            ('updated', a.pubDate),
            ('alternate',
             [OrderedDict([
                 ('href', uesc(a.link)),
                 ('type', 'text/html'),
             ])]),
            ('content', {
                'content': u(content),
            }),
            ('origin',
             OrderedDict([
                 ('streamId', outnum),
                 ('title', uesc(feed_title)),
                 ('htmlUrl', uesc(html_url)),
                 ('feedUrl', uesc(feed_url)),
             ])),
        ])

    dump_data = OrderedDict([
        ('id', 'akregator2zip/dump/feed/{}'.format(outnum)),
        ('title', u'List of {} articles'.format(uesc(feed_title))),
        ('author', 'akregator2zip dumper'),
        ('items', list(articles.itervalues())),
    ])

    dump_file = 'feed_{}_{:04d}.json'.format(
        datetime.today().strftime('%Y-%m-%d'), outnum)
    with codecs.open(os.path.join(output_dir, dump_file), 'w', 'utf-8') as f:
        json.dump(dump_data,
                  f,
                  ensure_ascii=False,
                  indent=4,
                  separators=(',', ': '))

    return len(articles)