Beispiel #1
0
def cmd_stats(NAME, settings):

    """
    """

    host = init_host(settings)

    if not SqlBase.metadata.bind:
        sa = Locator.get_session('default', settings.dbref)
        #sa = get_session(settings.dbref)
    assert sa, "XXX"

    dbRef = Locator.find(ref=settings.dbref)
    if not dbRef:
        dbRef = Locator(global_id=settings.dbref)
        dbRef.init_defaults()
        dbRef.commit()

    for l, v in (
        ( 'Host', host ),
        ( 'Net', host.net.name ),
        #( 'Domain', domain),
        ( 'DBRef', settings.dbref ),
        #( "Number of nodes", sa.query(Node).count())
    ):
        log.std('{green}%s{default}: {bwhite}%s{default}', l, v)
Beispiel #2
0
def init_host(settings):
    """
    Reinitialize or initialize host instance from environment and stored settings.
    """
    host = dict( name = get_hostname() )
    hostnameId = host['name'].lower()
    if hostnameId not in settings.nodes:
        ifaces = {}
        for iface, mac, spec in domain.inet_ifaces():
            if mac in settings.interfaces:
                raise Exception("Found existing interface", mac)
            else:
                settings.interfaces[mac] = ifaces[mac] = dict(
                    node = hostnameId
                )
        if settings.interactive:
            name = Prompt.raw_input("Give a name for this node", host['name'])
        host.update(dict(
            unid = str(uuid.uuid4()),
            interfaces = ifaces.keys()
        ))
        settings.nodes[hostnameId] = host
        open(hostIdFile, 'w+').write(" ".join((host['unid'], host['name'])))
        log.std("{bwhite}Wrote new host, {green}%s {default}<{bblack}%s{default}>",
            host['name'], host['unid'])
        settings.commit()
    else:
        host = get_current_host(settings)
        log.std("{bwhite}Found host, {green}%s {default}<{bblack}%s{default}>",
            host['name'], host['unid'])
    return host
Beispiel #3
0
def cmd_stats(settings):
    sa = get_session(settings.dbref)
    for stat, label in (
                (sa.query(Locator).count(), "Number of URLs: %s"),
                #(sa.query(Bookmark).count(), "Number of bookmarks: %s"),
                (sa.query(Domain).count(), "Number of domains: %s"),
                (sa.query(Tag).count(), "Number of tags: %s"),
            ):
        log.std(label, stat)
Beispiel #4
0
def print_Task(task):
    log.std(
"{blue}%s{bblack}. {bwhite}%s {bblack}[{magenta}%s {green}%s{bblack}]{default}" % (
                task.task_id,
                task.title,
                task.requiredFor_id and task.requiredFor_id or '',
                task.partOf_id and task.partOf_id or ''
            )
        )
Beispiel #5
0
def cmd_info(settings):
    if opts.flags.database_tables:
        reload_metadata(settings)
        log.std("{yellow}Loaded tables from DB{default}")
    for l, v in (
            ( 'DBRef', settings.dbref ),
            ( "Tables in schema", ", ".join(metadata.tables.keys()) ),
    ):
        log.std('{green}%s{default}: {bwhite}%s{default}', l, v)
Beispiel #6
0
def cmd_href(NAME, settings):
    sa = Locator.get_session(settings.session_name, settings.dbref)
    if NAME:
        rs = Locator.search(ref=NAME)
    else:
        rs = Locator.all()
    if not rs:
        log.std("Nothing")
    for r in rs:
        print r.ref
Beispiel #7
0
def cmd_domain(NAME, settings):
    sa = Domain.get_session(settings.session_name, settings.dbref)
    if NAME:
        rs = Domain.search(name=NAME)
    else:
        rs = Domain.all()
    if not rs:
        log.std("Nothing")
    for r in rs:
        print r.name
Beispiel #8
0
def format_Space_item(space):
    log.std(
"{blue}%s{bblack}. {bwhite}%s {bblack}[ about:{magenta}%s {bblack}] %s %s %s{default}" % (
                space.space_id,
                space.global_id,
                space.classes,

                str(space.date_added).replace(' ', 'T'),
                str(space.date_updated).replace(' ', 'T'),
                str(space.date_deleted).replace(' ', 'T')
            )
        )
Beispiel #9
0
def format_Topic_item(topic):
    log.std(
"{blue}%s{bblack}. {bwhite}%s {bblack}[ about:{magenta}%s {bblack}] %s %s %s{default}" % (
                topic.topic_id,
                topic.name,
                topic.about_id,

                str(topic.date_added).replace(' ', 'T'),
                str(topic.date_updated).replace(' ', 'T'),
                str(topic.date_deleted).replace(' ', 'T')
            )
        )
Beispiel #10
0
def cmd_init(settings):
    sa = Host.get_session("default", settings.dbref)
    host_dict = init_host(settings)
    name = host_dict["name"]
    record = Host.fetch(filters=(Host.name == name,), sa=sa, exists=False)
    if not record:
        host = Host(name=name, date_added=datetime.now(), date_updated=datetime.now())
        sa.add(host)
        sa.commit()
        log.std("{bwhite}Added host %s record{default}", name)
    else:
        host = record
    print "host at", host_dict.path(), ":", host
Beispiel #11
0
def print_Folder(folder):
    log.std(
"{blue}%s{bblack}. {bwhite}%s {bblack}[ type:{magenta}%s {bblack} parent:{cyan}%s ] %s %s %s{default}" % (
                folder.folder_id,
                folder.name,
                folder.ntype,
                folder.root,

                str(folder.date_added).replace(' ', 'T'),
                str(folder.date_updated).replace(' ', 'T'),
                str(folder.date_deleted).replace(' ', 'T')
            )
        )
Beispiel #12
0
def cmd_new(NAME, REF, settings):
    sa = Folder.start_session('folder', settings.dbref)
    folder = Folder.byName(NAME)
    if folder:
        log.std("Found existing folder %s, created %s", folder.name,
                folder.date_added)
    else:
        folder = Folder(name=NAME)
        folder.init_defaults()
        sa.add(folder)
        sa.commit()
        log.std("Added new folder %s", folder.name)
    print_Folder(folder)
Beispiel #13
0
def cmd_info(opts):
    """
        Dump settings dict.
    """
    for l, v in (
            ( "Tasks Document", opts.flags.tasks_file ),
            ( "Comment File", opts.flags.grep_file ),
            ( "Project Name Id Slug", opts.flags.project_slug ),
            ( "Key Type", opts.flags.key_type ),
            ( "Key Argument", opts.flags.key_arg ),
            ( "Key Separator", opts.flags.key_sep ),
    ):
        log.std('{green}%s{default}: {bwhite}%s{default}', l, v)
Beispiel #14
0
def cmd_memdebug(settings):
    # peak memory usage (bytes on OS X, kilobytes on Linux)
    res_usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
    if os.uname()[0] == 'Linux':
        res_usage /= 1024; # kilobytes?
    # XXX: http://stackoverflow.com/questions/938733/total-memory-used-by-python-process
    #res_usage /= resource.getpagesize()

    db_size = os.path.getsize(os.path.realpath(settings.dbref[10:]))
    for l, v in (
            ( 'Storage Size', lib.human_readable_bytesize( db_size ) ),
            ( 'Resource Usage', lib.human_readable_bytesize(res_usage) ),
        ):
            log.std('{green}%s{default}: {bwhite}%s{default}', l, v)
Beispiel #15
0
def cmd_info(settings):

    """
    Print info.
    """

    host = init_host(settings)

    for l, v in (
        ( 'Host', host ),
        ( 'Net', host.net.name ),
        #( 'Domain', domain),
        ( 'DBRef', settings.dbref ),
    ):
        log.std('{green}%s{default}: {bwhite}%s{default}', l, v)
Beispiel #16
0
def cmd_stats(settings, opts):
    """
    Print table record stats.
    """
    global metadata
    sa = schema.get_session(settings.dbref, metadata=metadata)
    if opts.flags.all_tables or opts.flags.database_tables:
        if opts.flags.database_tables:
            reload_metadata(settings)
            log.info("{yellow}Loaded tables from DB{default}")
        for t in metadata.tables:
            try:
                log.std("{blue}%s{default}: {bwhite}%s{default}",
                        t, sa.query(metadata.tables[t].count().alias("cnt")).all()[0][0])
            except Exception, e:
                log.err("Count failed for %s: %s", t, e)
        log.std("%i tables, done.", len(metadata.tables))
Beispiel #17
0
def cmd_info(settings):
    sa = Project.get_session('default', settings.dbref)
    #sa = get_session(settings.dbref)
    pwd = os.getcwd()
    name = os.path.basename(pwd)
    workdir = Workdir.find(pwd)
    if not workdir:
        print("Not in a metadata workdir!")
    rs = Project.search(_sa=sa, name=name)
    if not rs:
        print("No project found for %r" % name)
        return 1
    proj=rs[0]
    try:
        hosts = proj.hosts
    except Exception as e:
        print(settings.dbref, Project.metadata.bind)
        log.std("Error proj.hosts %s", e)
        hosts = []
    print(proj.name, hosts, proj.repositories[0].vc_type, proj.date_added)
Beispiel #18
0
def cmd_info(settings):

    """
        Verify DB connection is working. Print some settings and storage stats.
    """

    for l, v in (
            ( 'Settings Raw', pformat(settings.todict()) ),
            ( 'DBRef', settings.dbref ),

            ( "Tables in schema", ", ".join(metadata.tables.keys()) ),
            ( "Table lengths", "" ),
    ):
        log.std('{green}%s{default}: {bwhite}%s{default}', l, v)

    sa = get_session(settings.dbref, metadata=metadata)

    for t in metadata.tables:
        try:
            log.std("  {blue}%s{default}: {bwhite}%s{default}",
                    t, sa.query(metadata.tables[t].count()).all()[0][0])
        except Exception, e:
            log.err("Count failed for %s: %s", t, e)
Beispiel #19
0
def cmd_info(settings):
    for l, v in (("DBRef", settings.dbref), ("Tables in schema", ", ".join(metadata.tables.keys()))):
        log.std("{green}%s{default}: {bwhite}%s{default}", l, v)
    # try to connect
    try:
        sa = Node.get_session("default", settings.dbref)
        log.std("{magenta} * {bwhite}DB Connection {default}[{green}OK{default}]")
    except Exception, e:
        log.std("{magenta} * {bwhite}DB Connection {default}[{red}Error{default}]: %s", e)
Beispiel #20
0
def cmd_new(NAME, REF, settings):
    #store = Topic.start_master_session()
    #print store
    #topic = store.Topic.byName(NAME)
    #if topic:
    #    pass
    #else:
    #    topic = store.Topic.forge(name=NAME)
    #    store.commit()
    #reporter.stdout.Topic(topic)

    # XXX: old
    sa = Topic.get_session('default', settings.dbref)
    topic = Topic.byName(NAME)
    if topic:
        log.std("Found existing topic %s, created %s", topic.name,
                topic.date_added)
    else:
        topic = Topic(name=NAME)
        topic.init_defaults()
        sa.add(topic)
        sa.commit()
        log.std("Added new topic %s", topic.name)
    reporter.stdout.Topic(topic)
Beispiel #21
0
def cmd_account_add(props, settings, name):
    sa = get_session(settings.dbref)
    acc = Account(name=name)
    sa.add(acc)
    sa.commit()
    log.std("Added account %s", name)
Beispiel #22
0
def cmd_show(settings):
    for name, table in metadata.tables.items():
        log.std('{green}%s{default}: {bwhite}%s{default}',
                name, "{default}, {bwhite}".join(table.columns.keys()))
Beispiel #23
0
def cmd_info(settings):
    for l, v in (
            ( 'DBRef', settings.dbref ),
            ( "Tables in schema", ", ".join(metadata.tables.keys()) ),
    ):
        log.std('{green}%s{default}: {bwhite}%s{default}', l, v)
Beispiel #24
0
    # peak memory usage (bytes on OS X, kilobytes on Linux)
    res_usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
    if os.uname()[0] == 'Linux':
        res_usage /= 1024; # kilobytes?
    # XXX: http://stackoverflow.com/questions/938733/total-memory-used-by-python-process
    #res_usage /= resource.getpagesize()

    # FIXME: does not use dbref according to settings, may fail/report wrong file
    db_size = os.path.getsize(os.path.expanduser(__db__))

    for l, v in (
            ( 'Storage Size', lib.human_readable_bytesize( db_size ) ),
            ( 'Resource Usage', lib.human_readable_bytesize(res_usage) ),
        ):
            log.std('{green}%s{default}: {bwhite}%s{default}', l, v)

    log.std('\n{green}info {bwhite}OK{default}')


def cmd_list(settings):
    """
        List to root tags.
    """
    sa = get_session(settings.dbref, metadata=metadata)
    roots = sa.query(Tag).filter(Tag.contexts == None).all()
    for root in roots:
        print(root.name)


def cmd_find(settings, LIKE):
Beispiel #25
0
def format_Node_item(node):
    log.std("{blue}%s{bblack}. {bwhite}%s {bblack}[ {bblack}] {default}" % (node.node_id, node.name))
Beispiel #26
0
def cmd_dlcs_import(opts, settings):
    """
    TODO: built into generic import/export (ie. complete set)  so heuristics can
        update all stats each import.. or find some way to fragment dataset.
    """
    importFile = opts.args.FILE
    data = dlcs_parse_xml(open(importFile).read())
    sa = Locator.get_session('default', opts.flags.dbref)
    #sa = model.get_session(opts.flags.dbref, metadata=SqlBase.metadata)
    tags_stat = {}
    domains_stat = {}
    # first pass: validate, track stats and create Locator records where missing
    for post in data['posts']:
        href = post['href']
        dt = datetime.strptime(post['time'], ISO_8601_DATETIME)
# validate URL
        url = urlparse(href)
        domain = url[1]
        if not domain:
            log.std("Ignored domainless (non-net?) URIRef: %s", href)
            continue
        assert re.match('[a-z0-9]+(\.[a-z0-9]+)*', domain), domain
# get/init Locator
        lctr = Locator.fetch((Locator.ref == href,), exists=False)
        if lctr:
            if lctr.date_added != dt:
                lctr.date_added = dt
                sa.add(lctr)
        else:
            lctr = Locator(
                    global_id=href,
                    ref=href,
                    date_added=datetime.strptime(post['time'], ISO_8601_DATETIME)
                )
            lctr.init_defaults()
            log.std("new: %s", lctr)
            sa.add(lctr)
# get/init Bookmark
        bm = Bookmark.fetch((Bookmark.ref_id == lctr.lctr_id,), exists=False)
        if bm:
            if bm.date_added != dt:
                bm.date_added = dt
                sa.add(bm)
            if bm.ref_id != lctr.lctr_id:
                bm.ref = lctr
                sa.add(bm)
        else:
            bm = Bookmark.fetch((Bookmark.name == post['description'],), exists=False)
            if bm:
                log.std("Name already exists: %r" % post['description'])
                continue
            bm = Bookmark(
                    ref=lctr,
                    name=post['description'],
                    extended=post['extended'],
                    tags=post['tag'].replace(' ', ', '),
                    date_added=datetime.strptime(post['time'], ISO_8601_DATETIME)
                )
            bm.init_defaults()
            log.std("new: %s", bm)
            sa.add(bm)
# track domain frequency
        if domain in domains_stat:
            domains_stat[domain] += 1
        else:
            domains_stat[domain] = 1
# track tag frequency
        for tag in post['tag'].split(' '):
            if tag in tags_stat:
                tags_stat[tag] += 1
            else:
                tags_stat[tag] = 1
    log.std("Checked %i locator references", len(data['posts']))
    sa.commit()
# Prepare domain stats
    avgDomainFreq = sum(domains_stat.values())/(len(domains_stat)*1.0)
    hiDomainFreq = max(domains_stat.values())
    log.std("Found domain usage (max/avg): %i/%i", hiDomainFreq, avgDomainFreq)
    domains = 0
    domainOffset = int(opts.flags.domain_offset)
    if domainOffset == 0:
        domainOffset = hiFreq
    elif domainOffset == -1:
        domainOffset = round(hiDomainFreq * 0.2)
    log.std("Setting domain-offset: %i", domainOffset)
# get/init Domains
    for domain in domains_stat:
        freq = domains_stat[domain]
        if freq >= domainOffset:
            domains += 1
            domain_record = Domain.fetch((Domain.name == domain,), exists=False)
            if not domain_record:
                domain_record = Domain(name=domain)
                domain_record.init_defaults()
                sa.add(domain_record)
    sa.commit()
    log.std("Checked %i domains", len(domains_stat))
    log.std("Tracking %i domains", domains)
# Prepare tag stats
    avgFreq = sum(tags_stat.values())/(len(tags_stat)*1.0)
    hiFreq = max(tags_stat.values())
    log.std("Found tag usage (max/avg): %i/%i", hiFreq, avgFreq)
    tagOffset = int(opts.flags.tag_offset)
    if tagOffset == 0:
        tagOffset = hiFreq
    elif tagOffset == -1:
        tagOffset = round(hiFreq * 0.1)
    log.std("Setting tag-offset: %i", tagOffset)
# get/init Tags
    tags = 0
    for tag in tags_stat:
        freq = tags_stat[tag]
        if not re.match('[A-Za-z0-9-]+', tag):
            log.std("Non-std tag %s", tag)
        if freq >= tagOffset:
            tags += 1
            t = Node.fetch((Node.name == tag,), exists=False)
            if not t:
                t = Tag(name=tag)
                t.init_defaults()
                log.std("new: %s", t)
                sa.add(t)
            # store frequencies
            # TODO tags_freq
    log.std("Checked %i tags", len(tags_stat))
    log.std("Tracking %i tags", tags)
    sa.commit()
Beispiel #27
0
def cmd_mutation_import(opts, settings):

    """
    Import mutations from CSV, create accounts as needed.
    Indx with Year, Month.
    """

    sa = get_session(settings.dbref)
    if settings.reset or lib.Prompt.ask("Purge mutations?"):
        sa.query(Mutation).delete()
        log.std("Purged all previous mutations")

    assert settings.input_format == 'csv', settings.input_format
    cache = confparse.Values(dict(
        accounts={}, years={}, months={}
    ))
    for csvfile in opts.args.file:
        reader = csv_reader(csvfile, [
            'line', 'date', 'accnr', 'amount', 'destacc', 'cat',
            'destname', 'descr', 'descr2' ])
        for line, date, accnr, amount, destacc, cat, destname, descr, descr2 in reader:
            from_account, to_account = None, None
            assert accnr, (line, date, amount, cat)
            # from_account
            if accnr not in cache.accounts:
                from_account = Account.for_nr(sa, accnr)
                if not from_account:
                    from_account = Account(name=ACCOUNT_CREDIT+':'+accnr)
                    from_account.init_defaults()
                from_account.set_nr(accnr)
                sa.add(from_account)
                sa.commit()
                cache.accounts[accnr] = from_account
            else:
                from_account = cache.accounts[accnr]
            assert from_account.account_id, (str(from_account), line, accnr, date, amount, cat)

            # credit account
            if not destacc:
                if cat == 'ba':
                    # payment card checkout
                    to_account = Account.for_checkout(sa,
                            ACCOUNT_EXPENSES+':ba:'+descr)
                elif cat == 'ga':
                    # atm withdrawal
                    to_account = Account.for_withdrawal(sa,
                            ACCOUNT_EXPENSES+':ga:'+descr)
                elif cat == 'db':
                    # debet interest
                    to_account = Account.for_name_type(sa, ACCOUNT_ACCOUNTING)
                else:
                    print line, date, accnr, amount, cat, descr, descr2
                    assert not destname, (cat, destname, cat)
                    continue
            # billing account
            elif destacc not in cache.accounts:
                to_account = Account.for_nr(sa, destacc)
                if not to_account:
                    to_account = Account(name=ACCOUNT_EXPENSES+':'+destname)
                    to_account.init_defaults()
                to_account.set_nr(destacc)
                sa.add(to_account)
                sa.commit()
                cache.accounts[destacc] = to_account
            else:
                to_account = cache.accounts[destacc]
            # get Year/Month
            y, m, d = map(int, ( date[:4], date[4:6], date[6:]))
            if y not in cache.years:
                pass
            mut = Mutation(
                    from_account=from_account.account_id,
                    to_account=to_account.account_id,
                    year=y, month=m, day=d,
                    amount=amount, description=descr+'\t'+descr2,
                    category=cat)
            sa.add(mut)
            sa.commit()

    log.std("Import ready")

    cmd_balance_commit(settings)
Beispiel #28
0
def cmd_account_rm(settings, name):
    sa = get_session(settings.dbref)
    acc = sa.query(Account).filter(Account.name == name).one()
    sa.delete(acc)
    sa.commit()
    log.std("Dropped account %s", name)
Beispiel #29
0
                log.std("{blue}%s{default}: {bwhite}%s{default}",
                        t, sa.query(metadata.tables[t].count().alias("cnt")).all()[0][0])
            except Exception, e:
                log.err("Count failed for %s: %s", t, e)
        log.std("%i tables, done.", len(metadata.tables))
    else:
        if hasattr(schema, 'models'):
            models = schema.models
        else:
            models = [
                    getattr(schema, x) for x in dir(schema)
                    if inspect.isclass(getattr(schema, x))
                        and issubclass( getattr(schema, x), schema.SqlBase ) ]
        for m in models:
            try:
                log.std("{blue}%s{default}: {bwhite}%s{default}",
                        m.__name__, sa.query(m).count())
            except Exception, e:
                log.err("Count failed for %s: %s", m, e)
        log.std("%i models, done.", len(models))

def cmd_info(settings):
    if opts.flags.database_tables:
        reload_metadata(settings)
        log.std("{yellow}Loaded tables from DB{default}")
    for l, v in (
            ( 'DBRef', settings.dbref ),
            ( "Tables in schema", ", ".join(metadata.tables.keys()) ),
    ):
        log.std('{green}%s{default}: {bwhite}%s{default}', l, v)

def cmd_show(settings):