parser = OptionParser()
parser.add_option("-S",
"--no-schema",
action="store_true",
dest="public",
default=False,
help="don't configure the default schema")
parser.add_option("-s",
"--schema",
dest="schema",
default='musicbrainz',
help="default schema")
options, args = parser.parse_args()
config = Config(os.path.dirname(__file__) + '/mbslave.conf')
args = ['psql']
args.append('-U')
args.append(config.get('DATABASE', 'user'))
if config.has_option('DATABASE', 'host'):
args.append('-h')
args.append(config.get('DATABASE', 'host'))
if config.has_option('DATABASE', 'port'):
args.append('-p')
args.append(config.get('DATABASE', 'port'))
args.append(config.get('DATABASE', 'name'))
if not options.public:
schema = config.schema.name(options.schema)
os.environ['PGOPTIONS'] = '-c search_path=%s,public' % schema
def download_packet(base_url, replication_seq):
url = base_url + "/replication-%d.tar.bz2" % replication_seq
print "Downloading", url
try:
data = urllib2.urlopen(url)
except urllib2.HTTPError, e:
if e.code == 404:
return None
raise
tmp = tempfile.NamedTemporaryFile(suffix='.tar.bz2')
shutil.copyfileobj(data, tmp)
data.close()
tmp.seek(0)
return tmp
config = Config(os.path.dirname(__file__) + '/mbslave.conf')
db = connect_db(config)
base_url = config.get('MUSICBRAINZ', 'base_url')
ignored_tables = set(config.get('TABLES', 'ignore').split(','))
if config.solr.enabled:
from mbslave.search import SolrReplicationHook
hook_class = SolrReplicationHook
else:
hook_class = ReplicationHook
cursor = db.cursor()
cursor.execute("SELECT current_schema_sequence, current_replication_sequence FROM %s.replication_control" % config.schema.name('musicbrainz'))
schema_seq, replication_seq = cursor.fetchone()
#!/usr/bin/env python
import os
import itertools
from lxml import etree as ET
from lxml.builder import E
from mbslave import Config, connect_db
from mbslave.search import fetch_all
cfg = Config(os.path.join(os.path.dirname(__file__), 'mbslave.conf'))
db = connect_db(cfg)
print '<add>'
for doc in fetch_all(cfg, db):
print ET.tostring(doc)
print '</add>'
if token:
url += '?token=' + token
print "Downloading", url
try:
data = urllib2.urlopen(url, timeout=60)
except urllib2.HTTPError, e:
if e.code == 404:
return None
raise
tmp = tempfile.NamedTemporaryFile(suffix='.tar.bz2')
shutil.copyfileobj(data, tmp)
data.close()
tmp.seek(0)
return tmp
config = Config(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'mbslave.conf'))
db = connect_db(config)
base_url = config.get('MUSICBRAINZ', 'base_url')
if config.has_option('MUSICBRAINZ', 'token'):
token = config.get('MUSICBRAINZ', 'token')
else:
token = None
ignored_schemas = set(config.get('schemas', 'ignore').split(','))
ignored_tables = set(config.get('TABLES', 'ignore').split(','))
hook_class = ReplicationHook
cursor = db.cursor()
cursor.execute("SELECT current_schema_sequence, current_replication_sequence FROM %s.replication_control" % config.schema.name('musicbrainz'))
schema_seq, replication_seq = cursor.fetchone()
def download_packet(base_url, replication_seq):
url = base_url + "/replication-%d.tar.bz2" % replication_seq
print "Downloading", url
try:
data = urllib2.urlopen(url)
except urllib2.HTTPError, e:
if e.code == 404:
return None
raise
tmp = tempfile.NamedTemporaryFile(suffix='.tar.bz2')
shutil.copyfileobj(data, tmp)
data.close()
tmp.seek(0)
return tmp
config = Config(os.path.dirname(__file__) + '/mbslave.conf')
db = connect_db(config)
schema = config.get('DATABASE', 'schema')
base_url = config.get('MUSICBRAINZ', 'base_url')
ignored_tables = set(config.get('TABLES', 'ignore').split(','))
if config.solr.enabled:
from mbslave.search import SolrReplicationHook
hook_class = SolrReplicationHook
else:
hook_class = ReplicationHook
cursor = db.cursor()
cursor.execute("SELECT current_schema_sequence, current_replication_sequence FROM %s.replication_control" % schema)
schema_seq, replication_seq = cursor.fetchone()
if table in ignored_tables:
print " - Ignoring", name
continue
if not check_table_exists(db, schema, table):
print " - Skipping %s (table %s does not exist)" % (name, fulltable)
continue
cursor.execute("SELECT 1 FROM %s LIMIT 1" % fulltable)
if cursor.fetchone():
print " - Skipping %s (table %s already contains data)" % (name, fulltable)
continue
print " - Loading %s to %s" % (name, fulltable)
cursor.copy_from(tar.extractfile(member), fulltable)
db.commit()
config = Config(os.path.dirname(__file__) + '/mbslave.conf')
db = connect_db(config)
ignored_tables = set(config.get('TABLES', 'ignore').split(','))
for filename in sys.argv[1:]:
load_tar(filename, db, config, ignored_tables)
########NEW FILE########
__FILENAME__ = mbslave-psql
#!/usr/bin/env python
import os
from optparse import OptionParser
from mbslave import Config, connect_db
#!/usr/bin/env python2
import re
import os
import sys
from mbslave import Config
config = Config(os.path.dirname(__file__) + '/mbslave.conf')
def update_search_path(m):
schemas = m.group(2).replace("'", '').split(',')
schemas = [config.schema.name(s.strip()) for s in schemas]
return m.group(1) + ', '.join(schemas) + ';'
def update_schema(m):
return m.group(1) + config.schema.name(m.group(2)) + m.group(3)
for line in sys.stdin:
line = re.sub(r'(SET search_path = )(.+?);', update_search_path, line)
line = re.sub(r'(\b)(\w+)(\.)', update_schema, line)
line = re.sub(r'( SCHEMA )(\w+)(;)', update_schema, line)
sys.stdout.write(line)
url += '?token=' + token
print "Downloading", url
try:
data = urllib2.urlopen(url, timeout=60)
except urllib2.HTTPError, e:
if e.code == 404:
return None
raise
tmp = tempfile.NamedTemporaryFile(suffix='.tar.bz2')
shutil.copyfileobj(data, tmp)
data.close()
tmp.seek(0)
return tmp
config = Config(
os.path.join(os.path.dirname(os.path.abspath(__file__)), 'mbslave.conf'))
db = connect_db(config)
base_url = config.get('MUSICBRAINZ', 'base_url')
if config.has_option('MUSICBRAINZ', 'token'):
token = config.get('MUSICBRAINZ', 'token')
else:
token = None
ignored_schemas = set(config.get('schemas', 'ignore').split(','))
ignored_tables = set(config.get('TABLES', 'ignore').split(','))
hook_class = ReplicationHook
cursor = db.cursor()
cursor.execute(
"SELECT current_schema_sequence, current_replication_sequence FROM %s.replication_control"
#!/usr/bin/env python
import os
from optparse import OptionParser
from mbslave import Config, connect_db
parser = OptionParser()
parser.add_option("-S", "--no-schema", action="store_true", dest="public", default=False, help="don't configure the default schema")
parser.add_option("-s", "--schema", dest="schema", default='musicbrainz', help="default schema")
options, args = parser.parse_args()
config = Config(os.path.dirname(__file__) + '/mbslave.conf')
args = ['psql']
args.append('-U')
args.append(config.get('DATABASE', 'user'))
if config.has_option('DATABASE', 'host'):
args.append('-h')
args.append(config.get('DATABASE', 'host'))
if config.has_option('DATABASE', 'port'):
args.append('-p')
args.append(config.get('DATABASE', 'port'))
args.append(config.get('DATABASE', 'name'))
if not options.public:
schema = config.schema.name(options.schema)
os.environ['PGOPTIONS'] = '-c search_path=%s' % schema
if config.has_option('DATABASE', 'password'):
os.environ['PGPASSWORD'] = config.get('DATABASE', 'password')
os.execvp("psql", args)
tar = tarfile.open(filename, 'r:bz2')
cursor = db.cursor()
for member in tar:
if not member.name.startswith('mbdump/'):
continue
name = member.name.split('/')[1].replace('_sanitised', '')
schema, table = parse_name(config, name)
fulltable = fqn(schema, table)
if table in ignored_tables:
print " - Ignoring", name
continue
if not check_table_exists(db, schema, table):
print " - Skipping %s (table %s does not exist)" % (name, fulltable)
continue
cursor.execute("SELECT 1 FROM %s LIMIT 1" % fulltable)
if cursor.fetchone():
print " - Skipping %s (table %s already contains data)" % (name, fulltable)
continue
print " - Loading %s to %s" % (name, fulltable)
cursor.copy_from(tar.extractfile(member), fulltable)
db.commit()
config = Config(os.path.dirname(__file__) + '/mbslave.conf')
db = connect_db(config)
ignored_tables = set(config.get('TABLES', 'ignore').split(','))
for filename in sys.argv[1:]:
load_tar(filename, db, config, ignored_tables)
if not member.name.startswith('mbdump/'):
continue
name = member.name.split('/')[1].replace('_sanitised', '')
schema, table = parse_name(config, name)
fulltable = fqn(schema, table)
if schema in ignored_schemas:
print " - Ignoring", name
continue
if table in ignored_tables:
print " - Ignoring", name
continue
if not check_table_exists(db, schema, table):
print " - Skipping %s (table %s does not exist)" % (name, fulltable)
continue
cursor.execute("SELECT 1 FROM %s LIMIT 1" % fulltable)
if cursor.fetchone():
print " - Skipping %s (table %s already contains data)" % (name, fulltable)
continue
print " - Loading %s to %s" % (name, fulltable)
cursor.copy_from(tar.extractfile(member), fulltable)
db.commit()
config = Config(os.path.dirname(__file__) + '/mbslave.conf')
db = connect_db(config)
ignored_schemas = set(config.get('schemas', 'ignore').split(','))
ignored_tables = set(config.get('TABLES', 'ignore').split(','))
for filename in sys.argv[1:]:
load_tar(filename, db, config, ignored_schemas, ignored_tables)
def download_packet(base_url, replication_seq):
url = base_url + "/replication-%d.tar.bz2" % replication_seq
print "Downloading", url
try:
data = urllib2.urlopen(url)
except urllib2.HTTPError, e:
if e.code == 404:
return None
raise
tmp = tempfile.NamedTemporaryFile(suffix='.tar.bz2')
shutil.copyfileobj(data, tmp)
data.close()
tmp.seek(0)
return tmp
config = Config(os.path.dirname(__file__) + '/mbslave.conf')
db = connect_db(config)
schema = config.get('DATABASE', 'schema')
base_url = config.get('MUSICBRAINZ', 'base_url')
ignored_tables = set(config.get('TABLES', 'ignore').split(','))
if config.has_option('TABLES', 'accept'):
accepted_tables = set([x.strip() for x in config.get('TABLES', 'accept').split(',')])
else:
accepted_tables = None
if config.solr.enabled:
from mbslave.search import SolrReplicationHook
hook_class = SolrReplicationHook
else:
hook_class = ReplicationHook
url = base_url + "/replication-%d.tar.bz2" % replication_seq
print "Downloading", url
try:
data = urllib2.urlopen(url)
except urllib2.HTTPError, e:
if e.code == 404:
return None
raise
tmp = tempfile.NamedTemporaryFile(suffix='.tar.bz2')
shutil.copyfileobj(data, tmp)
data.close()
tmp.seek(0)
return tmp
config = Config(
os.path.join(os.path.dirname(os.path.abspath(__file__)), 'mbslave.conf'))
db = connect_db(config)
base_url = config.get('MUSICBRAINZ', 'base_url')
ignored_tables = set(config.get('TABLES', 'ignore').split(','))
hook_class = ReplicationHook
cursor = db.cursor()
cursor.execute(
"SELECT current_schema_sequence, current_replication_sequence FROM %s.replication_control"
% config.schema.name('musicbrainz'))
schema_seq, replication_seq = cursor.fetchone()
status = StatusReport(schema_seq, replication_seq)
if config.monitoring.enabled:
def download_packet(base_url, replication_seq):
url = base_url + "/replication-%d.tar.bz2" % replication_seq
print "Downloading", url
try:
data = urllib2.urlopen(url)
except urllib2.HTTPError, e:
if e.code == 404:
return None
raise
tmp = tempfile.NamedTemporaryFile(suffix='.tar.bz2')
shutil.copyfileobj(data, tmp)
data.close()
tmp.seek(0)
return tmp
config = Config(os.path.dirname(__file__) + '/mbslave.conf')
db = connect_db(config)
cursor = db.cursor()
schema = config.get('DATABASE', 'schema')
base_url = config.get('MUSICBRAINZ', 'base_url')
cursor.execute("SELECT array_to_string((SELECT array_agg(table_name::TEXT) FROM information_schema.tables WHERE table_schema = '%s')::text[], ',');" % schema)
not_ignored_tables = cursor.fetchall()[0][0].split(',')
if config.solr.enabled:
from mbslave.search import SolrReplicationHook
hook_class = SolrReplicationHook
else:
hook_class = ReplicationHook
cursor.execute("SELECT current_schema_sequence, current_replication_sequence FROM %s.replication_control" % schema)
schema_seq, replication_seq = cursor.fetchone()