try: data = urllib2.urlopen(url) except urllib2.HTTPError, e: if e.code == 404: return None raise tmp = tempfile.NamedTemporaryFile(suffix='.tar.bz2') shutil.copyfileobj(data, tmp) data.close() tmp.seek(0) return tmp config = Config(os.path.dirname(__file__) + '/mbslave.conf') db = connect_db(config) base_url = config.get('MUSICBRAINZ', 'base_url') ignored_tables = set(config.get('TABLES', 'ignore').split(',')) if config.solr.enabled: from mbslave.search import SolrReplicationHook hook_class = SolrReplicationHook else: hook_class = ReplicationHook cursor = db.cursor() cursor.execute("SELECT current_schema_sequence, current_replication_sequence FROM %s.replication_control" % config.schema.name('musicbrainz')) schema_seq, replication_seq = cursor.fetchone() status = StatusReport(schema_seq, replication_seq) if config.monitoring.enabled: status.load(config.monitoring.status_file)
action="store_true", dest="public", default=False, help="don't configure the default schema") parser.add_option("-s", "--schema", dest="schema", default='musicbrainz', help="default schema") options, args = parser.parse_args() config = Config(os.path.dirname(__file__) + '/mbslave.conf') args = ['psql'] args.append('-U') args.append(config.get('DATABASE', 'user')) if config.has_option('DATABASE', 'host'): args.append('-h') args.append(config.get('DATABASE', 'host')) if config.has_option('DATABASE', 'port'): args.append('-p') args.append(config.get('DATABASE', 'port')) args.append(config.get('DATABASE', 'name')) if not options.public: schema = config.schema.name(options.schema) os.environ['PGOPTIONS'] = '-c search_path=%s,public' % schema if config.has_option('DATABASE', 'password'): os.environ['PGPASSWORD'] = config.get('DATABASE', 'password') os.execvp("psql", args)
try: data = urllib2.urlopen(url, timeout=60) except urllib2.HTTPError, e: if e.code == 404: return None raise tmp = tempfile.NamedTemporaryFile(suffix='.tar.bz2') shutil.copyfileobj(data, tmp) data.close() tmp.seek(0) return tmp config = Config(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'mbslave.conf')) db = connect_db(config) base_url = config.get('MUSICBRAINZ', 'base_url') if config.has_option('MUSICBRAINZ', 'token'): token = config.get('MUSICBRAINZ', 'token') else: token = None ignored_schemas = set(config.get('schemas', 'ignore').split(',')) ignored_tables = set(config.get('TABLES', 'ignore').split(',')) hook_class = ReplicationHook cursor = db.cursor() cursor.execute("SELECT current_schema_sequence, current_replication_sequence FROM %s.replication_control" % config.schema.name('musicbrainz')) schema_seq, replication_seq = cursor.fetchone() status = StatusReport(schema_seq, replication_seq) if config.monitoring.enabled:
try: data = urllib2.urlopen(url) except urllib2.HTTPError, e: if e.code == 404: return None raise tmp = tempfile.NamedTemporaryFile(suffix='.tar.bz2') shutil.copyfileobj(data, tmp) data.close() tmp.seek(0) return tmp config = Config(os.path.dirname(__file__) + '/mbslave.conf') db = connect_db(config) schema = config.get('DATABASE', 'schema') base_url = config.get('MUSICBRAINZ', 'base_url') ignored_tables = set(config.get('TABLES', 'ignore').split(',')) if config.solr.enabled: from mbslave.search import SolrReplicationHook hook_class = SolrReplicationHook else: hook_class = ReplicationHook cursor = db.cursor() cursor.execute("SELECT current_schema_sequence, current_replication_sequence FROM %s.replication_control" % schema) schema_seq, replication_seq = cursor.fetchone() status = StatusReport(schema_seq, replication_seq) if config.monitoring.enabled:
#!/usr/bin/env python import os from optparse import OptionParser from mbslave import Config, connect_db parser = OptionParser() parser.add_option("-S", "--no-schema", action="store_true", dest="public", default=False, help="don't configure the default schema") parser.add_option("-s", "--schema", dest="schema", default='musicbrainz', help="default schema") options, args = parser.parse_args() config = Config(os.path.dirname(__file__) + '/mbslave.conf') args = ['psql'] args.append('-U') args.append(config.get('DATABASE', 'user')) if config.has_option('DATABASE', 'host'): args.append('-h') args.append(config.get('DATABASE', 'host')) if config.has_option('DATABASE', 'port'): args.append('-p') args.append(config.get('DATABASE', 'port')) args.append(config.get('DATABASE', 'name')) if not options.public: schema = config.schema.name(options.schema) os.environ['PGOPTIONS'] = '-c search_path=%s' % schema if config.has_option('DATABASE', 'password'): os.environ['PGPASSWORD'] = config.get('DATABASE', 'password') os.execvp("psql", args)
if not check_table_exists(db, schema, table): print " - Skipping %s (table %s does not exist)" % (name, fulltable) continue cursor.execute("SELECT 1 FROM %s LIMIT 1" % fulltable) if cursor.fetchone(): print " - Skipping %s (table %s already contains data)" % (name, fulltable) continue print " - Loading %s to %s" % (name, fulltable) cursor.copy_from(tar.extractfile(member), fulltable) db.commit() config = Config(os.path.dirname(__file__) + '/mbslave.conf') db = connect_db(config) ignored_tables = set(config.get('TABLES', 'ignore').split(',')) for filename in sys.argv[1:]: load_tar(filename, db, config, ignored_tables) ########NEW FILE######## __FILENAME__ = mbslave-psql #!/usr/bin/env python import os from optparse import OptionParser from mbslave import Config, connect_db parser = OptionParser() parser.add_option("-S", "--no-schema", action="store_true", dest="public", default=False, help="don't configure the default schema") parser.add_option("-s", "--schema", dest="schema", default='musicbrainz', help="default schema")
tar = tarfile.open(filename, 'r:bz2') cursor = db.cursor() for member in tar: if not member.name.startswith('mbdump/'): continue name = member.name.split('/')[1].replace('_sanitised', '') schema, table = parse_name(config, name) fulltable = fqn(schema, table) if table in ignored_tables: print " - Ignoring", name continue if not check_table_exists(db, schema, table): print " - Skipping %s (table %s does not exist)" % (name, fulltable) continue cursor.execute("SELECT 1 FROM %s LIMIT 1" % fulltable) if cursor.fetchone(): print " - Skipping %s (table %s already contains data)" % (name, fulltable) continue print " - Loading %s to %s" % (name, fulltable) cursor.copy_from(tar.extractfile(member), fulltable) db.commit() config = Config(os.path.dirname(__file__) + '/mbslave.conf') db = connect_db(config) ignored_tables = set(config.get('TABLES', 'ignore').split(',')) for filename in sys.argv[1:]: load_tar(filename, db, config, ignored_tables)
except urllib2.HTTPError, e: if e.code == 404: return None raise tmp = tempfile.NamedTemporaryFile(suffix='.tar.bz2') shutil.copyfileobj(data, tmp) data.close() tmp.seek(0) return tmp config = Config( os.path.join(os.path.dirname(os.path.abspath(__file__)), 'mbslave.conf')) db = connect_db(config) base_url = config.get('MUSICBRAINZ', 'base_url') if config.has_option('MUSICBRAINZ', 'token'): token = config.get('MUSICBRAINZ', 'token') else: token = None ignored_schemas = set(config.get('schemas', 'ignore').split(',')) ignored_tables = set(config.get('TABLES', 'ignore').split(',')) hook_class = ReplicationHook cursor = db.cursor() cursor.execute( "SELECT current_schema_sequence, current_replication_sequence FROM %s.replication_control" % config.schema.name('musicbrainz')) schema_seq, replication_seq = cursor.fetchone()
if not member.name.startswith('mbdump/'): continue name = member.name.split('/')[1].replace('_sanitised', '') schema, table = parse_name(config, name) fulltable = fqn(schema, table) if schema in ignored_schemas: print " - Ignoring", name continue if table in ignored_tables: print " - Ignoring", name continue if not check_table_exists(db, schema, table): print " - Skipping %s (table %s does not exist)" % (name, fulltable) continue cursor.execute("SELECT 1 FROM %s LIMIT 1" % fulltable) if cursor.fetchone(): print " - Skipping %s (table %s already contains data)" % (name, fulltable) continue print " - Loading %s to %s" % (name, fulltable) cursor.copy_from(tar.extractfile(member), fulltable) db.commit() config = Config(os.path.dirname(__file__) + '/mbslave.conf') db = connect_db(config) ignored_schemas = set(config.get('schemas', 'ignore').split(',')) ignored_tables = set(config.get('TABLES', 'ignore').split(',')) for filename in sys.argv[1:]: load_tar(filename, db, config, ignored_schemas, ignored_tables)
cursor = db.cursor() for member in tar: if not member.name.startswith('mbdump/'): continue name = member.name.split('/')[1].replace('_sanitised', '') schema, table = parse_name(config, name) fulltable = fqn(schema, table) if table in ignored_tables: print " - Ignoring", name continue if not check_table_exists(db, schema, table): print " - Skipping %s (table %s does not exist)" % (name, fulltable) continue cursor.execute("SELECT 1 FROM %s LIMIT 1" % fulltable) if cursor.fetchone(): print " - Skipping %s (table %s already contains data)" % ( name, fulltable) continue print " - Loading %s to %s" % (name, fulltable) cursor.copy_from(tar.extractfile(member), fulltable) db.commit() config = Config(os.path.dirname(__file__) + '/mbslave.conf') db = connect_db(config) ignored_tables = set(config.get('TABLES', 'ignore').split(',')) for filename in sys.argv[1:]: load_tar(filename, db, config, ignored_tables)
try: data = urllib2.urlopen(url) except urllib2.HTTPError, e: if e.code == 404: return None raise tmp = tempfile.NamedTemporaryFile(suffix='.tar.bz2') shutil.copyfileobj(data, tmp) data.close() tmp.seek(0) return tmp config = Config(os.path.dirname(__file__) + '/mbslave.conf') db = connect_db(config) schema = config.get('DATABASE', 'schema') base_url = config.get('MUSICBRAINZ', 'base_url') ignored_tables = set(config.get('TABLES', 'ignore').split(',')) if config.has_option('TABLES', 'accept'): accepted_tables = set([x.strip() for x in config.get('TABLES', 'accept').split(',')]) else: accepted_tables = None if config.solr.enabled: from mbslave.search import SolrReplicationHook hook_class = SolrReplicationHook else: hook_class = ReplicationHook cursor = db.cursor() cursor.execute("SELECT current_schema_sequence, current_replication_sequence FROM %s.replication_control" % schema) schema_seq, replication_seq = cursor.fetchone()
except urllib2.HTTPError, e: if e.code == 404: return None raise tmp = tempfile.NamedTemporaryFile(suffix='.tar.bz2') shutil.copyfileobj(data, tmp) data.close() tmp.seek(0) return tmp config = Config( os.path.join(os.path.dirname(os.path.abspath(__file__)), 'mbslave.conf')) db = connect_db(config) base_url = config.get('MUSICBRAINZ', 'base_url') ignored_tables = set(config.get('TABLES', 'ignore').split(',')) hook_class = ReplicationHook cursor = db.cursor() cursor.execute( "SELECT current_schema_sequence, current_replication_sequence FROM %s.replication_control" % config.schema.name('musicbrainz')) schema_seq, replication_seq = cursor.fetchone() status = StatusReport(schema_seq, replication_seq) if config.monitoring.enabled: status.load(config.monitoring.status_file) while True:
data = urllib2.urlopen(url) except urllib2.HTTPError, e: if e.code == 404: return None raise tmp = tempfile.NamedTemporaryFile(suffix='.tar.bz2') shutil.copyfileobj(data, tmp) data.close() tmp.seek(0) return tmp config = Config(os.path.dirname(__file__) + '/mbslave.conf') db = connect_db(config) cursor = db.cursor() schema = config.get('DATABASE', 'schema') base_url = config.get('MUSICBRAINZ', 'base_url') cursor.execute("SELECT array_to_string((SELECT array_agg(table_name::TEXT) FROM information_schema.tables WHERE table_schema = '%s')::text[], ',');" % schema) not_ignored_tables = cursor.fetchall()[0][0].split(',') if config.solr.enabled: from mbslave.search import SolrReplicationHook hook_class = SolrReplicationHook else: hook_class = ReplicationHook cursor.execute("SELECT current_schema_sequence, current_replication_sequence FROM %s.replication_control" % schema) schema_seq, replication_seq = cursor.fetchone() status = StatusReport(schema_seq, replication_seq) if config.monitoring.enabled: status.load(config.monitoring.status_file)