def connection():
utils.create_schema()
connection = utils.db_connection()
yield connection
utils.drop_schema(connection)
connection.close()
def register():
login = request.args.get('login')
if not login:
return '/register?login=YOUR_LOGIN'
auth_table = get_table('tech', 'auth')
existed = check_auth(login=login)
if not existed:
existed = {'login': login, 'password': ''.join(random.choice(string.ascii_letters) for _ in range(10))}
create_schema(login)
auth_table.insert(existed)
else:
existed = existed[0]
return jsonify(existed)
def calculate_artist_credit_similarities():
'''
Calculate artist credit relations by fetching recordings for various artist albums and then
using the fact that two artists are on teh same album as a vote that they are related.
'''
relations = {}
release_id = 0
artist_credits = []
with psycopg2.connect(config.DB_CONNECT) as conn:
create_schema(conn)
with conn.cursor() as curs:
count = 0
print(asctime(), "query for various artist recordings")
curs.execute(
"""SELECT DISTINCT r.id as release_id, ac.id as artist_id
FROM release r
JOIN medium m ON m.release = r.id AND r.artist_credit = 1
JOIN track t ON t.medium = m.id
JOIN artist_credit ac ON t.artist_credit = ac.id
JOIN artist_credit_name acm ON acm.artist_credit = ac.id
JOIN artist a ON acm.artist = a.id""")
print(asctime(), "load various artist recordings")
while True:
row = curs.fetchone()
if not row:
break
if row[1] in ARTIST_CREDIT_IDS_TO_EXCLUDE:
continue
if release_id != row[0] and artist_credits:
insert_artist_pairs(artist_credits, relations)
artist_credits = []
artist_credits.append(row[1])
release_id = row[0]
count += 1
print(asctime(), "save relations to new table")
create_or_truncate_table(conn)
dump_similarities(conn,
"relations.artist_credit_artist_credit_relations",
relations)
print(asctime(), "create indexes")
create_indexes(conn)
print(asctime(), "done!")
def run(self):
try:
input_files = get_input_files(self.input_path)
input_files.insert(0, get_pioneer_file(self.ds_name))
gfs_file_path = get_supported_datasets()[self.ds_name]
num_processes = 2
cur = get_db_cur(self.con_details)
for schema in [self.schema, self.schema + '_tmp']:
if not create_schema(cur, schema):
print 'Failed to create schema %s' % schema
self.quit(1)
return
pg_source = 'PG:dbname=\'%s\' host=\'%s\' port=\'%d\' active_schema=%s user=\'%s\'' % \
(self.con_details['database'], self.con_details['host'], self.con_details['port'],
self.schema + '_tmp', self.con_details['user'])
for arg in build_args(input_files, gfs_file_path, pg_source):
self.im.add(arg)
print 'Importing...'
self.im.start(num_processes)
except:
print
print 'Translation failed:'
print
print '%s\n\n' % traceback.format_exc()
print
self.quit(1)
return
def __init__(self):
config = get_config()
self.dbpw = config.get('oblige', 'dbpw')
self.start_time = time.time()
self.debug = False
# should use sqlalchemy, not the connection
create_schema()
conn = MySQLdb.connect(
host="localhost",
user="******",
#passwd = self.dbpw,
db="melange")
cursor = conn.cursor()
if self.debug:
cursor.execute("describe interfaces")
desc = cursor.fetchall()
for i, v in enumerate(desc):
print("{}: {}".format(i, v[0]))
print
cursor.execute("select * from interfaces")
ifaces = cursor.fetchall()
self.interfaces = {}
for interface in ifaces:
self.interfaces.update({
interface[0]:
melange.Interface(id=interface[0],
vif_id_on_device=interface[1],
device_id=interface[2],
tenant_id=interface[3],
created_at=interface[4])
})
print("Got {} interfaces...".format(len(self.interfaces)))
if self.debug:
cursor.execute("describe ip_addresses")
desc = cursor.fetchall()
for i, v in enumerate(desc):
print("{}: {}".format(i, v[0]))
print
cursor.execute("select * from ip_addresses")
self.ip_addresses = {}
ipaddrs = cursor.fetchall()
for ip in ipaddrs:
self.ip_addresses.update({
ip[0]:
melange.IpAddress(
id=ip[0],
address=ip[1],
interface_id=ip[2],
ip_block_id=ip[3],
used_by_tenant_id=ip[4],
created_at=ip[5], # 6 = updated_at
marked_for_deallocation=handle_null(ip[7]),
deallocated_at=ip[8],
allocated=handle_null(ip[9]))
})
print("Got {} ip_addresses...".format(len(self.ip_addresses)))
if self.debug:
cursor.execute("describe mac_addresses")
desc = cursor.fetchall()
for i, v in enumerate(desc):
print("{}: {}".format(i, v[0]))
print
cursor.execute("select * from mac_addresses")
self.mac_addresses = {}
macaddrs = cursor.fetchall()
for mac in macaddrs:
self.mac_addresses.update({
mac[0]:
melange.MacAddress(id=mac[0],
address=mac[1],
mac_address_range_id=mac[2],
interface_id=mac[3],
created_at=mac[4],
updated_at=mac[5])
})
self.interfaces[mac[3]].mac_address = mac[1]
print("Got {} mac_addresses...".format(len(self.mac_addresses)))
if self.debug:
cursor.execute("describe mac_address_ranges")
desc = cursor.fetchall()
for i, v in enumerate(desc):
print("{}: {}".format(i, v[0]))
print
cursor.execute("select * from mac_address_ranges")
self.mac_address_ranges = {}
mars = cursor.fetchall()
for mar in mars:
self.mac_address_ranges.update({
mar[0]:
melange.MacAddressRange(id=mar[0],
cidr=mar[1],
next_address=mar[2],
created_at=mar[3])
})
print("Got {} mac_address_ranges...".format(
len(self.mac_address_ranges)))
if self.debug:
cursor.execute("describe allocatable_macs")
desc = cursor.fetchall()
for i, v in enumerate(desc):
print("{}: {}".format(i, v[0]))
print
cursor.execute("select * from allocatable_macs")
self.allocatable_macs = {}
allmacs = cursor.fetchall()
for amac in allmacs:
self.allocatable_macs.update({
amac[0]:
melange.AllocatableMac(id=amac[0],
mac_address_range_id=amac[1],
address=amac[2],
created_at=amac[3])
})
print("Got {} allocatable_macs...".format(len(self.allocatable_macs)))
if self.debug:
cursor.execute("describe ip_blocks")
desc = cursor.fetchall()
for i, v in enumerate(desc):
print("{}: {}".format(i, v[0]))
print
cursor.execute("select * from ip_blocks")
ipb = cursor.fetchall()
self.ip_blocks = {}
for ip_block in ipb:
self.ip_blocks.update({
ip_block[0]:
melange.IpBlock(id=ip_block[0],
network_id=ip_block[1],
cidr=ip_block[2],
created_at=ip_block[3],
updated_at=ip_block[4],
_type=ip_block[5],
tenant_id=ip_block[6],
gateway=ip_block[7],
dns1=ip_block[8],
dns2=ip_block[9],
allocatable_ip_counter=ip_block[10],
is_full=handle_null(ip_block[11]),
policy_id=ip_block[12],
parent_id=ip_block[13],
network_name=ip_block[14],
omg_do_not_use=handle_null(ip_block[15]),
max_allocation=ip_block[16])
})
print("Got {} ip_blocks...".format(len(self.ip_blocks)))
if self.debug:
cursor.execute("describe ip_routes")
desc = cursor.fetchall()
for i, v in enumerate(desc):
print("{}: {}".format(i, v[0]))
print
cursor.execute("select * from ip_routes")
self.ip_routes = {}
iprs = cursor.fetchall()
for ipr in iprs:
self.ip_routes.update({
ipr[0]:
melange.IpRoute(id=ipr[0],
destination=ipr[1],
netmask=ipr[2],
gateway=ipr[3],
source_block_id=ipr[4],
created_at=ipr[5])
})
print("Got {} ip_routes...".format(len(self.ip_routes)))
if self.debug:
cursor.execute("describe policies")
desc = cursor.fetchall()
for i, v in enumerate(desc):
print("{}: {}".format(i, v[0]))
print
cursor.execute("select * from policies")
self.policies = {}
pols = cursor.fetchall()
for pol in pols:
self.policies.update({
pol[0]:
melange.Policy(id=pol[0],
name=pol[1],
tenant_id=pol[2],
description=pol[3],
created_at=pol[4])
})
print("Got {} policies...".format(len(self.policies)))
if self.debug:
cursor.execute("describe ip_octets")
desc = cursor.fetchall()
for i, v in enumerate(desc):
print("{}: {}".format(i, v[0]))
print
cursor.execute("select * from ip_octets")
self.ip_octets = {}
ipoc = cursor.fetchall()
for ipo in ipoc:
self.ip_octets.update({
ipo[0]:
melange.IpOctet(id=ipo[0],
octet=ipo[1],
policy_id=ipo[2],
created_at=ipo[3])
})
print("Got {} ip_octets...".format(len(self.ip_octets)))
if self.debug:
cursor.execute("describe ip_ranges")
desc = cursor.fetchall()
for i, v in enumerate(desc):
print("{}: {}".format(i, v[0]))
print
cursor.execute("select * from ip_ranges")
self.ip_ranges = {}
iprns = cursor.fetchall()
for iprn in iprns:
self.ip_ranges.update({
iprn[0]:
melange.IpRange(id=iprn[0],
offset=iprn[1],
length=iprn[2],
policy_id=iprn[3],
created_at=iprn[4])
})
print("Got {} ip_ranges...".format(len(self.ip_ranges)))
cursor.close()
conn.close()
self.quark_ip_addresses = {}
self.quark_port_ip_address_associations = {}
self.quark_ports = {}
self.quark_mac_addresses = {}
self.quark_mac_address_ranges = {}
self.quark_tags = {}
self.quark_networks = {}
self.quark_tag_associations = {}
self.quark_subnets = {}
self.quark_dns_nameservers = {}
self.quark_routes = {}
self.quark_ip_policies = {}
self.quark_ip_policy_cidrs = {}
self.quark_nvp_stuff = {}
self.quark_quotas = {}
self.policy_ids = {}
self.interface_network = {}
self.interface_ip = {}
self.port_cache = {}
self.interface_tenant = {}
print("\tInitialization complete in {:.2f} seconds.".format(
time.time() - self.start_time))
def connection():
utils.create_schema()
with utils.db_connection() as connection:
yield connection
utils.drop_schema(connection)
def __init__(self):
config = get_config()
self.dbpw = config.get('oblige', 'dbpw')
self.start_time = time.time()
self.debug = False
# should use sqlalchemy, not the connection
create_schema()
conn = MySQLdb.connect(host = "localhost",
user = "******",
#passwd = self.dbpw,
db = "melange")
cursor = conn.cursor()
if self.debug:
cursor.execute("describe interfaces")
desc = cursor.fetchall()
for i, v in enumerate(desc):
print("{}: {}".format(i, v[0]))
print
cursor.execute("select * from interfaces")
ifaces = cursor.fetchall()
self.interfaces = {}
for interface in ifaces:
self.interfaces.update({interface[0]: melange.Interface(
id=interface[0],
vif_id_on_device=interface[1],
device_id=interface[2],
tenant_id=interface[3],
created_at=interface[4])})
print("Got {} interfaces...".format(len(self.interfaces)))
if self.debug:
cursor.execute("describe ip_addresses")
desc = cursor.fetchall()
for i, v in enumerate(desc):
print("{}: {}".format(i, v[0]))
print
cursor.execute("select * from ip_addresses")
self.ip_addresses = {}
ipaddrs = cursor.fetchall()
for ip in ipaddrs:
self.ip_addresses.update({ip[0]: melange.IpAddress(
id = ip[0],
address = ip[1],
interface_id = ip[2],
ip_block_id = ip[3],
used_by_tenant_id = ip[4],
created_at = ip[5], # 6 = updated_at
marked_for_deallocation = handle_null(ip[7]),
deallocated_at = ip[8],
allocated = handle_null(ip[9]))})
print("Got {} ip_addresses...".format(len(self.ip_addresses)))
if self.debug:
cursor.execute("describe mac_addresses")
desc = cursor.fetchall()
for i, v in enumerate(desc):
print("{}: {}".format(i, v[0]))
print
cursor.execute("select * from mac_addresses")
self.mac_addresses = {}
macaddrs = cursor.fetchall()
for mac in macaddrs:
self.mac_addresses.update({mac[0]: melange.MacAddress(
id = mac[0],
address = mac[1],
mac_address_range_id = mac[2],
interface_id = mac[3],
created_at = mac[4],
updated_at = mac[5])})
self.interfaces[mac[3]].mac_address = mac[1]
print("Got {} mac_addresses...".format(len(self.mac_addresses)))
if self.debug:
cursor.execute("describe mac_address_ranges")
desc = cursor.fetchall()
for i, v in enumerate(desc):
print("{}: {}".format(i, v[0]))
print
cursor.execute("select * from mac_address_ranges")
self.mac_address_ranges = {}
mars = cursor.fetchall()
for mar in mars:
self.mac_address_ranges.update({mar[0]: melange.MacAddressRange(
id = mar[0],
cidr = mar[1],
next_address = mar[2],
created_at = mar[3])})
print("Got {} mac_address_ranges...".format(len(self.mac_address_ranges)))
if self.debug:
cursor.execute("describe allocatable_macs")
desc = cursor.fetchall()
for i, v in enumerate(desc):
print("{}: {}".format(i, v[0]))
print
cursor.execute("select * from allocatable_macs")
self.allocatable_macs = {}
allmacs = cursor.fetchall()
for amac in allmacs:
self.allocatable_macs.update({amac[0]: melange.AllocatableMac(
id = amac[0],
mac_address_range_id = amac[1],
address = amac[2],
created_at = amac[3])})
print("Got {} allocatable_macs...".format(len(self.allocatable_macs)))
if self.debug:
cursor.execute("describe ip_blocks")
desc = cursor.fetchall()
for i, v in enumerate(desc):
print("{}: {}".format(i, v[0]))
print
cursor.execute("select * from ip_blocks")
ipb = cursor.fetchall()
self.ip_blocks = {}
for ip_block in ipb:
self.ip_blocks.update({ip_block[0]: melange.IpBlock(
id=ip_block[0],
network_id=ip_block[1],
cidr=ip_block[2],
created_at=ip_block[3],
updated_at = ip_block[4],
_type=ip_block[5],
tenant_id=ip_block[6],
gateway=ip_block[7],
dns1=ip_block[8],
dns2=ip_block[9],
allocatable_ip_counter=ip_block[10],
is_full=handle_null(ip_block[11]),
policy_id=ip_block[12],
parent_id=ip_block[13],
network_name=ip_block[14],
omg_do_not_use=handle_null(ip_block[15]),
max_allocation=ip_block[16])})
print("Got {} ip_blocks...".format(len(self.ip_blocks)))
if self.debug:
cursor.execute("describe ip_routes")
desc = cursor.fetchall()
for i, v in enumerate(desc):
print("{}: {}".format(i, v[0]))
print
cursor.execute("select * from ip_routes")
self.ip_routes = {}
iprs = cursor.fetchall()
for ipr in iprs:
self.ip_routes.update({ipr[0]: melange.IpRoute(
id = ipr[0],
destination = ipr[1],
netmask = ipr[2],
gateway = ipr[3],
source_block_id = ipr[4],
created_at = ipr[5])})
print("Got {} ip_routes...".format(len(self.ip_routes)))
if self.debug:
cursor.execute("describe policies")
desc = cursor.fetchall()
for i, v in enumerate(desc):
print("{}: {}".format(i, v[0]))
print
cursor.execute("select * from policies")
self.policies = {}
pols = cursor.fetchall()
for pol in pols:
self.policies.update({pol[0]: melange.Policy(
id = pol[0],
name = pol[1],
tenant_id = pol[2],
description = pol[3],
created_at = pol[4])})
print("Got {} policies...".format(len(self.policies)))
if self.debug:
cursor.execute("describe ip_octets")
desc = cursor.fetchall()
for i, v in enumerate(desc):
print("{}: {}".format(i, v[0]))
print
cursor.execute("select * from ip_octets")
self.ip_octets = {}
ipoc = cursor.fetchall()
for ipo in ipoc:
self.ip_octets.update({ipo[0]: melange.IpOctet(
id = ipo[0],
octet = ipo[1],
policy_id = ipo[2],
created_at = ipo[3])})
print("Got {} ip_octets...".format(len(self.ip_octets)))
if self.debug:
cursor.execute("describe ip_ranges")
desc = cursor.fetchall()
for i, v in enumerate(desc):
print("{}: {}".format(i, v[0]))
print
cursor.execute("select * from ip_ranges")
self.ip_ranges = {}
iprns = cursor.fetchall()
for iprn in iprns:
self.ip_ranges.update({iprn[0]: melange.IpRange(
id = iprn[0],
offset = iprn[1],
length = iprn[2],
policy_id = iprn[3],
created_at = iprn[4])})
print("Got {} ip_ranges...".format(len(self.ip_ranges)))
cursor.close()
conn.close()
self.quark_ip_addresses = {}
self.quark_port_ip_address_associations = {}
self.quark_ports = {}
self.quark_mac_addresses = {}
self.quark_mac_address_ranges = {}
self.quark_tags = {}
self.quark_networks = {}
self.quark_tag_associations = {}
self.quark_subnets = {}
self.quark_dns_nameservers = {}
self.quark_routes = {}
self.quark_ip_policies = {}
self.quark_ip_policy_cidrs = {}
self.quark_nvp_stuff = {}
self.quark_quotas = {}
self.policy_ids = {}
self.interface_network = {}
self.interface_ip = {}
self.port_cache = {}
self.interface_tenant = {}
print("\tInitialization complete in {:.2f} seconds.".format(
time.time() - self.start_time))
