def handle(self, filename, fields='', delimiter=b',', **options):
fields = [field.strip() for field in fields.split(',')]
with open(filename, 'rb') as f:
for row in UnicodeReader(f, delimiter=delimiter):
data = dict(zip(fields, row))
try:
dev_id = int(data['id'])
except (ValueError, KeyError):
continue
if not dev_id:
continue
try:
dev = Device.objects.get(id=dev_id)
except Device.DoesNotExist:
sys.stderr.write("Device with id=%r doesn't exist!\n" %
dev_id)
continue
for field, value in data.iteritems():
if field in ('id', ''):
continue
if value in ('None', ''):
value = None
if value is None and field in ('remarks', ):
value = ''
print('%r.%s = %r' % (dev, field, value))
setattr(dev, field, value)
dev.save(priority=50)
def clean_csv(self):
csv_string = self.cleaned_data['csv'].strip().lower()
rows = UnicodeReader(cStringIO.StringIO(csv_string))
parsed_macs = set()
for row_number, cols in enumerate(rows, start=1):
_validate_cols_count(6, cols, row_number)
mac = cols[0].strip()
_validate_mac(mac, parsed_macs, row_number)
_validate_deploy_children(mac, row_number)
parsed_macs.add(mac)
management_ip = cols[1].strip()
_validate_management_ip(management_ip, row_number)
network_name = cols[2].strip()
if not (is_mac_address_known(mac) and network_name == ''):
# Allow empty network when the device already exists.
_validate_network_name(network_name, row_number)
venture_symbol = cols[3].strip()
venture_role = cols[4].strip()
_validate_venture_and_role(
venture_symbol,
venture_role,
row_number,
)
preboot = cols[5].strip()
_validate_preboot(preboot, row_number)
return csv_string
def handle_single(self, filename):
print('Importing Network from {}...'.format(filename))
with open(filename, 'rb') as f:
for i, value in enumerate(UnicodeReader(f), 1):
if len(value) != 6:
raise IncorrectLengthRowError(
'CSV row {} have {} elements, should be 6'.format(
i, len(value)))
(name, address, terminator_name, data_center_name,
environment_name, rack_name) = value
if not all((
name,
address,
terminator_name,
data_center_name,
environment_name,
rack_name,
)):
raise EmptyRecordValueError(
'Record fields can not be empty', )
self.create_network(
name,
address,
terminator_name,
data_center_name,
environment_name,
rack_name,
i,
)
def clean_csv(self):
csv_string = self.cleaned_data['csv'].strip().lower()
rows = UnicodeReader(cStringIO.StringIO(csv_string))
parsed_macs = set()
for row_number, cols in enumerate(rows, start=1):
_validate_cols_count(8, cols, row_number,
1 if self.assets_enabled else 0)
mac = cols[0].strip()
_validate_mac(mac, parsed_macs, row_number)
_validate_deploy_children(mac, row_number)
parsed_macs.add(mac)
management_ip = cols[1].strip()
_validate_management_ip(management_ip, row_number)
network_name = cols[2].strip()
if not (is_mac_address_known(mac) and network_name == ''):
# Allow empty network when the device already exists.
_validate_network_name(network_name, row_number)
venture_symbol = cols[3].strip()
venture_role = cols[4].strip()
_validate_venture_and_role(
venture_symbol,
venture_role,
row_number,
)
service = cols[5].strip()
_validate_service(service, row_number)
environment = cols[6].strip()
_validate_environment(environment, row_number)
preboot = cols[7].strip()
_validate_preboot(preboot, row_number)
if self.assets_enabled and len(cols) == 9:
asset_identity = cols[8].strip()
_validate_asset_identity(asset_identity, row_number, mac)
return csv_string
def import_csv(self, group, uploaded_file):
if uploaded_file.size > 4 * 1024 * 1024:
messages.error(self.request, "File too big to import.")
return
f = cStringIO.StringIO(uploaded_file.read())
rows = iter(UnicodeReader(f))
header = list(rows.next())
if header[0].strip() != 'sn':
messages.error(
self.request,
"The first row should have 'sn' followed by variable names."
)
return
variables = [name.strip() for name in header[1:]]
devices = {}
try:
for row in rows:
if not row:
continue
devices[row[0]] = dict(zip(
variables,
(decimal.Decimal(v) for v in row[1:]),
))
except ValueError as e:
messages.error(self.request, "Invalid value: %s." % e)
return
variable_dict = {}
for name in variables:
v = PricingVariable(name=name, group=group)
v.save()
variable_dict[name] = v
for sn, values in devices.iteritems():
try:
device = Device.objects.get(sn=sn)
except Device.DoesNotExist:
messages.warning(
self.request,
"Serial number %s not found, skipping." % sn
)
continue
group.devices.add(device)
for name, value in values.iteritems():
PricingValue(
variable=variable_dict[name],
device=device,
value=value,
).save()
group.save()
def handle_single(self, filename, **options):
if options['create_ptr']:
create_ptr = True
else:
create_ptr = False
print('Importing DNS records from {}...'.format(filename))
with open(filename, 'rb') as f:
for i, value in enumerate(UnicodeReader(f), 1):
if len(value) != 3:
raise IncorrectLengthRowError(
'CSV row {} has {} elements, should be 3'.format(
i,
len(value),
))
name, type, content = value
if not all((name, type, content)):
raise EmptyRecordValueError(
'Record fields can not be empty', )
if type not in RECORD_TYPES:
raise DisallowedRecordTypeError(
'Record type {} is not allowed. Use:\n {}'.format(
type,
RECORD_TYPES,
), )
if type == 'A':
try:
ipaddr.IPv4Address(content)
except ValueError:
raise InvalidAddressError(
'Record {} has invalid IP address ({}).'.format(
name,
content,
))
domain = get_domain(name)
if not domain:
raise DomainDoesNotExistError(
'Domain for {} does not exist in Ralph.'.format(name))
self.create_record(domain, name, type, content)
if create_ptr:
if type != 'A':
raise DisallowedRecordTypeError(
'PTR record can only be created for record type A.'
)
revname = '.'.join(reversed(
content.split('.'))) + '.in-addr.arpa'
# we need to create SOA domain/record first
today = datetime.date.today().strftime('%Y%m%d')
soa_name = '.'.join(revname.split('.')[1:])
soa_domain, created = Domain.objects.get_or_create(
name=soa_name, )
if created:
# last 2 digits == version for a given day
soa_domain.notified_serial = today + '01'
soa_domain.type = 'MASTER'
soa_domain.save()
print('Domain {} of type {} created (sn: {}).'.format(
soa_domain.name,
soa_domain.type,
soa_domain.notified_serial,
))
type = 'SOA'
if options['soa_content']:
content = options['soa_content']
else:
content = settings.DEFAULT_SOA_RECORD_CONTENT
self.create_record(soa_domain, soa_name, type, content)
type = 'PTR'
content = name
self.create_record(soa_domain, revname, type, content)
def get(self, *args, **kwargs):
mass_deployment = get_object_or_404(
MassDeploymentModel,
id=kwargs.get('deployment'),
is_done=False,
)
reserved_hostnames = []
reserved_ip_addresses = []
new_csv_rows = []
csv_rows = UnicodeReader(
cStringIO.StringIO(mass_deployment.csv.strip()))
for raw_cols in csv_rows:
cols = []
for col in raw_cols:
cols.append(" %s " % col.strip())
hostname = ""
ip = ""
rack = None
asset_identity = None
if self.assets_enabled and len(cols) == 7:
asset_identity = cols[6].strip()
device = _find_device(cols[0], asset_identity)
try:
network, ip = _find_network_ip(
cols[2].strip(),
reserved_ip_addresses,
device,
)
except Network.DoesNotExist:
pass
else:
hostname = _find_hostname(
network,
reserved_hostnames,
device,
ip,
)
for rack in network.racks.filter(
deleted=False).order_by('name')[:1]:
break
cols[2] = " %s " % network.name
cols.insert(0, " %s " % rack.sn if rack else " ")
cols.insert(0, " %s " % ip)
cols.insert(0, " %s " % hostname)
new_csv_rows.append(cols)
if device:
self.actions.append((
'warning',
"An old device %s will be re-used. Make sure it's not "
"used in production anymore!" % device,
))
if device.deleted:
self.actions.append((
'info',
"Device %s will be undeleted." % device,
))
if device.ipaddress_set.exists():
self.actions.append((
'info',
"All DNS entries for IP addresses [%s] will be "
"deleted." %
', '.join(ip.address
for ip in device.ipaddress_set.all()),
))
self.actions.append((
'info',
"All DHCP entries for IP addresses [%s] "
"will be deleted." %
(', '.join(ip.address
for ip in device.ipaddress_set.all()), ),
))
if device.ethernet_set.exists():
self.actions.append((
'info',
"All DHCP entries for "
"MAC addresses [%s] will be deleted." %
(', '.join(eth.mac
for eth in device.ethernet_set.all()), ),
))
if device.disksharemount_set.exists():
self.actions.append((
'info',
"All disk shares mounted on %s will be disconnected "
"from it." % device,
))
self.actions.append((
'info',
"The uptime, operating system and software list for %s "
"will be reset." % device,
))
else:
if hostname:
self.actions.append((
'success',
"A new device %s will be created." % hostname,
))
if hostname and ip:
self.actions.append((
'info',
"An A DNS entry for %s and %s will be created." %
(hostname, ip),
))
self.actions.append((
'info',
"A PTR DNS entry for %s and %s will be created." %
(hostname, ip),
))
if cols[0].strip() and ip:
self.actions.append((
'info',
"A DHCP entry for %s and %s will be created." %
(cols[3], ip),
))
csv_string = cStringIO.StringIO()
UnicodeWriter(csv_string).writerows(new_csv_rows)
self.form = MassDeploymentForm(initial={'csv': csv_string.getvalue()})
return super(MassDeployment, self).get(*args, **kwargs)
def clean_csv(self):
csv_string = self.cleaned_data['csv'].strip().lower()
rows = UnicodeReader(cStringIO.StringIO(csv_string))
cleaned_csv = []
parsed_hostnames = set()
parsed_ip_addresses = set()
parsed_macs = set()
for row_number, cols in enumerate(rows, start=1):
_validate_cols_count(9, cols, row_number)
_validate_cols_not_empty(cols, row_number)
mac = cols[3].strip()
_validate_mac(mac, parsed_macs, row_number)
parsed_macs.add(mac)
hostname = cols[0].strip()
_validate_hostname(hostname, mac, parsed_hostnames, row_number)
if not clean_hostname(hostname):
raise forms.ValidationError("Invalid hostname")
parsed_hostnames.add(hostname)
network_name = cols[5].strip()
try:
network = Network.objects.get(name=network_name)
except Network.DoesNotExist:
raise forms.ValidationError(
"Row %s: Network '%s' doesn't exists." %
(row_number, network_name))
rack_sn = cols[2].strip()
if re.match(r"^[0-9]+$", rack_sn):
rack_sn = "Rack %s %s" % (
rack_sn,
network.data_center.name.upper(),
)
if not rack_exists(rack_sn):
raise forms.ValidationError(
"Row %s: Rack with serial number '%s' doesn't exists." %
(row_number, rack_sn))
try:
network.racks.get(sn=rack_sn)
except Device.DoesNotExist:
raise forms.ValidationError(
"Row %s: Rack '%s' isn't connected to "
"network '%s'." % (row_number, rack_sn, network.name))
ip = cols[1].strip()
_validate_ip_address(ip, network, parsed_ip_addresses, row_number)
_validate_ip_owner(ip, mac, row_number)
parsed_ip_addresses.add(ip)
management_ip = cols[4].strip()
_validate_management_ip(management_ip, row_number)
try:
venture_role = VentureRole.objects.get(
venture__symbol=cols[6].strip().upper(),
name=cols[7].strip())
venture = venture_role.venture
except VentureRole.DoesNotExist:
raise forms.ValidationError(
"Row %s: "
"Couldn't find venture with symbol %s and role %s" %
(row_number, cols[6].strip(), cols[7].strip()))
try:
preboot = Preboot.objects.get(name=cols[8].strip())
except Preboot.DoesNotExist:
raise forms.ValidationError(
"Row %s: Couldn't find preboot %s" %
(row_number, cols[8].strip()))
cleaned_csv.append({
'hostname': hostname,
'ip': ip,
'mac': mac,
'rack_sn': rack_sn,
'venture': venture,
'venture_role': venture_role,
'preboot': preboot,
'management_ip': management_ip,
'network': network
})
return cleaned_csv
def handle_single(self, filename, **options):
if options['connect_business_segment']:
connect_business_segment = True
else:
connect_business_segment = False
print('Importing information from {}...'.format(filename))
with open(filename, 'rb') as f:
self.not_found_ventures = []
self.not_found_profit_centers = set()
self.not_found_business_segments = set()
for i, value in enumerate(UnicodeReader(f), 1):
if len(value) != 3:
raise IncorrectLengthRowError(
'CSV row {} have {} elements, should be 3'.format(
i, len(value)
)
)
if not connect_business_segment:
venture_id, profit_center, description = value
venture = self.get_venture(venture_id, i)
if not venture:
continue
else:
profit_center, created = ProfitCenter.objects.get_or_create( # noqa
name=profit_center or 'Other'
)
profit_center.description = description or 'Other'
profit_center.save()
venture.profit_center = profit_center
venture.save()
print(
'profit center {} joined to venture {}'.format(
profit_center, venture.name
)
)
else:
venture_id, profit_center_name, business_segment_name = value # noqa
venture = self.get_venture(venture_id, i)
if not venture:
continue
else:
profit_center = self.get_profit_center(
profit_center_name, i,
)
business_segment = self.get_business_segment(
business_segment_name, i,
)
if profit_center:
venture.profit_center = profit_center
if business_segment:
venture.business_segment = business_segment
venture.save()
print("Ventures with ids {} don't exist".format(
[v for v in self.not_found_ventures]
))
if connect_business_segment:
print("Business Segments with name {} does not exist".format(
[v for v in self.not_found_business_segments]
))
print("Profit Center with name {} does not exist".format(
[v for v in self.not_found_profit_centers]
))