def test_csv_export_api(self):
response = self.forced_auth_req(
'get',
reverse('partners_api:agreement-list'),
user=self.unicef_staff,
data={"format": "csv"},
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
dataset = Dataset().load(response.content.decode('utf-8'), 'csv')
self.assertEqual(dataset.height, 1)
self.assertEqual(dataset._get_headers(), [
'Reference Number', 'Status', 'Partner Name', 'Partner Number',
'Agreement Type', 'Start Date', 'End Date', 'Signed By Partner',
'Signed By Partner Date', 'Signed By UNICEF',
'Signed By UNICEF Date', 'Partner Authorized Officer',
'Amendments', 'URL'
])
exported_agreement = dataset[0]
self.assertEqual(
exported_agreement,
(self.agreement.agreement_number, str(
self.agreement.status), str(self.agreement.partner.name),
str(self.agreement.partner.vendor_number),
self.agreement.agreement_type, '{}'.format(self.agreement.start),
'{}'.format(self.agreement.end), u'', '{}'.format(
self.agreement.signed_by_partner_date),
self.unicef_staff.get_full_name(),
'{}'.format(self.agreement.signed_by_unicef_date), ', '.join([
sm.get_full_name()
for sm in self.agreement.authorized_officers.all()
]), u'', u'https://testserver/pmp/agreements/{}/details/'.format(
self.agreement.id)))
def test_intervention_export_api(self):
response = self.forced_auth_req(
'get',
'/api/partners/{}/interventions/export/'.format(self.partner.id),
user=self.unicef_staff)
self.assertEquals(response.status_code, status.HTTP_200_OK)
dataset = Dataset().load(response.content, 'csv')
self.assertEqual(dataset.height, 1)
self.assertEqual(dataset._get_headers(), [
'title', 'reference_number', 'status', 'partner__name',
'partnership_type', 'sectors', 'start_date', 'end_date',
'result_structure__name', 'locations', 'initiation_date',
'submission_date', 'review_date', 'days_from_submission_to_signed',
'days_from_review_to_signed', 'signed_by_partner_date',
'partner_manager_name', 'signed_by_unicef_date',
'unicef_manager_name', 'total_unicef_cash', 'supplies',
'total_budget', 'planned_visits'
])
self.assertEqual(
dataset[0],
('To save the galaxy from the Empire',
self.intervention.reference_number, 'in_process',
self.partner.name, 'PD', '', '', '', '', '',
self.intervention.initiation_date.strftime('%Y-%m-%d'), '', '',
'Not Submitted', 'Not Reviewed', '', '', '', '', '0', '', '0',
'0'))
def test_csv_export(self):
response = self.forced_auth_req('get',
self.url,
user=self.unicef_staff,
data={"format": "csv"})
self.assertEqual(response.status_code, status.HTTP_200_OK)
dataset = Dataset().load(response.content.decode('utf-8'), 'csv')
self.assertEqual(dataset.height, 1)
self.assertEqual(len(dataset._get_headers()), 33)
self.assertEqual(len(dataset[0]), 33)
def test_export_csv(self):
HactHistoryFactory(partner=self.partner,
year=2017,
partner_values=self.hact_data)
response = self.forced_auth_req("get",
self.url,
user=self.unicef_user,
data={"format": "csv"})
self.assertEqual(response.status_code, status.HTTP_200_OK)
dataset = Dataset().load(response.content.decode('utf-8'), "csv")
self.assertEqual(dataset.height, 1)
self.assertEqual(dataset._get_headers(), [
"Implementing Partner",
"Partner Type",
"Shared",
"Shared IP",
"TOTAL for current CP cycle",
"PLANNED for current year",
"Current Year (1 Oct - 30 Sep)",
"Micro Assessment",
"Risk Rating",
"Expiring Threshold",
"Approach Threshold",
"Programmatic Visits Planned",
"Programmatic Visits M.R",
"Programmatic Visits Done",
"Spot Checks M.R",
"Spot Checks Done",
"Audits M.R",
"Audits Done",
"Flag for Follow up",
])
self.assertEqual(
dataset[0],
(
"Partner Name",
PartnerType.UN_AGENCY,
"with UNFPA",
PartnerOrganization.AGENCY_CHOICES.UN,
"200.00",
"300.00",
"150.00",
"Yes",
"High",
"False",
"False",
"10", # programmatic visits
"8",
"5",
"3", # spot checks
"2",
"4", # audits
"2",
"No",
))
def as_tablib_dataset(report, parent_context):
render_context = _report(report, parent_context)
dataset = Dataset()
for element in render_context["elements"].values():
table = element["table"]
dataset.append_separator(element["title"])
for i, row in enumerate(table.as_values()):
dataset.append(row)
return dataset
def _prepare_table(string):
dataset = Dataset()
for i, line in enumerate(string.split('\n')[1:]):
if '====' in line or not line:
continue
row = line.split()
if i == 1:
dataset.headers = row
else:
dataset.append([numberfy(num) for num in row])
return dataset
def render_csv_report(self):
dataset = Dataset(*self._build_report_data(),
headers=self._build_report_headers())
response = HttpResponse(
dataset.csv,
'text/csv',
None,
'text/csv; charset=utf-8',
)
filename = self.get_filename().encode('utf-8')
response['Content-Disposition'] = 'attachment; filename=%s' % filename
return response
def test_agreement_export_api(self):
response = self.forced_auth_req(
'get',
'/api/v2/agreements/',
user=self.unicef_staff,
data={"format": "csv"},
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
dataset = Dataset().load(response.content.decode('utf-8'), 'csv')
self.assertEqual(dataset.height, 2)
self.assertEqual(dataset._get_headers(), [
'Reference Number',
'Status',
'Partner Name',
'Partner Number',
'Agreement Type',
'Start Date',
'End Date',
'Signed By Partner',
'Signed By Partner Date',
'Signed By UNICEF',
'Signed By UNICEF Date',
'Partner Authorized Officer',
'Amendments',
'URL',
'Special Conditions PCA',
])
# we're interested in the first agreement, so it will be last in the exported list
exported_agreement = dataset[-1]
self.assertEqual(exported_agreement, (
self.agreement.agreement_number,
str(self.agreement.status),
str(self.agreement.partner.name),
str(self.agreement.partner.vendor_number),
self.agreement.agreement_type,
'{}'.format(self.agreement.start),
'{}'.format(self.agreement.end),
'',
'{}'.format(self.agreement.signed_by_partner_date),
self.unicef_staff.get_full_name(),
'{}'.format(self.agreement.signed_by_unicef_date),
', '.join([
sm.get_full_name()
for sm in self.agreement.authorized_officers.all()
]),
'',
'https://testserver/pmp/agreements/{}/details/'.format(
self.agreement.id),
'No',
))
def test_csv_export_api(self):
response = self.forced_auth_req(
'get',
reverse('partners_api:partner-assessment'),
user=self.unicef_staff,
data={"format": "csv"},
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
dataset = Dataset().load(response.content.decode('utf-8'), 'csv')
self.assertEqual(dataset.height, 1)
self.assertEqual(len(dataset._get_headers()), 18)
self.assertEqual(len(dataset[0]), 18)
def test_csv_flat_export_api(self):
response = self.forced_auth_req(
'get',
reverse('reports:applied-indicator'),
user=self.unicef_staff,
data={"format": "csv_flat"},
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
dataset = Dataset().load(response.content.decode('utf-8'), 'csv')
self.assertEqual(dataset.height, 1)
self.assertEqual(len(dataset._get_headers()), 34)
self.assertEqual(len(dataset[0]), 34)
def _prepare_cluters(string):
dataset = Dataset()
string = string.replace('PULSE HEIGHT',
'PULSE-HEIGHT').replace(' +/-', '_+/-')
for i, line in enumerate(string.split('\n')[3:]):
if '*****' in line or not line:
continue
line = line.split()
if i == 0:
dataset.headers = line
else:
dataset.append(
[numberfy(cell.replace('_', ' ')) for cell in line])
return dataset
def _prepare_table(string):
dataset = Dataset()
for i, line in enumerate(string.split('\n')[1:]):
if '*****' in line or not line:
continue
row = line.split()
if i == 1:
row[4] += ' ' + row[5]
dataset.headers = row[:5]
else:
dataset.append([numberfy(i) for i in row])
return dataset
def _prepare_table(string):
result = []
for line in string.split('\n'):
if '*****' in line or not line:
continue
line = line.lstrip().strip('*').split('*')
row = []
for cell in line:
cell = cell.strip()
row.append(cell)
result.append(row)
dataset = Dataset()
dataset.headers = result[0]
for row in result[1:]:
dataset.append([numberfy(i) for i in row])
return dataset
def projectsImport(self, fname):
start = datetime.datetime.now()
imported_data = Dataset().load(open(fname).read())
project_resource = ProjectResource()
result = project_resource.import_data(imported_data, dry_run=False)
if result.has_errors():
logger.info('IMPORT COMPLETED WITH ERRORS: ')
logger.info(result)
print(result)
return
else:
end = datetime.datetime.now()
delta = end - start
logger.info('IMPORT SUCCESSFULLY COMPLETED IN TIME: ' + str(delta))
def test_partners_export_api(self):
response = self.forced_auth_req(
'get',
'/api/v2/partners/',
user=self.unicef_staff,
data={"format": "csv"},
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
dataset = Dataset().load(response.content.decode('utf-8'), 'csv')
self.assertEqual(dataset.height, 2)
self.assertEqual(dataset._get_headers(), [
'Vendor Number', 'Organizations Full Name', 'Short Name',
'Alternate Name', 'Partner Type', 'Shared Partner', 'Address',
'Phone Number', 'Email Address', 'Risk Rating',
'Date Last Assessed Against Core Values',
'Actual Cash Transfer for CP (USD)',
'Actual Cash Transfer for Current Year (USD)',
'Marked for Deletion', 'Blocked', 'Assessment Type',
'Date Assessed', 'Assessment Type (Date Assessed)',
'Staff Members', 'URL'
])
deleted_flag = "Yes" if self.partner.deleted_flag else "No"
blocked = "Yes" if self.partner.blocked else "No"
test_option = [
e for e in dataset if e[0] == self.partner.vendor_number
][0]
self.assertEqual(
test_option,
(self.partner.vendor_number, six.text_type(self.partner.name),
self.partner.short_name, self.partner.alternate_name, "{}".format(
self.partner.partner_type), u', '.join([
x for x in self.partner.shared_with
]), self.partner.address, self.partner.phone_number,
self.partner.email, self.partner.rating, u'{}'.format(
self.partner.core_values_assessment_date), u'{:.2f}'.format(
self.partner.total_ct_cp), u'{:.2f}'.format(
self.partner.total_ct_cy), deleted_flag, blocked,
self.partner.type_of_assessment, u'{}'.format(
self.partner.last_assessment_date), u'', ', '.join([
"{} ({})".format(sm.get_full_name(), sm.email)
for sm in self.partner.staff_members.filter(
active=True).all()
]), u'https://testserver/pmp/partners/{}/details/'.format(
self.partner.id)))
def _prepare_clusters_str(self, data):
dataset = Dataset()
stringRep = data.replace('PULSE HEIGHT',
'PULSE-HEIGHT').replace(' +/-', '_+/-')
for i, line in enumerate(stringRep.split('\n')[3:]):
if '*****' in line or not line or 'GEANT' in line:
continue
if 'NO. PULSE-HEIGHT' not in line and '+/-' not in line:
continue
parts = line.split()
if i == 0:
dataset.headers = parts
else:
if '+/-' in line:
dataset.append(
[numberfy(cell.replace('_', ' ')) for cell in parts])
return dataset
def _prepare_table(string):
dataset = Dataset()
good = None
for i, line in enumerate(string.split('\n')[1:]):
if '====' in line or not line:
continue
row = line.split()
if i == 1:
dataset.headers = row
else:
if len(row) == len(dataset.headers):
good = line
dataset.append([numberfy(num) for num in row])
else:
print(good)
print(row)
return dataset
def test_csv_flat_export_api_hact_value_string(self):
partner = self.partner
partner.pk = None
partner.vendor_number = "Vendor New Num"
partner.hact_values = json.dumps('{"key": "random string"}')
partner.save()
response = self.forced_auth_req(
'get',
reverse('partners_api:partner-list'),
user=self.unicef_staff,
data={"format": "csv_flat"},
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
dataset = Dataset().load(response.content.decode('utf-8'), 'csv')
self.assertEqual(dataset.height, 2)
self.assertEqual(len(dataset._get_headers()), 49)
self.assertEqual(len(dataset[0]), 49)
def test_agreement_export_api(self):
response = self.forced_auth_req(
'get',
'/api/partners/{}/agreements/export/'.format(self.partner.id),
user=self.unicef_staff)
self.assertEquals(response.status_code, status.HTTP_200_OK)
dataset = Dataset().load(response.content, 'csv')
self.assertEqual(dataset.height, 1)
self.assertEqual(dataset._get_headers(), [
'reference_number', 'partner__vendor_number', 'partner__name',
'partner__short_name', 'start_date', 'end_date',
'signed_by_partner', 'signed_by_partner_date', 'signed_by_unicef',
'signed_by_unicef_date', 'authorized_officers'
])
self.assertEqual(dataset[0],
(self.agreement.reference_number, '',
self.partner.name, '', '', '', '', '', '', '', ''))
def test_csv_flat_export_api(self):
response = self.forced_auth_req(
'get',
reverse('funds:funds-donor'),
user=self.unicef_staff,
data={"format": "csv_flat"},
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
dataset = Dataset().load(response.content.decode('utf-8'), 'csv')
self.assertEqual(dataset.height, 1)
six.assertCountEqual(self, dataset._get_headers(), [
"Grant",
"ID",
"Name",
"created",
"modified",
])
self.assertEqual(len(dataset[0]), 5)
def test_export_csv_empty_shared_with(self):
"""If partner shared_with value is empty
make sure we handle that gracefully
"""
partner = PartnerFactory(name="Partner Name",
partner_type=PartnerType.UN_AGENCY,
shared_with=None,
rating="High",
total_ct_cp=200.0,
total_ct_cy=150.0)
HactHistoryFactory(partner=partner,
year=2017,
partner_values=self.hact_data)
response = self.forced_auth_req("get",
self.url,
user=self.unicef_user,
data={"format": "csv"})
self.assertEqual(response.status_code, status.HTTP_200_OK)
dataset = Dataset().load(response.content.decode('utf-8'), "csv")
self.assertEqual(dataset.height, 1)
self.assertEqual(
dataset[0],
(
"Partner Name",
PartnerType.UN_AGENCY,
"with UNFPA",
PartnerOrganization.AGENCY_CHOICES.UN,
"200.00",
"300.00",
"150.00",
"Yes",
"High",
"False",
"False",
"10", # programmatic visits
"8",
"5",
"3", # spot checks
"2",
"4", # audits
"2",
"No",
))
def test_government_export_api(self):
response = self.forced_auth_req(
'get',
'/api/partners/{}/government_interventions/export/'.format(
self.partner.id),
user=self.unicef_staff)
self.assertEquals(response.status_code, status.HTTP_200_OK,
response.content)
dataset = Dataset().load(response.content, 'csv')
self.assertEqual(dataset.height, 1)
self.assertEqual(dataset._get_headers(), [
'number', 'partner__name', 'result_structure__name', 'sectors',
'cash_transfer', 'year'
])
self.assertEqual(dataset[0],
('RefNumber', self.partner.name,
self.government_intervention.result_structure.name,
'', '0', datetime.now().strftime('%Y')))
def test_partner_export_api(self):
response = self.forced_auth_req('get',
'/api/partners/export/',
user=self.unicef_staff)
self.assertEquals(response.status_code, status.HTTP_200_OK)
dataset = Dataset().load(response.content, 'csv')
self.assertEqual(dataset.height, 2)
self.assertEqual(dataset._get_headers(), [
'vendor_number', 'vision_synced', 'deleted_flag', 'name',
'short_name', 'alternate_id', 'alternate_name', 'partner_type',
'cso_type', 'shared_partner', 'address', 'email', 'phone_number',
'risk_rating', 'type_of_assessment', 'last_assessment_date',
'total_ct_cp', 'total_ct_cy', 'agreement_count',
'intervention_count', 'active_staff_members'
])
self.assertEqual(
dataset[0],
('', '0', '0', self.partner.name, '', '', '', '', '', 'No', '', '',
'', '', '', '', '', '', '1', '1', 'Mace Windu'))
def before_import(self, dataset, dry_run, **kwargs):
stream = io.StringIO(dataset.csv.replace(replace[0], replace[1]))
dataset = Dataset().load(stream.read(), format='csv')
return dataset
def test_csv_export_api(self):
response = self.forced_auth_req(
'get',
reverse('partners_api:partner-list'),
user=self.unicef_staff,
data={"format": "csv"},
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
dataset = Dataset().load(response.content.decode('utf-8'), 'csv')
self.assertEqual(dataset.height, 1)
self.assertEqual(dataset._get_headers(), [
'Vendor Number', 'Organizations Full Name', 'Short Name',
'Alternate Name', 'Partner Type', 'Shared Partner', 'Address',
'Phone Number', 'Email Address', 'Risk Rating',
'Date Last Assessed Against Core Values',
'Actual Cash Transfer for CP (USD)',
'Actual Cash Transfer for Current Year (USD)',
'Marked for Deletion', 'Blocked', 'Assessment Type',
'Date Assessed', 'Assessment Type (Date Assessed)',
'Staff Members', 'URL', 'Planned Programmatic Visits'
])
deleted_flag = "Yes" if self.partner.deleted_flag else "No"
blocked = "Yes" if self.partner.blocked else "No"
test_option = [
e for e in dataset if e[0] == self.partner.vendor_number
][0]
# the order of staff members in the results is hard to determine
# so just ensuring that all relevant staff members are in the results
for sm in self.partner.staff_members.filter(active=True).all():
member = "{} ({})".format(sm.get_full_name(), sm.email)
self.assertIn(member, test_option[18])
self.assertEqual(test_option, (
self.partner.vendor_number,
str(self.partner.name),
self.partner.short_name,
self.partner.alternate_name,
"{}".format(self.partner.partner_type),
', '.join([x for x in self.partner.shared_with]),
self.partner.address,
self.partner.phone_number,
self.partner.email,
self.partner.rating,
'{}'.format(self.partner.core_values_assessment_date),
'{:.2f}'.format(self.partner.total_ct_cp),
'{:.2f}'.format(self.partner.total_ct_ytd),
deleted_flag,
blocked,
self.partner.type_of_assessment,
'{}'.format(self.partner.last_assessment_date),
'',
test_option[18],
'https://testserver/pmp/partners/{}/details/'.format(
self.partner.id),
'{} (Q1:{} Q2:{}, Q3:{}, Q4:{})'.format(
self.planned_visit.year,
self.planned_visit.programmatic_q1,
self.planned_visit.programmatic_q2,
self.planned_visit.programmatic_q3,
self.planned_visit.programmatic_q4,
),
))
def test_intervention_export_api(self):
response = self.forced_auth_req(
'get',
'/api/v2/interventions/',
user=self.unicef_staff,
data={"format": "csv"},
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
dataset = Dataset().load(response.content.decode('utf-8'), 'csv')
self.assertEqual(dataset.height, 1)
self.assertEqual(dataset._get_headers(), [
"Partner",
"Vendor Number",
"Status",
"Partner Type",
"Agreement",
"Country Programme",
"Document Type",
"Reference Number",
"Document Title",
"Start Date",
"End Date",
"UNICEF Office",
"Sections",
"Locations",
"Contingency PD",
"Cluster",
"UNICEF Focal Points",
"CSO Authorized Officials",
"Budget Currency",
"Total CSO Contribution",
"UNICEF Cash",
"UNICEF Supply",
"Total PD/SSFA Budget",
"FR Number(s)",
"FR Currency",
"FR Posting Date",
"FR Amount",
"FR Actual CT",
"Outstanding DCT",
"Document Submission Date by CSO",
"Submission Date to PRC",
"Review Date by PRC",
"Signed by Partner",
"Signed by Partner Date",
"Signed by UNICEF",
"Signed by UNICEF Date",
"Days from Submission to Signed",
"Days from Review to Signed",
"Total no. of amendments",
"Last amendment date",
"Attachment type",
"# of attachments",
"CP Outputs",
"URL",
])
self.assertEqual(dataset[0], (
str(self.intervention.agreement.partner.name),
str(self.intervention.agreement.partner.vendor_number),
self.intervention.status,
self.intervention.agreement.partner.partner_type,
self.intervention.agreement.agreement_number,
str(self.intervention.country_programme.name),
self.intervention.document_type,
self.intervention.number,
str(self.intervention.title),
'{}'.format(self.intervention.start),
'{}'.format(self.intervention.end),
u'',
u'',
u'',
str("Yes" if self.intervention.contingency_pd else "No"),
u'',
u'',
u'',
str(self.ib.currency),
u'{:.2f}'.format(self.intervention.total_partner_contribution),
u'{:.2f}'.format(self.intervention.total_unicef_cash),
u'{:.2f}'.format(self.intervention.total_in_kind_amount),
u'{:.2f}'.format(self.intervention.total_budget),
u', '.join([fr.fr_numbers for fr in self.intervention.frs.all()]),
u'',
u'',
u'',
u'',
u'',
'{}'.format(self.intervention.submission_date),
'{}'.format(self.intervention.submission_date_prc),
'{}'.format(self.intervention.review_date_prc),
u'{}'.format(self.intervention.
partner_authorized_officer_signatory.get_full_name()),
'{}'.format(self.intervention.signed_by_partner_date),
self.unicef_staff.get_full_name(),
'{}'.format(self.intervention.signed_by_unicef_date),
'{}'.format(self.intervention.days_from_submission_to_signed),
'{}'.format(self.intervention.days_from_review_to_signed),
str(self.intervention.amendments.count()),
u'',
str(', '.join([
'{}'.format(att.type.name)
for att in self.intervention.attachments.all()
])),
str(self.intervention.attachments.count()),
u'',
u'https://testserver/pmp/interventions/{}/details/'.format(
self.intervention.id),
))
number_of_decays = 0
event_id = 0
progress = 0
if __name__ == '__main__':
with GimelSession(user=USER_NAME,
password=PASSWORD,
output_file=SESSION_FILE) as g:
g.start_gimmel()
# Insert the name of the particle here:
g.send_particle_in_bulk('pi-0', momentum, times)
with open(SESSION_FILE) as f:
text = f.read()
events = parse(text)
raw = Dataset()
raw.headers = ('Event ID', 'P Parent Particle', 'Cluster1 y', 'Cluster1 z',
'Pulse Height 1', 'Cluster2 y', 'Cluster2 z',
'Pulse Height 2')
for event in events:
sp.call('cls', shell=True)
event_id += 1
progress = int(100 * event_id / times)
print('Processing..', str(progress) + '% completed.')
if len(event.raw.strip()) == 0:
event_id -= 1
# elif len(event.calorimeter.clusters.clusters) == 1 and len(event.tracks.tracks) == 0:
PASSWORD = data["password"]
IDNum = data["id"]
with GimelSession(user=USER_NAME,
password=PASSWORD,
output_file=SESSION_FILE) as g:
g.start_gimmel()
g.send_command(IDNum)
g.send_particles_ascending_energies(particle, minimum_energy,
step_size, number_of_injections,
per_energy)
with open(SESSION_FILE) as f:
text = f.read()
if particle is 'photon':
events = parse(text)
dataset = Dataset()
dataset.headers = ('P', 'pulseheight', 'x', 'dx', 'y', 'dy', 'z', 'dz',
'ywidth', 'zwidth')
for event in events:
row = []
if len(event.clusters.clusters.clusters) is 1:
row.append(event.energy)
row.append(event.clusters.clusters.clusters[0].pulse_height)
row.append(event.clusters.clusters.clusters[0].x.value)
row.append(event.clusters.clusters.clusters[0].x.error)
row.append(event.clusters.clusters.clusters[0].y.value)
row.append(event.clusters.clusters.clusters[0].y.error)
row.append(event.clusters.clusters.clusters[0].z.value)
row.append(event.clusters.clusters.clusters[0].z.error)
row.append(event.clusters.clusters.clusters[0].ywidth)
row.append(event.clusters.clusters.clusters[0].zwidth)
