def encode_plot(P,
pad=None,
pad_inches=0.1,
bbox_inches=None,
remove_axes=False,
transparent=False,
axes_pad=None):
"""
Convert a plot object to base64-encoded png format.
pad is passed down to matplotlib's tight_layout; pad_inches and bbox_inches to savefig.
The resulting object is a base64-encoded version of the png
formatted plot, which can be displayed in web pages with no
further intervention.
"""
from io import BytesIO as IO
from matplotlib.backends.backend_agg import FigureCanvasAgg
from base64 import b64encode
from urllib.parse import quote
virtual_file = IO()
fig = P.matplotlib(axes_pad=axes_pad)
fig.set_canvas(FigureCanvasAgg(fig))
if remove_axes:
for a in fig.axes:
a.axis('off')
if pad is not None:
fig.tight_layout(pad=pad)
fig.savefig(virtual_file,
format='png',
pad_inches=pad_inches,
bbox_inches=bbox_inches,
transparent=transparent)
virtual_file.seek(0)
buf = virtual_file.getbuffer()
return "data:image/png;base64," + quote(b64encode(buf))
def zipper(response):
accept_encoding = request.headers.get('Accept-Encoding', '')
if 'gzip' not in accept_encoding.lower():
return response
response.direct_passthrough = False
if (response.status_code < 200 or
response.status_code >= 300 or
'Content-Encoding' in response.headers):
return response
gzip_buffer = IO()
gzip_file = gzip.GzipFile(mode='wb',
fileobj=gzip_buffer)
gzip_file.write(response.data)
gzip_file.close()
response.data = gzip_buffer.getvalue()
response.headers['Content-Encoding'] = 'gzip'
response.headers['Vary'] = 'Accept-Encoding'
response.headers['Content-Length'] = len(response.data)
return response
def export_submit(request):
if request.BODY['type'] == 'excel':
byte_io = IO()
excel_file = pd.DataFrame(request.BODY['inventory'])
excel_file = excel_file[request.BODY['columns']]
writer = pd.ExcelWriter(byte_io, engine='xlsxwriter')
excel_file.to_excel(writer, sheet_name='Inventory')
writer.save()
writer.close()
byte_io.seek(0)
response = HttpResponse(
byte_io.read(),
content_type=
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
)
response['Content-Disposition'] = 'attachment; filename=inventory.xlsx'
return response
else:
return JsonResponse({}, safe=False)
def get(self, request, **kwargs):
try:
data = Data.objects.filter(name='analytics_data')[0]
# Let's do the analytics
df = pd.read_excel(data.file, sheet_name='Raw Data')
df.columns = [
col.replace('(min)', '').lower() for col in df.columns
]
df.columns = [col.strip() for col in df.columns]
df.columns = [col.replace(' ', '_').lower() for col in df.columns]
df = df.assign(retention_time_roundoff=(df.retention_time))
df.retention_time_roundoff = df.retention_time_roundoff.astype(
float).round().astype(int)
retention_time_roundoff = df['retention_time_roundoff']
excel_file = IO()
xlwriter = pd.ExcelWriter(excel_file, engine='xlsxwriter')
df.to_excel(xlwriter, 'raw_data_1')
retention_time_roundoff.to_excel(xlwriter, 'raw_data_2')
xlwriter.save()
xlwriter.close()
excel_file.seek(0)
response = HttpResponse(
excel_file.read(),
content_type=
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
)
response[
'Content-Disposition'] = 'attachment; filename=output2.xlsx'
return response
except DatabaseError as e:
return Response({"Error": str(e)},
status=status.HTTP_304_NOT_MODIFIED)
def stats_to_csv(stats):
if sys.version_info[0] >= 3:
from io import StringIO as IO # pragma: no cover
else:
from cStringIO import StringIO as IO # pragma: no cover
import csv
csv_fh = IO()
keys = set()
for stat in stats:
for key in list(stat.keys()):
keys.add(key)
fieldnames = sorted(list(keys), key=str)
csvwriter = csv.DictWriter(csv_fh, delimiter=str(","), fieldnames=fieldnames)
csvwriter.writerow(dict((fn, fn) for fn in fieldnames))
for row in stats:
csvwriter.writerow(row)
contents = csv_fh.getvalue()
csv_fh.close()
return contents
def download(request):
samples_list = samples.objects.none()
df1 = convert_to_dataframe2(samples_list,
fields=[
'transaction_GlCode', 'transaction_date',
'transaction_number', 'transaction_value',
'remarks', 'action', 'Area',
'Financial_Year', 'Client'
])
frames = [df1]
result = pd.concat(frames)
excel_file = IO()
xlwriter = pd.ExcelWriter(excel_file, engine='openpyxl')
result.to_excel(xlwriter, 'samples')
xlwriter.save()
xlwriter.close()
excel_file.seek(0)
response = HttpResponse(
excel_file.read(),
content_type=
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
response['content-Disposition'] = 'attachment; filename=sample_format.xlsx'
return response
def Advance_Amount_year_action(request):
if request.method == "POST":
#monthvarnum = request.GET.get('sel')
#capturing values from user input
yearvarnum = request.POST.get('year')
yearvarnum = int(yearvarnum)
#for all data show i dataframe
pd.set_option('display.max_colwidth', -1)
try:
from io import BytesIO as IO # for modern python
except ImportError:
from io import StringIO as IO # for legacy python
#Attendance table copy as it is and filter as per not_recived and sent status
copy1 = Advance_amount.objects.all().values_list(
'Employee_name',
'Advance_Amount_Taken', 'year', 'Cleared_or_Notcleared').filter(
year=yearvarnum) #get either not_recieved or sent
dframe00 = pd.DataFrame(copy1,
columns=[
'Employee_name',
'Total_Advance_Amount_Taken', 'year',
'Advance_amount_Cleared?'
])
#group by non repeating for sum of items_names and not recieved number
dframe1 = dframe00.groupby([
'Employee_name',
]).agg({
'Total_Advance_Amount_Taken': 'sum',
}).reset_index()
#insertion logic of year value repeatedly in dataframe
This_year_name = yearvarnum
repeat_var_year = list(
itertools.repeat(This_year_name, dframe1.Employee_name.count())
) #list generation of repeated values it = itertools.repeat(no of items,no of times)
dframe1['Year'] = repeat_var_year
#insertion logic of cleared or not
dframe00 = dframe00.drop([
'Total_Advance_Amount_Taken',
'year',
],
axis=1) #dropping few column names
dframe00 = dframe00.groupby('Employee_name').prod(
) #grouping data and multplication on cleared_not_cleared column
for col in dframe00.columns[dframe00.dtypes == 'bool']:
dframe00[col] = dframe00[col].map({True: 'Yes', False: 'No'})
#group by non repeating for sum of items_names and not recieved number
#join two dataframes on attribute=Employee name
#combined two dataframes same columns
dframe2 = reduce(
lambda x, y: pd.merge(x, y, on=[
'Employee_name',
], how='outer'), [dframe00, dframe1])
#add column Whole_Or_Partial_Amount_Paid_in_Middle into dframe2
copy2 = Advance_amount.objects.all().values_list(
'Employee_name', 'Whole_Or_Partial_Amount_Paid_in_Middle').filter(
year=yearvarnum) #get either not_recieved or sent
dframetemp = pd.DataFrame(copy2,
columns=[
'Employee_name',
'Whole_Or_Partial_Amount_Paid_in_Middle',
])
dframetemp = dframetemp.groupby([
'Employee_name',
]).agg({
'Whole_Or_Partial_Amount_Paid_in_Middle': 'sum',
}).reset_index()
#join two dataframes on attribute=Employee name
#combined two dataframes same columns
dframe3 = reduce(
lambda x, y: pd.merge(x, y, on=[
'Employee_name',
], how='outer'), [dframe2, dframetemp])
dframe3[
'Remaining_Amt'] = dframe3['Total_Advance_Amount_Taken'] - dframe3[
'Whole_Or_Partial_Amount_Paid_in_Middle']
#rearanging dataframe columns
dframe3 = dframe3[[
'Employee_name',
'Year',
'Total_Advance_Amount_Taken',
'Whole_Or_Partial_Amount_Paid_in_Middle',
'Remaining_Amt',
'Advance_amount_Cleared?',
]]
# my "Excel" file, which is an in-memory output file (buffer)
# for the new workbook
excel_file = IO()
xlwriter = pd.ExcelWriter(excel_file, engine='xlsxwriter')
dframe3.to_excel(xlwriter, 'Advanced_Amount_Statistics')
workbook = xlwriter.book
worksheet = xlwriter.sheets['Advanced_Amount_Statistics']
worksheet.set_column('B:B', 25)
worksheet.set_column('C:C', 6)
worksheet.set_column('D:E', 30)
worksheet.set_column('E:E', 40)
worksheet.set_column('F:G', 28)
xlwriter.save()
xlwriter.close()
# important step, rewind the buffer or when it is read() you'll get nothing
# but an error message when you try to open your zero length file in Excel
excel_file.seek(0)
response = HttpResponse(
excel_file.read(),
content_type=
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
)
# set the file name in the Content-Disposition header
response[
'Content-Disposition'] = 'attachment; filename= Yearwise_Statistics_Of_Advanced_Amount.xlsx'
return response
def Export_Database(request):
names = [
'id',
'id_SAP',
'id_user__username',
'id_checker__username',
'id_rechecker__username',
'id_retitle__username',
'status',
'division_status',
'created_date',
'modified_date',
'comment',
'time_tracking',
'check_time_tracking',
]
workdata = Work_data.objects.all().values_list(*names)
workdata_column_names = [
field.verbose_name for field in Work_data._meta.fields
]
extractsap = ExtractSAP.objects.all().values_list()
extractsap_column_names = [
field.verbose_name for field in ExtractSAP._meta.fields
]
extractsap_column_names[0] = "id SAP"
df_work = pd.DataFrame.from_records(workdata,
columns=workdata_column_names)
df_sap = pd.DataFrame.from_records(extractsap,
columns=extractsap_column_names)
df_sap = df_sap.set_index('id SAP')
df = df_work.join(df_sap, on='id SAP', rsuffix='_sap')
df = df.set_index('ID')
df.sort_values(by=['id SAP'], inplace=True)
# print(df_work)
# print(df_sap)
# print(df)
Extractsap_filter = [
'ID', 'Site', 'Div.', 'Ancien numéro Ordre', 'Ordre', 'typ',
'Intitulé du type de document', 'Num.', 'Folio', 'Rev', 'ID Document',
'Dernière version', 'titre du projet', 'Transfert vers ordre',
'Lien vers le serveur', 'Titre du document',
'Division section sous section client',
'Division section sous section AUSY', 'Catégorie de document',
'Date d émission', 'Provenance', 'Auteur', 'Vérificateur',
'Approbateur', 'Validateur', 'Entrepreneur/Fournisseur',
'Référence externe', 'Ancien numéro de plan', 'Numéro Cadastre ENG',
'Révision Cadastre Eng', 'TAG', 'Poste technique #1',
'Libellé Poste technique #1', 'Poste technique #2',
'Libellé Poste technique #2', 'Poste technique #3',
'Libellé Poste technique #3', 'Poste technique #4',
'Libellé Poste technique #4', 'Poste technique #5',
'Libellé Poste technique #5', 'Poste technique #6',
'Libellé Poste technique #6', 'Poste technique',
'Libellé Poste Technique', 'Remarque', 'N° d imputation',
'N° de bon de travail', 'Existance fichier tif/pdf/dwg', 'id SAP',
'id user', 'id checker', 'id rechecker', 'id retitle', 'created date',
'modified date', 'comment', 'time tracking', 'check time tracking',
'id SAP_sap', 'status', 'status_sap', 'division status'
]
# my "Excel" file, which is an in-memory output file (buffer)
# for the new workbook
excel_file = IO()
df = df.filter(items=Extractsap_filter)
# df = df.style.format({"Date d émission": lambda t: t.strftime("%d/%m/%Y")})
writer = pd.ExcelWriter(excel_file,
engine='xlsxwriter',
date_format='dd/mm/yyy')
df.to_excel(writer, 'Extract_SAP')
writer.save()
writer.close()
# important step, rewind the buffer or when it is read() you'll get nothing
# but an error message when you try to open your zero length file in Excel
excel_file.seek(0)
response = HttpResponse(
excel_file.read(),
content_type=
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
response['Content-Disposition'] = 'attachment; filename="Export_SAP.xlsx"'
return response
def deserialize(self, value):
audio = value[value.find(",")+1:]
audio = base64.decodestring(audio.encode('utf8'))
buffer = IO(audio)
data, sr = soundfile.read(buffer)
return data, sr
def parse_string(self, string):
"""Supply file-like object containing the string."""
file = IO(string.encode('utf-8'))
return self.run(file=file)
body2 = """
--spam
Content-Disposition: form-data; name="param1"; filename=blob
Content-Type: binary/octet-stream
value1
--spam--
"""
body = body_file
headers = {
'content-type': 'multipart/form-data; boundary=spam;',
'content-length': len(body)
}
headers2 = {
'content-type':
'multipart/form-data; boundary=---------------------------265001916915724',
'content-length': len(body)
}
headers3 = {
'content-type':
'multipart/form-data; boundary=---------------------------146043902153',
'content-length': len(body)
}
environ = {'REQUEST_METHOD': 'POST'}
input_headers = [('Host', '127.0.0.1:8192'), ('User-Agent', 'Mozilla/5.0')]
parsed = cgi.FieldStorage(IO(body.encode('utf-8')),
headers=headers3,
environ=environ)
# parsed2 = cgi.parse_multipart(IO(body.encode('utf-8')), pdict = headers2)
def exportPandas(
request, report,
cloud): #function which export data of selected report to excel
from io import BytesIO as IO
if report == str(1):
output = [{
'Tenant Name': x.name,
'Parent Name': x.parentName,
'Products': productsToString(x.products),
'Amount of Users': x.users.count(),
'Amount of Apps': x.apps.count()
} for x in Tenant.objects.all()]
df = pd.DataFrame(output)
df = df[[
'Parent Name', 'Tenant Name', 'Products', 'Amount of Users',
'Amount of Apps'
]]
excel_file = IO()
xlwriter = pd.ExcelWriter(excel_file, engine='xlsxwriter')
df.to_excel(xlwriter, 'sheetname', index=False)
xlwriter.save()
xlwriter.close()
# important step, rewind the buffer or when it is read() you'll get nothing
# but an error message when you try to open your zero length file in Excel
excel_file.seek(0)
# set the mime type so that the browser knows what to do with the file
response = HttpResponse(
excel_file.read(),
content_type=
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
)
# set the file name in the Content-Disposition header
response['Content-Disposition'] = 'attachment; filename=' + str(
cloud) + '_report_' + str(report) + '.xlsx'
return response
elif report == str(2):
context = getContext2(cloud)
output = [{
'Product Name': x['name'],
'Active': x['active'],
'Inactive': x['inactive'],
'All': str(int(x['active']) + int(x['inactive']))
} for x in context['products']]
df = pd.DataFrame(output)
df = df[['Product Name', 'Active', 'Inactive', 'All']]
excel_file = IO()
xlwriter = pd.ExcelWriter(excel_file, engine='xlsxwriter')
df.to_excel(xlwriter, 'sheetname', index=False)
xlwriter.save()
xlwriter.close()
# important step, rewind the buffer or when it is read() you'll get nothing
# but an error message when you try to open your zero length file in Excel
excel_file.seek(0)
# set the mime type so that the browser knows what to do with the file
response = HttpResponse(
excel_file.read(),
content_type=
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
)
# set the file name in the Content-Disposition header
response['Content-Disposition'] = 'attachment; filename=' + str(
cloud) + '_report_' + str(report) + '.xlsx'
return response
elif report == str(3):
context = getContext3(cloud)
excel_file = IO()
xlwriter = pd.ExcelWriter(excel_file, engine='xlsxwriter')
for product in context['products']:
newContext = [] # special format for excel
for tenant in product['active']:
dictToExcel = {
'tenantName': tenant,
'status': 'active',
}
newContext.append(dictToExcel)
for tenant in product['inactive']:
dictToExcel = {
'tenantName': tenant,
'status': 'inactive',
}
newContext.append(dictToExcel)
output = [{
'Tenant Name': x['tenantName'].name,
'Status': x['status']
} for x in newContext]
df = pd.DataFrame(output)
df = df[['Tenant Name', 'Status']]
df.to_excel(xlwriter,
str(product['name'].replace('/', '_')),
index=False)
xlwriter.save()
xlwriter.close()
excel_file.seek(0)
response = HttpResponse(
excel_file.read(),
content_type=
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
)
# set the file name in the Content-Disposition header
response['Content-Disposition'] = 'attachment; filename=' + str(
cloud) + '_report_' + str(report) + '.xlsx'
return response
elif report == str(4):
context = getContext4(cloud)
newContext = []
for x in context['tenants']:
if str(x['serviceBox']).upper() != 'None'.upper():
newContext.append(x)
output = [{
'Tenant Name': x['name'],
'Parent Name': x['parentName'],
'Demo': x['demoTenant'],
'Service Box': x['serviceBox'],
'Installed Version': x['installedVersion'],
'Last Updated': x['lastUpdate'],
'Products': productsToString(x['products'])
} for x in newContext]
df = pd.DataFrame(output)
df = df[[
'Parent Name', 'Tenant Name', 'Demo', 'Service Box',
'Installed Version', 'Last Updated', 'Products'
]]
excel_file = IO()
xlwriter = pd.ExcelWriter(excel_file, engine='xlsxwriter')
df.to_excel(xlwriter, 'sheetname', index=False)
xlwriter.save()
xlwriter.close()
# important step, rewind the buffer or when it is read() you'll get nothing
# but an error message when you try to open your zero length file in Excel
excel_file.seek(0)
# set the mime type so that the browser knows what to do with the file
response = HttpResponse(
excel_file.read(),
content_type=
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
)
# set the file name in the Content-Disposition header
response['Content-Disposition'] = 'attachment; filename=' + str(
cloud) + '_report_' + str(report) + '.xlsx'
return response
def heat_mpl(df, id_prop="Compound_Id", cmap="bwr",
show=True, colorbar=True, biosim=False, chemsim=False, method="dist_corr",
sort_parm=False, parm_dict=None,
plot_cache=None):
# try to load heatmap from cache:
if plot_cache is not None and op.isfile(plot_cache):
result = open(plot_cache).read()
return result
if "dist" in method.lower():
profile_sim = cpt.profile_sim_dist_corr
else:
profile_sim = cpt.profile_sim_tanimoto
df_len = len(df)
img_size = 15 if show else 17
plt.style.use("seaborn-white")
plt.style.use("seaborn-pastel")
plt.style.use("seaborn-talk")
plt.rcParams['axes.labelsize'] = 25
# plt.rcParams['legend.fontsize'] = 20
plt.rcParams['figure.figsize'] = (img_size, 1.1 + 0.47 * (df_len - 1))
plt.rcParams['axes.labelsize'] = 25
plt.rcParams['ytick.labelsize'] = 20
plt.rcParams['xtick.labelsize'] = 15
fs_text = 18
y_labels = []
fp_list = []
max_val = 3 # using a fixed color range now
min_val = -3
ylabel_templ = "{}{}{}"
ylabel_cs = ""
ylabel_bs = ""
id_prop_list = []
for ctr, (_, rec) in enumerate(df.iterrows()):
if sort_parm:
if ctr == 0:
compartments = ["Median_Cells", "Median_Cytoplasm", "Median_Nuclei"]
parm_list = []
for comp in compartments:
parm_comp = [x for x in ACT_PROF_PARAMETERS if x.startswith(comp)]
val_list = [rec[x] for x in parm_comp]
parm_sorted = [x for _, x in sorted(zip(val_list, parm_comp))]
parm_list.extend(parm_sorted)
else:
parm_list = ACT_PROF_PARAMETERS
fp = [rec[x] for x in ACT_PROF_PARAMETERS]
fp_view = [rec[x] for x in parm_list]
fp_list.append(fp_view)
id_prop_list.append(rec[id_prop])
if chemsim:
if ctr == 0:
mol = mol_from_smiles(rec.get("Smiles", "*"))
if len(mol.GetAtoms()) > 1:
ylabel_cs = "Chem | "
mol_fp = Chem.GetMorganFingerprint(mol, 2) # ECFC4
else: # no Smiles present in the DataFrame
ylabel_cs = ""
chemsim = False
else:
q = rec.get("Smiles", "*")
if len(q) < 2:
ylabel_cs = " | "
else:
sim = cpt.chem_sim(mol_fp, q) * 100
ylabel_cs = "{:3.0f}% | ".format(sim)
if biosim:
if ctr == 0:
prof_ref = fp
ylabel_bs = " Bio | "
else:
sim = profile_sim(prof_ref, fp) * 100
ylabel_bs = "{:3.0f}% | ".format(sim)
ylabel = ylabel_templ.format(ylabel_cs, ylabel_bs, rec[id_prop])
y_labels.append(ylabel)
# m_val = max(fp) # this was the calculation of the color range
# if m_val > max_val:
# max_val = m_val
# m_val = min(fp)
# if m_val < min_val:
# min_val = m_val
if isinstance(parm_dict, dict):
parm_dict["Parameter"] = parm_list
for i in range(len(id_prop_list)):
parm_dict[str(id_prop_list[i])] = fp_list[i].copy()
# calc the colorbar range
max_val = max(abs(min_val), max_val)
# invert y axis:
y_labels = y_labels[::-1]
fp_list = fp_list[::-1]
Z = np.asarray(fp_list)
plt.xticks(XTICKS)
plt.yticks(np.arange(df_len) + 0.5, y_labels)
plt.pcolor(Z, vmin=-max_val, vmax=max_val, cmap=cmap)
plt.text(XTICKS[1] // 2, -1.1, "Cells",
horizontalalignment='center', fontsize=fs_text)
plt.text(XTICKS[1] + ((XTICKS[2] - XTICKS[1]) // 2), -1.1,
"Cytoplasm", horizontalalignment='center', fontsize=fs_text)
plt.text(XTICKS[2] + ((XTICKS[3] - XTICKS[2]) // 2), -1.1,
"Nuclei", horizontalalignment='center', fontsize=fs_text)
if colorbar and len(df) > 3:
plt.colorbar()
plt.tight_layout()
if show:
plt.show()
else:
img_file = IO()
plt.savefig(img_file, bbox_inches='tight', format="jpg")
result = img_tag(img_file, format="jpg",
options='style="width: 900px;"')
img_file.close()
# important, otherwise the plots will accumulate and fill up memory:
plt.clf()
plt.close()
gc.collect()
if plot_cache is not None: # cache the plot
open(plot_cache, "w").write(result)
return result
def makefile(self, *args, **kwargs):
return IO(bytes(self.path, 'utf-8'))
def display_mol(self):
highlight_lipinski = self.check_lipinski.isChecked()
if self.curr_sdf_mol_index == 0:
self.btn_prev.setEnabled(False)
else:
self.btn_prev.setEnabled(True)
if self.curr_sdf_mol_index == self.curr_sdf_num_of_mols - 1:
self.btn_next.setEnabled(False)
else:
self.btn_next.setEnabled(True)
img_file = IO() # for structure depiction
img = sdft.autocrop(
Draw.MolToImage(self.curr_sdf[self.curr_sdf_mol_index]), "white")
img.save(img_file, format='PNG')
# qimg = QtGui.QImage.fromData(img_file.getvalue())
self.qpixmap = QtGui.QPixmap()
self.qpixmap.loadFromData(img_file.getvalue(), "PNG")
self.label_molimage.setPixmap(self.qpixmap)
self.le_recnumber.setText("{} of {}".format(
self.curr_sdf_mol_index + 1, self.curr_sdf_num_of_mols))
if self.SDF_CHANGED:
# self.SDF_CHANGED = False
# self.selected_fields = self.get_selected_fields()
self.table_props.clear()
self.table_props.setHorizontalHeaderLabels(["prop", "value"])
for row, prop in enumerate(self.curr_sdf_fields):
tbl_item = QtGui.QTableWidgetItem(prop[2:])
# tbl_item.setFlags(QtCore.Qt.ItemIsEnabled)
self.table_props.setItem(row, 0, tbl_item)
if self.SDF_CHANGED:
if self.selected_fields and prop in self.selected_fields:
self.table_props.setItemSelected(tbl_item, True)
if prop in self.curr_sdf[self.curr_sdf_mol_index].GetPropNames():
value = self.curr_sdf[self.curr_sdf_mol_index].GetProp(prop)
tbl_item = QtGui.QTableWidgetItem(value)
# QtCore.Qt.ItemIsEditable is required to edit the cells
# tbl_item.setFlags(QtCore.Qt.ItemIsEnabled | QtCore.Qt.ItemIsEditable)
tbl_item.setFlags(QtCore.Qt.ItemIsEnabled
| QtCore.Qt.ItemIsEditable)
if highlight_lipinski and prop in self.highlight_dict:
if float(value) > self.highlight_dict[prop]:
tbl_item.setBackgroundColor(QtGui.QColor(255, 0, 0))
else:
tbl_item.setBackgroundColor(QtGui.QColor(0, 255, 0))
self.table_props.setItem(row, 1, tbl_item)
else:
tbl_item = QtGui.QTableWidgetItem("n.d.")
# see above for flag QtCore.Qt.ItemIsEditable
tbl_item.setFlags(QtCore.Qt.ItemIsEnabled
| QtCore.Qt.ItemIsEditable)
self.table_props.setItem(row, 1, tbl_item)
self.table_props.setRowHeight(row, 18)
self.SDF_CHANGED = False
if self.curr_sdf_mol_index in self.selected_recs:
self.check_rec_selected.setChecked(True)
else:
self.check_rec_selected.setChecked(False)
def main(request):
if request.method != 'POST':
return render(request, "main.html")
if request.method == 'POST':
d = dict(request.POST)
# getting all the data
field_name = d['fieldName']
field_type = d['fieldType']
percentage = ['percentage']
nrows = d['fname']
fileExport = d['fileExport']
ending = d['l_ending_style']
chkempty = d['Empty']
button_value = d["button"]
valempty = d['valEmpty']
rows = []
chk_index = 0
chk_arr = []
formula = (d['formulas'])
while chk_index < len(chkempty):
if chkempty[chk_index] == 'Yes':
chk_arr.append('on')
chk_index += 2
else:
chk_arr.append('off')
chk_index += 1
chkempty = chk_arr
valindex = 0
chk_index = 0
val_arr = []
while valindex < len(valempty) and chk_index < len(chkempty):
if chkempty[chk_index] == "on":
val_arr.append(valempty[valindex])
valindex += 2
else:
val_arr.append("0")
valindex += 1
chk_index += 1
valempty = val_arr
# generating the data using functions above
field_type.pop()
field_name.pop()
for i in field_type:
# GUID Generation
if i == 'GUID':
tempdata = []
for j in range(int(nrows[0])):
tempdata.append(id())
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# Row Number Generation
elif i == 'Row Number':
tempdata = []
for j in range(1, int(nrows[0]) + 1):
data = str(j)
string = formula[field_type.index(i)]
if 'this' in string:
if "if" in string:
print('yes it is there')
newValue = enforce_function(string, data, 3)
else:
newValue = enforce_function(string, data, 2)
tempdata.append(newValue)
else:
tempdata.append(data)
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# First Name Generation
elif i == 'First Name':
tempdata = []
for j in range(int(nrows[0])):
data = fname()
string = formula[field_type.index(i)]
if 'this' in string:
if "if" in string and "else" in string:
newValue = enforce_function(string, data, 3)
else:
datas = data
newValue = ''
for data in datas.split():
newValue = newValue + ' ' + enforce_function(
string, data, 1)
tempdata.append(newValue)
else:
tempdata.append(data)
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# last Name Generation
elif i == 'Last Name':
tempdata = []
for j in range(int(nrows[0])):
data = lname()
string = formula[field_type.index(i)]
if 'this' in string:
if "if" in string and "else" in string:
newValue = enforce_function(string, data, 3)
else:
datas = data
newValue = ''
for data in datas.split():
newValue = newValue + ' ' + enforce_function(
string, data, 1)
tempdata.append(newValue)
else:
tempdata.append(data)
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# full Name Generation
elif i == 'Full Name':
tempdata = []
for j in range(int(nrows[0])):
data = fullname()
string = formula[field_type.index(i)]
if 'this' in string:
if "if" in string and "else" in string:
newValue = enforce_function(string, data, 3)
else:
datas = data
newValue = ''
for data in datas.split():
newValue = newValue + ' ' + enforce_function(
string, data, 1)
tempdata.append(newValue)
else:
tempdata.append(data)
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# First Name male Generation
elif i == 'First Name(male)':
tempdata = []
for j in range(int(nrows[0])):
data = f_fname()
string = formula[field_type.index(i)]
if 'this' in string:
if "if" in string and "else" in string:
newValue = enforce_function(string, data, 3)
else:
datas = data
newValue = ''
for data in datas.split():
newValue = newValue + ' ' + enforce_function(
string, data, 1)
tempdata.append(newValue)
else:
tempdata.append(data)
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# First Name female Generation
elif i == 'First Name(female)':
tempdata = []
for j in range(int(nrows[0])):
data = m_fname()
string = formula[field_type.index(i)]
if 'this' in string:
if "if" in string and "else" in string:
newValue = enforce_function(string, data, 3)
else:
datas = data
newValue = ''
for data in datas.split():
newValue = newValue + ' ' + enforce_function(
string, data, 1)
tempdata.append(newValue)
else:
tempdata.append(data)
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# Email Generation
elif i == 'Email':
tempdata = []
for j in range(int(nrows[0])):
data = Email()
string = formula[field_type.index(i)]
if 'this' in string:
if "if" in string and "else" in string:
newValue = enforce_function(string, data, 3)
else:
datas = data
newValue = ''
for data in datas.split():
newValue = newValue + ' ' + enforce_function(
string, data, 1)
tempdata.append(newValue)
else:
tempdata.append(data)
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# username Generation
elif i == 'Username':
tempdata = []
for j in range(int(nrows[0])):
data = username()
string = formula[field_type.index(i)]
if 'this' in string:
if "if" in string and "else" in string:
newValue = enforce_function(string, data, 3)
else:
datas = data
newValue = ''
for data in datas.split():
newValue = newValue + ' ' + enforce_function(
string, data, 1)
tempdata.append(newValue)
else:
tempdata.append(data)
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# Comapany Name Generation
elif i == 'Comapny Name':
tempdata = []
for j in range(int(nrows[0])):
data = comapny()
string = formula[field_type.index(i)]
if 'this' in string:
if "if" in string and "else" in string:
newValue = enforce_function(string, data, 3)
else:
datas = data
newValue = ''
for data in datas.split():
newValue = newValue + ' ' + enforce_function(
string, data, 1)
tempdata.append(newValue)
else:
tempdata.append(data)
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# IP address v4 Generation
elif i == 'Ip address v4':
tempdata = []
for j in range(int(nrows[0])):
tempdata.append(ipv4())
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# IP address v6 Generation
elif i == 'Ip address v6':
tempdata = []
for j in range(int(nrows[0])):
tempdata.append(ipv6())
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# job title Generation
elif i == 'Job Title':
tempdata = []
for j in range(int(nrows[0])):
data = job()
string = formula[field_type.index(i)]
if 'this' in string:
if "if" in string and "else" in string:
newValue = enforce_function(string, data, 3)
else:
datas = data
newValue = ''
for data in datas.split():
newValue = newValue + ' ' + enforce_function(
string, data, 1)
tempdata.append(newValue)
else:
tempdata.append(data)
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# Language Generation
elif i == 'Language':
tempdata = []
for j in range(int(nrows[0])):
data = lang()
string = formula[field_type.index(i)]
if 'this' in string:
if "if" in string and "else" in string:
newValue = enforce_function(string, data, 3)
else:
datas = data
newValue = ''
for data in datas.split():
newValue = newValue + ' ' + enforce_function(
string, data, 1)
tempdata.append(newValue)
else:
tempdata.append(data)
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# programming Language Generation
elif i == 'Programming Language':
tempdata = []
for j in range(int(nrows[0])):
data = planguage()
string = formula[field_type.index(i)]
if 'this' in string:
if "if" in string and "else" in string:
newValue = enforce_function(string, data, 3)
else:
datas = data
newValue = ''
for data in datas.split():
newValue = newValue + ' ' + enforce_function(
string, data, 1)
tempdata.append(newValue)
else:
tempdata.append(data)
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# Currency
elif i == 'Currency':
tempdata = []
for j in range(int(nrows[0])):
data = Currency()
string = formula[field_type.index(i)]
if 'this' in string:
if "if" in string and "else" in string:
newValue = enforce_function(string, data, 3)
else:
datas = data
newValue = ''
for data in datas.split():
newValue = newValue + ' ' + enforce_function(
string, data, 1)
tempdata.append(newValue)
else:
tempdata.append(data)
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# Currency symbol
elif i == 'Currency Symbol':
tempdata = []
for j in range(int(nrows[0])):
tempdata.append(symbol())
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# Currency symbol
elif i == 'Gender':
tempdata = []
for j in range(int(nrows[0])):
data = gender()
string = formula[field_type.index(i)]
if 'this' in string:
if "if" in string and "else" in string:
newValue = enforce_function(string, data, 3)
else:
datas = data
newValue = ''
for data in datas.split():
newValue = newValue + ' ' + enforce_function(
string, data, 1)
tempdata.append(newValue)
else:
tempdata.append(data)
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# random number generation
elif i == 'Number':
tempdata = []
for j in range(int(nrows[0])):
data = number()
string = formula[field_type.index(i)]
if 'this' in string:
if "if" in string:
print('yes it is there')
newValue = enforce_function(string, data, 3)
else:
newValue = enforce_function(string, data, 2)
tempdata.append(newValue)
else:
tempdata.append(data)
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# phone number generation
elif i == 'Phone Number':
tempdata = []
for j in range(int(nrows[0])):
tempdata.append(phnumber())
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# domain name generation
elif i == 'Domain Name':
tempdata = []
for j in range(int(nrows[0])):
tempdata.append(domain())
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# date
elif i == 'Date':
tempdata = []
for j in range(int(nrows[0])):
tempdata.append(date_())
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# date with time
elif i == 'Date with time':
tempdata = []
for j in range(int(nrows[0])):
tempdata.append(datetime())
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# time zone generation
elif i == 'Time Zone':
tempdata = []
for j in range(int(nrows[0])):
data = timezone()
string = formula[field_type.index(i)]
if 'this' in string:
if "if" in string and "else" in string:
newValue = enforce_function(string, data, 3)
else:
datas = data
newValue = ''
for data in datas.split():
newValue = newValue + ' ' + enforce_function(
string, data, 1)
tempdata.append(newValue)
else:
tempdata.append(data)
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# boolean generation
elif i == 'Boolean':
tempdata = []
for j in range(int(nrows[0])):
data = boolean()
string = formula[field_type.index(i)]
if 'this' in string:
if "if" in string and "else" in string:
newValue = enforce_function(string, data, 3)
else:
datas = data
newValue = ''
for data in datas.split():
newValue = newValue + ' ' + enforce_function(
string, data, 1)
tempdata.append(newValue)
else:
tempdata.append(data)
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# money generation
elif i == 'Money':
tempdata = []
for j in range(int(nrows[0])):
tempdata.append(money())
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# mac address generation
elif i == 'MAC Address':
tempdata = []
for j in range(int(nrows[0])):
tempdata.append(mac())
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# name title generation
elif i == 'Title':
tempdata = []
for j in range(int(nrows[0])):
data = nametitle()
string = formula[field_type.index(i)]
if 'this' in string:
if "if" in string and "else" in string:
newValue = enforce_function(string, data, 3)
else:
datas = data
newValue = ''
for data in datas.split():
newValue = newValue + ' ' + enforce_function(
string, data, 1)
tempdata.append(newValue)
else:
tempdata.append(data)
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# gender
elif i == 'Gender':
tempdata = []
for j in range(int(nrows[0])):
data = gender()
string = formula[field_type.index(i)]
if 'this' in string:
if "if" in string and "else" in string:
newValue = enforce_function(string, data, 3)
else:
datas = data
newValue = ''
for data in datas.split():
newValue = newValue + ' ' + enforce_function(
string, data, 1)
tempdata.append(newValue)
else:
tempdata.append(data)
if chkempty[field_type.index(i)] == 'on':
df = pd.DataFrame(tempdata)
df = empty(df, valempty[field_type.index(i)])
tempdata = df['result'].values.tolist()
# if no match to input
else:
tempdata = []
for j in range(int(nrows[0])):
tempdata.append('Not valid Field Type')
rows.append(tempdata)
df = pd.DataFrame((list(zip(*rows))), columns=field_name, dtype=str)
r, c = df.shape
table_content = df.to_html(index=False,
justify='center',
classes="table")
if button_value[0] == 'Preview':
if c == 0:
return render(request, "main.html")
else:
context = {'table_content': table_content}
return JsonResponse(context)
elif button_value[0] == '':
if c == 0:
return render(request, "main.html")
else:
# csv file exporter
if fileExport[0] == 'CSV':
filename = "download/random_data.csv"
if ending[0] == 'Unix':
print("uix")
df.to_csv(filename,
index=False,
encoding="ascii",
line_terminator="\n")
else:
df.to_csv(filename,
index=False,
encoding="ascii",
line_terminator="\r\n")
path_to_file = os.path.realpath("download/random_data.csv")
f = open(path_to_file, 'rb')
myfile = File(f)
response = HttpResponse(myfile, content_type='text/csv')
response[
'Content-Disposition'] = 'attachment; filename=' + 'random_data.csv'
return response
# JSON file exporter
elif fileExport[0] == 'JSON':
filename = "download/random_data.json"
df.to_json(filename, orient='records', lines=True)
path_to_file = os.path.realpath(
"download/random_data.json")
f = open(path_to_file, 'r')
myfile = File(f)
response = HttpResponse(myfile,
content_type='application/json')
response[
'Content-Disposition'] = 'attachment; filename="result.json"'
return response
# TSV file exporter
elif fileExport[0] == 'TSV':
filename = "download/random_data.tsv"
if ending[0] == 'Unix':
df.to_csv(filename,
sep='\t',
index=False,
line_terminator="\n")
else:
df.to_csv(filename,
sep='\t',
index=False,
line_terminator="\r\n")
path_to_file = os.path.realpath(filename)
f = open(path_to_file, 'r')
myfile = File(f)
response = HttpResponse(myfile, content_type='text/tsv')
response[
'Content-Disposition'] = 'attachment; filename=' + 'random_data.tsv'
return response
# TSV file exporter
elif fileExport[0] == 'Excel':
from io import BytesIO as IO
excel_file = IO()
xlwriter = pd.ExcelWriter(excel_file, engine='xlsxwriter')
df.to_excel(xlwriter, 'sheetname')
xlwriter.save()
xlwriter.close()
excel_file.seek(0)
response = HttpResponse(
excel_file.read(),
content_type=
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
)
response[
'Content-Disposition'] = 'attachment; filename=download/random_data.xlsx'
return response
# XML file exporter
elif fileExport[0] == 'XML':
filename = "download/random_data.xml"
pd.DataFrame.to_xml = to_xml
to_xml(df, filename)
path_to_file = os.path.realpath(filename)
f = open(path_to_file, 'r')
myfile = File(f)
response = HttpResponse(myfile, content_type="text/xml")
response[
'Content-Disposition'] = 'attachment; filename=' + 'random_data.xml'
return response
if fileExport[0] == 'SQL':
r, c = df.shape
open("download/randomdata.txt", "w")
with open("download/randomdata.txt", "a") as file_prime:
for num in range(r):
file_prime.write(
"insert into MOCK_DATA (" +
", ".join(df.columns.values.tolist()) +
") values ('" + str("', '".join(
str(x)
for x in df.iloc[[num]].values.tolist()
[0])) + "');" + '\n')
path_to_file = os.path.realpath("download/randomdata.txt")
f = open(path_to_file, 'r')
with open("download/random_data.sql", 'w') as file:
file.write("".join(f))
path_to_file = os.path.realpath("download/random_data.sql")
f = open(path_to_file, 'r')
myfile = File(f)
response = HttpResponse(myfile, content_type="text/sql")
response[
'Content-Disposition'] = 'attachment; filename=' + 'random_data.sql'
return response
def capture(command, *args, **kwargs):
out, sys.stdout = sys.stdout, IO()
command(*args, **kwargs)
sys.stdout.seek(0)
yield sys.stdout.read()
sys.stdout = out
def test_request_logs(self):
test_cases = [
[
'',
[
'Invalid URL', 'No schema supplied', 'gettaddrinfo failed',
[
'nodename nor servname provided, or not known',
'Name or service not known'
]
]
],
[
'typo://plotly.acme.com',
[
'No connection adapters were found', 'gettaddrinfo failed',
[
'nodename nor servname provided, or not known',
'No connection adapters were found'
]
]
],
[
'https://doesntexist.plotly.systems',
[
'Failed to establish a new connection',
'gettaddrinfo failed',
[
'nodename nor servname provided, or not known',
'Name or service not known'
]
]
],
[
'https://expired.badssl.com',
[
'Caused by SSLError(SSLError("bad handshake: Error([(',
'SSL routines', 'tls_process_server_certificate',
'certificate verify failed', 'gettaddrinfo: ',
"'104.154.89.105', 443"
]
],
[
'https://self-signed.badssl.com',
[
'Caused by SSLError(SSLError("bad handshake: Error([(',
'SSL routines', 'tls_process_server_certificate',
'certificate verify failed', 'gettaddrinfo',
"'104.154.89.105', 443"
]
]
]
for url, expected_messages in test_cases:
os.environ['plotly_api_domain'] = url
f = IO()
with captured_output(f) as out:
try:
api_requests.post('/dash-apps')
except Exception:
pass
for expected_message in [url] + expected_messages:
stdout = out.getvalue()
if isinstance(expected_message, six.string_types):
self.assertTrue(
expected_message in stdout,
'url "{}"\nExpected "{}" to be in:\n{}\n'.format(
url, expected_message, stdout))
else:
self.assertTrue(
(expected_message[0] in stdout)
or (expected_message[1] in stdout),
'url "{}"\nExpected\n"{}"\nor "{}"\nto be in:\n{}\n'.
format(url, expected_message[0], expected_message[1],
stdout))
def makefile(self, *args, **kwargs):
return IO(b"GET /")
def setup(self):
super().setup()
self.wfile = IO()
def __init__(self, headers, body):
parsed = cgi.FieldStorage(IO(body.encode('utf-8')),
headers=headers3,
environ=environ)
def makefile(self, *args, **kwargs):
return IO(b"GET %s HTTP/1.0" % self._path)
def train_once(
sess,
step,
ops,
names=None,
gen_feed_dict_fn=None,
deal_results_fn=None,
interval_steps=100,
eval_ops=None,
eval_names=None,
gen_eval_feed_dict_fn=None,
deal_eval_results_fn=melt.print_results,
valid_interval_steps=100,
print_time=True,
print_avg_loss=True,
model_dir=None,
log_dir=None,
is_start=False,
num_steps_per_epoch=None,
metric_eval_fn=None,
metric_eval_interval_steps=0,
summary_excls=None,
fixed_step=None, # for epoch only, incase you change batch size
eval_loops=1,
learning_rate=None,
learning_rate_patience=None,
learning_rate_decay_factor=None,
num_epochs=None,
model_path=None,
use_horovod=False,
):
use_horovod = 'OMPI_COMM_WORLD_RANK' in os.environ
#is_start = False # force not to evaluate at first step
#print('-----------------global_step', sess.run(tf.train.get_or_create_global_step()))
timer = gezi.Timer()
if print_time:
if not hasattr(train_once, 'timer'):
train_once.timer = Timer()
train_once.eval_timer = Timer()
train_once.metric_eval_timer = Timer()
melt.set_global('step', step)
epoch = (fixed_step
or step) / num_steps_per_epoch if num_steps_per_epoch else -1
if not num_epochs:
epoch_str = 'epoch:%.3f' % (epoch) if num_steps_per_epoch else ''
else:
epoch_str = 'epoch:%.3f/%d' % (
epoch, num_epochs) if num_steps_per_epoch else ''
melt.set_global('epoch', '%.2f' % (epoch))
info = IO()
stop = False
if eval_names is None:
if names:
eval_names = ['eval/' + x for x in names]
if names:
names = ['train/' + x for x in names]
if eval_names:
eval_names = ['eval/' + x for x in eval_names]
is_eval_step = is_start or valid_interval_steps and step % valid_interval_steps == 0
summary_str = []
eval_str = ''
if is_eval_step:
# deal with summary
if log_dir:
if not hasattr(train_once, 'summary_op'):
#melt.print_summary_ops()
if summary_excls is None:
train_once.summary_op = tf.summary.merge_all()
else:
summary_ops = []
for op in tf.get_collection(tf.GraphKeys.SUMMARIES):
for summary_excl in summary_excls:
if not summary_excl in op.name:
summary_ops.append(op)
print('filtered summary_ops:')
for op in summary_ops:
print(op)
train_once.summary_op = tf.summary.merge(summary_ops)
#train_once.summary_train_op = tf.summary.merge_all(key=melt.MonitorKeys.TRAIN)
train_once.summary_writer = tf.summary.FileWriter(
log_dir, sess.graph)
tf.contrib.tensorboard.plugins.projector.visualize_embeddings(
train_once.summary_writer, projector_config)
# if eval ops then should have bee rank 0
if eval_ops:
#if deal_eval_results_fn is None and eval_names is not None:
# deal_eval_results_fn = lambda x: melt.print_results(x, eval_names)
for i in range(eval_loops):
eval_feed_dict = {} if gen_eval_feed_dict_fn is None else gen_eval_feed_dict_fn(
)
#eval_feed_dict.update(feed_dict)
# if use horovod let each rant use same sess.run!
if not log_dir or train_once.summary_op is None or gezi.env_has(
'EVAL_NO_SUMMARY') or use_horovod:
#if not log_dir or train_once.summary_op is None:
eval_results = sess.run(eval_ops, feed_dict=eval_feed_dict)
else:
eval_results = sess.run(eval_ops + [train_once.summary_op],
feed_dict=eval_feed_dict)
summary_str = eval_results[-1]
eval_results = eval_results[:-1]
eval_loss = gezi.get_singles(eval_results)
#timer_.print()
eval_stop = False
if use_horovod:
sess.run(hvd.allreduce(tf.constant(0)))
#if not use_horovod or hvd.local_rank() == 0:
# @TODO user print should also use logging as a must ?
#print(gezi.now_time(), epoch_str, 'eval_step: %d'%step, 'eval_metrics:', end='')
eval_names_ = melt.adjust_names(eval_loss, eval_names)
#if not use_horovod or hvd.rank() == 0:
# logging.info2('{} eval_step:{} eval_metrics:{}'.format(epoch_str, step, melt.parse_results(eval_loss, eval_names_)))
eval_str = 'valid:{}'.format(
melt.parse_results(eval_loss, eval_names_))
# if deal_eval_results_fn is not None:
# eval_stop = deal_eval_results_fn(eval_results)
assert len(eval_loss) > 0
if eval_stop is True:
stop = True
eval_names_ = melt.adjust_names(eval_loss, eval_names)
if not use_horovod or hvd.rank() == 0:
melt.set_global('eval_loss',
melt.parse_results(eval_loss, eval_names_))
elif interval_steps != valid_interval_steps:
#print()
pass
metric_evaluate = False
# if metric_eval_fn is not None \
# and (is_start \
# or (num_steps_per_epoch and step % num_steps_per_epoch == 0) \
# or (metric_eval_interval_steps \
# and step % metric_eval_interval_steps == 0)):
# metric_evaluate = True
if metric_eval_fn is not None \
and ((is_start or metric_eval_interval_steps \
and step % metric_eval_interval_steps == 0) or model_path):
metric_evaluate = True
if 'EVFIRST' in os.environ:
if os.environ['EVFIRST'] == '0':
if is_start:
metric_evaluate = False
else:
if is_start:
metric_evaluate = True
if step == 0 or 'QUICK' in os.environ:
metric_evaluate = False
#print('------------1step', step, 'pre metric_evaluate', metric_evaluate, hvd.rank())
if metric_evaluate:
if use_horovod:
print('------------metric evaluate step', step, model_path,
hvd.rank())
if not model_path or 'model_path' not in inspect.getargspec(
metric_eval_fn).args:
metric_eval_fn_ = metric_eval_fn
else:
metric_eval_fn_ = lambda: metric_eval_fn(model_path=model_path)
try:
l = metric_eval_fn_()
if isinstance(l, tuple):
num_returns = len(l)
if num_returns == 2:
evaluate_results, evaluate_names = l
evaluate_summaries = None
else:
assert num_returns == 3, 'retrun 1,2,3 ok 4.. not ok'
evaluate_results, evaluate_names, evaluate_summaries = l
else: #return dict
evaluate_results, evaluate_names = tuple(zip(*dict.items()))
evaluate_summaries = None
except Exception:
logging.info('Do nothing for metric eval fn with exception:\n',
traceback.format_exc())
if not use_horovod or hvd.rank() == 0:
#logging.info2('{} valid_step:{} {}:{}'.format(epoch_str, step, 'valid_metrics' if model_path is None else 'epoch_valid_metrics', melt.parse_results(evaluate_results, evaluate_names)))
logging.info2('{} valid_step:{} {}:{}'.format(
epoch_str, step, 'valid_metrics',
melt.parse_results(evaluate_results, evaluate_names)))
if learning_rate is not None and (learning_rate_patience
and learning_rate_patience > 0):
assert learning_rate_decay_factor > 0 and learning_rate_decay_factor < 1
valid_loss = evaluate_results[0]
if not hasattr(train_once, 'min_valid_loss'):
train_once.min_valid_loss = valid_loss
train_once.deacy_steps = []
train_once.patience = 0
else:
if valid_loss < train_once.min_valid_loss:
train_once.min_valid_loss = valid_loss
train_once.patience = 0
else:
train_once.patience += 1
logging.info2('{} valid_step:{} patience:{}'.format(
epoch_str, step, train_once.patience))
if learning_rate_patience and train_once.patience >= learning_rate_patience:
lr_op = ops[1]
lr = sess.run(lr_op) * learning_rate_decay_factor
train_once.deacy_steps.append(step)
logging.info2(
'{} valid_step:{} learning_rate_decay by *{}, learning_rate_decay_steps={}'
.format(epoch_str, step, learning_rate_decay_factor,
','.join(map(str, train_once.deacy_steps))))
sess.run(tf.assign(lr_op, tf.constant(lr, dtype=tf.float32)))
train_once.patience = 0
train_once.min_valid_loss = valid_loss
if ops is not None:
#if deal_results_fn is None and names is not None:
# deal_results_fn = lambda x: melt.print_results(x, names)
feed_dict = {} if gen_feed_dict_fn is None else gen_feed_dict_fn()
# NOTICE ops[2] should be scalar otherwise wrong!! loss should be scalar
#print('---------------ops', ops)
if eval_ops is not None or not log_dir or not hasattr(
train_once,
'summary_op') or train_once.summary_op is None or use_horovod:
feed_dict[K.learning_phase()] = 1
results = sess.run(ops, feed_dict=feed_dict)
else:
## TODO why below ?
#try:
feed_dict[K.learning_phase()] = 1
results = sess.run(ops + [train_once.summary_op],
feed_dict=feed_dict)
summary_str = results[-1]
results = results[:-1]
# except Exception:
# logging.info('sess.run(ops + [train_once.summary_op], feed_dict=feed_dict) fail')
# results = sess.run(ops, feed_dict=feed_dict)
#print('------------results', results)
# #--------trace debug
# if step == 210:
# run_metadata = tf.RunMetadata()
# results = sess.run(
# ops,
# feed_dict=feed_dict,
# options=tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE),
# run_metadata=run_metadata)
# from tensorflow.python.client import timeline
# trace = timeline.Timeline(step_stats=run_metadata.step_stats)
# trace_file = open('timeline.ctf.json', 'w')
# trace_file.write(trace.generate_chrome_trace_format())
#reults[0] assume to be train_op, results[1] to be learning_rate
learning_rate = results[1]
results = results[2:]
#@TODO should support aver loss and other avg evaluations like test..
if print_avg_loss:
if not hasattr(train_once, 'avg_loss'):
train_once.avg_loss = AvgScore()
#assume results[0] as train_op return, results[1] as loss
loss = gezi.get_singles(results)
train_once.avg_loss.add(loss)
steps_per_second = None
instances_per_second = None
hours_per_epoch = None
#step += 1
#if is_start or interval_steps and step % interval_steps == 0:
interval_ok = not use_horovod or hvd.local_rank() == 0
if interval_steps and step % interval_steps == 0 and interval_ok:
train_average_loss = train_once.avg_loss.avg_score()
if print_time:
duration = timer.elapsed()
duration_str = 'duration:{:.2f} '.format(duration)
melt.set_global('duration', '%.2f' % duration)
#info.write(duration_str)
elapsed = train_once.timer.elapsed()
steps_per_second = interval_steps / elapsed
batch_size = melt.batch_size()
num_gpus = melt.num_gpus()
instances_per_second = interval_steps * batch_size / elapsed
gpu_info = '' if num_gpus <= 1 else ' gpus:[{}]'.format(
num_gpus)
if num_steps_per_epoch is None:
epoch_time_info = ''
else:
hours_per_epoch = num_steps_per_epoch / interval_steps * elapsed / 3600
epoch_time_info = '1epoch:[{:.2f}h]'.format(
hours_per_epoch)
info.write(
'elapsed:[{:.2f}] batch_size:[{}]{} batches/s:[{:.2f}] insts/s:[{:.2f}] {} lr:[{:.6f}]'
.format(elapsed, batch_size, gpu_info, steps_per_second,
instances_per_second, epoch_time_info,
learning_rate))
if print_avg_loss:
#info.write('train_avg_metrics:{} '.format(melt.value_name_list_str(train_average_loss, names)))
names_ = melt.adjust_names(train_average_loss, names)
#info.write('train_avg_metric:{} '.format(melt.parse_results(train_average_loss, names_)))
info.write(' train:{} '.format(
melt.parse_results(train_average_loss, names_)))
#info.write('train_avg_loss: {} '.format(train_average_loss))
info.write(eval_str)
#print(gezi.now_time(), epoch_str, 'train_step:%d'%step, info.getvalue(), end=' ')
logging.info2('{} {} {}'.format(epoch_str, 'step:%d' % step,
info.getvalue()))
if deal_results_fn is not None:
stop = deal_results_fn(results)
summary_strs = gezi.to_list(summary_str)
if metric_evaluate:
if evaluate_summaries is not None:
summary_strs += evaluate_summaries
if step > 1:
if is_eval_step:
# deal with summary
if log_dir:
summary = tf.Summary()
if eval_ops is None:
if train_once.summary_op is not None:
for summary_str in summary_strs:
train_once.summary_writer.add_summary(
summary_str, step)
else:
for summary_str in summary_strs:
train_once.summary_writer.add_summary(
summary_str, step)
suffix = 'valid' if not eval_names else ''
# loss/valid
melt.add_summarys(summary,
eval_results,
eval_names_,
suffix=suffix)
if ops is not None:
try:
# loss/train_avg
melt.add_summarys(summary,
train_average_loss,
names_,
suffix='train_avg')
except Exception:
pass
##optimizer has done this also
melt.add_summary(summary, learning_rate, 'learning_rate')
melt.add_summary(summary,
melt.batch_size(),
'batch_size',
prefix='other')
melt.add_summary(summary,
melt.epoch(),
'epoch',
prefix='other')
if steps_per_second:
melt.add_summary(summary,
steps_per_second,
'steps_per_second',
prefix='perf')
if instances_per_second:
melt.add_summary(summary,
instances_per_second,
'instances_per_second',
prefix='perf')
if hours_per_epoch:
melt.add_summary(summary,
hours_per_epoch,
'hours_per_epoch',
prefix='perf')
if metric_evaluate:
#melt.add_summarys(summary, evaluate_results, evaluate_names, prefix='eval')
prefix = 'step_eval'
if model_path:
prefix = 'eval'
if not hasattr(train_once, 'epoch_step'):
train_once.epoch_step = 1
else:
train_once.epoch_step += 1
step = train_once.epoch_step
# eval/loss eval/auc ..
melt.add_summarys(summary,
evaluate_results,
evaluate_names,
prefix=prefix)
train_once.summary_writer.add_summary(summary, step)
train_once.summary_writer.flush()
return stop
elif metric_evaluate and log_dir:
summary = tf.Summary()
for summary_str in summary_strs:
train_once.summary_writer.add_summary(summary_str, step)
#summary.ParseFromString(evaluate_summaries)
summary_writer = train_once.summary_writer
prefix = 'step_eval'
if model_path:
prefix = 'eval'
if not hasattr(train_once, 'epoch_step'):
## TODO.. restart will get 1 again..
#epoch_step = tf.Variable(0, trainable=False, name='epoch_step')
#epoch_step += 1
#train_once.epoch_step = sess.run(epoch_step)
valid_interval_epochs = 1.
try:
valid_interval_epochs = FLAGS.valid_interval_epochs
except Exception:
pass
train_once.epoch_step = 1 if melt.epoch() <= 1 else int(
int(melt.epoch() * 10) /
int(valid_interval_epochs * 10))
logging.info('train_once epoch start step is',
train_once.epoch_step)
else:
#epoch_step += 1
train_once.epoch_step += 1
step = train_once.epoch_step
#melt.add_summarys(summary, evaluate_results, evaluate_names, prefix='eval')
melt.add_summarys(summary,
evaluate_results,
evaluate_names,
prefix=prefix)
summary_writer.add_summary(summary, step)
summary_writer.flush()
def makefile(self, *args, **kwargs):
return IO(b"PUT %s HTTP/1.0\nContent-Length: %d\n\n%s" %
(self._path, len(self._data), self._data))
def serialize(self, value):
data, sr = value
buffer = IO()
soundfile.write(buffer, data, sr, format='WAV', subtype='PCM_16')
body = base64.b64encode(buffer.getvalue()).decode('utf8')
return 'data:audio/wav;base64,{body}'.format(body=body)
def attendance_Of_Any_month1(request):
if request.method == "POST":
#monthvarnum = request.GET.get('sel')
#capturing values from user input
monthvarnum = request.POST.get('Select')
monthvarnum = int(monthvarnum)
yearvarnum = request.POST.get('year')
yearvarnum = int(yearvarnum)
#for all data show i dataframe
pd.set_option('display.max_colwidth', -1)
try:
from io import BytesIO as IO # for modern python
except ImportError:
from io import StringIO as IO # for legacy python
#Attendance table copy as it is and filter as per not_recived and sent status
copy1 = Attendance.objects.all().values_list(
'Employee_name',
'Attnd_status',
'year',
'month',
).filter(
Q(Attnd_status=1) & Q(year=yearvarnum)
& Q(month=monthvarnum)) #get either not_recieved or sent
dframe1 = pd.DataFrame(copy1,
columns=[
'Employee_name',
'Present_Days',
'year',
'month',
])
#group by non repeating for sum of items_names and not recieved number
dframe1 = dframe1.groupby([
'Employee_name',
]).agg({
'Present_Days': 'sum',
}).reset_index()
#Attendance table copy as it is and filter as per not_recived and sent status
copy2 = Attendance.objects.all().values_list(
'Employee_name',
'Attnd_status',
'year',
'month',
).filter(
Q(Attnd_status=0) & Q(year=yearvarnum)
& Q(month=monthvarnum)) #get either not_recieved or sent
dframe2 = pd.DataFrame(copy2,
columns=[
'Employee_name',
'Absent_Days',
'year',
'month',
])
dframe2['Absent_Days'] = dframe2['Absent_Days'].replace([
0,
], 1)
#group by non repeating for sum of items_names and not recieved number
dframe2 = dframe2.groupby([
'Employee_name',
]).agg({
'Absent_Days': 'sum',
}).reset_index()
#join two dataframes on attribute=Employee name
#combined two dataframes same columns
dframe3 = reduce(
lambda x, y: pd.merge(x, y, on=[
'Employee_name',
], how='outer'), [dframe2, dframe1])
#insertion logic of days of this month
# now = datetime.datetime
This_month_days = calendar.monthrange(yearvarnum, monthvarnum)[1]
repeat_var = list(
itertools.repeat(This_month_days, dframe3.Employee_name.count())
) #list generation of repeated values it = itertools.repeat(no of items,no of times)
dframe3['Total_Days_Of_This_Month'] = repeat_var
#adding 0 at missing entries
dframe3 = dframe3.fillna(0)
#rearanging dataframe columns
dframe3 = dframe3[[
'Employee_name', 'Total_Days_Of_This_Month', 'Present_Days',
'Absent_Days'
]]
# my "Excel" file, which is an in-memory output file (buffer)
# for the new workbook
excel_file = IO()
xlwriter = pd.ExcelWriter(excel_file, engine='xlsxwriter')
dframe3.to_excel(xlwriter, 'This_month_Attendance')
workbook = xlwriter.book
worksheet = xlwriter.sheets['This_month_Attendance']
worksheet.set_column('B:C', 25)
worksheet.set_column('D:E', 25)
xlwriter.save()
xlwriter.close()
# important step, rewind the buffer or when it is read() you'll get nothing
# but an error message when you try to open your zero length file in Excel
excel_file.seek(0)
response = HttpResponse(
excel_file.read(),
content_type=
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
)
# set the file name in the Content-Disposition header
response[
'Content-Disposition'] = 'attachment; filename= Custom_Attendance_Month_Salary.xlsx'
#dframe2.to_csv(path_or_buf=response,index=False,)
return response
def export(request, mode, tender_ids=None):
if tender_ids is None:
tender_objs = Tender.objects.all()
bid_objs = Bid.objects.all()
volume_objs = Volume.objects.all()
company_objs = Company.objects.all()
else:
list_tender_ids = tender_ids.split("|")
tender_objs = Tender.objects.filter(pk__in=list_tender_ids).distinct()
bid_objs = Bid.objects.filter(tender__in=tender_objs).distinct()
volume_objs = Volume.objects.filter(tender__in=tender_objs).distinct()
company_objs = Company.objects.filter(bids__in=bid_objs).distinct()
if mode == "volume":
tenders = pd.DataFrame(list(tender_objs.values()))
bids = pd.DataFrame(list(bid_objs.values()))
volumes = pd.DataFrame(list(volume_objs.values()))
companies = pd.DataFrame(list(company_objs.values()))
tenders.rename(columns={"id": "tender_id"},
inplace=True) # 修改列名，为之后merge匹配列做准备，下同
df = pd.merge(volumes, tenders, how="left",
on="tender_id") # 以volume为base匹配tender
bids.rename(columns={"id": "winner_id"}, inplace=True)
df = pd.merge(df, bids, how="left",
on="winner_id") # 以volume+tender为base匹配bid
companies.rename(columns={"id": "bidder_id"}, inplace=True)
df = pd.merge(df, companies, how="left",
on="bidder_id") # 以volume+tender+bid为base匹配company
# df["proc_percentage"] = df.apply(
# lambda x: Tender.objects.get(pk=x["tender_id"]).proc_percentage, axis=1
# ) # 添加列：集采比例
df["amount_contract"] = df.apply(
lambda x: Volume.objects.get(pk=x["id"]).amount_contract(),
axis=1) # 添加列：实际合同量
df = df[[
"vol",
"target",
"spec",
"tender_begin",
"ceiling_price",
"region",
"amount_reported",
# "proc_percentage",
"amount_contract",
"full_name",
"abbr_name",
"mnc_or_local",
"origin",
"bid_price",
"original_price",
]]
df.columns = [
"批次",
"标的",
"剂型剂量",
"标期开始时间",
"最高有效申报价",
"地区",
"区域报量",
# "集采比例",
"实际合同量",
"竞标公司全称",
"竞标公司简称",
"是否跨国公司",
"是否此标的原研",
"竞标价",
"集采前价格",
]
elif mode == "tender":
tenders = pd.DataFrame(list(tender_objs.values()))
bids = pd.DataFrame(list(bid_objs.values()))
companies = pd.DataFrame(list(company_objs.values()))
companies.rename(columns={"id": "bidder_id"}, inplace=True)
df = pd.merge(bids, companies, how="left",
on="bidder_id") # 以bid为base匹配company
tenders.rename(columns={"id": "tender_id"}, inplace=True)
df = pd.merge(df, tenders, how="left",
on="tender_id") # 以bid+company为base匹配tender
df["is_winner"] = df.apply(
lambda x: Bid.objects.get(pk=x["id"]).is_winner(),
axis=1) # 添加列：是否中标
df["specs"] = df.apply(
lambda x: ",".join(
list(Tender.objects.get(pk=x["tender_id"]).get_specs())),
axis=1,
) # 添加列：剂型剂量
df["total_std_volume_reported"] = df.apply(
lambda x: Tender.objects.get(pk=x["tender_id"]).
total_std_volume_reported(),
axis=1,
) # 添加列：标的官方报量
df["total_std_volume_contract"] = df.apply(
lambda x: Tender.objects.get(pk=x["tender_id"]).
total_std_volume_contract(),
axis=1,
) # 添加列：标的实际合同量
df["total_value_contract"] = df.apply(
lambda x: Tender.objects.get(pk=x["tender_id"]).
total_value_contract(),
axis=1,
) # 添加列：标的实际合同金额
df["specs"] = df.apply(
lambda x: ",".join(
list(Tender.objects.get(pk=x["tender_id"]).get_specs())),
axis=1,
) # 添加列：剂型剂量
df["regions_win"] = df.apply(
lambda x: ",".join(list(Bid.objects.get(pk=x["id"]).regions_win())
),
axis=1,
) # 添加中标区域
df["std_volume_win"] = df.apply(
lambda x: Bid.objects.get(pk=x["id"]).std_volume_win(),
axis=1) # 添加列：竞标者赢得实际合同量
df["value_win"] = df.apply(
lambda x: Bid.objects.get(pk=x["id"]).value_win(),
axis=1) # 添加列：竞标者赢得实际合同金额
df["tender_period"] = df.apply(
lambda x: Tender.objects.get(pk=x["tender_id"]).tender_period,
axis=1) # 添加列：标期
df["proc_percentage"] = df.apply(
lambda x: Tender.objects.get(pk=x["tender_id"]).proc_percentage,
axis=1) # 添加列：带量比例
df = df[[
"vol",
"target",
"specs",
"tender_begin",
"tender_period",
"ceiling_price",
"total_std_volume_reported",
"proc_percentage",
"total_std_volume_contract",
"total_value_contract",
"full_name",
"abbr_name",
"mnc_or_local",
"origin",
"bid_price",
"original_price",
"is_winner",
"std_volume_win",
"value_win",
"regions_win",
]]
df.columns = [
"批次",
"标的",
"标的剂型剂量",
"标期开始时间",
"标期",
"最高有效申报价",
"标的官方报量",
"标的带量比例",
"标的实际合同量",
"标的实际合同金额",
"竞标公司全称",
"竞标公司简称",
"是否跨国公司",
"是否此标的原研",
"竞标价",
"集采前价格",
"是否中标",
"竞标者赢得实际合同量",
"竞标者赢得实际合同金额",
"中标区域",
]
excel_file = IO()
xlwriter = pd.ExcelWriter(excel_file, engine="xlsxwriter")
df.to_excel(xlwriter, "data", index=False)
xlwriter.save()
xlwriter.close()
excel_file.seek(0)
# 设置浏览器mime类型
response = HttpResponse(
excel_file.read(),
content_type=
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
)
# 设置文件名
now = datetime.datetime.now().strftime(
"%Y%m%d%H%M%S") # 当前精确时间不会重复，适合用来命名默认导出文件
response["Content-Disposition"] = "attachment; filename=" + now + ".xlsx"
return response
def __init__(self, *components, **kwargs):
"""
"""
# Check input components
if len(components) < 2:
raise \
'\n\n +++ CliMT.federation: you must give me more than 1 component to federate!\n\n'
else:
for component in components:
assert type(component) is InstanceType, \
'\n\n +++CliMT.federation: Input item %s is not an instance.\n\n' % str(c)
# Re-order components: diagnostic, semi-implicit, explicit, implicit
components = list(components)
"""
for i in range(len(components)):
if len(components[i].Prognostic) > 0:
components.append(components.pop(i))
for scheme in ['semi-implicit', 'explicit', 'implicit']:
for i in range(len(components)):
if components[i].SteppingScheme == scheme:
components.append(components.pop(i))
"""
self.components = components
# Federation's Required is union of all components' Required;
# same for Prognostic and Diagnostic
self.Required = []
self.Prognostic = []
self.Diagnostic = []
for component in components:
self.Required = list(set(self.Required).union(component.Required))
self.Prognostic = list(
set(self.Prognostic).union(component.Prognostic))
self.Diagnostic = list(
set(self.Diagnostic).union(component.Diagnostic))
# Other attributes
self.Name = 'federation'
self.Extension = None
# Set LevType to None if all components are None, else p
self.LevType = None
for component in components:
if component.LevType == 'p': self.LevType = 'p'
# Initialize self.Fixed (subset of self.Prognostic which will NOT be time-marched)
if 'Fixed' in kwargs: self.Fixed = kwargs.pop('Fixed')
else: self.Fixed = []
# Instantiate I/O
self.Io = IO(self, **kwargs)
# Get values from restart file, if available
if 'RestartFile' in kwargs:
ParamNames = Parameters().value.keys()
FieldNames = self.Required
kwargs = self.Io.readRestart(FieldNames, ParamNames, kwargs)
# Initialize scalar parameters
self.Params = Parameters(**kwargs)
# Initialize State
self.State = State(self, **kwargs)
self.Grid = self.State.Grid
# Set some redundant attributes (mainly for backward compatibility)
self.nlon = self.Grid['nlon']
self.nlat = self.Grid['nlat']
self.nlev = self.Grid['nlev']
try:
self.o3 = self.State['o3']
except:
pass
# Check if components enforce axis dimensions, ensure consistency
for component in self.components:
for AxisName in ['lev', 'lat', 'lon']:
exec('n_fed = self.n%s' % AxisName)
try:
exec('n_com = component.Extension.get_n%s()' % AxisName)
except:
n_com = n_fed
assert n_com == n_fed, \
'\n\n ++++ CliMT.federation.init: recompile with %i %ss to run this federation\n'\
% (n_fed,AxisName)
# Dictionary to hold increments on prognos fields
self.Inc = {}
# Adjust components' attributes
for component in self.components:
component.Monitoring = False
component.Io.OutputFreq = self.Io.OutputFreq
component.Fixed.extend(self.Fixed)
if component.UpdateFreq == component['dt']:
component.UpdateFreq = self['dt']
component.Params = self.Params
component.Grid = self.State.Grid
component.State = self.State
component.Inc = {}
# insolation component gets special treatment because
# of need to set orb params in common block (yes, this is ugly)
try:
component.setOrbParams(**kwargs)
except:
pass
self.compute(ForcedCompute=True)
# Create output file
self.Io.createOutputFile(self.State, self.Params.value)
# Write out initial state
if not self.Io.Appending: self.write()
# Initialize plotting facilities
self.Plot = Plot()
# Initialize runtime monitor
self.Monitor = Monitor(self, **kwargs)
# Notify user of unused input quantities
self._checkUnused(kwargs)
def __init__(self, filename):
self.filename = filename
self.excel_file = IO()
self.xlwriter = pd.ExcelWriter(self.excel_file, engine='xlsxwriter')
def makefile(self, *args, **kwargs):
return IO(self.data)
