def upload(params):
Debug = DebugManager.DebugManager();
Debug.start();
Debug.trace('start');
dbManager = SharedMemoryManager.getInstance();
db = dbManager.query();
date = fn.getNestedElement(params, 'date');
path = fn.getNestedElement(params, 'path');
# url = fn.getNestedElement(params, 'callback_url'); # required params to handle callback_url
paths, should_reset = ModelUpload.getPath(params);
for idx in range(0, len(paths)):
p = paths[idx];
processed_filename = File.converExcelFileToCsv(p, ignore_index=True);
Logger.v('processed_filename', processed_filename);
Debug.trace('convert to json : path {0}'.format( processed_filename ) );
if idx == 0 and should_reset: #reset once at the beginning
Logger.v('Reset Database.');
reset(date); #reset stock_issue collection
ModelSIIntegrity.reset(date); #reset stock_issue_datalog by date given
File.readCsvFileInChunks(processed_filename, save, params, chunksize=chunksize);
Debug.trace('uploaded to mongo.');
generateIndex();
ModelSIIntegrity.generateIndex();
Debug.trace('indexing mongo collection.');
saveIssueOption();
Debug.trace('save option to json.');
trigger_params = copy.deepcopy(params);
trigger_params['result'] = 'data count: {0}'.format(params['data_count'][path]);
# Logger.v('trigger_params', trigger_params);
dbManager.executeBulkOperations(None); # Insert all the remaining job at once.
ReportStock.triggerOnComplete(trigger_params);
Debug.trace('trigger api on complete.');
Debug.end();
Debug.show('Stock.upload');
def save(params, chunk, chunks_info):
global collection_name, column_keymap;
upload_date = fn.getNestedElement(params, 'date');
data = File.readChunkData(chunk);
dbManager = SharedMemoryManager.getInstance();
db = dbManager.query();
current_index = fn.getNestedElement(chunks_info, 'current', 0);
total_index = fn.getNestedElement(chunks_info, 'total', len(data));
total_length = len(data);
queue_info = chunks_info['queue']
# Logger.v('Running Index:', chunks_info['queue']['running']);
chunks_info['queue']['current']+=1;
# Logger.v('Saving from... {0}/{1}, current package: {2}'.format(current_index, total_index, total_length) );
fn.printProgressBar(queue_info['current'], queue_info['total'], 'Processing Chunk Insertion');
for idx in range(0, total_length):
row = data[idx];
# Logger.v('row', row);
obj_ = transformToLowercase(row);
date_only = obj_['approved_date'].split(' ')[0];
# Logger.v('date_only', date_only);
obj_.update({
'approved_year_month': DateTime.getDateCategoryName(date=date_only, element='year_month_digit'),
'upload_date': upload_date,
});
dbManager.addBulkInsert(collection_name, obj_, batch=True);
ModelSIIntegrity.update(data=obj_);
retrieveIssueOption(obj_);
#ensure all data is save properly
dbManager.executeBulkOperations(collection_name);
return chunks_info;
def save(params, chunk, chunks_info):
global latest_collection_name, history_collection_name
data = File.readChunkData(chunk)
dbManager = SharedMemoryManager.getInstance()
db = dbManager.query()
current_index = fn.getNestedElement(chunks_info, 'current', 0)
total_index = fn.getNestedElement(chunks_info, 'total', len(data))
date = fn.getNestedElement(params, 'date')
datetime = DateTime.convertDateTimeFromString(date)
total_length = len(data)
queue_info = chunks_info['queue']
# Logger.v('Running Index:', chunks_info['queue']['running']);
chunks_info['queue']['current'] += 1
# Logger.v('Saving from... {0}/{1}, current package: {2}'.format(current_index, total_index, total_length) );
fn.printProgressBar(queue_info['current'], queue_info['total'],
'Processing Chunk Insertion')
for idx in range(0, total_length):
# insert stock_latest
row = data[idx]
obj_ = transformToLowercase(data=row, datetime=datetime)
ModelStockIntegrity.update(data=obj_)
dbManager.addBulkInsert(latest_collection_name, obj_, batch=True)
# dbManager.addBulkInsert(history_collection_name, obj_, batch=True); # temporary off (need 7 day data only)
# insert items
# d = data[idx];
ModelItem.saveItem(row)
# fn.printProgressBar(current_index+idx, total_index, 'Processing Item Insertion');
#ensure all data is save properly
# dbManager.executeBulkOperations(history_collection_name); # temporary off (need 7 day data only)
dbManager.executeBulkOperations(latest_collection_name)
return chunks_info
def server_run_segment(config: dict):
input_dir = config["input_dir"]
output_dir = config["output_dir"]
file_pattern = config["files"]
threshold_method = config["threshold_method"]
threshold_params = config["threshold_params"]
utils.check_dir_exists(input_dir)
utils.check_dir_exists(output_dir)
_save_config(config)
files_to_segment = glob.glob(f"{input_dir}/{file_pattern}")
logger.info(f"{files_to_segment}")
for i, file_name in enumerate(files_to_segment):
logger.info(f"{i+1}/{len(files_to_segment)}")
channel_file = File.from_tiff(file_name)
segmented_file = segmentation.segment_stack(channel_file,
threshold_method, 3, 90000,
**threshold_params)
segmented_file.save_to_tiff(output_dir)
import json
import sys
from lib import File
File = File()
secret_filename = raw_input('''
Enter the path of your secret json key file.
Example : /Users/Desktop/credentials.json
$: ''')
json_file = File.read(secret_filename)
data = json.loads(json_file)["web"]
client_id = data["client_id"]
project_id = data["project_id"]
auth_uri = data["auth_uri"]
token_uri = data["token_uri"]
auth_provider_x509_cert_url = data["auth_provider_x509_cert_url"]
client_secret = data["client_secret"]
from facebook_business.api import FacebookAdsApi
from facebook_business.adobjects.adaccount import AdAccount
from facebook_business.adobjects.ad import Ad
import json
import sys
from lib import File
File = File()
def arrayToTSVLine(array):
line = ""
for elem in array:
line += "\"" + elem + "\"" + "\t"
return line[:-1]
FORMAT_TEXT = '''
The format must be the following:
{
"app_id" : "<YOUR APP_ID HERE>",
"app_secret" : "<YOUR APP_SECRET HERE>",
"access_token" : "<YOUR ACCESS_TOKEN HERE>",
"ad_account" : "<YOUR_AD_ACCOUNT_HERE>"
}
'''
with open(stats_file) as f:
for line in f:
print(line, end='')
else:
print(stats)
print(useless_bar + "\n")
#################################
# MAIN ##
################################
if __name__ == '__main__':
print("\n")
define_arguments()
contentsFile = File.File(cmdargs)
print(" Architecture was set to " + contentsFile.get_arch())
print(" Repository URL is: " + contentsFile.get_url())
# clear all artifacts
if cmdargs.clear:
print(" Cleaning old files..")
plot_list = Path(CONFIG["statistics"]["plot_folder"]).glob("*.png")
csv_list = Path(CONFIG["statistics"]["plot_folder"]).glob("*.csv")
stat_list = Path(CONFIG["statistics"]["folder"]).glob("statistics_*")
down_list = Path(CONFIG["downloadFolder"]).glob("*")
for file in itertools.chain(plot_list, csv_list, stat_list, down_list):
file.unlink()
# check repo metadata to avoid downloading the same file twice
stats_file = file_utils.check_remote_file(contentsFile)
def getOption(params):
data = []
filename = '{0}/stock_issue_options.json'.format(reference_folder)
data = File.readJSONFile(filename)
return data
from lib import File, segmentation, image_adjustments
def reverse_palette(palette):
list_palette = [x for x in palette]
list_palette.reverse()
return list_palette
file_name = sys.argv[1]
RGreens7 = reverse_palette(Greens7)
ROrRd3 = reverse_palette(OrRd3)
channel_file = File.from_tiff(file_name)
segmented_image_stack = segmentation.segment_stack(
channel_file, "fixed", threshold=channel_file.image.mean())
segmented_image_stack = segmented_image_stack.labelled_image > 1
nominal_image_stack = image_adjustments.rescale_intensity(channel_file)
main_fig = figure(plot_height=500,
plot_width=500,
title="Segmentation",
sizing_mode="scale_both")
print((nominal_image_stack.max()))
layer = Slider(start=0,
end=len(nominal_image_stack),
value=0,
step=1,