def main(args):
"""
Main function - launches the program.
:param args:
The Parser arguments
:type args:
Parser object
:returns:
List
:example:
["Downloading files from the Compranet site."]
"""
if args:
if args.sample:
source_folder = settings.folder_sample_data
else:
# Use cached versions of the source data in csv format
source_folder = settings.folder_full_cache
check_create_folder(source_folder)
if args.download:
clean_folder(source_folder)
download_compranet(settings.years)
# Check if there are CSV files in the sample folder
pattern = os.path.join(source_folder, '*.csv')
source_data = list_files(pattern)
if source_data:
print "About to clean the data"
clean_df = clean.clean_csv(source_data)
print "About to store it in OCDS format"
ocds.generate_json(clean_df)
else:
return["No source data found. Make sure there is at least one CSV file in " + source_folder, 1]
return["Prepared and cleaned the files from the Compranet site.",0]
'gpu2': gpu2,
'min_reward': -env_params['stacksize'],
'max_reward': env_params['pot'] + env_params['stacksize']
}
validation_params = {
'actor_path': config.agent_params['actor_path'],
'epochs': args.valepochs,
'koth': args.koth
}
path = training_params['save_dir']
directory = os.path.dirname(path)
if not os.path.exists(directory):
os.mkdir(directory)
# Clean training_run folder
if not args.resume:
clean_folder(training_params['critic_path'])
clean_folder(training_params['actor_path'])
# Set processes
mp.set_start_method('spawn')
num_processes = min(mp.cpu_count(), 3)
print(f'Number of used processes {num_processes}')
print(f'Training {args.network_type} model')
if args.network_type == 'combined':
alphaPoker = CombinedNet(seed, nS, nA, nB, network_params).to(device)
if args.frozen:
# Load pretrained hand recognizer
copy_weights(alphaPoker,
network_params['actor_hand_recognizer_path'])
alphaPoker.summary
alphaPoker_optimizer = optim.Adam(alphaPoker.parameters(),
lr=config.agent_params['critic_lr'])
def generate_json(df):
"""
Generate OCDS record packages for each month
:param df:
Dataframe with all the contracts
:type args:
DataFrame
:returns:
:example:
"""
check_create_folder(settings.folder_ocds_json)
check_create_folder(settings.folder_tmp)
clean_folder(settings.folder_tmp)
# Group the Compranet by date
df['group_date'] = df[settings.grouping_date].convert_objects(convert_dates='coerce')
grouped_df = df.set_index('group_date').groupby(pd.TimeGrouper(freq='M'))
# Store the records for each month in a temporary CSV file
# The JSON files will be generated from these CSV files, which
# is much more performant than iterating over the rows in pandas
files = []
for month, records in grouped_df:
if not records.empty:
m = month.strftime("%Y%m%d")
file_name = os.path.join(settings.folder_tmp, m + '.csv')
files.append(file_name)
records.to_csv(file_name, index=False)
# Loop over each CSV file and create an OCDS record package
for f in files:
# Store the package meta-data
## ADD MONTH
package = {
"uri": os.path.join("http://example.com/" + get_filename(f) + '.json'),
"publishedDate": get_filename(f),
"records": [],
"publisher": {
"identifier": "100",
"name": "Compranet"
},
"packages": []
}
# Read the file and generate the records
with open(f, 'rb') as infile:
data = csv.DictReader(infile)
ocds_records = {}
for record in data:
record_id = record['NUMERO_PROCEDIMIENTO']
# Add the generic tender data for this record,
# if it's not there already
if not record_id in ocds_records:
ocds_records[record_id] = get_tender_data(record)
# The contract and award data needs to be added for each row
# OCDS expects a unique ID for every award. NUMERO_EXPEDIENTE is not unique, hence
# a custom ID
award_id = str(record['NUMERO_EXPEDIENTE']) + '-' + str(len(ocds_records[record_id]['awards']) + 1)
ocds_records[record_id]['awards'].append(get_award_data(record, award_id))
ocds_records[record_id]['contracts'].append(get_contract_data(record, award_id))
for key, value in ocds_records.iteritems():
package['records'].append(value)
ofn = os.path.join(settings.folder_ocds_json, get_filename(f) + '.json')
with open(ofn, 'w') as outfile:
json.dump(package, outfile)
