def main(): parser = argparse.ArgumentParser(prog='PROG') parser.add_argument('-w', '--absolute_path', required=True, help="The work space for build.") parser.add_argument('-p', '--previous_manifest', required=True, help="Path of the old manifest.") parser.add_argument('-c', '--current_manifest', required=True, help="Path of the new manifest.") parser.add_argument('-s', '--gerrit_host', required=True, help="The gerrit ip to query.") parser.add_argument('-t', '--gerrit_port', required=True, help="The gerrit port.") parser.add_argument('-u', '--gerrit_user', required=True, help="The gerrit user.") parser.add_argument('-r', '--release_subject', required=True, help="The subject of wiki page.") parser.add_argument('-o', '--wiki_content_file_path', required=True, help="The output of the wiki page content.") args = parser.parse_args() absolue_path_prefix = args.absolute_path manifest1 = args.previous_manifest manifest2 = args.current_manifest gerrit_ip = args.gerrit_host gerrit_user = args.gerrit_user gerrit_port = args.gerrit_port release_subject = args.release_subject wiki_content_file_path = args.wiki_content_file_path wiki_content_file = open('%s' % (wiki_content_file_path), 'w+') wiki_content_file.writelines('<wiki_page>\n') wiki_content_file.writelines('<title>%s</title>\n' % (release_subject)) wiki_content_file.writelines('<parent title="Releases"/>\n') wiki_content_file.writelines('<text>h1. %s \n\n' % (release_subject)) mp1 = ManifestParser(manifest1) mp2 = ManifestParser(manifest2) mp1_path_name_dic = mp1.get_path_name_dic() mp2_path_name_dic = mp2.get_path_name_dic() mp1_path_version_dic = mp1.get_path_version_dic() mp2_path_version_dic = mp2.get_path_version_dic() new_version_removed_list_path_name = {} new_version_removed_list_path_version = {} new_version_added_list_path_name = {} new_version_added_list_path_version = {} for k_path in mp1_path_name_dic.keys(): if k_path not in mp2_path_name_dic.keys(): new_version_removed_list_path_name[k_path] = mp1_path_name_dic[k_path] new_version_removed_list_path_version[k_path] = mp1_path_version_dic[k_path] del(mp1_path_version_dic[k_path]) wiki_content_file.writelines('|Deleted |%s |\n\n' % (k_path)) for k_path in mp2_path_name_dic.keys(): if k_path not in mp1_path_name_dic.keys(): new_version_added_list_path_name[k_path] = mp2_path_name_dic[k_path] new_version_added_list_path_version[k_path] = mp2_path_version_dic[k_path] del(mp2_path_version_dic[k_path]) wiki_content_file.writelines('|Added |%s |\n\n' % (k_path)) wiki_content_file.writelines('|_.GerritID |_.Subject |_.Submitter |_.Path |_.CommitID |\n') #By the above filter add or removed projects, the remained project shall be the same if len(mp1_path_version_dic) == len(mp2_path_version_dic): for k_path in mp1_path_version_dic.keys(): mp1_version = mp1_path_version_dic[k_path] mp2_version = mp2_path_version_dic[k_path] cmd_list_sha1 = 'cd %s/%s ; git log --pretty=oneline %s..%s' % (absolue_path_prefix, k_path, mp1_version, mp2_version) if mp1_version != mp2_version: cmd_list_output = subprocess.Popen(cmd_list_sha1, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) for line in cmd_list_output.stdout.readlines(): sha1 = line[:40] cmd_gerrit_query = 'ssh -p %s %s@%s gerrit query --format=JSON --commit-message %s --current-patch-set | sed 2d' % (gerrit_port, gerrit_user, gerrit_ip, sha1) #print cmd_gerrit_query cmd_gerrit_result = subprocess.Popen(cmd_gerrit_query, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) cmd_gerrit_output = cmd_gerrit_result.stdout.read() try: json_str = json.loads(cmd_gerrit_output) gerrit_id = json_str["number"] gerrit_url = json_str["url"] gerrit_subject = json_str["subject"] gerrit_submitter = json_str["currentPatchSet"]["uploader"]["email"] gerrit_path = k_path gerrit_sha1 = sha1 wiki_content_file.writelines(cgi.escape('|"%s":%s |%s |%s |%s |%s |\n' % (gerrit_id, gerrit_url, gerrit_subject, gerrit_submitter, gerrit_path, gerrit_sha1))) except Exception,e: pass
def lambda_handler(event, context): """ This is the main entry point for lambda function :param event: Event object that contains the bucket name and key of the manifest file that triggered the lambda :param context: Contains context information about the lambda function itself e.g. remaining time :return: Does not return anything """ # set environment exec_environment = os.environ['exec_environment'] etl_file_path = os.environ['etl_file_path'] manifest_file_path = os.environ['manifest_file_path'] manifest_file = os.environ['manifest_file'] log_uri = os.environ['log_uri'] # Download manifest file get_manifest_file(event, manifest_file_path, manifest_file) # Instantiate Connection conn = Connection() # Create an EMR connection conn_emr = conn.emr_connection() # Create an S3 connection conn_s3 = conn.s3_connection() # Instantiate S3Manager s3_manager = S3Manager() # Parse the manifest file and generate etl from ETL template wth placeholder values filled in. # Also gets the details about the EMR cluster to create. try: # Instantiate ManifestParser manifest_parser = ManifestParser() logger.info( "Generating new ETL file from ETL template wth placeholder values filled in" ) dest_etl_file = manifest_parser.parse_manifest_file( manifest_file_path, manifest_file, conn_s3, s3_manager, exec_environment) logger.info("Generated: {}".format(dest_etl_file)) except: logger.error( 'Failed while trying to generate a new ETL from s3://{}/{}'.format( manifest_parser.script_s3_bucket, manifest_parser.script_s3_key)) logging.error(sys.exc_info()) raise # Copy generated ETL to S3. This will then be submitted to EMR try: s3_manager.upload_object(conn_s3, etl_file_path, dest_etl_file, manifest_parser.script_s3_bucket, 'generated-etls', dest_etl_file) except: raise # Launch and submit jobs to EMR try: # Instantiate EMR Instance emr = EMRInstance() cluster_name = "{}_{}".format(exec_environment, manifest_parser.script_s3_key) cluster_id = emr.get_first_available_cluster(conn_emr) if manifest_parser.use_existing_cluster and cluster_id: instance_groups = emr.get_instance_groups(conn_emr, cluster_id) group_id = instance_groups['CORE'] instance_groups_count = emr.get_instance_groups_count( conn_emr, cluster_id) current_instance_count = instance_groups_count[group_id] if manifest_parser.instance_count > current_instance_count: emr.set_instance_count(conn_emr, cluster_id, group_id, manifest_parser.instance_count) # Allow 10 secs for resizing to start time.sleep(10) #submit job emr.submit_job( conn_emr, cluster_id, 's3://{}/generated-etls/{}'.format( manifest_parser.script_s3_bucket, dest_etl_file), dest_etl_file, 'cluster', 'CONTINUE') else: # Launch EMR cluster emr.launch_emr_and_submit_job( conn_emr, log_uri, 's3://{}/generated-etls/{}'.format( manifest_parser.script_s3_bucket, dest_etl_file), dest_etl_file, 'cluster', 'CONTINUE', '{}'.format(cluster_name), manifest_parser.terminate_cluster, manifest_parser.instance_type, manifest_parser.instance_count) logger.info("Submitted s3://{}/{} to process_{}".format( manifest_parser.script, dest_etl_file, cluster_name)) except: logger.error( "Failed while trying to launch EMR cluster. Details below:") raise
def main(): usage = """usage: %prog -p PATH -o OLD-MANIFEST -n NEW-MANIFEST List how many commits between two versions. """ parser = OptionParser(usage=usage) parser.add_option("-p", "--path", dest="ABS_PATH", help="The work space for build.") parser.add_option("-o", "--old", dest="OLD_MANIFEST", help="Path of the old manifest.") parser.add_option("-n", "--new", dest="NEW_MANIFEST", help="Path of the new manifest.") parser.add_option("-l", "--url", dest="GERRIT_URL", help="The gerrit url.") parser.add_option("-u", "--username", dest="GERRIT_USER", help="The gerrit username") options, extra = parser.parse_args() absolue_path_prefix = options.ABS_PATH manifest1 = options.OLD_MANIFEST manifest2 = options.NEW_MANIFEST gerrit_url = options.GERRIT_URL gerrit_user = options.GERRIT_USER (gerrit_ip, gerrit_port) = get_ssh_info(gerrit_url) mp1 = ManifestParser(manifest1) mp2 = ManifestParser(manifest2) mp1_path_name_dic = mp1.get_path_name_dic() mp2_path_name_dic = mp2.get_path_name_dic() mp1_path_version_dic = mp1.get_path_version_dic() mp2_path_version_dic = mp2.get_path_version_dic() print '<B><font color="blue">2. Repositories added/removed: </font></B><br>' for k_path in mp1_path_name_dic.keys(): if k_path not in mp2_path_name_dic.keys(): new_version_removed_list_path_name[k_path] = mp1_path_name_dic[k_path] new_version_removed_list_path_version[k_path] = mp1_path_version_dic[k_path] del(mp1_path_version_dic[k_path]) print '<B><font color="red">%s</B></font> was deleted.<br>' % (k_path) for k_path in mp2_path_name_dic.keys(): if k_path not in mp1_path_name_dic.keys(): new_version_added_list_path_name[k_path] = mp2_path_name_dic[k_path] new_version_added_list_path_version[k_path] = mp2_path_version_dic[k_path] del(mp2_path_version_dic[k_path]) print '<B><font color="red">%s</B></font> was added.<br>' % (k_path) print '<br><B><font color="blue">3. New commits found today:</font><B><br>' print '<table border="1" bordercolor="green">' print '<tr><th><B><font color="green">GerritID</font><B></th><th><B><font color="green">Submitter</font><B></th><th><B><font color="green">Subject</font><B></th><th><B><font color="green">Path</font><B></th><th><B><font color="green">SHA-1</font><B></th></tr>' #By the above filter add or removed projects, the remained project shall be the same if len(mp1_path_version_dic) == len(mp2_path_version_dic): for k_path in mp1_path_version_dic.keys(): mp1_version = mp1_path_version_dic[k_path] mp2_version = mp2_path_version_dic[k_path] cmd_list_sha1 = 'cd %s/%s ; git log --pretty=oneline %s..%s' % (absolue_path_prefix, k_path, mp1_version, mp2_version) if mp1_version != mp2_version: cmd_list_output = subprocess.Popen(cmd_list_sha1, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) for line in cmd_list_output.stdout.readlines(): sha1 = line[:40] cmd_gerrit_query = 'ssh -p %s %s@%s gerrit query --format=JSON --commit-message %s --current-patch-set | sed 2d' % (gerrit_port,gerrit_user,gerrit_ip,sha1) # print cmd_gerrit_query cmd_gerrit_result = subprocess.Popen(cmd_gerrit_query, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) cmd_gerrit_output = cmd_gerrit_result.stdout.read() try: json_str = json.loads(cmd_gerrit_output) gerrit_id = json_str["number"] gerrit_subject = json_str["subject"] gerrit_submitter = json_str["currentPatchSet"]["uploader"]["email"] gerrit_path = k_path gerrit_sha1 = sha1 print '<tr><th><a href="%s/%s">%s</a></th><th>%s</th><th>%s</th><th>%s</th><th>%s</th></tr>' % (gerrit_url,gerrit_id, gerrit_id, gerrit_submitter, gerrit_subject, gerrit_path, gerrit_sha1) except Exception,e: print '<tr><th>%s</th><th>%s</th><th>%s</th><th>%s</th><th><font color="red">Fatal: %s</font></th></tr>' % ('Unknown', 'Unknown', 'Unknown', k_path, sha1) print '</table>'