def main():
  parser = argparse.ArgumentParser(prog='PROG')
  parser.add_argument('-w', '--absolute_path', required=True, help="The work space for build.")
  parser.add_argument('-p', '--previous_manifest', required=True, help="Path of the old manifest.")
  parser.add_argument('-c', '--current_manifest', required=True, help="Path of the new manifest.")
  parser.add_argument('-s', '--gerrit_host', required=True, help="The gerrit ip to query.")
  parser.add_argument('-t', '--gerrit_port', required=True, help="The gerrit port.")
  parser.add_argument('-u', '--gerrit_user', required=True, help="The gerrit user.")
  parser.add_argument('-r', '--release_subject', required=True, help="The subject of wiki page.")
  parser.add_argument('-o', '--wiki_content_file_path', required=True, help="The output of the wiki page content.")
  args = parser.parse_args()

  absolue_path_prefix = args.absolute_path
  manifest1 = args.previous_manifest
  manifest2 = args.current_manifest
  gerrit_ip = args.gerrit_host
  gerrit_user = args.gerrit_user
  gerrit_port = args.gerrit_port
  release_subject = args.release_subject
  wiki_content_file_path = args.wiki_content_file_path
  
  wiki_content_file = open('%s' % (wiki_content_file_path), 'w+')
  wiki_content_file.writelines('<wiki_page>\n')
  wiki_content_file.writelines('<title>%s</title>\n' % (release_subject))
  wiki_content_file.writelines('<parent title="Releases"/>\n')
  wiki_content_file.writelines('<text>h1. %s \n\n' % (release_subject))

  mp1 = ManifestParser(manifest1)
  mp2 = ManifestParser(manifest2)

  mp1_path_name_dic = mp1.get_path_name_dic()
  mp2_path_name_dic = mp2.get_path_name_dic()
  mp1_path_version_dic = mp1.get_path_version_dic()
  mp2_path_version_dic = mp2.get_path_version_dic()

  new_version_removed_list_path_name = {}
  new_version_removed_list_path_version = {}
  new_version_added_list_path_name = {}
  new_version_added_list_path_version = {}

  for k_path in mp1_path_name_dic.keys():
    if k_path not in mp2_path_name_dic.keys():
      new_version_removed_list_path_name[k_path] = mp1_path_name_dic[k_path]
      new_version_removed_list_path_version[k_path] = mp1_path_version_dic[k_path]
      del(mp1_path_version_dic[k_path])
      wiki_content_file.writelines('|Deleted |%s |\n\n' % (k_path))

  for k_path in mp2_path_name_dic.keys():
    if k_path not in mp1_path_name_dic.keys():
      new_version_added_list_path_name[k_path] = mp2_path_name_dic[k_path]
      new_version_added_list_path_version[k_path] = mp2_path_version_dic[k_path]
      del(mp2_path_version_dic[k_path])
      wiki_content_file.writelines('|Added |%s |\n\n' % (k_path))

  wiki_content_file.writelines('|_.GerritID |_.Subject |_.Submitter |_.Path |_.CommitID |\n')
  #By the above filter add or removed projects, the remained project shall be the same
  if len(mp1_path_version_dic) == len(mp2_path_version_dic):
    for k_path in mp1_path_version_dic.keys():
      mp1_version = mp1_path_version_dic[k_path]
      mp2_version = mp2_path_version_dic[k_path]
      cmd_list_sha1 = 'cd %s/%s ; git log --pretty=oneline %s..%s' % (absolue_path_prefix, k_path, mp1_version, mp2_version)
      if mp1_version != mp2_version:
        cmd_list_output = subprocess.Popen(cmd_list_sha1, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        for line in cmd_list_output.stdout.readlines():
          sha1 = line[:40]
          cmd_gerrit_query = 'ssh -p %s %s@%s gerrit query --format=JSON --commit-message %s --current-patch-set | sed 2d' % (gerrit_port, gerrit_user, gerrit_ip, sha1)
          #print cmd_gerrit_query
          cmd_gerrit_result = subprocess.Popen(cmd_gerrit_query, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
          cmd_gerrit_output = cmd_gerrit_result.stdout.read()
          try:
            json_str = json.loads(cmd_gerrit_output)
            gerrit_id = json_str["number"]
            gerrit_url = json_str["url"]
            gerrit_subject = json_str["subject"]
            gerrit_submitter = json_str["currentPatchSet"]["uploader"]["email"]
            gerrit_path = k_path
            gerrit_sha1 = sha1
            wiki_content_file.writelines(cgi.escape('|"%s":%s |%s |%s |%s |%s |\n' % (gerrit_id, gerrit_url, gerrit_subject, gerrit_submitter, gerrit_path, gerrit_sha1)))
          except Exception,e:
            pass
def lambda_handler(event, context):
    """
    This is the main entry point for lambda function
    :param event: Event object that contains the bucket name and key of the manifest file that triggered the lambda
    :param context: Contains context information about the lambda function itself e.g. remaining time
    :return: Does not return anything
    """
    # set environment
    exec_environment = os.environ['exec_environment']
    etl_file_path = os.environ['etl_file_path']
    manifest_file_path = os.environ['manifest_file_path']
    manifest_file = os.environ['manifest_file']
    log_uri = os.environ['log_uri']

    # Download manifest file
    get_manifest_file(event, manifest_file_path, manifest_file)

    # Instantiate Connection
    conn = Connection()
    # Create an EMR connection
    conn_emr = conn.emr_connection()
    # Create an S3 connection
    conn_s3 = conn.s3_connection()
    # Instantiate S3Manager
    s3_manager = S3Manager()

    # Parse the manifest file and generate etl from ETL template wth placeholder values filled in.
    # Also gets the details about the EMR cluster to create.
    try:
        # Instantiate ManifestParser
        manifest_parser = ManifestParser()
        logger.info(
            "Generating new ETL file from ETL template wth placeholder values filled in"
        )
        dest_etl_file = manifest_parser.parse_manifest_file(
            manifest_file_path, manifest_file, conn_s3, s3_manager,
            exec_environment)
        logger.info("Generated: {}".format(dest_etl_file))
    except:
        logger.error(
            'Failed while trying to generate a new ETL from s3://{}/{}'.format(
                manifest_parser.script_s3_bucket,
                manifest_parser.script_s3_key))
        logging.error(sys.exc_info())
        raise

    # Copy generated ETL to S3. This will then be submitted to EMR
    try:
        s3_manager.upload_object(conn_s3, etl_file_path, dest_etl_file,
                                 manifest_parser.script_s3_bucket,
                                 'generated-etls', dest_etl_file)
    except:
        raise

    # Launch and submit jobs to EMR
    try:
        # Instantiate EMR Instance
        emr = EMRInstance()

        cluster_name = "{}_{}".format(exec_environment,
                                      manifest_parser.script_s3_key)
        cluster_id = emr.get_first_available_cluster(conn_emr)

        if manifest_parser.use_existing_cluster and cluster_id:
            instance_groups = emr.get_instance_groups(conn_emr, cluster_id)
            group_id = instance_groups['CORE']

            instance_groups_count = emr.get_instance_groups_count(
                conn_emr, cluster_id)
            current_instance_count = instance_groups_count[group_id]

            if manifest_parser.instance_count > current_instance_count:
                emr.set_instance_count(conn_emr, cluster_id, group_id,
                                       manifest_parser.instance_count)
                # Allow 10 secs for resizing to start
                time.sleep(10)

            #submit job
            emr.submit_job(
                conn_emr, cluster_id, 's3://{}/generated-etls/{}'.format(
                    manifest_parser.script_s3_bucket, dest_etl_file),
                dest_etl_file, 'cluster', 'CONTINUE')
        else:
            # Launch EMR cluster
            emr.launch_emr_and_submit_job(
                conn_emr, log_uri, 's3://{}/generated-etls/{}'.format(
                    manifest_parser.script_s3_bucket,
                    dest_etl_file), dest_etl_file, 'cluster', 'CONTINUE',
                '{}'.format(cluster_name), manifest_parser.terminate_cluster,
                manifest_parser.instance_type, manifest_parser.instance_count)

        logger.info("Submitted s3://{}/{} to process_{}".format(
            manifest_parser.script, dest_etl_file, cluster_name))
    except:
        logger.error(
            "Failed while trying to launch EMR cluster. Details below:")
        raise
def main():
    usage = """usage: %prog -p PATH -o OLD-MANIFEST -n NEW-MANIFEST
      List how many commits between two versions.
      """
    parser = OptionParser(usage=usage)
    parser.add_option("-p", "--path", dest="ABS_PATH",
                      help="The work space for build.")
    parser.add_option("-o", "--old", dest="OLD_MANIFEST",
                      help="Path of the old manifest.")
    parser.add_option("-n", "--new", dest="NEW_MANIFEST",
                      help="Path of the new manifest.")
    parser.add_option("-l", "--url", dest="GERRIT_URL",
                      help="The gerrit url.")
    parser.add_option("-u", "--username", dest="GERRIT_USER",
                      help="The gerrit username")
    options, extra = parser.parse_args()

    absolue_path_prefix = options.ABS_PATH
    manifest1 = options.OLD_MANIFEST
    manifest2 = options.NEW_MANIFEST
    gerrit_url = options.GERRIT_URL
    gerrit_user = options.GERRIT_USER
    (gerrit_ip, gerrit_port) = get_ssh_info(gerrit_url)

    mp1 = ManifestParser(manifest1)
    mp2 = ManifestParser(manifest2)

    mp1_path_name_dic = mp1.get_path_name_dic()
    mp2_path_name_dic = mp2.get_path_name_dic()

    mp1_path_version_dic = mp1.get_path_version_dic()
    mp2_path_version_dic = mp2.get_path_version_dic()

    print '<B><font color="blue">2. Repositories added/removed: </font></B><br>'
    for k_path in mp1_path_name_dic.keys():
      if k_path not in mp2_path_name_dic.keys():
        new_version_removed_list_path_name[k_path] = mp1_path_name_dic[k_path]
        new_version_removed_list_path_version[k_path] = mp1_path_version_dic[k_path]
        del(mp1_path_version_dic[k_path])
        print '<B><font color="red">%s</B></font> was deleted.<br>' % (k_path)

    for k_path in mp2_path_name_dic.keys():
      if k_path not in mp1_path_name_dic.keys():
        new_version_added_list_path_name[k_path] = mp2_path_name_dic[k_path]
        new_version_added_list_path_version[k_path] = mp2_path_version_dic[k_path]
        del(mp2_path_version_dic[k_path])
        print '<B><font color="red">%s</B></font> was added.<br>' % (k_path)

    print '<br><B><font color="blue">3. New commits found today:</font><B><br>'
    print '<table border="1" bordercolor="green">'
    print '<tr><th><B><font color="green">GerritID</font><B></th><th><B><font color="green">Submitter</font><B></th><th><B><font color="green">Subject</font><B></th><th><B><font color="green">Path</font><B></th><th><B><font color="green">SHA-1</font><B></th></tr>'
    #By the above filter add or removed projects, the remained project shall be the same
    if len(mp1_path_version_dic) == len(mp2_path_version_dic):
      for k_path in mp1_path_version_dic.keys():
        mp1_version = mp1_path_version_dic[k_path]
        mp2_version = mp2_path_version_dic[k_path]
        cmd_list_sha1 = 'cd %s/%s ; git log --pretty=oneline %s..%s' % (absolue_path_prefix, k_path, mp1_version, mp2_version)
        if mp1_version != mp2_version:
          cmd_list_output = subprocess.Popen(cmd_list_sha1, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
          for line in cmd_list_output.stdout.readlines():
            sha1 = line[:40]
            cmd_gerrit_query = 'ssh -p %s %s@%s gerrit query --format=JSON --commit-message %s --current-patch-set | sed 2d' % (gerrit_port,gerrit_user,gerrit_ip,sha1)
#            print cmd_gerrit_query
            cmd_gerrit_result = subprocess.Popen(cmd_gerrit_query, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
            cmd_gerrit_output = cmd_gerrit_result.stdout.read()
            try:
              json_str = json.loads(cmd_gerrit_output)
              gerrit_id = json_str["number"]
              gerrit_subject = json_str["subject"]
              gerrit_submitter = json_str["currentPatchSet"]["uploader"]["email"]
              gerrit_path = k_path
              gerrit_sha1 = sha1
              print '<tr><th><a href="%s/%s">%s</a></th><th>%s</th><th>%s</th><th>%s</th><th>%s</th></tr>' % (gerrit_url,gerrit_id, gerrit_id, gerrit_submitter, gerrit_subject, gerrit_path, gerrit_sha1)
            except Exception,e:
              print '<tr><th>%s</th><th>%s</th><th>%s</th><th>%s</th><th><font color="red">Fatal: %s</font></th></tr>' % ('Unknown', 'Unknown', 'Unknown', k_path, sha1)
      print '</table>'