def SetVersion():
input = raw_input('Please input version: ')
if not input:
print 'You canceled SetVersion.'
return
print 'Your version is `%s`.' % input
sys.argv = [sys.argv[0], '-V', input]
if appdir: sys.argv.append(appdir)
uploader.main()
def SetAppID():
input = raw_input('Please input your appid: ')
if not input:
print 'You canceled SetAppID.'
return
print 'Your appid is `%s`.' % input
sys.argv = [sys.argv[0], '-A', input]
if appdir: sys.argv.append(appdir)
uploader.main()
def test_success(self, endpoint, token, filename, user_id, tenant_id,
cookie):
# コマンドライン引数の初期化と設定
del sys.argv[1:]
# ランダムの数値を取得
random_name = random.randint(10000, 99999)
random_name_csv = str(random_name) + ".csv"
random_name_tar = str(random_name) + ".tar"
# headersとcookiesの設定
headers = {'X-Requested-With': 'XMLHttpRequest'}
cookies = dict(s=cookie)
# tarがアップロードされるディレクトリ名前を設定
bucket_name = 'sdeep-upload-dev'
# csvファイルをランダムの名前を付けてコピー
shutil.copy(filename, random_name_csv)
# tokenを取得 エラーの場合のみ値を渡す
if token == "":
token_request = requests.get(
'{}/v1/users/{}/datasets/create'.format(
endpoint, user_id, tenant_id),
headers=headers,
cookies=cookies)
token = json.loads(token_request.text)['encrypted_text']
# 環境変数を設定
sys.argv.append('-e' + endpoint)
sys.argv.append(token)
sys.argv.append(random_name_csv)
# uploaderを実行
uploader.main()
# tarがアップロードされているか確認
s3 = boto3.resource('s3')
bucket = s3.Bucket(bucket_name)
uploader_tar_folder_list = bucket.meta.client.list_objects(
Bucket=bucket.name, Delimiter='/')
tar_check = False
# tarがある階層を再帰的に取得して比較
for uploader_tar_folder in uploader_tar_folder_list.get(
'CommonPrefixes'):
uploader_tar = bucket.meta.client.list_objects(
Bucket=bucket.name,
Prefix=uploader_tar_folder.get('Prefix'),
Delimiter='/')
uploader_tar_name = uploader_tar['Contents']
if random_name_tar in uploader_tar_name[0]['Key']:
tar_check = True
# コピーしたcsvを削除
os.remove(random_name_csv)
self.assertTrue(tar_check)
def sheets_uploader():
logger.info('Starting the uploader...{0}'.format(os.getpid()))
main()
def main(file_details, geo_details, server_details, petrarch_version, logger_file=None, run_filter=None,
run_date='', version=''):
"""
Main function to run all the things.
Parameters
----------
file_details: Named tuple.
All the other config information not in ``server_details``.
geo_details: Named tuple.
Settings for geocoding.
server_details: Named tuple.
Config information specifically related to the remote
server for FTP uploading.
logger_file: String.
Path to a log file. Defaults to ``None`` and opens a
``PHOX_pipeline.log`` file in the current working
directory.
run_filter: String.
Whether to run the ``oneaday_formatter``. Takes True or False
(strings) as values.
run_date: String.
Date of the format YYYYMMDD. The pipeline will run using this
date. If not specified the pipeline will run with
``current_date`` minus one day.
"""
if logger_file:
utilities.init_logger(logger_file)
else:
utilities.init_logger('PHOX_pipeline.log')
# get a local copy for the pipeline
logger = logging.getLogger('pipeline_log')
if petrarch_version == '1':
from petrarch import petrarch
logger.info("Using original Petrarch version")
elif petrarch_version == '2':
from petrarch2 import petrarch2 as petrarch
logger.info("Using Petrarch2")
else:
logger.error("Invalid Petrarch version. Argument must be '1' or '2'")
print('\nPHOX.pipeline run:', datetime.datetime.utcnow())
if run_date:
process_date = dateutil.parser.parse(run_date)
date_string = '{:02d}{:02d}{:02d}'.format(process_date.year,
process_date.month,
process_date.day)
logger.info('Date string: {}'.format(date_string))
print('Date string:', date_string)
else:
process_date = datetime.datetime.utcnow() - datetime.timedelta(days=1)
date_string = '{:02d}{:02d}{:02d}'.format(process_date.year,
process_date.month,
process_date.day)
logger.info('Date string: {}'.format(date_string))
print('Date string:', date_string)
results, scraperfilename = scraper_connection.main(process_date,
file_details)
if geo_details.geo_service == "Mordecai":
dest = "{0}:{1}/places".format(geo_details.mordecai_host, geo_details.mordecai_port)
try:
out = requests.get(dest)
assert out.status_code == 200
except (AssertionError, requests.exceptions.ConnectionError):
print("Mordecai geolocation service not responding. Continuing anyway...")
elif geo_details.geo_service == "CLIFF":
print("CLIFF")
else:
print("Invalid geo service name. Must be 'CLIFF' or 'Mordecai'. Continuing...")
if scraperfilename:
logger.info("Scraper file name: " + scraperfilename)
print("Scraper file name:", scraperfilename)
logger.info("Running Mongo.formatter.py")
print("Running Mongo.formatter.py")
formatted = formatter.main(results, file_details,
process_date, date_string)
logger.info("Running PETRARCH")
file_details.fullfile_stem + date_string
if run_filter == 'False':
print('Running PETRARCH and writing to a file. No one-a-day.')
logger.info('Running PETRARCH and writing to a file. No one-a-day.')
# Command to write output to a file directly from PETR
# petrarch.run_pipeline(formatted,
# '{}{}.txt'.format(file_details.fullfile_stem,
# date_string), parsed=True)
petr_results = petrarch.run_pipeline(formatted, write_output=False,
parsed=True)
elif run_filter == 'True':
print('Running PETRARCH and returning output.')
logger.info('Running PETRARCH and returning output.')
petr_results = petrarch.run_pipeline(formatted, write_output=False,
parsed=True)
else:
print("""Can't run with the options you've specified. You need to fix
something.""")
logger.warning("Can't run with the options you've specified. Exiting.")
sys.exit()
if run_filter == 'True':
logger.info("Running oneaday_formatter.py")
print("Running oneaday_formatter.py")
formatted_results = oneaday_filter.main(petr_results)
else:
logger.info("Running result_formatter.py")
print("Running result_formatter.py")
formatted_results = result_formatter.main(petr_results)
logger.info("Running postprocess.py")
print("Running postprocess.py")
if version:
postprocess.main(formatted_results, date_string, version, file_details,
server_details, geo_details)
else:
print("Please specify a data version number. Program ending.")
logger.info("Running phox_uploader.py")
print("Running phox_uploader.py")
try:
uploader.main(date_string, server_details, file_details)
except Exception as e:
logger.warning("Error on the upload portion. {}".format(e))
print("""Error on the uploader. This step isn't absolutely necessary.
Valid events should still be generated.""")
logger.info('PHOX.pipeline end')
print('PHOX.pipeline end:', datetime.datetime.utcnow())
def main(file_details, server_details, logger_file=None, run_filter=None,
run_date=None):
"""
Main function to run all the things.
Parameters
----------
file_details: Named tuple.
All the other config information not in ``server_details``.
server_details: Named tuple.
Config information specifically related to the remote
server for FTP uploading.
logger_file: String.
Path to a log file. Defaults to ``None`` and opens a
``PHOX_pipeline.log`` file in the current working
directory.
run_filter: String.
Whether to run the ``oneaday_formatter``. Takes True or False
(strings) as values.
run_date: String.
Date of the format YYYYMMDD. The pipeline will run using this
date. If not specified the pipeline will run with
``current_date`` minus one day.
"""
if logger_file:
utilities.init_logger(logger_file)
else:
utilities.init_logger('PHOX_pipeline.log')
# get a local copy for the pipeline
logger = logging.getLogger('pipeline_log')
print('\nPHOX.pipeline run:', datetime.datetime.utcnow())
if run_date:
process_date = dateutil.parser.parse(run_date)
date_string = '{:02d}{:02d}{:02d}'.format(process_date.year,
process_date.month,
process_date.day)
logger.info('Date string: {}'.format(date_string))
print('Date string:', date_string)
else:
process_date = datetime.datetime.utcnow() - datetime.timedelta(days=1)
date_string = '{:02d}{:02d}{:02d}'.format(process_date.year,
process_date.month,
process_date.day)
logger.info('Date string: {}'.format(date_string))
print('Date string:', date_string)
results, scraperfilename = scraper_connection.main(process_date,
file_details)
if scraperfilename:
logger.info("Scraper file name: " + scraperfilename)
print("Scraper file name:", scraperfilename)
logger.info("Running Mongo.formatter.py")
print("Running Mongo.formatter.py")
formatted = formatter.main(results, file_details,
process_date, date_string)
logger.info("Running PETRARCH")
file_details.fullfile_stem + date_string
if run_filter == 'False':
print('Running PETRARCH and writing to a file. No one-a-day.')
logger.info('Running PETRARCH and writing to a file. No one-a-day.')
#Command to write output to a file directly from PETR
# petrarch.run_pipeline(formatted,
# '{}{}.txt'.format(file_details.fullfile_stem,
# date_string), parsed=True)
petr_results = petrarch.run_pipeline(formatted, write_output=False,
parsed=True)
elif run_filter == 'True':
print('Running PETRARCH and returning output.')
logger.info('Running PETRARCH and returning output.')
petr_results = petrarch.run_pipeline(formatted, write_output=False,
parsed=True)
else:
print("""Can't run with the options you've specified. You need to fix
something.""")
logger.warning("Can't run with the options you've specified. Exiting.")
sys.exit()
if run_filter == 'True':
logger.info("Running oneaday_formatter.py")
print("Running oneaday_formatter.py")
formatted_results = oneaday_filter.main(petr_results)
else:
logger.info("Running result_formatter.py")
print("Running result_formatter.py")
formatted_results = result_formatter.main(petr_results)
logger.info("Running postprocess.py")
print("Running postprocess.py")
postprocess.main(formatted_results, date_string, file_details)
logger.info("Running phox_uploader.py")
print("Running phox_uploader.py")
uploader.main(date_string, server_details, file_details)
logger.info('PHOX.pipeline end')
print('PHOX.pipeline end:', datetime.datetime.utcnow())
def main(file_details,
server_details,
logger_file=None,
run_filter=None,
run_date=None):
"""
Main function to run all the things.
Parameters
----------
file_details: Named tuple.
All the other config information not in ``server_details``.
server_details: Named tuple.
Config information specifically related to the remote
server for FTP uploading.
logger_file: String.
Path to a log file. Defaults to ``None`` and opens a
``PHOX_pipeline.log`` file in the current working
directory.
run_filter: String.
Whether to run the ``oneaday_formatter``. Takes True or False
(strings) as values.
run_date: String.
Date of the format YYYYMMDD. The pipeline will run using this
date. If not specified the pipeline will run with
``current_date`` minus one day.
"""
if logger_file:
utilities.init_logger(logger_file)
else:
utilities.init_logger('PHOX_pipeline.log')
# get a local copy for the pipeline
logger = logging.getLogger('pipeline_log')
print('\nPHOX.pipeline run:', datetime.datetime.utcnow())
if run_date:
process_date = dateutil.parser.parse(run_date)
date_string = '{:02d}{:02d}{:02d}'.format(process_date.year,
process_date.month,
process_date.day)
logger.info('Date string: {}'.format(date_string))
print('Date string:', date_string)
else:
process_date = datetime.datetime.utcnow() - datetime.timedelta(days=1)
date_string = '{:02d}{:02d}{:02d}'.format(process_date.year,
process_date.month,
process_date.day)
logger.info('Date string: {}'.format(date_string))
print('Date string:', date_string)
results, scraperfilename = scraper_connection.main(process_date,
file_details)
if scraperfilename:
logger.info("Scraper file name: " + scraperfilename)
print("Scraper file name:", scraperfilename)
logger.info("Running Mongo.formatter.py")
print("Running Mongo.formatter.py")
formatted = formatter.main(results, file_details, process_date,
date_string)
logger.info("Running PETRARCH")
file_details.fullfile_stem + date_string
if run_filter == 'False':
print('Running PETRARCH and writing to a file. No one-a-day.')
logger.info('Running PETRARCH and writing to a file. No one-a-day.')
#Command to write output to a file directly from PETR
# petrarch.run_pipeline(formatted,
# '{}{}.txt'.format(file_details.fullfile_stem,
# date_string), parsed=True)
petr_results = petrarch.run_pipeline(formatted,
write_output=False,
parsed=True)
elif run_filter == 'True':
print('Running PETRARCH and returning output.')
logger.info('Running PETRARCH and returning output.')
petr_results = petrarch.run_pipeline(formatted,
write_output=False,
parsed=True)
else:
print("""Can't run with the options you've specified. You need to fix
something.""")
logger.warning("Can't run with the options you've specified. Exiting.")
sys.exit()
if run_filter == 'True':
logger.info("Running oneaday_formatter.py")
print("Running oneaday_formatter.py")
formatted_results = oneaday_filter.main(petr_results)
else:
logger.info("Running result_formatter.py")
print("Running result_formatter.py")
formatted_results = result_formatter.main(petr_results)
logger.info("Running postprocess.py")
print("Running postprocess.py")
postprocess.main(formatted_results, date_string, file_details)
logger.info("Running phox_uploader.py")
print("Running phox_uploader.py")
uploader.main(date_string, server_details, file_details)
logger.info('PHOX.pipeline end')
print('PHOX.pipeline end:', datetime.datetime.utcnow())
def Rollback():
sys.argv = [sys.argv[0], 'rollback']
if appdir: sys.argv.append(appdir)
uploader.main()
def Upload():
sys.argv = [sys.argv[0], 'update']
if appdir: sys.argv.append(appdir)
uploader.main()
#!/usr/bin/env python
# Copyright (C) 2010-2011 | GNU GPLv3
__author__ = 'ZHRtYWppYUAxNjMuY29t'.decode('base64')
import sys, os, traceback
dir = os.path.abspath(os.path.dirname(sys.argv[0]))
sys.path.append(os.path.join(dir, 'src.zip'))
import uploader
if len(sys.argv)>1 and sys.argv[1] in ('-A','-V','update','rollback'):
uploader.main()
sys.exit(0)
appdir = None if len(sys.argv)==1 else sys.argv[1]
def Menu():
print '[1]SetAppID [2]SetVersion [3]Upload [4]Rollback(IfConflict) [5]SetProxy [6]Exit'
input = raw_input('Please select[1-6]: ')
try:
return int(input[0])
except:
return 0
def SetAppID():
input = raw_input('Please input your appid: ')
if not input:
print 'You canceled SetAppID.'
return
print 'Your appid is `%s`.' % input
sys.argv = [sys.argv[0], '-A', input]
if appdir: sys.argv.append(appdir)
def main(file_details, geo_details, server_details, petrarch_version, mongo_details, logger_file=None, run_filter=None,
run_date='', version=''):
"""
Main function to run all the things.
Parameters
----------
file_details: Named tuple.
All the other config information not in ``server_details``.
geo_details: Named tuple.
Settings for geocoding.
server_details: Named tuple.
Config information specifically related to the remote
server for FTP uploading.
petrarch_version: String.
Which version of Petrarch to use. Must be '1' or '2'
logger_file: String.
Path to a log file. Defaults to ``None`` and opens a
``PHOX_pipeline.log`` file in the current working
directory.
run_filter: String.
Whether to run the ``oneaday_formatter``. Takes True or False
(strings) as values.
run_date: String.
Date of the format YYYYMMDD. The pipeline will run using this
date. If not specified the pipeline will run with
``current_date`` minus one day.
"""
if logger_file:
utilities.init_logger(logger_file)
else:
utilities.init_logger('PHOX_pipeline.log')
# get a local copy for the pipeline
logger = logging.getLogger('pipeline_log')
if petrarch_version == '1':
from petrarch import petrarch
logger.info("Using original Petrarch version")
elif petrarch_version == '2':
from petrarch2 import petrarch2 as petrarch
logger.info("Using Petrarch2")
else:
logger.error("Invalid Petrarch version. Argument must be '1' or '2'")
print('\nPHOX.pipeline run:', datetime.datetime.utcnow())
if run_date:
process_date = dateutil.parser.parse(run_date)
date_string = '{:02d}{:02d}{:02d}'.format(process_date.year,
process_date.month,
process_date.day)
logger.info('Date string: {}'.format(date_string))
print('Date string:', date_string)
else:
process_date = datetime.datetime.utcnow() - datetime.timedelta(days=1)
date_string = '{:02d}{:02d}{:02d}'.format(process_date.year,
process_date.month,
process_date.day)
logger.info('Date string: {}'.format(date_string))
print('Date string:', date_string)
results, scraperfilename = scraper_connection.main(process_date,
file_details)
if geo_details.geo_service == "Mordecai":
dest = "{0}:{1}/places".format(geo_details.mordecai_host, geo_details.mordecai_port)
try:
out = requests.get(dest)
assert out.status_code == 200
except (AssertionError, requests.exceptions.ConnectionError):
print("Mordecai geolocation service not responding. Continuing anyway...")
elif geo_details.geo_service == "CLIFF":
print("CLIFF")
else:
print("Invalid geo service name. Must be 'CLIFF' or 'Mordecai'. Continuing...")
if scraperfilename:
logger.info("Scraper file name: " + scraperfilename)
print("Scraper file name:", scraperfilename)
logger.info("Running Mongo.formatter.py")
print("Running Mongo.formatter.py")
formatted = formatter.main(results, file_details,
process_date, date_string)
logger.info("Running PETRARCH")
file_details.fullfile_stem + date_string
if run_filter == 'False':
print('Running PETRARCH and writing to a file. No one-a-day.')
logger.info('Running PETRARCH and writing to a file. No one-a-day.')
# Command to write output to a file directly from PETR
# petrarch.run_pipeline(formatted,
# '{}{}.txt'.format(file_details.fullfile_stem,
# date_string), parsed=True)
petr_results = petrarch.run_pipeline(formatted, config = "petr_config.ini", write_output=False,
parsed=True)
elif run_filter == 'True':
print('Running PETRARCH and returning output.')
logger.info('Running PETRARCH and returning output.')
petr_results = petrarch.run_pipeline(formatted, config = "petr_config.ini", write_output=False,
parsed=True)
else:
print("""Can't run with the options you've specified. You need to fix
something.""")
logger.warning("Can't run with the options you've specified. Exiting.")
sys.exit()
if run_filter == 'True':
logger.info("Running oneaday_formatter.py")
print("Running oneaday_formatter.py")
formatted_results = oneaday_filter.main(petr_results)
else:
logger.info("Running result_formatter.py")
print("Running result_formatter.py")
formatted_results = result_formatter.main(petr_results)
logger.info("Running postprocess.py")
print("Running postprocess.py")
if version:
postprocess.main(formatted_results, date_string, version, file_details,
server_details, geo_details)
else:
print("Please specify a data version number. Program ending.")
logger.info("Running phox_uploader.py")
print("Running phox_uploader.py")
try:
uploader.main(date_string, server_details, file_details)
except Exception as e:
logger.warning("Error on the upload portion. {}".format(e))
print("""Error on the uploader. This step isn't absolutely necessary.
Valid events should still be generated.""")
logger.info('PHOX.pipeline end')
print('PHOX.pipeline end:', datetime.datetime.utcnow())
#! /usr/bin/python import retriever import uploader retriever.main(3) uploader.main()
