def iveGotToMoveIt(source,destination, array):
for x in range(len(array)):
src = (source+array[x])
dest = (destination+array[x])
shutil.copytree(src,dest)
collectionRes = (src + " =>" + dest)
log.logger(collectionRes)
def get_hist_data(code, ktype, start=None, end=None):
"""
股票历史数据
:param code: 股票代码
:param ktype: K线类型
:param start: 开始时间
:param end: 结束时间
:return:
"""
hist_data_list = []
df = ts.get_hist_data(code=code, start=start, end=end, ktype=ktype)
df.reset_index(inplace=True)
if len(df.index) == 0:
log.logger().info("no data found for: " + code + " start date: " +
start + "end date: " + end + "K type: " + ktype)
return
indicator(df)
size = df.iloc[:, 0].size
if size == 0:
return
df = df.replace([np.inf, -np.inf], np.nan)
df = df.astype(object).where(pd.notnull(df), None)
return hist_data_list
def view(self, avid, page=1):
params = {"id": avid, "page": page}
sign = self.get_sign(params)
log.logger().info("get: " + self.__urls["view"] + "?" + sign)
res = self.opener.open(self.__urls["view"] + "?" + sign).read()
return Subject(json.loads(res.decode()))
def push_to_queue(self, movies):
for movie_json in movies:
movie = model.MovieQueue.select().where(model.MovieQueue.douban_id == movie_json["id"]).limit(1).first()
if movie is None:
model.MovieQueue.create(douban_id=movie_json["id"])
log.logger().info("{0}, {1} push into queue".format(movie_json["id"], movie_json["title"]))
else:
log.logger().info("movie: {0}, {1} exist".format(movie_json["id"], movie_json["title"]))
def download(file_name, ftp, output, download_path):
TempFile = open(output+'/'+file_name, "wb")
ftp.cwd(download_path)
start = time.time()
ftp.retrbinary('RETR '+ file_name, TempFile.write)
end = time.time()
TempFile.close()
log.logger("*****Download operation*****", end-start,file_name, ftp.size(file_name), datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
print file_name+" is downloaded!"
def upload(file_name, ftp, upload_path):
ftp.cwd(upload_path)
TempFile = open(file_name,"rb")
start = time.time()
ftp.storbinary("STOR "+file_name,TempFile)
end = time.time()
log.logger("*****Upload operation*****", end-start,file_name, os.stat(file_name).st_size, datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
TempFile.close()
print file_name + " is uploaded!"
def on_message(self, client, userdata, msg):
msg_topic = msg.topic
try:
msg_payload = msg.payload.decode('utf-8')
except UnicodeDecodeError:
msg_payload = f'binary:{dump_hex(msg.payload)}'
# Using log to record.
logger().info(f"Received topic: {msg_topic}")
logger().info(f"Received Msg: {msg_payload}")
def register():
name_entry = name.get()
age_entry = age.get()
sex_entry = sex.get()
email_entry = email.get()
pass1_entry = pass1.get()
registerData = [
name_entry, age_entry, sex_entry, email_entry, pass1_entry
]
logger(registerData)
login_open()
def scanFolder(folder):
badFiles = []
#flag = 0
for dirpath, dirnames, filenames in os.walk(folder):
for filename in [f for f in filenames if f.endswith(find)]:
#flag = 1
toLog = (os.path.join(dirpath, filename))
log.logger(toLog)
#badFiles.append((dirpath, filename))
badFiles.append(toLog)
#print(badFiles)
if badFiles:
return badFiles
def start(self, settings, workflow=None):
# Log
identity = "starter_%s" % int(random.random() * 1000)
logFile = "starter.log"
#logFile = None
logger = log.logger(logFile, settings.setLevel, identity)
# Simple connect
conn = boto.swf.layer1.Layer1(settings.aws_access_key_id, settings.aws_secret_access_key)
if workflow is not None:
(workflow_id, workflow_name, workflow_version, \
child_policy, execution_start_to_close_timeout, \
input) = self.get_workflow_params(workflow, settings)
# Start a workflow execution
try:
response = conn.start_workflow_execution(settings.domain, workflow_id, \
workflow_name, workflow_version, \
settings.default_task_list, child_policy, \
execution_start_to_close_timeout, input)
logger.info('got response: \n%s' % json.dumps(response, sort_keys=True, indent=4))
except boto.swf.exceptions.SWFWorkflowExecutionAlreadyStartedError:
# There is already a running workflow with that ID, cannot start another
message = ('SWFWorkflowExecutionAlreadyStartedError: There is already ' +
'a running workflow with ID %s' % workflow_id)
print message
logger.info(message)
def test(self, command):
"""Should the alias be triggered by the text?"""
match = self.re_alias.search(command)
if match:
log = logger("client")
log.debug("Executing the alias {}".format(
repr(self.alias)))
engine = self.sharp_engine
if "args" not in engine.locals:
engine.locals["args"] = {}
args = engine.locals["args"]
# Copy the groups of this match
i = 0
for group in match.groups():
i += 1
args[str(i)] = group
# Copy the named groups
for name, group in match.groupdict().items():
engine.locals[name] = group
# Execute the alias
self.execute()
return True
return False
def start(self,
ENV="dev",
all=None,
last_updated_since=None,
docs=None,
doi_id=None):
# Specify run environment settings
settings = settingsLib.get_settings(ENV)
# Log
identity = "starter_%s" % int(random.random() * 1000)
logFile = "starter.log"
#logFile = None
logger = log.logger(logFile, settings.setLevel, identity)
# Simple connect
conn = boto.swf.layer1.Layer1(settings.aws_access_key_id,
settings.aws_secret_access_key)
if (all == True):
# Publish all articles, use SimpleDB as the source
docs = self.get_docs_from_SimpleDB(ENV)
elif (doi_id is not None):
docs = self.get_docs_from_SimpleDB(ENV, doi_id=doi_id)
elif (last_updated_since is not None):
# Publish only articles since the last_modified date, use SimpleDB as the source
docs = self.get_docs_from_SimpleDB(ENV, last_updated_since)
if (docs):
for doc in docs:
document = doc["document"]
elife_id = doc["elife_id"]
id_string = elife_id
# Start a workflow execution
workflow_id = "PublishArticle_%s" % (id_string)
workflow_name = "PublishArticle"
workflow_version = "1"
child_policy = None
execution_start_to_close_timeout = None
input = '{"data": ' + json.dumps(doc) + '}'
try:
response = conn.start_workflow_execution(
settings.domain, workflow_id, workflow_name,
workflow_version, settings.default_task_list,
child_policy, execution_start_to_close_timeout, input)
logger.info('got response: \n%s' %
json.dumps(response, sort_keys=True, indent=4))
except boto.swf.exceptions.SWFWorkflowExecutionAlreadyStartedError:
# There is already a running workflow with that ID, cannot start another
message = 'SWFWorkflowExecutionAlreadyStartedError: There is already a running workflow with ID %s' % workflow_id
print message
logger.info(message)
def start(self, ENV = "dev"):
# Specify run environment settings
settings = settingsLib.get_settings(ENV)
ping_marker_id = "cron_NewS3FullArticle"
# Log
logFile = "starter.log"
logger = log.logger(logFile, settings.setLevel, ping_marker_id)
# Data provider
db = dblib.SimpleDB(settings)
db.connect()
# SWF meta data provider
swfmeta = swfmetalib.SWFMeta(settings)
swfmeta.connect()
last_startTimestamp = swfmeta.get_last_completed_workflow_execution_startTimestamp(workflow_id = ping_marker_id)
# Start a ping workflow as a marker
self.start_ping_marker(ping_marker_id, ENV)
# Check for S3 XML files that were updated since the last run
date_format = "%Y-%m-%dT%H:%M:%S.000Z"
# Quick hack - subtract 15 minutes,
# the time between S3Monitor running and this cron starter
if last_startTimestamp is not None:
last_startTimestamp_minus_15 = last_startTimestamp - (60*15)
else:
# On the first run ever the last start timestamp will be unavailable
last_startTimestamp_minus_15 = time.gmtime() - (60*15)
time_tuple = time.gmtime(last_startTimestamp_minus_15)
last_startDate = time.strftime(date_format, time_tuple)
logger.info('last run %s' % (last_startDate))
#
file_list = db.elife_get_production_final_delivery_S3_file_items(last_updated_since = last_startDate)
logger.info('Full Article files updated since %s: %s' % (last_startDate, str(len(file_list))))
if(len(file_list) <= 0):
# No new XML
pass
else:
# Found new XML files
# Start a PackagePOA starter
try:
starter_name = "starter_PublishFullArticleZip"
self.import_starter_module(starter_name, logger)
s = self.get_starter_module(starter_name, logger)
s.start(ENV = ENV, last_updated_since = last_startDate)
except:
logger.info('Error: %s starting %s' % (ping_marker_id, starter_name))
logger.exception('')
def main():
os.chdir(os.path.dirname(__file__))
logger = log.logger()
info('Starting ETL of LEIE data.')
# Figure out where we put data
datadir = get_datadir()
dbdir = get_dbdir()
# Get a database connection, create db if needed
conn = model.LEIE("development",
db_conf_file=os.path.join(dbdir, "dbconf.yml"))
# Make sure the db schema is up to date, create tables, etc.
conn.migrate()
assert os.path.exists(datadir)
# Do our ETL
download(datadir, conn)
excl = Exclusions(conn)
excl.etl_from_dir(datadir)
rein = Reinstatements(conn)
rein.etl_from_dir(datadir)
# Close the db connection
conn.close()
info('Finished ETL of LEIE data.')
def __init__(self, version):
self.version = version
self.stopped = False
self.tc = 0
self.th = {}
self.sjid = u'%s@%s/%s' % (config.USER, config.SERVER, config.RESOURCE)
self.jid = jid.JID(self.sjid)
self.onauthd = None
self.a = XMPPAuthenticator(self.jid, config.PASSWD)
self.c = ClientFactory(self.a, self)
self.c.maxRetries = 0
self.c.addBootstrap(xmlstream.STREAM_AUTHD_EVENT, self.authd)
self.c.addBootstrap(xmlstream.INIT_FAILED_EVENT, self.initfailed)
self.c.addBootstrap(xmlstream.STREAM_CONNECTED_EVENT, self.onConnected)
self.c.addBootstrap(xmlstream.STREAM_END_EVENT, self.onDisconnected)
self.x = None
self.log = log.logger()
self.msghandlers = []
port = config.PORT
if config.CONNECT_SERVER: server = config.CONNECT_SERVER
else: server = config.SERVER
if config.USE_SSL:
from twisted.internet import ssl
reactor.connectSSL(server, port, self.c, ssl.ClientContextFactory())
else: reactor.connectTCP(server, port, self.c)
def start(self, ENV = "dev", workflow = "Ping"):
# Specify run environment settings
settings = settingsLib.get_settings(ENV)
# Log
identity = "starter_%s" % int(random.random() * 1000)
logFile = "starter.log"
#logFile = None
logger = log.logger(logFile, settings.setLevel, identity)
# Simple connect
conn = boto.swf.layer1.Layer1(settings.aws_access_key_id, settings.aws_secret_access_key)
if(workflow):
(workflow_id, workflow_name, workflow_version, child_policy, execution_start_to_close_timeout, input) = self.get_workflow_params(workflow)
logger.info('Starting workflow: %s' % workflow_id)
try:
response = conn.start_workflow_execution(settings.domain, workflow_id, workflow_name, workflow_version, settings.default_task_list, child_policy, execution_start_to_close_timeout, input)
logger.info('got response: \n%s' % json.dumps(response, sort_keys=True, indent=4))
except boto.swf.exceptions.SWFWorkflowExecutionAlreadyStartedError:
# There is already a running workflow with that ID, cannot start another
message = 'SWFWorkflowExecutionAlreadyStartedError: There is already a running workflow with ID %s' % workflow_id
print message
logger.info(message)
def main():
parser = OptionParser()
parser.add_option("-e", "--env", default="dev", action="store", type="string", dest="env",
help="set the environment to run, either dev or live")
(options, args) = parser.parse_args()
if options.env:
env = options.env
global settings
settings = settings_lib.get_settings(env)
log_file = "process_dashboard_queue.log"
global logger
logger = log.logger(log_file, settings.log_level)
# Simple connect
queue = get_queue()
pool = Pool(settings.event_queue_pool_size)
while True:
messages = queue.get_messages(num_messages=settings.event_queue_message_count, visibility_timeout=60,
wait_time_seconds=20)
if messages is not None:
logger.info(str(len(messages)) + " message received")
pool.map(process_message, messages)
else:
logger.info("No messages received")
def start(self, settings, info=None):
# TODO : much of this is common to many starters and could probably be streamlined
# Log
identity = "starter_%s" % int(random.random() * 1000)
log_file = "starter.log"
# logFile = None
logger = log.logger(log_file, settings.setLevel, identity)
if info.file_name is None:
logger.error("Did not get a filename")
return
# Simple connect
conn = boto.swf.layer1.Layer1(settings.aws_access_key_id, settings.aws_secret_access_key)
# Start a workflow execution
workflow_id = "NewS3File_%s" % info.file_name + str(int(random.random() * 1000))
workflow_name = "NewS3File"
workflow_version = "1"
child_policy = None
execution_start_to_close_timeout = None
workflow_input = json.dumps(info, default=lambda ob: ob.__dict__)
try:
response = conn.start_workflow_execution(settings.domain, workflow_id, workflow_name, workflow_version,
settings.default_task_list, child_policy,
execution_start_to_close_timeout, workflow_input)
logger.info('got response: \n%s' % json.dumps(response, sort_keys=True, indent=4))
except boto.swf.exceptions.SWFWorkflowExecutionAlreadyStartedError:
# There is already a running workflow with that ID, cannot start another
message = 'SWFWorkflowExecutionAlreadyStartedError: There is already a running workflow with ID %s' % workflow_id
logger.info(message)
def __init__(self, loginId=None, password=None):
self._loginId = loginId
self._password = password
self._logined = False
self.logger = log.logger(loginId)
self.opener = buildOpener("resource/cookie_%s.log" % loginId)
self.ignore_error = False
def __init__(self, name, config=None, options=None, args=None, logname=None, **kwargs):
self.name = name
self.__dict__.update(kwargs)
if not options:
options, args = _parser.parser('cskpUPufmvVFw').parse_args()
args = [_decode(arg) for arg in args]
self.options, self.args = options, args
self.name = name
self.logname = logname
self.ql = None
config2 = _config.CONFIG.copy()
if config:
config2.update(config)
if getattr(options, 'config_file', None):
options.config_file = os.path.abspath(options.config_file) # XXX useful during testing. could be generalized with optparse callback?
if not getattr(options, 'service', True):
options.foreground = True
self.config = _config.Config(config2, service=name, options=options)
self.config.data['server_socket'] = os.getenv("KOPANO_SOCKET") or self.config.data['server_socket']
if getattr(options, 'worker_processes', None):
self.config.data['worker_processes'] = options.worker_processes
self.log = _log.logger(self.logname or self.name, options=self.options, config=self.config) # check that this works here or daemon may die silently XXX check run_as_user..?
for msg in self.config.warnings:
self.log.warn(msg)
if self.config.errors:
for msg in self.config.errors:
self.log.error(msg)
sys.exit(1)
self.stats = collections.defaultdict(int, {'errors': 0})
self._server = None
def decide(ENV = "dev"):
# Specify run environment settings
settings = settingsLib.get_settings(ENV)
# Decider event history length requested
maximum_page_size = 100
# Log
identity = "decider_%s" % int(random.random() * 1000)
logFile = "decider.log"
#logFile = None
logger = log.logger(logFile, settings.setLevel, identity)
# Simple connect
conn = boto.swf.layer1.Layer1(settings.aws_access_key_id, settings.aws_secret_access_key)
token = None
# Poll for a decision task
while(True):
if(token == None):
logger.info('polling for decision...')
decision = conn.poll_for_decision_task(settings.domain, settings.default_task_list, identity, maximum_page_size)
# Check for a nextPageToken and keep polling until all events are pulled
decision = get_all_paged_events(decision, conn, settings.domain, settings.default_task_list, identity, maximum_page_size)
token = get_taskToken(decision)
logger.info('got decision: [json omitted], token %s' % token)
#logger.info('got decision: \n%s' % json.dumps(decision, sort_keys=True, indent=4))
if(token != None):
# Get the workflowType and attempt to do the work
workflowType = get_workflowType(decision)
if(workflowType != None):
logger.info('workflowType: %s' % workflowType)
# Instantiate and object for the workflow using eval
# Build a string for the object name
workflow_name = get_workflow_name(workflowType)
# Attempt to import the module for the workflow
if(import_workflow_class(workflow_name)):
# Instantiate the workflow object
workflow_object = get_workflow_object(workflow_name, settings, logger, conn, token, decision, maximum_page_size)
# Process the workflow
success = workflow_object.do_workflow()
# Print the result to the log
logger.info('%s success %s' % (workflow_name, success))
else:
logger.info('error: could not load object %s\n' % workflow_name)
# Reset and loop
token = None
def start(self, ENV = "dev"):
# Specify run environment settings
settings = settingsLib.get_settings(ENV)
ping_marker_id = "cron_NewS3FiguresPDF"
# Log
logFile = "starter.log"
logger = log.logger(logFile, settings.setLevel, ping_marker_id)
# Data provider
db = dblib.SimpleDB(settings)
db.connect()
# SWF meta data provider
swfmeta = swfmetalib.SWFMeta(settings)
swfmeta.connect()
# Default, if cron never run before
last_startTimestamp = 0
# Get the last time this cron was run
last_startTimestamp = swfmeta.get_last_completed_workflow_execution_startTimestamp(workflow_id = ping_marker_id)
# Start a ping workflow as a marker
self.start_ping_marker(ping_marker_id, ENV)
# Check for S3 PDF files that were updated since the last run
date_format = "%Y-%m-%dT%H:%M:%S.000Z"
# Quick hack - subtract 30 minutes to not ignore the top of the hour
# the time between S3Monitor running and this cron starter
last_startTimestamp_minus_30 = last_startTimestamp - (60*30)
if(last_startTimestamp_minus_30 < 0):
last_startTimestamp_minus_30 = 0
time_tuple = time.gmtime(last_startTimestamp_minus_30)
last_startDate = time.strftime(date_format, time_tuple)
logger.info('last run %s' % (last_startDate))
S3_item_list = db.elife_get_article_S3_file_items(file_data_type = "figures", latest = True, last_updated_since = last_startDate)
logger.info('Figures PDF files updated since %s: %s' % (last_startDate, str(len(S3_item_list))))
if(len(S3_item_list) <= 0):
# No new PDF
pass
else:
# Found new PDF files
# Start a PublishPDF starter
try:
starter_name = "starter_PublishFiguresPDF"
self.import_starter_module(starter_name, logger)
s = self.get_starter_module(starter_name, logger)
s.start(ENV = ENV, last_updated_since = last_startDate)
except:
logger.info('Error: %s starting %s' % (ping_marker_id, starter_name))
logger.exception('')
def start(self, ENV = "dev", workflow = "S3Monitor"):
# Specify run environment settings
settings = settingsLib.get_settings(ENV)
# Log
identity = "starter_%s" % int(random.random() * 1000)
logFile = "starter.log"
#logFile = None
logger = log.logger(logFile, settings.setLevel, identity)
# Simple connect
conn = boto.swf.layer1.Layer1(settings.aws_access_key_id, settings.aws_secret_access_key)
if(workflow):
(workflow_id, workflow_name, workflow_version, child_policy, execution_start_to_close_timeout, input) = self.get_workflow_params(workflow, settings)
logger.info('Starting workflow: %s' % workflow_id)
try:
response = conn.start_workflow_execution(settings.domain, workflow_id, workflow_name, workflow_version, settings.default_task_list, child_policy, execution_start_to_close_timeout, input)
logger.info('got response: \n%s' % json.dumps(response, sort_keys=True, indent=4))
except boto.swf.exceptions.SWFWorkflowExecutionAlreadyStartedError:
# There is already a running workflow with that ID, cannot start another
message = 'SWFWorkflowExecutionAlreadyStartedError: There is already a running workflow with ID %s' % workflow_id
print message
logger.info(message)
def detach(database):
print("Detaching ", database + ".dbo")
try:
cursor2.execute(sqlNoResultsExpected)
cursor.execute(sqlDETACH + database + sqlDETACHend)
#print(sqlDETACH+database+sqlDETACHend)
#print ("Great sucess?")
collectionLog = (database, "successfully detached")
log.logger(collectionLog)
print(database, "succesfully detached")
except pyodbc.ProgrammingError as err:
print(err)
print("The Database is already detached")
collErr = (database, "Already appears to be detached see err \n", err)
log.logger(collErr)
def start(self, ENV = "dev"):
# Specify run environment settings
settings = settingsLib.get_settings(ENV)
# Log
identity = "starter_%s" % int(random.random() * 1000)
logFile = "starter.log"
#logFile = None
logger = log.logger(logFile, settings.setLevel, identity)
# Simple connect
conn = boto.swf.layer1.Layer1(settings.aws_access_key_id, settings.aws_secret_access_key)
start = True
# Start a workflow execution
workflow_id = "LensIndexPublish"
workflow_name = "LensIndexPublish"
workflow_version = "1"
child_policy = None
execution_start_to_close_timeout = str(60*45)
input = None
if(start):
response = conn.start_workflow_execution(settings.domain, workflow_id, workflow_name, workflow_version, settings.default_task_list, child_policy, execution_start_to_close_timeout, input)
logger.info('got response: \n%s' % json.dumps(response, sort_keys=True, indent=4))
def __init__(self, version):
#self.lastsent = {}
#self.queues = {}
self.version = version
self.tc = 0
self.th = {}
self.sjid = u'%s@%s/%s' % (config.USER, config.SERVER, config.RESOURCE)
self.jid = jid.JID(self.sjid)
self.onauthd = None
self.tryingSRV = True
self.tryingNonSASL = False
self.tryingSASL = True
#self.c = client.basicClientFactory(self.jid, config.PASSWD)
a = XMPPAndGoogleAuthenticator(self.jid, config.PASSWD, self)
self.c = ClientFactory(a,self)
self.c.maxRetries = 0
self.c.addBootstrap(xmlstream.STREAM_AUTHD_EVENT, self.authd)
self.c.addBootstrap(xmlstream.INIT_FAILED_EVENT, self.initfailed)
self.c.addBootstrap(xmlstream.STREAM_CONNECTED_EVENT, self.onConnected)
self.c.addBootstrap(xmlstream.STREAM_END_EVENT, self.onDisconnected)
self.x = None
self.log = log.logger()
self.handlers = []
self.msghandlers = []
#self.send_from_queue(True)
#self.clean_queue()
port = config.PORT
server = config.CONNECT_SERVER
if not server: server = config.SERVER
self.connector = XMPPClientConnector(reactor, server, self.c, port)
self.connector.connect()
def start(self, ENV = "dev"):
# Specify run environment settings
settings = settingsLib.get_settings(ENV)
# Log
identity = "starter_%s" % int(random.random() * 1000)
logFile = "starter.log"
#logFile = None
logger = log.logger(logFile, settings.setLevel, identity)
# Simple connect
conn = boto.swf.layer1.Layer1(settings.aws_access_key_id, settings.aws_secret_access_key)
start = True
# Start a workflow execution
workflow_id = "LensIndexPublish"
workflow_name = "LensIndexPublish"
workflow_version = "1"
child_policy = None
execution_start_to_close_timeout = str(60*45)
input = None
if(start):
response = conn.start_workflow_execution(settings.domain, workflow_id, workflow_name, workflow_version, settings.default_task_list, child_policy, execution_start_to_close_timeout, input)
logger.info('got response: \n%s' % json.dumps(response, sort_keys=True, indent=4))
def start(self, settings, limit=None):
# Log
identity = "starter_%s" % int(random.random() * 1000)
logFile = "starter.log"
logger = log.logger(logFile, settings.setLevel, identity)
# Simple connect
conn = boto.swf.layer1.Layer1(settings.aws_access_key_id, settings.aws_secret_access_key)
# Start a workflow execution
workflow_id = "SendQueuedEmail"
workflow_name = "SendQueuedEmail"
workflow_version = "1"
child_policy = None
execution_start_to_close_timeout = None
if limit:
input = '{"data": {"limit": "' + limit + '"}}'
else:
input = None
try:
response = conn.start_workflow_execution(settings.domain, workflow_id, workflow_name,
workflow_version, settings.default_task_list,
child_policy, execution_start_to_close_timeout,
input)
logger.info('got response: \n%s' % json.dumps(response, sort_keys=True, indent=4))
except boto.swf.exceptions.SWFWorkflowExecutionAlreadyStartedError:
# There is already a running workflow with that ID, cannot start another
message = ('SWFWorkflowExecutionAlreadyStartedError: There is already ' +
'a running workflow with ID %s' % workflow_id)
print message
logger.info(message)
def test(self, command):
"""Should the alias be triggered by the text?"""
match = self.re_alias.search(command)
if match:
log = logger("client")
log.debug("Executing the alias {}".format(repr(self.alias)))
engine = self.sharp_engine
if "args" not in engine.locals:
engine.locals["args"] = {}
args = engine.locals["args"]
# Copy the groups of this match
i = 0
for group in match.groups():
i += 1
args[str(i)] = group
# Copy the named groups
for name, group in match.groupdict().items():
engine.locals[name] = group
# Execute the alias
self.execute()
return True
return False
def start(self, ENV = "dev"):
# Specify run environment settings
settings = settingsLib.get_settings(ENV)
ping_marker_id = "cron_NewS3Suppl"
# Log
logFile = "starter.log"
logger = log.logger(logFile, settings.setLevel, ping_marker_id)
# Data provider
db = dblib.SimpleDB(settings)
db.connect()
# SWF meta data provider
swfmeta = swfmetalib.SWFMeta(settings)
swfmeta.connect()
# Default, if cron never run before
last_startTimestamp = 0
# Get the last time this cron was run
last_startTimestamp = swfmeta.get_last_completed_workflow_execution_startTimestamp(workflow_id = ping_marker_id)
# Start a ping workflow as a marker
self.start_ping_marker(ping_marker_id, ENV)
# Check for S3 Suppl files that were updated since the last run
date_format = "%Y-%m-%dT%H:%M:%S.000Z"
# Quick hack - subtract 30 minutes to not ignore the top of the hour
# the time between S3Monitor running and this cron starter
last_startTimestamp_minus_30 = last_startTimestamp - (60*30)
if(last_startTimestamp_minus_30 < 0):
last_startTimestamp_minus_30 = 0
time_tuple = time.gmtime(last_startTimestamp_minus_30)
last_startDate = time.strftime(date_format, time_tuple)
logger.info('last run %s' % (last_startDate))
S3_item_list = db.elife_get_article_S3_file_items(file_data_type = "suppl", latest = True, last_updated_since = last_startDate)
logger.info('Suppl files updated since %s: %s' % (last_startDate, str(len(S3_item_list))))
if(len(S3_item_list) <= 0):
# No new SVG
pass
else:
# Found new SVG files
# Start a PublishSVG starter
try:
starter_name = "starter_PublishSuppl"
self.import_starter_module(starter_name, logger)
s = self.get_starter_module(starter_name, logger)
s.start(ENV = ENV, last_updated_since = last_startDate)
except:
logger.info('Error: %s starting %s' % (ping_marker_id, starter_name))
logger.exception('')
def handle(self, msg):
self.chat_id = msg['chat']['id'] #WTF?
self.command = msg['text']
print(self.command, self.chat_id)
data_logger = log.logger()
data_logger.do_log(self.chat_id, self.command)
#Check if this ID is registered
if (self.password.check_id(self.chat_id) == True):
print("Id rilevato in lista")
self.id_check = 1
else:
self.id_check = 0
if (self.id_check == 0):
if (self.command == local_opt.PSWD):
self.bot.sendMessage(self.chat_id, 'Chat registrata!')
self.password.insert_db_file(self.chat_id)
self.password.load_db_file()
else:
self.bot.sendMessage(self.chat_id,
'Chat non registrata. Inserire password')
elif self.id_check == 1:
if self.command == '/get':
self.img_take(self.chat_id)
if self.command == '/start':
self.send_periodic = True
if self.command == '/stop':
self.send_periodic = False
def start(self, settings):
# Log
identity = "starter_%s" % int(random.random() * 1000)
logFile = "starter.log"
#logFile = None
logger = log.logger(logFile, settings.setLevel, identity)
# Simple connect
conn = boto.swf.layer1.Layer1(settings.aws_access_key_id,
settings.aws_secret_access_key)
# Start a workflow execution
workflow_id = "DepositCrossref"
workflow_name = "DepositCrossref"
workflow_version = "1"
child_policy = None
execution_start_to_close_timeout = None
input = None
try:
response = conn.start_workflow_execution(
settings.domain, workflow_id, workflow_name, workflow_version,
settings.default_task_list, child_policy,
execution_start_to_close_timeout, input)
logger.info('got response: \n%s' %
json.dumps(response, sort_keys=True, indent=4))
except boto.swf.exceptions.SWFWorkflowExecutionAlreadyStartedError:
# There is already a running workflow with that ID, cannot start another
message = (
'SWFWorkflowExecutionAlreadyStartedError: There is already ' +
'a running workflow with ID %s' % workflow_id)
print message
logger.info(message)
def main(flag):
global settings
global env
parser = OptionParser()
parser.add_option("-e", "--env", default="dev", action="store", type="string", dest="env",
help="set the environment to run, either dev or live")
(options, args) = parser.parse_args()
if options.env:
env = options.env
settings = settings_lib.get_settings(env)
env = env
log_file = "queue_workflow_starter.log"
global logger
logger = log.logger(log_file, settings.setLevel)
# Simple connect
queue = get_queue()
while flag.green():
messages = queue.get_messages(1, visibility_timeout=60,
wait_time_seconds=20)
if messages:
logger.info(str(len(messages)) + " message received")
logger.info('message contents: %s', messages[0])
process_message(messages[0])
else:
logger.debug("No messages received")
logger.info("graceful shutdown")
def start(self, settings, document=None, last_updated_since=None):
# Log
identity = "starter_%s" % int(random.random() * 1000)
logFile = "starter.log"
#logFile = None
logger = log.logger(logFile, settings.setLevel, identity)
# Simple connect
conn = boto.swf.layer1.Layer1(settings.aws_access_key_id,
settings.aws_secret_access_key)
docs = None
if document is not None:
docs = []
doc = {}
doc["document"] = document
docs.append(doc)
elif last_updated_since is not None:
# Publish only articles since the last_modified date, use SimpleDB as the source
docs = self.get_docs_from_SimpleDB(
settings, last_updated_since=last_updated_since)
if docs:
for doc in docs:
document = doc["document"]
# Get a unique id from the document name for the workflow_id
id_string = None
try:
id_string = document.split("_")[0]
except:
id_string = "000"
# Start a workflow execution
workflow_id = "PackagePOA_%s" % (id_string)
workflow_name = "PackagePOA"
workflow_version = "1"
child_policy = None
execution_start_to_close_timeout = None
input = '{"data": ' + json.dumps(doc) + '}'
try:
response = conn.start_workflow_execution(
settings.domain, workflow_id, workflow_name,
workflow_version, settings.default_task_list,
child_policy, execution_start_to_close_timeout, input)
logger.info('got response: \n%s' %
json.dumps(response, sort_keys=True, indent=4))
except boto.swf.exceptions.SWFWorkflowExecutionAlreadyStartedError:
# There is already a running workflow with that ID, cannot start another
message = (
'SWFWorkflowExecutionAlreadyStartedError: There is already '
+ 'a running workflow with ID %s' % workflow_id)
print message
logger.info(message)
def start(self, settings, document=None, last_updated_since=None):
# Log
identity = "starter_%s" % int(random.random() * 1000)
logFile = "starter.log"
#logFile = None
logger = log.logger(logFile, settings.setLevel, identity)
# Simple connect
conn = boto.swf.layer1.Layer1(settings.aws_access_key_id, settings.aws_secret_access_key)
docs = None
if document is not None:
docs = []
doc = {}
doc["document"] = document
docs.append(doc)
elif last_updated_since is not None:
# Publish only articles since the last_modified date, use SimpleDB as the source
docs = self.get_docs_from_SimpleDB(settings, last_updated_since=last_updated_since)
if docs:
for doc in docs:
document = doc["document"]
# Get a unique id from the document name for the workflow_id
id_string = None
try:
id_string = document.split("_")[0]
except:
id_string = "000"
# Start a workflow execution
workflow_id = "PackagePOA_%s" % (id_string)
workflow_name = "PackagePOA"
workflow_version = "1"
child_policy = None
execution_start_to_close_timeout = None
input = '{"data": ' + json.dumps(doc) + '}'
try:
response = conn.start_workflow_execution(settings.domain, workflow_id,
workflow_name, workflow_version,
settings.default_task_list,
child_policy,
execution_start_to_close_timeout,
input)
logger.info('got response: \n%s' %
json.dumps(response, sort_keys=True, indent=4))
except boto.swf.exceptions.SWFWorkflowExecutionAlreadyStartedError:
# There is already a running workflow with that ID, cannot start another
message = ('SWFWorkflowExecutionAlreadyStartedError: There is already ' +
'a running workflow with ID %s' % workflow_id)
print message
logger.info(message)
def get_movies(self, start_num, tag, data = {
'type': 'movie',
'sort': 'recommend',
'page_limit': options.page_limit,
}):
data['tag'] = tag
for x in range(start_num, options.max_page):
self.x = x
data['page_start'] = x * options.page_limit
params = urllib.parse.urlencode(data)
log.logger().info("get: " + self.__urls["search"] + "?" + params)
res = self.opener.open(self.__urls["search"] + "?" + params, timeout = self.timeout)
result = res.read()
subjects_json = json.loads(result.decode())
self.push_to_queue(subjects_json["subjects"])
sleep(1)
def execute(self):
"""Execute the alias."""
try:
self.sharp_engine.execute(self.action, variables=True)
except Exception:
log = logger("client")
log.exception("An error occurred while executing the alias " \
"{}".format(repr(self.alias)))
def execute(self):
"""Execute the alias."""
try:
self.sharp_engine.execute(self.action, variables=True)
except Exception:
log = logger("client")
log.exception("An error occurred while executing the alias " \
"{}".format(repr(self.alias)))
def __init__(self):
self.logger = logger("")
begin()
self.settings = Settings(self)
self.worlds = {}
self.default_world = None
self.level = Level.engine
self.logger.info("CocoMUD engine started")
def scanFolderIgnore(folder):
#flag = 0
badFiles = []
for dirpath, dirs, filenames in os.walk(folder, topdown=True):
dirs[:] = [d for d in dirs if d not in settings.exclude]
for filename in [f for f in filenames if f.endswith(find)]:
#flag =1
toLog = (os.path.join(dirpath, filename))
log.logger(toLog)
#badFolder = (os.path.join(dirpath))
#badFiles.append((dirpath,filename))
badFiles.append(toLog)
#print(badFiles)
if badFiles:
return badFiles
def main(dirname=None):
logger = log.logger()
logger.info('Running model.py directly to produce schema/goose output.')
conn = SQL(connect=False)
fnames = conn.goose_write(dirname)
logger.info(
'Finished running model.py directly to produce schema/goose output.')
return fnames
def main():
global logger
cl_opts = parse_command_line()
log.setup(cl_opts.job_dir)
logger = log.logger()
logger.info('=== New session ===')
j = job.Job(cl_opts.job_dir, cl_opts.restart)
j.send_all()
def start(self, ENV = "dev", bucket = None, document = None):
# Specify run environment settings
settings = settingsLib.get_settings(ENV)
# Log
identity = "starter_%s" % int(random.random() * 1000)
logFile = "starter.log"
#logFile = None
logger = log.logger(logFile, settings.setLevel, identity)
# Simple connect
conn = boto.swf.layer1.Layer1(settings.aws_access_key_id, settings.aws_secret_access_key)
docs = None
if(document is not None):
docs = []
doc = {}
doc["document"] = document
if bucket is not None:
doc["bucket"] = bucket
docs.append(doc)
if(docs):
for doc in docs:
document = doc["document"]
# Get a unique id from the document name for the workflow_id
id_string = None
try:
id_string = ''
document_file = document.split("/")[-1]
if "bucket" in doc:
id_string += doc['bucket'] + '_'
id_string += document_file.split("_")[0]
except:
id_string = "000"
# Start a workflow execution
workflow_id = "PMCDeposit_%s" % (id_string)
workflow_name = "PMCDeposit"
workflow_version = "1"
child_policy = None
execution_start_to_close_timeout = None
input = '{"data": ' + json.dumps(doc) + '}'
try:
response = conn.start_workflow_execution(settings.domain, workflow_id, workflow_name, workflow_version, settings.default_task_list, child_policy, execution_start_to_close_timeout, input)
logger.info('got response: \n%s' % json.dumps(response, sort_keys=True, indent=4))
except boto.swf.exceptions.SWFWorkflowExecutionAlreadyStartedError:
# There is already a running workflow with that ID, cannot start another
message = 'SWFWorkflowExecutionAlreadyStartedError: There is already a running workflow with ID %s' % workflow_id
print message
logger.info(message)
def start(self, settings):
ping_marker_id = "cron_NewS3POA"
# Log
logFile = "starter.log"
logger = log.logger(logFile, settings.setLevel, ping_marker_id)
# Data provider
db = dblib.SimpleDB(settings)
db.connect()
# SWF meta data provider
swfmeta = swfmetalib.SWFMeta(settings)
swfmeta.connect()
last_startTimestamp = swfmeta.get_last_completed_workflow_execution_startTimestamp(
workflow_id=ping_marker_id)
# Start a ping workflow as a marker
self.start_ping_marker(ping_marker_id, settings)
# Check for S3 XML files that were updated since the last run
date_format = "%Y-%m-%dT%H:%M:%S.000Z"
# Quick hack - subtract 15 minutes,
# the time between S3Monitor running and this cron starter
last_startTimestamp_minus_15 = last_startTimestamp - (60 * 15)
time_tuple = time.gmtime(last_startTimestamp_minus_15)
last_startDate = time.strftime(date_format, time_tuple)
logger.info('last run %s' % (last_startDate))
xml_item_list = db.elife_get_POA_delivery_S3_file_items(
last_updated_since=last_startDate)
logger.info('POA files updated since %s: %s' %
(last_startDate, str(len(xml_item_list))))
if len(xml_item_list) <= 0:
# No new XML
pass
else:
# Found new XML files
# Start a PackagePOA starter
try:
starter_name = "starter_PackagePOA"
self.import_starter_module(starter_name, logger)
s = self.get_starter_module(starter_name, logger)
s.start(settings=settings, last_updated_since=last_startDate)
except:
logger.info('Error: %s starting %s' %
(ping_marker_id, starter_name))
logger.exception('')
def create_student(self): #创建学生账号
with open(settings.USER_FILE, 'rb') as f:
user_dict = pickle.loads(f.read())
username = input('输入学员账号：').strip()
if user_dict.get(username):
print('用户名已存在')
return
password = input('为学员账号创建密码：').strip()
username_obj = student(username)
user_dict[username] = {
'pwd': password,
'status': 0,
'role': 'student',
'username_obj': username_obj
}
with open(settings.USER_FILE, 'wb') as f:
f.write(pickle.dumps(user_dict))
print('创建学员{}账号成功'.format(username))
logger('{} create an account {}'.format(self.name, username))
def __init__(self, sharp, reaction, action):
self.sharp_engine = sharp
self.reaction = reaction
self.re_reaction = self.find_regex(reaction)
self.action = dedent(action.strip("\n"))
self.mute = False
self.logger = logger("sharp")
# Set the trigger's level
self.level = sharp.engine.level
def OnInput(self, message):
"""Some text has been sent from the input."""
if self.world:
self.world.reset_autocompletion()
try:
self.client.write(message)
except Exception:
log = logger("client")
log.exception("An error occurred when sending a message")
def main(dirname=None):
"""Main entrypoint that builds and stores our models."""
logger = log.logger()
logger.info('Running model.py directly to produce schema/goose output.')
conn = FBO(connect=False)
fnames = conn.goose_write(dirname)
logger.info(
'Finished running model.py directly to produce schema/goose output.')
return fnames
def get_movie_url_from_bilibili():
betelgeuse = Betelgeuse()
bellatrix = Bellatrix(options.appkey, options.appsecret)
while True:
available = model.MovieQueue.select().where(model.MovieQueue.state == 1).count()
if available > 0:
prepare_movies = model.MovieQueue.select().where(model.MovieQueue.state == 1).limit(100)
for m in prepare_movies:
movie = Alnilam.get_first_movie(m.douban_id)
try:
search_reault = bellatrix.search(movie.title)
if search_reault.is_done:
match_movie = False
for sr in search_reault.result:
if allow_type(sr["typename"]):
match_movie = True
detail = bellatrix.view(sr["aid"])
pages = detail.pages
for page in range(1, pages+1):
if hasattr(detail, 'cid'):
detail.download, detail.expires = bellatrix.build_download_url(detail.cid)
with model.database.atomic() as txn:
try:
betelgeuse.build_bilibili(sr, m.douban_id)
betelgeuse.replenish_bilibili(sr["aid"], detail)
m.state = 2
m.save()
except Exception as e:
log.logger().error(e)
txn.rollback()
detail = bellatrix.view(sr["aid"], page + 1)
sleep(2)
sleep(2)
if not match_movie:
m.state = 3
m.save()
sleep(2)
except Exception as e:
log.logger().error(e)
else:
sleep(2)
def start(self, ENV="dev"):
# Specify run environment settings
settings = settingsLib.get_settings(ENV)
ping_marker_id = "cron_FiveMinute"
# Log
logFile = "starter.log"
logger = log.logger(logFile, settings.setLevel, ping_marker_id)
# Data provider
db = dblib.SimpleDB(settings)
db.connect()
# SWF meta data provider
swfmeta = swfmetalib.SWFMeta(settings)
swfmeta.connect()
last_startTimestamp = swfmeta.get_last_completed_workflow_execution_startTimestamp(
workflow_id=ping_marker_id)
# Start a ping workflow as a marker
self.start_ping_marker(ping_marker_id, ENV)
# Check for S3 XML files that were updated since the last run
date_format = "%Y-%m-%dT%H:%M:%S.000Z"
# Date conversion
time_tuple = time.gmtime(last_startTimestamp)
last_startDate = time.strftime(date_format, time_tuple)
logger.info('last run %s %s' % (ping_marker_id, last_startDate))
# A conditional start for SendQueuedEmail
# Only start a workflow if there are emails in the queue ready to send
item_list = db.elife_get_email_queue_items(
query_type="count", date_scheduled_before=last_startDate)
try:
if (int(item_list[0]["Count"]) > 0):
# More than one email in the queue, start a workflow
try:
starter_name = "starter_SendQueuedEmail"
self.import_starter_module(starter_name, logger)
s = self.get_starter_module(starter_name, logger)
s.start(ENV=ENV)
except:
logger.info('Error: %s starting %s' %
(ping_marker_id, starter_name))
logger.exception('')
except:
# Some error
logger.info('Exception encountered starting %s: %s' %
(ping_marker_id, last_startDate))
def run(self):
"""Run the thread."""
# Try to connect to the specified host and port
host = self.factory.world.hostname
port = self.factory.world.port
protocol = self.factory.world.protocol.lower()
protocol = "SSL" if protocol == "ssl" else "telnet"
log = logger("client")
log.info("Connecting {protocol} client for {host}:{port}".format(
protocol=protocol, host=host, port=port))
self.running = True
def connectionLost(self, reason):
"""The connection was lost."""
self.send_queue()
host = self.transport.getPeer().host
port = self.transport.getPeer().port
log = logger("client")
log.info("Lost Connection on {host}:{port}: {reason}".format(
host=host, port=port, reason=reason.type))
wx.CallAfter(pub.sendMessage, "disconnect", client=self, reason=reason)
if reason.type is ConnectionDone:
self.factory.stopTrying()
def start(self, settings):
ping_marker_id = "cron_NewS3POA"
# Log
logFile = "starter.log"
logger = log.logger(logFile, settings.setLevel, ping_marker_id)
# Data provider
db = dblib.SimpleDB(settings)
db.connect()
# SWF meta data provider
swfmeta = swfmetalib.SWFMeta(settings)
swfmeta.connect()
last_startTimestamp = swfmeta.get_last_completed_workflow_execution_startTimestamp(
workflow_id=ping_marker_id)
# Start a ping workflow as a marker
self.start_ping_marker(ping_marker_id, settings)
# Check for S3 XML files that were updated since the last run
date_format = "%Y-%m-%dT%H:%M:%S.000Z"
# Quick hack - subtract 15 minutes,
# the time between S3Monitor running and this cron starter
last_startTimestamp_minus_15 = last_startTimestamp - (60 * 15)
time_tuple = time.gmtime(last_startTimestamp_minus_15)
last_startDate = time.strftime(date_format, time_tuple)
logger.info('last run %s' % (last_startDate))
xml_item_list = db.elife_get_POA_delivery_S3_file_items(
last_updated_since=last_startDate)
logger.info('POA files updated since %s: %s' % (last_startDate, str(len(xml_item_list))))
if len(xml_item_list) <= 0:
# No new XML
pass
else:
# Found new XML files
# Start a PackagePOA starter
try:
starter_name = "starter_PackagePOA"
self.import_starter_module(starter_name, logger)
s = self.get_starter_module(starter_name, logger)
s.start(settings=settings, last_updated_since=last_startDate)
except:
logger.info('Error: %s starting %s' % (ping_marker_id, starter_name))
logger.exception('')
def __init__(self, use_cache: bool):
"""
:param use_cache: will use cache or not
:type <bool>
"""
self.use_cache = use_cache
self.logger = logger('state', 'states.log', 'INFO')
if use_cache:
self.cache = Cache(use_cache)
self.logger.info(
f"Now param use_cache in Cache class instance is {self.use_cache}")
def __init__(self):
self.tc = 0
self.th = {}
self.jid = jid.JID("%s@%s/%s" % (config.USER, config.SERVER, config.RESOURCE))
self.onauthd = None
self.c = client.basicClientFactory(self.jid, config.PASSWD)
self.c.addBootstrap(xmlstream.STREAM_AUTHD_EVENT, self.authd);
self.x = None
self.log = log.logger()
self.handlers = []
self.msghandlers = []
reactor.connectTCP(config.SERVER, 5222, self.c)
def start(self, ENV = "dev"):
# Specify run environment settings
settings = settingsLib.get_settings(ENV)
ping_marker_id = "cron_FiveMinute"
# Log
logFile = "starter.log"
logger = log.logger(logFile, settings.setLevel, ping_marker_id)
# Data provider
db = dblib.SimpleDB(settings)
db.connect()
# SWF meta data provider
swfmeta = swfmetalib.SWFMeta(settings)
swfmeta.connect()
last_startTimestamp = swfmeta.get_last_completed_workflow_execution_startTimestamp(workflow_id = ping_marker_id)
# Start a ping workflow as a marker
self.start_ping_marker(ping_marker_id, ENV)
# Check for S3 XML files that were updated since the last run
date_format = "%Y-%m-%dT%H:%M:%S.000Z"
# Date conversion
time_tuple = time.gmtime(last_startTimestamp)
last_startDate = time.strftime(date_format, time_tuple)
logger.info('last run %s %s' % (ping_marker_id, last_startDate))
# A conditional start for SendQueuedEmail
# Only start a workflow if there are emails in the queue ready to send
item_list = db.elife_get_email_queue_items(
query_type = "count",
date_scheduled_before = last_startDate)
try:
if(int(item_list[0]["Count"]) > 0):
# More than one email in the queue, start a workflow
try:
starter_name = "starter_SendQueuedEmail"
self.import_starter_module(starter_name, logger)
s = self.get_starter_module(starter_name, logger)
s.start(ENV = ENV)
except:
logger.info('Error: %s starting %s' % (ping_marker_id, starter_name))
logger.exception('')
except:
# Some error
logger.info('Exception encountered starting %s: %s' % (ping_marker_id, last_startDate))
def run(self, filename):
"""Play the audio file."""
log = logger("sharp")
log.debug("#play {}".format(filename))
filename = self.find_abs_filename(filename)
if os.path.exists(filename):
log.debug("#play playing {}".format(filename))
else:
log.warning("#play cannot find the file at {}".format(
filename))
sound = mixer.Sound(filename)
sound.play()
def get_movie_detail():
meissa = Meissa()
alnilam = Alnilam()
while True:
available = model.MovieQueue.select().where(model.MovieQueue.state == 0).count()
if available > 0:
prepare_movies = model.MovieQueue.select().where(model.MovieQueue.state == 0).limit(100)
for m in prepare_movies:
detail = meissa.get_detail(m.douban_id)
with model.database.atomic() as txn:
try:
alnilam.generate_movie(detail)
m.state = 1
m.save()
log.logger().info(detail.id + " " + detail.title)
except Exception as e:
log.logger().error(e)
txn.rollback()
sleep(2)
else:
sleep(2)
def start(self, ENV = "dev", all = None, last_updated_since = None, docs = None, doi_id = None):
# Specify run environment settings
settings = settingsLib.get_settings(ENV)
# Log
identity = "starter_%s" % int(random.random() * 1000)
logFile = "starter.log"
#logFile = None
logger = log.logger(logFile, settings.setLevel, identity)
# Simple connect
conn = boto.swf.layer1.Layer1(settings.aws_access_key_id, settings.aws_secret_access_key)
if(all == True):
# Publish all articles, use SimpleDB as the source
docs = self.get_docs_from_SimpleDB(ENV)
elif(doi_id is not None):
docs = self.get_docs_from_SimpleDB(ENV, doi_id = doi_id)
elif(last_updated_since is not None):
# Publish only articles since the last_modified date, use SimpleDB as the source
docs = self.get_docs_from_SimpleDB(ENV, last_updated_since = last_updated_since)
if(docs):
for doc in docs:
document = doc["document"]
elife_id = doc["elife_id"]
id_string = elife_id
# Start a workflow execution
workflow_id = "PublishSuppl_%s" % (id_string)
workflow_name = "PublishSuppl"
workflow_version = "1"
child_policy = None
execution_start_to_close_timeout = str(60*30)
input = '{"data": ' + json.dumps(doc) + '}'
try:
response = conn.start_workflow_execution(settings.domain, workflow_id, workflow_name, workflow_version, settings.default_task_list, child_policy, execution_start_to_close_timeout, input)
logger.info('got response: \n%s' % json.dumps(response, sort_keys=True, indent=4))
except boto.swf.exceptions.SWFWorkflowExecutionAlreadyStartedError:
# There is already a running workflow with that ID, cannot start another
message = 'SWFWorkflowExecutionAlreadyStartedError: There is already a running workflow with ID %s' % workflow_id
print message
logger.info(message)
